diff --git a/.gitignore b/.gitignore index e0b0dde..4b66020 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.css -*.css.map \ No newline at end of file +*.css.map +*~ +.sass-cache \ No newline at end of file diff --git a/celeryconfig.py b/celeryconfig.py index 79f0c0a..043dc22 100644 --- a/celeryconfig.py +++ b/celeryconfig.py @@ -7,7 +7,9 @@ CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] CELERYBEAT_SCHEDULE = { 'tick-every-5-minutes': { - 'task': 'tasks.tick', - 'schedule': datetime.timedelta(minutes=5), + 'task': 'woodwind.tasks.tick', + 'schedule': datetime.timedelta(seconds=30), } } +# recommended to disable if not using -- introduces a lot of complexity +CELERY_DISABLE_RATE_LIMITS = True diff --git a/config.py.template b/config.py.template new file mode 100644 index 0000000..3449bd9 --- /dev/null +++ b/config.py.template @@ -0,0 +1,6 @@ +import os + + +class Config: + SECRET_KEY = 'super secret key' + SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.getcwd() + '/db.sqlite' diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 75c98d2..9876326 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -1,30 +1,31 @@ -from woodwind.extensions import db from woodwind.models import Feed, Entry - +from config import Config import celery -import requests import celery.utils.log import feedparser import mf2py import mf2util -import requests import time import urllib.parse import datetime +import sqlalchemy +import sqlalchemy.orm UPDATE_INTERVAL = datetime.timedelta(hours=1) -queue = celery.Celery('woodwind') -queue.config_from_object('celeryconfig') +app = celery.Celery('woodwind') +app.config_from_object('celeryconfig') logger = celery.utils.log.get_task_logger(__name__) +engine = sqlalchemy.create_engine(Config.SQLALCHEMY_DATABASE_URI) +session = sqlalchemy.orm.Session(bind=engine) -@queue.task +@app.task def tick(): now = datetime.datetime.utcnow() logger.debug('Tick {}'.format(now)) - for feed in Feed.query.all(): + for feed in session.query(Feed).all(): logger.debug('Feed {} last checked {}'.format( feed, feed.last_checked)) if (not feed.last_checked @@ -32,9 +33,9 @@ def tick(): update_feed.delay(feed.id) -@queue.task +@app.task def update_feed(feed_id): - feed = Feed.query.get(feed_id) + feed = session.query(Feed).get(feed_id) logger.info('Updating {}'.format(feed)) new_entries = process_feed_for_new_entries(feed) for entry in new_entries: @@ -55,7 +56,7 @@ def process_feed_for_new_entries(feed): feed.last_checked = now if result: feed.last_updated = now - db.session.commit() + session.commit() def process_xml_feed_for_new_entries(feed): @@ -70,7 +71,7 @@ def process_xml_feed_for_new_entries(feed): default_author_photo = feed_props.get('logo') all_uids = [e.id or e.link for e in parsed.entries] - preexisting = set(row[0] for row in db.session.query(Entry.uid) + preexisting = set(row[0] for row in session.query(Entry.uid) .filter(Entry.uid.in_(all_uids)) .filter(Entry.feed == feed)) @@ -118,8 +119,8 @@ def process_xml_feed_for_new_entries(feed): author_photo=default_author_photo or fallback_photo(feed.origin)) - db.session.add(entry) - db.session.commit() + session.add(entry) + session.commit() yield entry @@ -132,7 +133,7 @@ def process_html_feed_for_new_entries(feed): hfeed = parsed.get('entries', []) all_uids = [e.get('uid') or e.get('url') for e in hfeed] - preexisting = set(row[0] for row in db.session.query(Entry.uid) + preexisting = set(row[0] for row in session.query(Entry.uid) .filter(Entry.uid.in_(all_uids)) .filter(Entry.feed == feed)) @@ -161,8 +162,8 @@ def process_html_feed_for_new_entries(feed): author_photo=hentry.get('author', {}).get('photo') or fallback_photo(feed.origin), author_url=hentry.get('author', {}).get('url')) - db.session.add(entry) - db.session.commit() + session.add(entry) + session.commit() logger.debug('saved entry: %s', entry.permalink) yield entry diff --git a/woodwind/templates/subscribe.jinja2 b/woodwind/templates/subscribe.jinja2 index 2d7dc87..7e2dd2d 100644 --- a/woodwind/templates/subscribe.jinja2 +++ b/woodwind/templates/subscribe.jinja2 @@ -1,7 +1,7 @@ {% extends "base.jinja2" %} {% block body %}
-
+
diff --git a/woodwind/views.py b/woodwind/views.py index b65d1c3..58fc5c0 100644 --- a/woodwind/views.py +++ b/woodwind/views.py @@ -15,6 +15,9 @@ ui = flask.Blueprint('ui', __name__) @ui.route('/') def index(): + import os + print('cwd', os.getcwd(), 'db', db) + if flask_login.current_user.is_authenticated(): feed_ids = [f.id for f in flask_login.current_user.feeds] entries = Entry.query.filter( @@ -149,28 +152,25 @@ def subscribe(): def add_subscription(origin, feed, type): + feed = None if type == 'html': parsed = mf2util.interpret_feed(mf2py.parse(url=feed), feed) name = parsed.get('name') if not name or len(name) > 140: p = urllib.parse.urlparse(origin) name = p.netloc + p.path - feed = Feed(user=flask_login.current_user, name=name, origin=origin, feed=feed, type=type) - - db.session.add(feed) - db.session.commit() - return feed - elif type == 'xml': parsed = feedparser.parse(feed) feed = Feed(user=flask_login.current_user, name=parsed.feed.title, origin=origin, feed=feed, type=type) - + if feed: db.session.add(feed) db.session.commit() + # go ahead and update the fed + tasks.update_feed.delay(feed.id) return feed