From ddb75f5993604fa280dd5543fedf568499866c52 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Fri, 4 Mar 2016 01:21:31 +0000 Subject: [PATCH 01/18] broken: trying to put all entries into one transaction and now getting all sorts of detached exceptions. rolling back --- woodwind/app.py | 6 +++- woodwind/tasks.py | 78 ++++++++++++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/woodwind/app.py b/woodwind/app.py index 5b35112..27bde0e 100644 --- a/woodwind/app.py +++ b/woodwind/app.py @@ -36,7 +36,11 @@ def configure_logging(app): return app.logger.setLevel(logging.DEBUG) - app.logger.addHandler(logging.StreamHandler(sys.stdout)) + + handler = logging.StreamHandler(sys.stdout) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + app.logger.addHandler(handler) recipients = app.config.get('ADMIN_EMAILS') if recipients: diff --git a/woodwind/tasks.py b/woodwind/tasks.py index c6c60f7..95d18d2 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -4,6 +4,7 @@ from redis import StrictRedis from woodwind import util from woodwind.extensions import db from woodwind.models import Feed, Entry +import sqlalchemy import bs4 import datetime import feedparser @@ -112,12 +113,12 @@ def update_feed(feed_id, content=None, with flask_app() as app: feed = Feed.query.get(feed_id) - current_app.logger.info('Updating {}'.format(feed)) + current_app.logger.info('Updating {}'.format(str(feed)[:32])) now = datetime.datetime.utcnow() - new_ids = [] - updated_ids = [] + new_entries = [] + updated_entries = [] reply_pairs = [] try: @@ -125,7 +126,7 @@ def update_feed(feed_id, content=None, current_app.logger.info('using provided content. size=%d', len(content)) else: - current_app.logger.info('fetching feed: %s', feed) + current_app.logger.info('fetching feed: %s', str(feed)[:32]) try: response = util.requests_get(feed.feed) @@ -163,24 +164,30 @@ def update_feed(feed_id, content=None, result = [] for entry in result: + current_app.logger.debug('searching for entry with uid=%s', entry.uid) old = Entry.query\ .filter(Entry.feed == feed)\ .filter(Entry.uid == entry.uid)\ .order_by(Entry.id.desc())\ .first() + current_app.logger.debug('done searcing: %s', 'found' if old else 'not found') + # have we seen this post before if not old: + current_app.logger.debug('this is a new post, saving a new entry') # set a default value for published if none is provided entry.published = entry.published or now in_reply_tos = entry.get_property('in-reply-to', []) + db.session.add(entry) feed.entries.append(entry) - db.session.commit() - new_ids.append(entry.id) + new_entries.append(entry) for irt in in_reply_tos: - reply_pairs.append((entry.id, irt)) + reply_pairs.append((entry, irt)) elif not is_content_equal(old, entry): + current_app.logger.debug('this post content has changed, updating entry') + entry.published = entry.published or old.published in_reply_tos = entry.get_property('in-reply-to', []) # we're updating an old entriy, use the original @@ -190,28 +197,38 @@ def update_feed(feed_id, content=None, # punt on deleting for now, learn about cascade # and stuff later # session.delete(old) + db.session.add(entry) feed.entries.append(entry) - db.session.commit() - updated_ids.append(entry.id) + updated_entries.append(entry) for irt in in_reply_tos: - reply_pairs.append((entry.id, irt)) + reply_pairs.append((entry, irt)) else: current_app.logger.debug( 'skipping previously seen post %s', old.permalink) - for entry_id, in_reply_to in reply_pairs: - fetch_reply_context(entry_id, in_reply_to, now) + for entry, in_reply_to in reply_pairs: + fetch_reply_context(entry, in_reply_to, now) + + db.session.commit() + except: + db.session.rollback() + raise finally: if is_polling: feed.last_checked = now - if new_ids or updated_ids: + if new_entries or updated_entries: feed.last_updated = now db.session.commit() - if new_ids: - notify_feed_updated(app, feed_id, new_ids) + + if new_entries: + + for e in new_entries: + current_app.logger.debug('entry %s state: %s', e.uid, sqlalchemy.inspect(e)) + + notify_feed_updated(app, feed_id, new_entries) def check_push_subscription(feed, response): @@ -267,6 +284,8 @@ def check_push_subscription(feed, response): if ((expiry and expiry - datetime.datetime.utcnow() <= UPDATE_INTERVAL_PUSH) or hub != old_hub or topic != old_topic or not feed.push_verified): + current_app.logger.debug('push subscription expired or hub/topic changed') + feed.push_hub = hub feed.push_topic = topic feed.push_verified = False @@ -274,29 +293,24 @@ def check_push_subscription(feed, response): db.session.commit() if old_hub and old_topic and hub != old_hub and topic != old_topic: + current_app.logger.debug('unsubscribing hub=%s, topic=%s', old_hub, old_topic) send_request('unsubscribe', old_hub, old_topic) if hub and topic: + current_app.logger.debug('subscribing hub=%s, topic=%s', hub, topic) send_request('subscribe', hub, topic) db.session.commit() -def notify_feed_updated(app, feed_id, entry_ids): +def notify_feed_updated(app, feed_id, entries): """Render the new entries and publish them to redis """ from flask import render_template import flask.ext.login as flask_login - current_app.logger.debug( - 'notifying feed updated for entries %r', entry_ids) + current_app.logger.debug('notifying feed updated: %s', feed_id) feed = Feed.query.get(feed_id) - entries = Entry.query\ - .filter(Entry.id.in_(entry_ids))\ - .order_by(Entry.retrieved.desc(), - Entry.published.desc())\ - .all() - for s in feed.subscriptions: with app.test_request_context(): flask_login.login_user(s.user, remember=True) @@ -345,7 +359,7 @@ def is_content_equal(e1, e2): def process_xml_feed_for_new_entries(feed, content, backfill, now): - current_app.logger.debug('fetching xml feed: %s', feed) + current_app.logger.debug('fetching xml feed: %s', str(feed)[:32]) parsed = feedparser.parse(content, response_headers={ 'content-location': feed.feed, }) @@ -354,12 +368,11 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now): default_author_name = feed_props.get('author_detail', {}).get('name') default_author_photo = feed_props.get('logo') - current_app.logger.debug('found {} entries'.format(len(parsed.entries))) + current_app.logger.debug('found %d entries', len(parsed.entries)) # work from the bottom up (oldest first, usually) for p_entry in reversed(parsed.entries): - current_app.logger.debug('processing entry {}'.format( - str(p_entry)[:256])) + current_app.logger.debug('processing entry %s', str(p_entry)[:32]) permalink = p_entry.get('link') uid = p_entry.get('id') or permalink @@ -406,6 +419,8 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now): video = VIDEO_ENCLOSURE_TMPL.format(href=link.get('href')) content = (content or '') + video + current_app.logger.debug('building entry') + entry = Entry( published=published, updated=updated, @@ -422,6 +437,8 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now): author_photo=default_author_photo or fallback_photo(feed.origin)) + current_app.logger.debug('yielding entry') + yield entry @@ -527,9 +544,8 @@ def hentry_to_entry(hentry, feed, backfill, now): return entry -def fetch_reply_context(entry_id, in_reply_to, now): +def fetch_reply_context(entry, in_reply_to, now): with flask_app(): - entry = Entry.query.get(entry_id) context = Entry.query\ .join(Entry.feed)\ .filter(Entry.permalink==in_reply_to, Feed.type == 'html')\ @@ -542,10 +558,10 @@ def fetch_reply_context(entry_id, in_reply_to, now): mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to) if parsed: context = hentry_to_entry(parsed, None, False, now) + db.session.add(context) if context: entry.reply_context.append(context) - db.session.commit() def proxy_url(url): From 4f11bb9f27cad77c2eee6da32c999c33c2afb0b4 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Fri, 4 Mar 2016 08:53:15 -0800 Subject: [PATCH 02/18] remove extra app context from fetch_reply_contexts was causing detached state issues in previous commit --- woodwind/tasks.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 95d18d2..e7cc256 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -17,6 +17,7 @@ import requests import rq import sys import time +import traceback import urllib.parse # normal update interval for polling feeds @@ -94,6 +95,8 @@ def tick(): q.enqueue(update_feed, feed.id) + + def update_feed(feed_id, content=None, content_type=None, is_polling=True): @@ -224,10 +227,6 @@ def update_feed(feed_id, content=None, db.session.commit() if new_entries: - - for e in new_entries: - current_app.logger.debug('entry %s state: %s', e.uid, sqlalchemy.inspect(e)) - notify_feed_updated(app, feed_id, new_entries) @@ -545,23 +544,22 @@ def hentry_to_entry(hentry, feed, backfill, now): def fetch_reply_context(entry, in_reply_to, now): - with flask_app(): - context = Entry.query\ - .join(Entry.feed)\ - .filter(Entry.permalink==in_reply_to, Feed.type == 'html')\ - .first() + context = Entry.query\ + .join(Entry.feed)\ + .filter(Entry.permalink==in_reply_to, Feed.type == 'html')\ + .first() - if not context: - current_app.logger.info('fetching in-reply-to url: %s', - in_reply_to) - parsed = mf2util.interpret( - mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to) - if parsed: - context = hentry_to_entry(parsed, None, False, now) - db.session.add(context) + if not context: + current_app.logger.info('fetching in-reply-to url: %s', + in_reply_to) + parsed = mf2util.interpret( + mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to) + if parsed: + context = hentry_to_entry(parsed, None, False, now) + db.session.add(context) - if context: - entry.reply_context.append(context) + if context: + entry.reply_context.append(context) def proxy_url(url): From cd10fde3a06795e566bffd182eeccdcccfaa7009 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Sat, 5 Mar 2016 08:10:55 -0800 Subject: [PATCH 03/18] do not try to add null context to the session --- woodwind/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index e7cc256..39a3e01 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -556,9 +556,9 @@ def fetch_reply_context(entry, in_reply_to, now): mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to) if parsed: context = hentry_to_entry(parsed, None, False, now) - db.session.add(context) if context: + db.session.add(context) entry.reply_context.append(context) From 37c556fcbb9ca17c25b6c89f2dd8452916155330 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 22:43:28 +0000 Subject: [PATCH 04/18] micropub: set the access token in the query string too, for sites that don't support header parameters for some reason --- woodwind/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/woodwind/api.py b/woodwind/api.py index 1ddfefd..1cf9cea 100644 --- a/woodwind/api.py +++ b/woodwind/api.py @@ -16,6 +16,7 @@ def publish(): data = { 'h': 'entry', 'syndicate-to[]': syndicate_to, + 'access_token': flask_login.current_user.access_token, } if action == 'like': From 2786ab4770b670234b1248e38027971942cc5dd4 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 22:43:43 +0000 Subject: [PATCH 05/18] add a footer line with a link to Github --- woodwind/static/style.css | 26 ++++++++++++++------------ woodwind/static/style.scss | 7 ++++++- woodwind/templates/base.jinja2 | 4 ++++ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/woodwind/static/style.css b/woodwind/static/style.css index 11386c2..fb74b29 100644 --- a/woodwind/static/style.css +++ b/woodwind/static/style.css @@ -361,7 +361,7 @@ th { body { font: 12pt Helvetica, Arial, sans-serif; line-height: 1.4em; - background: #ECEBF0; + background: #ecebf0; /*background: #f4f4f4;*/ padding-top: 1em; } @@ -377,13 +377,17 @@ p:first-child { p:last-child { margin-bottom: 0; } -header, main { +header, main, .footer { max-width: 800px; margin: 0 auto; } header { margin-bottom: 1em; } +.footer { + font-size: 0.8em; + text-align: center; } + ul#navigation { list-style-type: none; margin: 0; @@ -399,8 +403,8 @@ ul#navigation { .button-link a { padding: 0.5em; background-color: #353129; - color: #ECEBF0; - border: 1px solid #ECEBF0; + color: #ecebf0; + border: 1px solid #ecebf0; border-radius: 4px; display: inline-block; } @@ -412,7 +416,7 @@ ul#navigation { article { margin-bottom: 1em; - box-shadow: 0 0 2px #687D77; + box-shadow: 0 0 2px #687d77; background-color: white; border-radius: 3px; padding: 0.5em; } @@ -434,8 +438,8 @@ article { article img, article video { max-width: 100%; } article header { - color: #484A47; - border-bottom: 1px solid #687D77; + color: #484a47; + border-bottom: 1px solid #687d77; margin-bottom: 0.5em; overflow: auto; } article header img { @@ -531,11 +535,11 @@ button { .reply-area .reply-link { display: inline-block; padding: 0.2em; - border: 1px solid #687D77; + border: 1px solid #687d77; border-radius: 4px; - background-color: #ECEBF0; + background-color: #ecebf0; text-decoration: none; - color: #484A47; + color: #484a47; min-width: 50px; text-align: center; } @@ -548,5 +552,3 @@ button { max-height: 1.2em; min-width: inherit; min-height: inherit; } } - -/*# sourceMappingURL=style.css.map */ diff --git a/woodwind/static/style.scss b/woodwind/static/style.scss index c7abbc6..6bec1ad 100644 --- a/woodwind/static/style.scss +++ b/woodwind/static/style.scss @@ -43,7 +43,7 @@ p { } } -header, main { +header, main, .footer { max-width: 800px; margin: 0 auto; } @@ -52,6 +52,11 @@ header { margin-bottom: 1em; } +.footer { + font-size: 0.8em; + text-align: center; +} + ul#navigation { list-style-type: none; margin: 0; diff --git a/woodwind/templates/base.jinja2 b/woodwind/templates/base.jinja2 index c525c80..330d82a 100644 --- a/woodwind/templates/base.jinja2 +++ b/woodwind/templates/base.jinja2 @@ -73,6 +73,10 @@ {% block foot %}{% endblock %} +
+ Woodwind is on GitHub. Have any problems? File an issue or come chat in #indiewebcamp on Freenode IRC. +
+ {% endif %} - + {% if current_user and current_user.settings and current_user.settings.get('reply-method') == 'indie-config' %} From eef7416af381096d377a41baa34a24f6825da02c Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 15:25:20 -0800 Subject: [PATCH 07/18] upgrade certifi in requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a972cbc..fc89ee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ asyncio-redis==0.14.2 beautifulsoup4==4.4.1 bleach==1.4.2 blinker==1.4 -certifi==14.05.14 # rq.filter: <=2015.04.28 +certifi==2015.04.28 # rq.filter: <=2015.04.28 cffi==1.5.0 click==6.2 cryptography==1.2.1 From 43925d91197cd567a9fd89214aa8e6a707b5e458 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 15:26:51 -0800 Subject: [PATCH 08/18] cache busting for style.css --- woodwind/templates/base.jinja2 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/woodwind/templates/base.jinja2 b/woodwind/templates/base.jinja2 index 330d82a..f39cc67 100644 --- a/woodwind/templates/base.jinja2 +++ b/woodwind/templates/base.jinja2 @@ -8,14 +8,12 @@ - + - - {% block head %}{% endblock %} From 3d506a6ab331f496cbd752655292d9b7f9a51d8c Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 16:06:46 -0800 Subject: [PATCH 09/18] show start/end for events --- woodwind/tasks.py | 6 ++++++ woodwind/templates/_entry.jinja2 | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 198011d..1c84ce7 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -534,6 +534,12 @@ def hentry_to_entry(hentry, feed, backfill, now): if value: entry.set_property(prop, value) + if 'start-str' in hentry: + entry.set_property('start', hentry.get('start-str')) + + if 'end-str' in hentry: + entry.set_property('end', hentry.get('end-str')) + # set a flag for events so we can show RSVP buttons if hentry.get('type') == 'event': entry.set_property('event', True) diff --git a/woodwind/templates/_entry.jinja2 b/woodwind/templates/_entry.jinja2 index 62a2c9f..abd3f57 100644 --- a/woodwind/templates/_entry.jinja2 +++ b/woodwind/templates/_entry.jinja2 @@ -46,6 +46,18 @@

{{ entry.title }}

{% endif %} + {% if entry.get_property('event') %} +

+ {% if entry.get_property('start') %} + start: {{ entry.get_property('start') }} + {% endif %} +
+ {% if entry.get_property('end') %} + end: {{ entry.get_property('end') }} + {% endif %} +

+ {% endif %} + {% set photo = entry.get_property('photo') %} {% if photo and (not entry.content or ' From 5180d151395eaa59fe850c57db139774d87a5309 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 16:12:16 -0800 Subject: [PATCH 10/18] bugfix: events with a name and no content should not have their title removed --- woodwind/tasks.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 1c84ce7..7e394bd 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -349,12 +349,14 @@ def is_content_equal(e1, e2): content = COMMENT_RE.sub('', content) return content - return (e1.title == e2.title - and normalize(e1.content) == normalize(e2.content) - and e1.author_name == e2.author_name - and e1.author_url == e2.author_url - and e1.author_photo == e2.author_photo - and e1.properties == e2.properties) + return ( + e1.title == e2.title + and normalize(e1.content) == normalize(e2.content) + and e1.author_name == e2.author_name + and e1.author_url == e2.author_url + and e1.author_photo == e2.author_photo + and e1.properties == e2.properties + ) def process_xml_feed_for_new_entries(feed, content, backfill, now): @@ -478,7 +480,7 @@ def hentry_to_entry(hentry, feed, backfill, now): title = hentry.get('name') content = hentry.get('content') - if not content: + if not content and hentry.get('type') == 'entry': content = title title = None From 8e02f6b1b4e4c00ba63509c1c169a160a4bedac3 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 22:28:18 -0800 Subject: [PATCH 11/18] catch SSLErrors when fetching reply contexts, just output a warning --- woodwind/tasks.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 7e394bd..3b5e8e7 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -562,12 +562,15 @@ def fetch_reply_context(entry, in_reply_to, now): .first() if not context: - current_app.logger.info('fetching in-reply-to url: %s', - in_reply_to) - parsed = mf2util.interpret( - mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to) - if parsed: - context = hentry_to_entry(parsed, None, False, now) + current_app.logger.info('fetching in-reply-to: %s', in_reply_to) + try: + proxied_reply_url = proxy_url(in_reply_to) + parsed = mf2util.interpret( + mf2py.parse(url=proxied_reply_url), in_reply_to) + if parsed: + context = hentry_to_entry(parsed, None, False, now) + except requests.exceptions.SSLError: + current_app.logger.warn('SSLError fetching: %s', proxied_reply_url) if context: db.session.add(context) From 1e2210d32456473a499d384116cfe04c13aa8e6c Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Tue, 8 Mar 2016 22:56:29 -0800 Subject: [PATCH 12/18] hush more exceptions --- woodwind/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 3b5e8e7..be5de53 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -569,8 +569,10 @@ def fetch_reply_context(entry, in_reply_to, now): mf2py.parse(url=proxied_reply_url), in_reply_to) if parsed: context = hentry_to_entry(parsed, None, False, now) - except requests.exceptions.SSLError: - current_app.logger.warn('SSLError fetching: %s', proxied_reply_url) + except requests.exceptions.RequestException as err: + current_app.logger.warn( + '%s fetching reply context: %s for entry: %s', + type(err).__name__, proxied_reply_url, entry.permalink) if context: db.session.add(context) From 0c88cbf7c1e5d24be38f3f86a07bcecc506df4dc Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Fri, 18 Mar 2016 16:33:03 +0000 Subject: [PATCH 13/18] upgrade flask-micropub to support HTTP Link headers --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc89ee1..aef91ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ feedparser==5.2.1 Flask==0.10.1 Flask-DebugToolbar==0.10.0 Flask-Login==0.3.2 -Flask-Micropub==0.2.5 +Flask-Micropub==0.2.6 Flask-SQLAlchemy==2.1 html5lib==0.9999999 idna==2.0 From 860756a0f2f9977f7635e8a93b1eb569894e812a Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Fri, 18 Mar 2016 16:33:20 +0000 Subject: [PATCH 14/18] better response codes and log information for failed push requests --- woodwind/push.py | 33 ++++++++++++++++++++++----------- woodwind/tasks.py | 14 +++++--------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/woodwind/push.py b/woodwind/push.py index 2a2c3c0..4645c61 100644 --- a/woodwind/push.py +++ b/woodwind/push.py @@ -31,13 +31,14 @@ def notify(feed_id): if not feed: current_app.logger.warn( 'could not find feed corresponding to %d', feed_id) - abort(404) + return make_response('no feed with id %d' % feed_id, 400) if topic != feed.push_topic: current_app.logger.warn( 'feed topic (%s) does not match subscription request (%s)', feed.push_topic, topic) - abort(404) + return make_response( + 'topic %s does not match subscription request %s' % (feed.push_topic, topic), 400) current_app.logger.debug( 'PuSH verify subscribe for feed=%r, topic=%s', feed, topic) @@ -48,18 +49,28 @@ def notify(feed_id): db.session.commit() return challenge - elif mode == 'unsubscribe' and (not feed or topic != feed.push_topic): - current_app.logger.debug( - 'PuSH verify unsubscribe for feed=%r, topic=%s', feed, topic) - return challenge - current_app.logger.debug('PuSH cannot verify %s for feed=%r, topic=%s', - mode, feed, topic) - abort(404) + elif mode == 'unsubscribe': + if not feed or topic != feed.push_topic: + current_app.logger.debug( + 'PuSH verify unsubscribe for feed=%r, topic=%s', feed, topic) + return challenge + else: + current_app.logger.debug( + 'PuSH denying unsubscribe for feed=%r, topic=%s', feed, topic) + return make_response('unsubscribe denied', 400) + + elif mode: + current_app.logger.debug('PuSH request with unknown mode %s', mode) + return make_response('unrecognized hub.mode=%s' % mode, 400) + + else: + current_app.logger.debug('PuSH request with no mode') + return make_response('missing requred parameter hub.mode', 400) if not feed: current_app.logger.warn( 'could not find feed corresponding to %d', feed_id) - abort(404) + return make_response('no feed with id %d' % feed_id, 400) # could it be? an actual push notification!? current_app.logger.debug( @@ -83,7 +94,7 @@ def notify(feed_id): content = request.data.decode('utf-8') tasks.q_high.enqueue(tasks.update_feed, feed.id, - content=content, content_type=content_type, + content=content, content_type=content_type, is_polling=False) feed.last_pinged = datetime.datetime.utcnow() db.session.commit() diff --git a/woodwind/tasks.py b/woodwind/tasks.py index be5de53..3745db3 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from flask import current_app +from flask import current_app, url_for from redis import StrictRedis from woodwind import util from woodwind.extensions import db @@ -231,21 +231,17 @@ def update_feed(feed_id, content=None, def check_push_subscription(feed, response): - def build_callback_url(): - return '{}://{}/_notify/{}'.format( - getattr(current_app.config, 'PREFERRED_URL_SCHEME', 'http'), - current_app.config['SERVER_NAME'], - feed.id) - def send_request(mode, hub, topic): hub = urllib.parse.urljoin(feed.feed, hub) topic = urllib.parse.urljoin(feed.feed, topic) + callback = url_for('push.notify', feed_id=feed.id, _external=True) current_app.logger.debug( - 'sending %s request for hub=%r, topic=%r', mode, hub, topic) + 'sending %s request for hub=%r, topic=%r, callback=%r', + mode, hub, topic, callback) r = requests.post(hub, data={ 'hub.mode': mode, 'hub.topic': topic, - 'hub.callback': build_callback_url(), + 'hub.callback': callback, 'hub.secret': feed.get_or_create_push_secret(), 'hub.verify': 'sync', # backcompat with 0.3 }) From 21a1351ddeabec7f215b1472d92fdedf916fea49 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Thu, 7 Apr 2016 13:25:46 -0700 Subject: [PATCH 15/18] bump version of mf2util to avoid crashes on weird input --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aef91ff..c4017d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ itsdangerous==0.24 Jinja2==2.8 MarkupSafe==0.23 mf2py==1.0.2 -mf2util==0.3.2 +mf2util==0.3.3 psycopg2==2.6.1 pyasn1==0.1.9 pycparser==2.14 From 093ad551dfedc3658fbfd65ea2ed5be85f297419 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Thu, 21 Apr 2016 12:13:07 -0700 Subject: [PATCH 16/18] start supporting new jf2 style micropub endpoints --- woodwind/tasks.py | 2 -- woodwind/views.py | 28 ++++++++++++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 3745db3..fcd23ec 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -95,8 +95,6 @@ def tick(): q.enqueue(update_feed, feed.id) - - def update_feed(feed_id, content=None, content_type=None, is_polling=True): diff --git a/woodwind/views.py b/woodwind/views.py index 894577f..ea8c3ef 100644 --- a/woodwind/views.py +++ b/woodwind/views.py @@ -306,6 +306,23 @@ def micropub_callback(resp): @flask_login.login_required def update_micropub_syndicate_to(): + + def adapt_expanded(exp): + """Backcompat support for old-style "syndicate-to-expanded" properties, + e.g., + { + "id": "twitter::kylewmahan", + "name": "@kylewmahan", + "service": "Twitter" + } + """ + if isinstance(exp, dict): + return { + 'uid': exp.get('id'), + 'name': '{} on {}'.format(exp.get('name'), exp.get('service')), + } + return exp + endpt = flask_login.current_user.micropub_endpoint token = flask_login.current_user.access_token if not endpt or not token: @@ -331,9 +348,10 @@ def update_micropub_syndicate_to(): if content_type == 'application/json': blob = resp.json() - syndicate_tos = blob.get('syndicate-to-expanded') + syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded')) if not syndicate_tos: syndicate_tos = blob.get('syndicate-to') + else: # try to parse query string syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', []) if isinstance(syndicate_tos, list): @@ -735,17 +753,15 @@ def font_awesome_class_for_service(service): @views.app_template_filter('syndication_target_id') def syndication_target_id(target): if isinstance(target, dict): - return target.get('id') + return target.get('uid') or target.get('id') return target @views.app_template_filter('render_syndication_target') def render_syndication_target(target): if isinstance(target, dict): - service = target.get('service') - name = target.get('name') - return ' {}'.format( - font_awesome_class_for_service(service), name) + full_name = target.get('name') + return full_name return '{} {}'.format( favicon_for_url(target), target, prettify_url(target)) From d6d5e5056c1320417c4faa6b8d20850671fc94fd Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Thu, 21 Apr 2016 19:59:29 +0000 Subject: [PATCH 17/18] add link to update micropub syndication targets without re-authorizing --- woodwind/templates/settings_micropub.jinja2 | 3 +++ woodwind/views.py | 22 ++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/woodwind/templates/settings_micropub.jinja2 b/woodwind/templates/settings_micropub.jinja2 index 8be5b31..232ad87 100644 --- a/woodwind/templates/settings_micropub.jinja2 +++ b/woodwind/templates/settings_micropub.jinja2 @@ -12,6 +12,9 @@ {% if current_user.micropub_endpoint or current_user.access_token %} +

+ Update Syndication Targets +

Reauthorize Micropub

diff --git a/woodwind/views.py b/woodwind/views.py index ea8c3ef..043191d 100644 --- a/woodwind/views.py +++ b/woodwind/views.py @@ -304,10 +304,18 @@ def micropub_callback(resp): return flask.redirect(resp.next_url or flask.url_for('.index')) +@flask_login.login_required +@views.route('/micropub-update') +def micropub_update(): + update_micropub_syndicate_to() + return flask.redirect(flask.request.args.get('next') + or flask.url_for('.index')) + + @flask_login.login_required def update_micropub_syndicate_to(): - def adapt_expanded(exp): + def adapt_expanded(targets): """Backcompat support for old-style "syndicate-to-expanded" properties, e.g., { @@ -316,12 +324,12 @@ def update_micropub_syndicate_to(): "service": "Twitter" } """ - if isinstance(exp, dict): - return { - 'uid': exp.get('id'), - 'name': '{} on {}'.format(exp.get('name'), exp.get('service')), - } - return exp + if targets: + return [{ + 'uid': t.get('id'), + 'name': '{} on {}'.format(t.get('name'), t.get('service')), + } for t in targets] + return targets endpt = flask_login.current_user.micropub_endpoint token = flask_login.current_user.access_token From af6bcdfcf6c97c1abb2d34a1a475c6fd925f79c0 Mon Sep 17 00:00:00 2001 From: Kyle Mahan Date: Thu, 21 Apr 2016 20:07:52 +0000 Subject: [PATCH 18/18] flash a notice so the user knows their syndication links have actually updated --- woodwind/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/woodwind/views.py b/woodwind/views.py index 043191d..2ec325c 100644 --- a/woodwind/views.py +++ b/woodwind/views.py @@ -308,6 +308,12 @@ def micropub_callback(resp): @views.route('/micropub-update') def micropub_update(): update_micropub_syndicate_to() + + syndicate_to = flask_login.current_user.get_setting('syndicate-to', []) + + flask.flash('Updated syndication targets: {}'.format(', '.join([ + t.get('name') if isinstance(t, dict) else t for t in syndicate_to]))) + return flask.redirect(flask.request.args.get('next') or flask.url_for('.index'))