jeena/woodwind
1
0
Fork 0
Code Issues Pull requests Projects Packages Wiki Activity Actions

Merge branch 'master' into requires-io-master

Browse source
This commit is contained in:
Kyle Mahan 2016-04-21 13:22:54 -07:00
commit c85430e071
14 changed files with 240 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

7
woodwind/api.py
View file

@ -16,9 +16,14 @@ def publish():
data = {
'h': 'entry',
'syndicate-to[]': syndicate_to,
'access_token': flask_login.current_user.access_token,
}
if action == 'like':
if action.startswith('rsvp-'):
data['in-reply-to'] = target
data['content'] = content
data['rsvp'] = action.split('-', 1)[-1]
elif action == 'like':
data['like-of'] = target
elif action == 'repost':
data['repost-of'] = target

6
woodwind/app.py
View file

@ -36,7 +36,11 @@ def configure_logging(app):
return
app.logger.setLevel(logging.DEBUG)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
app.logger.addHandler(handler)
recipients = app.config.get('ADMIN_EMAILS')
if recipients:

33
woodwind/push.py
View file

@ -31,13 +31,14 @@ def notify(feed_id):
if not feed:
current_app.logger.warn(
'could not find feed corresponding to %d', feed_id)
abort(404)
return make_response('no feed with id %d' % feed_id, 400)
if topic != feed.push_topic:
current_app.logger.warn(
'feed topic (%s) does not match subscription request (%s)',
feed.push_topic, topic)
abort(404)
return make_response(
'topic %s does not match subscription request %s' % (feed.push_topic, topic), 400)
current_app.logger.debug(
'PuSH verify subscribe for feed=%r, topic=%s', feed, topic)
@ -48,18 +49,28 @@ def notify(feed_id):
db.session.commit()
return challenge
elif mode == 'unsubscribe' and (not feed or topic != feed.push_topic):
current_app.logger.debug(
'PuSH verify unsubscribe for feed=%r, topic=%s', feed, topic)
return challenge
current_app.logger.debug('PuSH cannot verify %s for feed=%r, topic=%s',
mode, feed, topic)
abort(404)
elif mode == 'unsubscribe':
if not feed or topic != feed.push_topic:
current_app.logger.debug(
'PuSH verify unsubscribe for feed=%r, topic=%s', feed, topic)
return challenge
else:
current_app.logger.debug(
'PuSH denying unsubscribe for feed=%r, topic=%s', feed, topic)
return make_response('unsubscribe denied', 400)
elif mode:
current_app.logger.debug('PuSH request with unknown mode %s', mode)
return make_response('unrecognized hub.mode=%s' % mode, 400)
else:
current_app.logger.debug('PuSH request with no mode')
return make_response('missing requred parameter hub.mode', 400)
if not feed:
current_app.logger.warn(
'could not find feed corresponding to %d', feed_id)
abort(404)
return make_response('no feed with id %d' % feed_id, 400)
# could it be? an actual push notification!?
current_app.logger.debug(
@ -83,7 +94,7 @@ def notify(feed_id):
content = request.data.decode('utf-8')
tasks.q_high.enqueue(tasks.update_feed, feed.id,
content=content, content_type=content_type,
content=content, content_type=content_type,
is_polling=False)
feed.last_pinged = datetime.datetime.utcnow()
db.session.commit()

18
woodwind/static/feed.js
View file

@ -39,20 +39,30 @@ $(function(){
function submitMicropubForm(evt) {
evt.preventDefault();
var form = $(this).closest('form');
var button = this;
var form = $(button).closest('form');
var replyArea = form.parent();
var endpoint = form.attr('action');
var responseArea = $('.micropub-response', replyArea);
var formData = form.serializeArray();
formData.push({name: this.name, value: this.value});
formData.push({name: button.name, value: button.value});
$.post(
form.attr('action'),
endpoint,
formData,
function(result) {
if (Math.floor(result.code / 100) == 2) {
responseArea.html('<a target="_blank" href="' + result.location + '">Success!</a>');
$(".micropub-form textarea").val("");
$("textarea", form).val("");
if (button.value === 'rsvp-yes') {
$(".rsvps", form).html('✓ Going');
} else if (button.value === 'rsvp-maybe') {
$(".rsvps", form).html('? Interested');
} else if (button.value === 'rsvp-no') {
$(".rsvps", form).html('✗ Not Going');
}
} else {
responseArea.html('Failure');
}

41
woodwind/static/style.css
View file

@ -361,7 +361,7 @@ th {
body {
font: 12pt Helvetica, Arial, sans-serif;
line-height: 1.4em;
background: #ECEBF0;
background: #ecebf0;
/*background: #f4f4f4;*/
padding-top: 1em; }
@ -377,13 +377,17 @@ p:first-child {
p:last-child {
margin-bottom: 0; }
header, main {
header, main, .footer {
max-width: 800px;
margin: 0 auto; }
header {
margin-bottom: 1em; }
.footer {
font-size: 0.8em;
text-align: center; }
ul#navigation {
list-style-type: none;
margin: 0;
@ -399,8 +403,8 @@ ul#navigation {
.button-link a {
padding: 0.5em;
background-color: #353129;
color: #ECEBF0;
border: 1px solid #ECEBF0;
color: #ecebf0;
border: 1px solid #ecebf0;
border-radius: 4px;
display: inline-block; }
@ -412,7 +416,7 @@ ul#navigation {
article {
margin-bottom: 1em;
box-shadow: 0 0 2px #687D77;
box-shadow: 0 0 2px #687d77;
background-color: white;
border-radius: 3px;
padding: 0.5em; }
@ -434,8 +438,8 @@ article {
article img, article video {
max-width: 100%; }
article header {
color: #484A47;
border-bottom: 1px solid #687D77;
color: #484a47;
border-bottom: 1px solid #687d77;
margin-bottom: 0.5em;
overflow: auto; }
article header img {
@ -494,14 +498,17 @@ button {
margin: 0;
vertical-align: middle; }
.micropub-form button {
height: 24px;
width: 24px;
padding: 4px;
background-color: #eee;
border-radius: 3px;
border: 0;
line-height: 1;
vertical-align: middle; }
vertical-align: middle;
border: 0; }
.micropub-form button.small {
width: 24px;
height: 24px;
padding: 4px;
line-height: 1; }
.micropub-form .rsvps {
text-align: center; }
.syndication-toggle {
display: inline-block; }
@ -531,11 +538,11 @@ button {
.reply-area .reply-link {
display: inline-block;
padding: 0.2em;
border: 1px solid #687D77;
border: 1px solid #687d77;
border-radius: 4px;
background-color: #ECEBF0;
background-color: #ecebf0;
text-decoration: none;
color: #484A47;
color: #484a47;
min-width: 50px;
text-align: center; }
@ -548,5 +555,3 @@ button {
max-height: 1.2em;
min-width: inherit;
min-height: inherit; } }
/*# sourceMappingURL=style.css.map */

2
woodwind/static/style.css.map
View file

File diff suppressed because one or more lines are too long

22
woodwind/static/style.scss
View file

@ -43,7 +43,7 @@ p {
}
}
header, main {
header, main, .footer {
max-width: 800px;
margin: 0 auto;
}
@ -52,6 +52,11 @@ header {
margin-bottom: 1em;
}
.footer {
font-size: 0.8em;
text-align: center;
}
ul#navigation {
list-style-type: none;
margin: 0;
@ -201,13 +206,20 @@ button {
}
button {
height: 24px;width: 24px;
padding: 4px;
background-color: #eee;
border-radius: 3px;
border: 0;
line-height: 1;
vertical-align: middle;
border: 0;
&.small {
width: 24px;
height: 24px;
padding: 4px;
line-height: 1;
}
}
.rsvps {
text-align: center;
}
}

141
woodwind/tasks.py
View file

@ -1,9 +1,10 @@
from contextlib import contextmanager
from flask import current_app
from flask import current_app, url_for
from redis import StrictRedis
from woodwind import util
from woodwind.extensions import db
from woodwind.models import Feed, Entry
import sqlalchemy
import bs4
import datetime
import feedparser
@ -16,6 +17,7 @@ import requests
import rq
import sys
import time
import traceback
import urllib.parse
# normal update interval for polling feeds
@ -112,12 +114,12 @@ def update_feed(feed_id, content=None,
with flask_app() as app:
feed = Feed.query.get(feed_id)
current_app.logger.info('Updating {}'.format(feed))
current_app.logger.info('Updating {}'.format(str(feed)[:32]))
now = datetime.datetime.utcnow()
new_ids = []
updated_ids = []
new_entries = []
updated_entries = []
reply_pairs = []
try:
@ -125,7 +127,7 @@ def update_feed(feed_id, content=None,
current_app.logger.info('using provided content. size=%d',
len(content))
else:
current_app.logger.info('fetching feed: %s', feed)
current_app.logger.info('fetching feed: %s', str(feed)[:32])
try:
response = util.requests_get(feed.feed)
@ -163,24 +165,30 @@ def update_feed(feed_id, content=None,
result = []
for entry in result:
current_app.logger.debug('searching for entry with uid=%s', entry.uid)
old = Entry.query\
.filter(Entry.feed == feed)\
.filter(Entry.uid == entry.uid)\
.order_by(Entry.id.desc())\
.first()
current_app.logger.debug('done searcing: %s', 'found' if old else 'not found')
# have we seen this post before
if not old:
current_app.logger.debug('this is a new post, saving a new entry')
# set a default value for published if none is provided
entry.published = entry.published or now
in_reply_tos = entry.get_property('in-reply-to', [])
db.session.add(entry)
feed.entries.append(entry)
db.session.commit()
new_ids.append(entry.id)
new_entries.append(entry)
for irt in in_reply_tos:
reply_pairs.append((entry.id, irt))
reply_pairs.append((entry, irt))
elif not is_content_equal(old, entry):
current_app.logger.debug('this post content has changed, updating entry')
entry.published = entry.published or old.published
in_reply_tos = entry.get_property('in-reply-to', [])
# we're updating an old entriy, use the original
@ -190,46 +198,48 @@ def update_feed(feed_id, content=None,
# punt on deleting for now, learn about cascade
# and stuff later
# session.delete(old)
db.session.add(entry)
feed.entries.append(entry)
db.session.commit()
updated_ids.append(entry.id)
updated_entries.append(entry)
for irt in in_reply_tos:
reply_pairs.append((entry.id, irt))
reply_pairs.append((entry, irt))
else:
current_app.logger.debug(
'skipping previously seen post %s', old.permalink)
for entry_id, in_reply_to in reply_pairs:
fetch_reply_context(entry_id, in_reply_to, now)
for entry, in_reply_to in reply_pairs:
fetch_reply_context(entry, in_reply_to, now)
db.session.commit()
except:
db.session.rollback()
raise
finally:
if is_polling:
feed.last_checked = now
if new_ids or updated_ids:
if new_entries or updated_entries:
feed.last_updated = now
db.session.commit()
if new_ids:
notify_feed_updated(app, feed_id, new_ids)
if new_entries:
notify_feed_updated(app, feed_id, new_entries)
def check_push_subscription(feed, response):
def build_callback_url():
return '{}://{}/_notify/{}'.format(
getattr(current_app.config, 'PREFERRED_URL_SCHEME', 'http'),
current_app.config['SERVER_NAME'],
feed.id)
def send_request(mode, hub, topic):
hub = urllib.parse.urljoin(feed.feed, hub)
topic = urllib.parse.urljoin(feed.feed, topic)
callback = url_for('push.notify', feed_id=feed.id, _external=True)
current_app.logger.debug(
'sending %s request for hub=%r, topic=%r', mode, hub, topic)
'sending %s request for hub=%r, topic=%r, callback=%r',
mode, hub, topic, callback)
r = requests.post(hub, data={
'hub.mode': mode,
'hub.topic': topic,
'hub.callback': build_callback_url(),
'hub.callback': callback,
'hub.secret': feed.get_or_create_push_secret(),
'hub.verify': 'sync', # backcompat with 0.3
})
@ -267,6 +277,8 @@ def check_push_subscription(feed, response):
if ((expiry and expiry - datetime.datetime.utcnow()
<= UPDATE_INTERVAL_PUSH)
or hub != old_hub or topic != old_topic or not feed.push_verified):
current_app.logger.debug('push subscription expired or hub/topic changed')
feed.push_hub = hub
feed.push_topic = topic
feed.push_verified = False
@ -274,29 +286,24 @@ def check_push_subscription(feed, response):
db.session.commit()
if old_hub and old_topic and hub != old_hub and topic != old_topic:
current_app.logger.debug('unsubscribing hub=%s, topic=%s', old_hub, old_topic)
send_request('unsubscribe', old_hub, old_topic)
if hub and topic:
current_app.logger.debug('subscribing hub=%s, topic=%s', hub, topic)
send_request('subscribe', hub, topic)
db.session.commit()
def notify_feed_updated(app, feed_id, entry_ids):
def notify_feed_updated(app, feed_id, entries):
"""Render the new entries and publish them to redis
"""
from flask import render_template
import flask.ext.login as flask_login
current_app.logger.debug(
'notifying feed updated for entries %r', entry_ids)
current_app.logger.debug('notifying feed updated: %s', feed_id)
feed = Feed.query.get(feed_id)
entries = Entry.query\
.filter(Entry.id.in_(entry_ids))\
.order_by(Entry.retrieved.desc(),
Entry.published.desc())\
.all()
for s in feed.subscriptions:
with app.test_request_context():
flask_login.login_user(s.user, remember=True)
@ -336,16 +343,18 @@ def is_content_equal(e1, e2):
content = COMMENT_RE.sub('', content)
return content
return (e1.title == e2.title
and normalize(e1.content) == normalize(e2.content)
and e1.author_name == e2.author_name
and e1.author_url == e2.author_url
and e1.author_photo == e2.author_photo
and e1.properties == e2.properties)
return (
e1.title == e2.title
and normalize(e1.content) == normalize(e2.content)
and e1.author_name == e2.author_name
and e1.author_url == e2.author_url
and e1.author_photo == e2.author_photo
and e1.properties == e2.properties
)
def process_xml_feed_for_new_entries(feed, content, backfill, now):
current_app.logger.debug('fetching xml feed: %s', feed)
current_app.logger.debug('fetching xml feed: %s', str(feed)[:32])
parsed = feedparser.parse(content, response_headers={
'content-location': feed.feed,
})
@ -354,12 +363,11 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now):
default_author_name = feed_props.get('author_detail', {}).get('name')
default_author_photo = feed_props.get('logo')
current_app.logger.debug('found {} entries'.format(len(parsed.entries)))
current_app.logger.debug('found %d entries', len(parsed.entries))
# work from the bottom up (oldest first, usually)
for p_entry in reversed(parsed.entries):
current_app.logger.debug('processing entry {}'.format(
str(p_entry)[:256]))
current_app.logger.debug('processing entry %s', str(p_entry)[:32])
permalink = p_entry.get('link')
uid = p_entry.get('id') or permalink
@ -406,6 +414,8 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now):
video = VIDEO_ENCLOSURE_TMPL.format(href=link.get('href'))
content = (content or '') + video
current_app.logger.debug('building entry')
entry = Entry(
published=published,
updated=updated,
@ -422,6 +432,8 @@ def process_xml_feed_for_new_entries(feed, content, backfill, now):
author_photo=default_author_photo
or fallback_photo(feed.origin))
current_app.logger.debug('yielding entry')
yield entry
@ -462,7 +474,7 @@ def hentry_to_entry(hentry, feed, backfill, now):
title = hentry.get('name')
content = hentry.get('content')
if not content:
if not content and hentry.get('type') == 'entry':
content = title
title = None
@ -518,6 +530,16 @@ def hentry_to_entry(hentry, feed, backfill, now):
if value:
entry.set_property(prop, value)
if 'start-str' in hentry:
entry.set_property('start', hentry.get('start-str'))
if 'end-str' in hentry:
entry.set_property('end', hentry.get('end-str'))
# set a flag for events so we can show RSVP buttons
if hentry.get('type') == 'event':
entry.set_property('event', True)
# does it look like a jam?
plain = hentry.get('content-plain')
if plain and JAM_RE.match(plain):
@ -527,25 +549,28 @@ def hentry_to_entry(hentry, feed, backfill, now):
return entry
def fetch_reply_context(entry_id, in_reply_to, now):
with flask_app():
entry = Entry.query.get(entry_id)
context = Entry.query\
.join(Entry.feed)\
.filter(Entry.permalink==in_reply_to, Feed.type == 'html')\
.first()
def fetch_reply_context(entry, in_reply_to, now):
context = Entry.query\
.join(Entry.feed)\
.filter(Entry.permalink==in_reply_to, Feed.type == 'html')\
.first()
if not context:
current_app.logger.info('fetching in-reply-to url: %s',
in_reply_to)
if not context:
current_app.logger.info('fetching in-reply-to: %s', in_reply_to)
try:
proxied_reply_url = proxy_url(in_reply_to)
parsed = mf2util.interpret(
mf2py.parse(url=proxy_url(in_reply_to)), in_reply_to)
mf2py.parse(url=proxied_reply_url), in_reply_to)
if parsed:
context = hentry_to_entry(parsed, None, False, now)
except requests.exceptions.RequestException as err:
current_app.logger.warn(
'%s fetching reply context: %s for entry: %s',
type(err).__name__, proxied_reply_url, entry.permalink)
if context:
entry.reply_context.append(context)
db.session.commit()
if context:
db.session.add(context)
entry.reply_context.append(context)
def proxy_url(url):

12
woodwind/templates/_entry.jinja2
View file

@ -46,6 +46,18 @@
<h1>{{ entry.title }}</h1>
{% endif %}
{% if entry.get_property('event') %}
<p>
{% if entry.get_property('start') %}
<strong>start:</strong> {{ entry.get_property('start') }}
{% endif %}
<br/>
{% if entry.get_property('end') %}
<strong>end:</strong> {{ entry.get_property('end') }}
{% endif %}
</p>
{% endif %}
{% set photo = entry.get_property('photo') %}
{% if photo and (not entry.content or '<img' not in entry.content) %}
<div class="photo">

15
woodwind/templates/_reply.jinja2
View file

@ -4,11 +4,20 @@
{% if reply_method == 'micropub' and current_user.micropub_endpoint %}
<form class="micropub-form" action="{{ url_for('api.publish') }}" method="POST">
<input type="hidden" name="target" value="{{ entry.permalink }}"/>
{% if entry.get_property('event') %}
<div class="rsvps">
<button type="submit" name="action" value="rsvp-yes">Going</button>
<button type="submit" name="action" value="rsvp-maybe">Interested</button>
<button type="submit" name="action" value="rsvp-no">Not Going</button>
</div>
{% endif %}
<div>
<textarea class="content" name="content"></textarea>
<button type="submit" name="action" value="reply"><i class="fa fa-reply"></i></button>
<button type="submit" name="action" value="repost"><i class="fa fa-retweet"></i></button>
<button type="submit" name="action" value="like"><i class="fa fa-star"></i></button>
<button type="submit" name="action" value="reply" class="small"><i class="fa fa-reply"></i></button>
<button type="submit" name="action" value="repost" class="small"><i class="fa fa-retweet"></i></button>
<button type="submit" name="action" value="like" class="small"><i class="fa fa-star"></i></button>
</div>
<div class="syndication-toggles">
{% for target in current_user.get_setting('syndicate-to', []) %}

8
woodwind/templates/base.jinja2
View file

@ -8,14 +8,12 @@
<link rel="shortcut icon" href="{{ url_for('static', filename='logo.png') }}"/>
<link rel="apple-touch-icon" href="{{ url_for('static', filename='logo.png') }}"/>
<link rel="stylesheet" href="{{ url_for('static', filename='style.css', version='2016-01-31') }}"/>
<link rel="stylesheet" href="{{ url_for('static', filename='style.css', version='2016-03-08') }}"/>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css"/>
<script src="//code.jquery.com/jquery-2.1.3.min.js"></script>
<script src="{{ url_for('static', filename='moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='cassis.js') }}"></script>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css">
{% block head %}{% endblock %}
</head>
<body>
@ -73,6 +71,10 @@
</main>
{% block foot %}{% endblock %}
<div class="footer">
Woodwind is <a href="https://github.com/kylewm/woodwind">on GitHub</a>. Have any problems? File an issue or come chat in <code>#indiewebcamp</code> on Freenode IRC.
</div>
<script>
$("input[type='url']").blur(function() {
if (this.value.trim() != '') {

2
woodwind/templates/feed.jinja2
View file

@ -4,7 +4,7 @@
{% if ws_topic %}
<script>var WS_TOPIC = "{{ ws_topic }}";</script>
{% endif %}
<script src="{{url_for('static', filename='feed.js', version='2015-12-24')}}"></script>
<script src="{{url_for('static', filename='feed.js', version='2016-03-08')}}"></script>
{% if current_user and current_user.settings
and current_user.settings.get('reply-method') == 'indie-config' %}

3
woodwind/templates/settings_micropub.jinja2
View file

@ -12,6 +12,9 @@
{% if current_user.micropub_endpoint or current_user.access_token %}
<input type="text" value="{{ current_user.micropub_endpoint }}" readonly />
<input type="text" value="{{ current_user.access_token }}" readonly />
<p>
<a href="{{url_for('.micropub_update', next=request.path)}}">Update Syndication Targets</a>
</p>
<p>
<a href="{{url_for('.authorize', next=request.path)}}">Reauthorize Micropub</a>
</p>

42
woodwind/views.py
View file

@ -304,8 +304,39 @@ def micropub_callback(resp):
return flask.redirect(resp.next_url or flask.url_for('.index'))
@flask_login.login_required
@views.route('/micropub-update')
def micropub_update():
update_micropub_syndicate_to()
syndicate_to = flask_login.current_user.get_setting('syndicate-to', [])
flask.flash('Updated syndication targets: {}'.format(', '.join([
t.get('name') if isinstance(t, dict) else t for t in syndicate_to])))
return flask.redirect(flask.request.args.get('next')
or flask.url_for('.index'))
@flask_login.login_required
def update_micropub_syndicate_to():
def adapt_expanded(targets):
"""Backcompat support for old-style "syndicate-to-expanded" properties,
e.g.,
{
"id": "twitter::kylewmahan",
"name": "@kylewmahan",
"service": "Twitter"
}
"""
if targets:
return [{
'uid': t.get('id'),
'name': '{} on {}'.format(t.get('name'), t.get('service')),
} for t in targets]
return targets
endpt = flask_login.current_user.micropub_endpoint
token = flask_login.current_user.access_token
if not endpt or not token:
@ -331,9 +362,10 @@ def update_micropub_syndicate_to():
if content_type == 'application/json':
blob = resp.json()
syndicate_tos = blob.get('syndicate-to-expanded')
syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded'))
if not syndicate_tos:
syndicate_tos = blob.get('syndicate-to')
else: # try to parse query string
syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', [])
if isinstance(syndicate_tos, list):
@ -735,17 +767,15 @@ def font_awesome_class_for_service(service):
@views.app_template_filter('syndication_target_id')
def syndication_target_id(target):
if isinstance(target, dict):
return target.get('id')
return target.get('uid') or target.get('id')
return target
@views.app_template_filter('render_syndication_target')
def render_syndication_target(target):
if isinstance(target, dict):
service = target.get('service')
name = target.get('name')
return '<i class="{}"></i>&nbsp;{}'.format(
font_awesome_class_for_service(service), name)
full_name = target.get('name')
return full_name
return '<img src="{}" alt="{}" />&nbsp;{}'.format(
favicon_for_url(target), target, prettify_url(target))