add reply-context to posts

This commit is contained in:
Kyle Mahan 2015-02-19 09:07:52 -08:00
parent 367ae82f0a
commit bc53c9483f
6 changed files with 162 additions and 35 deletions

View file

@ -4,6 +4,9 @@ import binascii
from .extensions import db
import re
from sqlalchemy.ext.orderinglist import ordering_list
from sqlalchemy.ext.associationproxy import association_proxy
bleach.ALLOWED_TAGS += ['a', 'img', 'p', 'br', 'marquee', 'blink',
'audio', 'video', 'table', 'tbody', 'td', 'tr']
@ -39,6 +42,12 @@ users_to_feeds = db.Table(
db.Column('feed_id', db.Integer, db.ForeignKey('feed.id'), index=True))
entry_to_reply_context = db.Table(
'entry_to_reply_context', db.Model.metadata,
db.Column('entry_id', db.Integer, db.ForeignKey('entry.id'), index=True),
db.Column('context_id', db.Integer, db.ForeignKey('entry.id'), index=True))
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
url = db.Column(db.String(256))
@ -120,12 +129,42 @@ class Entry(db.Model):
author_photo = db.Column(db.String(512))
title = db.Column(db.String(512))
content = db.Column(db.Text)
# other properties
properties = db.Column(JsonType)
# # association with the InReplyTo objects
# irt = db.relationship(
# 'InReplyTo', order_by='InReplyTo.list_index',
# collection_class=ordering_list('list_index'))
# # proxy for just the urls
# in_reply_to = association_proxy(
# 'irt', 'url', creator=lambda url: InReplyTo(url=url))
reply_context = db.relationship(
'Entry', secondary='entry_to_reply_context',
primaryjoin=id == entry_to_reply_context.c.entry_id,
secondaryjoin=id == entry_to_reply_context.c.context_id)
def content_cleaned(self):
if self.content:
text = self.content
text = re.sub('<script>.*?</script>', '', text, flags=re.DOTALL)
text = re.sub('<script.*?</script>', '', text, flags=re.DOTALL)
return bleach.clean(text, strip=True)
def get_property(self, key, default=None):
if self.properties is None:
return default
return self.properties.get(key, default)
def set_property(self, key, value):
self.properties = ({} if self.properties is None
else dict(self.settings))
self.properties[key] = value
def __repr__(self):
return '<Entry:{},{}>'.format(self.title, (self.content or '')[:140])
# class InReplyTo(db.Model):
# id = db.Column(db.Integer, primary_key=True)
# entry_id = db.Column(db.Integer, db.ForeignKey(Entry.id))
# url = db.Column(db.String(512))
# list_index = db.Column(db.Integer)

View file

@ -394,6 +394,9 @@ article {
box-shadow: 0 0 2px #687D77;
background-color: white;
padding: 0.5em; }
article.reply-context {
margin-bottom: 0.5em;
background-color: #f3f3f3; }
article div {
overflow: auto; }
article img {

View file

@ -1,6 +1,6 @@
{
"version": 3,
"mappings": ";;;;;;AAQA,IAAK;EACH,WAAW,EAAE,UAAU;;EACvB,oBAAoB,EAAE,IAAI;;EAC1B,wBAAwB,EAAE,IAAI;;;;;;AAOhC,IAAK;EACH,MAAM,EAAE,CAAC;;;;;;;;;;AAaX;;;;;;;;;;;;OAYQ;EACN,OAAO,EAAE,KAAK;;;;;;AAQhB;;;KAGM;EACJ,OAAO,EAAE,YAAY;;EACrB,cAAc,EAAE,QAAQ;;;;;;;AAQ1B,qBAAsB;EACpB,OAAO,EAAE,IAAI;EACb,MAAM,EAAE,CAAC;;;;;;AAQX;QACS;EACP,OAAO,EAAE,IAAI;;;;;;;AAUf,CAAE;EACA,gBAAgB,EAAE,WAAW;;;;;AAO/B;OACQ;EACN,OAAO,EAAE,CAAC;;;;;;;AAUZ,WAAY;EACV,aAAa,EAAE,UAAU;;;;;AAO3B;MACO;EACL,WAAW,EAAE,IAAI;;;;;AAOnB,GAAI;EACF,UAAU,EAAE,MAAM;;;;;;AAQpB,EAAG;EACD,SAAS,EAAE,GAAG;EACd,MAAM,EAAE,QAAQ;;;;;AAOlB,IAAK;EACH,UAAU,EAAE,IAAI;EAChB,KAAK,EAAE,IAAI;;;;;AAOb,KAAM;EACJ,SAAS,EAAE,GAAG;;;;;AAOhB;GACI;EACF,SAAS,EAAE,GAAG;EACd,WAAW,EAAE,CAAC;EACd,QAAQ,EAAE,QAAQ;EAClB,cAAc,EAAE,QAAQ;;AAG1B,GAAI;EACF,GAAG,EAAE,MAAM;;AAGb,GAAI;EACF,MAAM,EAAE,OAAO;;;;;;;AAUjB,GAAI;EACF,MAAM,EAAE,CAAC;;;;;AAOX,cAAe;EACb,QAAQ,EAAE,MAAM;;;;;;;AAUlB,MAAO;EACL,MAAM,EAAE,QAAQ;;;;;AAOlB,EAAG;EACD,eAAe,EAAE,WAAW;EAC5B,UAAU,EAAE,WAAW;EACvB,MAAM,EAAE,CAAC;;;;;AAOX,GAAI;EACF,QAAQ,EAAE,IAAI;;;;;AAOhB;;;IAGK;EACH,WAAW,EAAE,oBAAoB;EACjC,SAAS,EAAE,GAAG;;;;;;;;;;;;;;AAkBhB;;;;QAIS;EACP,KAAK,EAAE,OAAO;;EACd,IAAI,EAAE,OAAO;;EACb,MAAM,EAAE,CAAC;;;;;;AAOX,MAAO;EACL,QAAQ,EAAE,OAAO;;;;;;;;AAUnB;MACO;EACL,cAAc,EAAE,IAAI;;;;;;;;;AAWtB;;;oBAGqB;EACnB,kBAAkB,EAAE,MAAM;;EAC1B,MAAM,EAAE,OAAO;;;;;;AAOjB;oBACqB;EACnB,MAAM,EAAE,OAAO;;;;;AAOjB;uBACwB;EACtB,MAAM,EAAE,CAAC;EACT,OAAO,EAAE,CAAC;;;;;;AAQZ,KAAM;EACJ,WAAW,EAAE,MAAM;;;;;;;;;AAWrB;mBACoB;EAClB,UAAU,EAAE,UAAU;;EACtB,OAAO,EAAE,CAAC;;;;;;;;AASZ;+CACgD;EAC9C,MAAM,EAAE,IAAI;;;;;;;AASd,oBAAqB;EACnB,kBAAkB,EAAE,SAAS;;EAC7B,eAAe,EAAE,WAAW;EAC5B,kBAAkB,EAAE,WAAW;;EAC/B,UAAU,EAAE,WAAW;;;;;;;AASzB;+CACgD;EAC9C,kBAAkB,EAAE,IAAI;;;;;AAO1B,QAAS;EACP,MAAM,EAAE,iBAAiB;EACzB,MAAM,EAAE,KAAK;EACb,OAAO,EAAE,qBAAqB;;;;;;AAQhC,MAAO;EACL,MAAM,EAAE,CAAC;;EACT,OAAO,EAAE,CAAC;;;;;;AAOZ,QAAS;EACP,QAAQ,EAAE,IAAI;;;;;;AAQhB,QAAS;EACP,WAAW,EAAE,IAAI;;;;;;;AAUnB,KAAM;EACJ,eAAe,EAAE,QAAQ;EACzB,cAAc,EAAE,CAAC;;AAGnB;EACG;EACD,OAAO,EAAE,CAAC;;;;ACxZZ,IAAK;EACD,IAAI,EAAE,iCAAe;EACrB,UAAU,EATA,OAAO;;AAarB,YAAa;EACT,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAGlB,MAAO;EACH,aAAa,EAAE,GAAG;;AAGtB,aAAc;EACV,eAAe,EAAE,IAAI;EACrB,MAAM,EAAE,CAAC;EACT,OAAO,EAAE,CAAC;EACV,KAAK,EAAE,KAAK;EAEZ,gBAAG;IACC,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,GAAG;;AAIpB,MAAO;EACH,UAAU,EAAE,MAAM;EAClB,MAAM,EAAE,KAAK;EAEb,QAAE;IACE,OAAO,EAAE,KAAK;IACd,gBAAgB,EAtCZ,OAAO;IAuCX,KAAK,EAzCC,OAAO;IA0Cb,MAAM,EAAE,iBAAsB;IAC9B,aAAa,EAAE,GAAG;IAClB,OAAO,EAAE,YAAY;;AAI7B,OAAQ;EACJ,aAAa,EAAE,GAAG;EAClB,UAAU,EA9CD,eAAgB;EA+CzB,gBAAgB,EAAE,KAAK;EACvB,OAAO,EAAE,KAAK;EAEd,WAAI;IACA,QAAQ,EAAE,IAAI;EAGlB,WAAK;IACD,SAAS,EAAE,IAAI;EAGnB,cAAO;IAUH,KAAK,EA1EC,OAAO;IA2Eb,aAAa,EAAE,iBAAkB;IACjC,aAAa,EAAE,KAAK;IAXpB,kBAAI;MACA,SAAS,EAAE,IAAI;MACf,UAAU,EAAE,IAAI;MAChB,SAAS,EAAE,IAAI;MACf,UAAU,EAAE,IAAI;MAChB,WAAW,EAAE,KAAK;MAClB,KAAK,EAAE,IAAI;MACX,aAAa,EAAE,GAAG;EAO1B,cAAO;IACH,UAAU,EAAE,KAAK;IACjB,aAAa,EAAE,CAAC;EAGpB,UAAG;IACC,SAAS,EAAE,KAAK;IAChB,WAAW,EAAE,IAAI;;AAIzB,KAAM;EACF,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,GAAG;EAClB,SAAS,EAAE,IAAI;;AAGnB,+CAAgD;EAC5C,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,QAAQ;EAEhB,0EAAW;IACP,KAAK,EAAE,GAAG;EAEd,0EAAW;IACP,KAAK,EAAE,GAAG;EAEd,0EAAW;IACP,KAAK,EAAE,GAAG;;AAKd,uBAAS;EACL,MAAM,EAAE,GAAG;EACX,KAAK,EAAE,yBAAyB;EAChC,OAAO,EAAE,GAAG;EACZ,MAAM,EAAE,CAAC;EACT,cAAc,EAAE,MAAM;AAG1B,qBAAO;EACH,MAAM,EAAE,IAAI;EAAC,KAAK,EAAE,IAAI;EACxB,OAAO,EAAE,GAAG;EACZ,gBAAgB,EAAE,IAAI;EACtB,aAAa,EAAE,GAAG;EAClB,MAAM,EAAE,CAAC;EACT,WAAW,EAAE,CAAC;EACd,cAAc,EAAE,MAAM;;AAI9B,mBAAoB;EAChB,OAAO,EAAE,YAAY;EACrB,UAAU,EAAE,MAAM;EAElB,yBAAM;IACF,OAAO,EAAE,IAAI;EAGjB,yBAAM;IACF,OAAO,EAAE,YAAY;IACrB,cAAc,EAAE,MAAM;IACtB,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,aAAa,EAAE,GAAG;IAClB,gBAAgB,EAAE,IAAI;IACtB,MAAM,EAAE,CAAC;IAET,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,OAAO;IAEf,6BAAI;MACA,UAAU,EAAE,IAAI;MAChB,SAAS,EAAE,IAAI;EAIvB,yCAAsB;IAClB,gBAAgB,EAAE,OAAO;IACzB,KAAK,EAAE,IAAI;;AAKnB,WAAY;EACR,UAAU,EAAE,MAAM;EAClB,UAAU,EAAE,KAAK;EAEjB,uBAAY;IACR,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,KAAK;IACd,MAAM,EAAE,iBAAkB;IAC1B,aAAa,EAAE,GAAG;IAClB,gBAAgB,EA7KV,OAAO;IA8Kb,eAAe,EAAE,IAAI;IACrB,KAAK,EAjLC,OAAO;IAkLb,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;;AAI1B,yCAA0C;EAG9B,kBAAI;IACA,cAAc,EAAE,WAAW;IAC3B,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,MAAM;IACf,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,OAAO;IAClB,UAAU,EAAE,OAAO",
"mappings": ";;;;;;AAQA,IAAK;EACH,WAAW,EAAE,UAAU;;EACvB,oBAAoB,EAAE,IAAI;;EAC1B,wBAAwB,EAAE,IAAI;;;;;;AAOhC,IAAK;EACH,MAAM,EAAE,CAAC;;;;;;;;;;AAaX;;;;;;;;;;;;OAYQ;EACN,OAAO,EAAE,KAAK;;;;;;AAQhB;;;KAGM;EACJ,OAAO,EAAE,YAAY;;EACrB,cAAc,EAAE,QAAQ;;;;;;;AAQ1B,qBAAsB;EACpB,OAAO,EAAE,IAAI;EACb,MAAM,EAAE,CAAC;;;;;;AAQX;QACS;EACP,OAAO,EAAE,IAAI;;;;;;;AAUf,CAAE;EACA,gBAAgB,EAAE,WAAW;;;;;AAO/B;OACQ;EACN,OAAO,EAAE,CAAC;;;;;;;AAUZ,WAAY;EACV,aAAa,EAAE,UAAU;;;;;AAO3B;MACO;EACL,WAAW,EAAE,IAAI;;;;;AAOnB,GAAI;EACF,UAAU,EAAE,MAAM;;;;;;AAQpB,EAAG;EACD,SAAS,EAAE,GAAG;EACd,MAAM,EAAE,QAAQ;;;;;AAOlB,IAAK;EACH,UAAU,EAAE,IAAI;EAChB,KAAK,EAAE,IAAI;;;;;AAOb,KAAM;EACJ,SAAS,EAAE,GAAG;;;;;AAOhB;GACI;EACF,SAAS,EAAE,GAAG;EACd,WAAW,EAAE,CAAC;EACd,QAAQ,EAAE,QAAQ;EAClB,cAAc,EAAE,QAAQ;;AAG1B,GAAI;EACF,GAAG,EAAE,MAAM;;AAGb,GAAI;EACF,MAAM,EAAE,OAAO;;;;;;;AAUjB,GAAI;EACF,MAAM,EAAE,CAAC;;;;;AAOX,cAAe;EACb,QAAQ,EAAE,MAAM;;;;;;;AAUlB,MAAO;EACL,MAAM,EAAE,QAAQ;;;;;AAOlB,EAAG;EACD,eAAe,EAAE,WAAW;EAC5B,UAAU,EAAE,WAAW;EACvB,MAAM,EAAE,CAAC;;;;;AAOX,GAAI;EACF,QAAQ,EAAE,IAAI;;;;;AAOhB;;;IAGK;EACH,WAAW,EAAE,oBAAoB;EACjC,SAAS,EAAE,GAAG;;;;;;;;;;;;;;AAkBhB;;;;QAIS;EACP,KAAK,EAAE,OAAO;;EACd,IAAI,EAAE,OAAO;;EACb,MAAM,EAAE,CAAC;;;;;;AAOX,MAAO;EACL,QAAQ,EAAE,OAAO;;;;;;;;AAUnB;MACO;EACL,cAAc,EAAE,IAAI;;;;;;;;;AAWtB;;;oBAGqB;EACnB,kBAAkB,EAAE,MAAM;;EAC1B,MAAM,EAAE,OAAO;;;;;;AAOjB;oBACqB;EACnB,MAAM,EAAE,OAAO;;;;;AAOjB;uBACwB;EACtB,MAAM,EAAE,CAAC;EACT,OAAO,EAAE,CAAC;;;;;;AAQZ,KAAM;EACJ,WAAW,EAAE,MAAM;;;;;;;;;AAWrB;mBACoB;EAClB,UAAU,EAAE,UAAU;;EACtB,OAAO,EAAE,CAAC;;;;;;;;AASZ;+CACgD;EAC9C,MAAM,EAAE,IAAI;;;;;;;AASd,oBAAqB;EACnB,kBAAkB,EAAE,SAAS;;EAC7B,eAAe,EAAE,WAAW;EAC5B,kBAAkB,EAAE,WAAW;;EAC/B,UAAU,EAAE,WAAW;;;;;;;AASzB;+CACgD;EAC9C,kBAAkB,EAAE,IAAI;;;;;AAO1B,QAAS;EACP,MAAM,EAAE,iBAAiB;EACzB,MAAM,EAAE,KAAK;EACb,OAAO,EAAE,qBAAqB;;;;;;AAQhC,MAAO;EACL,MAAM,EAAE,CAAC;;EACT,OAAO,EAAE,CAAC;;;;;;AAOZ,QAAS;EACP,QAAQ,EAAE,IAAI;;;;;;AAQhB,QAAS;EACP,WAAW,EAAE,IAAI;;;;;;;AAUnB,KAAM;EACJ,eAAe,EAAE,QAAQ;EACzB,cAAc,EAAE,CAAC;;AAGnB;EACG;EACD,OAAO,EAAE,CAAC;;;;ACxZZ,IAAK;EACD,IAAI,EAAE,iCAAe;EACrB,UAAU,EATA,OAAO;;AAarB,YAAa;EACT,SAAS,EAAE,KAAK;EAChB,MAAM,EAAE,MAAM;;AAGlB,MAAO;EACH,aAAa,EAAE,GAAG;;AAGtB,aAAc;EACV,eAAe,EAAE,IAAI;EACrB,MAAM,EAAE,CAAC;EACT,OAAO,EAAE,CAAC;EACV,KAAK,EAAE,KAAK;EAEZ,gBAAG;IACC,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,GAAG;;AAIpB,MAAO;EACH,UAAU,EAAE,MAAM;EAClB,MAAM,EAAE,KAAK;EAEb,QAAE;IACE,OAAO,EAAE,KAAK;IACd,gBAAgB,EAtCZ,OAAO;IAuCX,KAAK,EAzCC,OAAO;IA0Cb,MAAM,EAAE,iBAAsB;IAC9B,aAAa,EAAE,GAAG;IAClB,OAAO,EAAE,YAAY;;AAI7B,OAAQ;EACJ,aAAa,EAAE,GAAG;EAClB,UAAU,EA9CD,eAAgB;EA+CzB,gBAAgB,EAAE,KAAK;EACvB,OAAO,EAAE,KAAK;EAEd,qBAAgB;IACZ,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,OAAO;EAG7B,WAAI;IACA,QAAQ,EAAE,IAAI;EAGlB,WAAK;IACD,SAAS,EAAE,IAAI;EAGnB,cAAO;IAUH,KAAK,EA/EC,OAAO;IAgFb,aAAa,EAAE,iBAAkB;IACjC,aAAa,EAAE,KAAK;IAXpB,kBAAI;MACA,SAAS,EAAE,IAAI;MACf,UAAU,EAAE,IAAI;MAChB,SAAS,EAAE,IAAI;MACf,UAAU,EAAE,IAAI;MAChB,WAAW,EAAE,KAAK;MAClB,KAAK,EAAE,IAAI;MACX,aAAa,EAAE,GAAG;EAO1B,cAAO;IACH,UAAU,EAAE,KAAK;IACjB,aAAa,EAAE,CAAC;EAGpB,UAAG;IACC,SAAS,EAAE,KAAK;IAChB,WAAW,EAAE,IAAI;;AAIzB,KAAM;EACF,WAAW,EAAE,IAAI;EACjB,OAAO,EAAE,YAAY;EACrB,aAAa,EAAE,GAAG;EAClB,SAAS,EAAE,IAAI;;AAGnB,+CAAgD;EAC5C,KAAK,EAAE,IAAI;EACX,MAAM,EAAE,QAAQ;EAEhB,0EAAW;IACP,KAAK,EAAE,GAAG;EAEd,0EAAW;IACP,KAAK,EAAE,GAAG;EAEd,0EAAW;IACP,KAAK,EAAE,GAAG;;AAKd,uBAAS;EACL,MAAM,EAAE,GAAG;EACX,KAAK,EAAE,yBAAyB;EAChC,OAAO,EAAE,GAAG;EACZ,MAAM,EAAE,CAAC;EACT,cAAc,EAAE,MAAM;AAG1B,qBAAO;EACH,MAAM,EAAE,IAAI;EAAC,KAAK,EAAE,IAAI;EACxB,OAAO,EAAE,GAAG;EACZ,gBAAgB,EAAE,IAAI;EACtB,aAAa,EAAE,GAAG;EAClB,MAAM,EAAE,CAAC;EACT,WAAW,EAAE,CAAC;EACd,cAAc,EAAE,MAAM;;AAI9B,mBAAoB;EAChB,OAAO,EAAE,YAAY;EACrB,UAAU,EAAE,MAAM;EAElB,yBAAM;IACF,OAAO,EAAE,IAAI;EAGjB,yBAAM;IACF,OAAO,EAAE,YAAY;IACrB,cAAc,EAAE,MAAM;IACtB,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,GAAG;IACZ,aAAa,EAAE,GAAG;IAClB,gBAAgB,EAAE,IAAI;IACtB,MAAM,EAAE,CAAC;IAET,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,OAAO;IAEf,6BAAI;MACA,UAAU,EAAE,IAAI;MAChB,SAAS,EAAE,IAAI;EAIvB,yCAAsB;IAClB,gBAAgB,EAAE,OAAO;IACzB,KAAK,EAAE,IAAI;;AAKnB,WAAY;EACR,UAAU,EAAE,MAAM;EAClB,UAAU,EAAE,KAAK;EAEjB,uBAAY;IACR,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,KAAK;IACd,MAAM,EAAE,iBAAkB;IAC1B,aAAa,EAAE,GAAG;IAClB,gBAAgB,EAlLV,OAAO;IAmLb,eAAe,EAAE,IAAI;IACrB,KAAK,EAtLC,OAAO;IAuLb,SAAS,EAAE,IAAI;IACf,UAAU,EAAE,MAAM;;AAI1B,yCAA0C;EAG9B,kBAAI;IACA,cAAc,EAAE,WAAW;IAC3B,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,MAAM;IACf,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,OAAO;IAClB,UAAU,EAAE,OAAO",
"sources": ["normalize.scss","style.scss"],
"names": [],
"file": "style.css"

View file

@ -62,6 +62,11 @@ article {
background-color: white;
padding: 0.5em;
&.reply-context {
margin-bottom: 0.5em;
background-color: #f3f3f3;
}
div {
overflow: auto;
}

View file

@ -1,18 +1,21 @@
from woodwind.models import Feed, Entry
from config import Config
from contextlib import contextmanager
from woodwind.models import Feed, Entry
import celery
import celery.utils.log
import datetime
import feedparser
import mf2py
import mf2util
import time
import urllib.parse
import datetime
import re
import sqlalchemy
import sqlalchemy.orm
import time
import urllib.parse
UPDATE_INTERVAL = datetime.timedelta(hours=1)
TWITTER_RE = re.compile(r'https?://(?:www\.|mobile\.)?twitter\.com/(\w+)/status(?:es)?/(\w+)')
app = celery.Celery('woodwind')
app.config_from_object('celeryconfig')
@ -84,6 +87,10 @@ def process_feed_for_new_entries(session, feed):
feed.entries.append(entry)
session.commit()
for in_reply_to in entry.get_property('in-reply-to', []):
fetch_reply_context.delay(entry.id, in_reply_to)
found_new = True
else:
logger.info('skipping previously seen post %s', old.permalink)
@ -103,7 +110,8 @@ def is_content_equal(e1, e2):
and e1.content == e2.content
and e1.author_name == e2.author_name
and e1.author_url == e2.author_url
and e1.author_photo == e2.author_photo)
and e1.author_photo == e2.author_photo
and e1.properties == e2.properties)
def process_xml_feed_for_new_entries(session, feed):
@ -173,43 +181,90 @@ def process_xml_feed_for_new_entries(session, feed):
def process_html_feed_for_new_entries(session, feed):
logger.info('fetching html feed: %s', feed)
now = datetime.datetime.utcnow()
parsed = mf2util.interpret_feed(
mf2py.Parser(url=feed.feed).to_dict(), feed.feed)
hfeed = parsed.get('entries', [])
for hentry in hfeed:
permalink = url = hentry.get('url')
uid = hentry.get('uid') or url
logger.debug('processing permalink %s. uid %s', permalink, uid)
if not uid:
continue
entry = hentry_to_entry(hentry, feed)
if entry:
logger.debug('built entry: %s', entry.permalink)
yield entry
# hentry = mf2util.interpret(mf2py.Parser(url=url).to_dict(), url)
# permalink = hentry.get('url') or url
# uid = hentry.get('uid') or uid
title = hentry.get('name')
content = hentry.get('content')
if not content:
content = title
title = None
def hentry_to_entry(hentry, feed):
now = datetime.datetime.utcnow()
permalink = url = hentry.get('url')
uid = hentry.get('uid') or url
if not uid:
return
entry = Entry(
uid=uid,
retrieved=now,
permalink=permalink,
published=hentry.get('published'),
updated=hentry.get('updated'),
title=title,
content=content,
author_name=hentry.get('author', {}).get('name'),
author_photo=hentry.get('author', {}).get('photo')
or fallback_photo(feed.origin),
author_url=hentry.get('author', {}).get('url'))
# hentry = mf2util.interpret(mf2py.Parser(url=url).to_dict(), url)
# permalink = hentry.get('url') or url
# uid = hentry.get('uid') or uid
title = hentry.get('name')
content = hentry.get('content')
if not content:
content = title
title = None
entry = Entry(
uid=uid,
retrieved=now,
permalink=permalink,
published=hentry.get('published'),
updated=hentry.get('updated'),
title=title,
content=content,
author_name=hentry.get('author', {}).get('name'),
author_photo=hentry.get('author', {}).get('photo')
or (feed and fallback_photo(feed.origin)),
author_url=hentry.get('author', {}).get('url'))
in_reply_to = hentry.get('in-reply-to')
if in_reply_to:
entry.set_property('in-reply-to', in_reply_to)
return entry
@app.task
def fetch_reply_context(entry_id, in_reply_to):
with session_scope() as session:
entry = session.query(Entry).get(entry_id)
context = session.query(Entry)\
.filter_by(permalink=in_reply_to).first()
if not context:
logger.info('fetching in-reply-to url: %s', in_reply_to)
parsed = mf2util.interpret(
mf2py.Parser(url=proxy_url(in_reply_to)).to_dict(),
in_reply_to)
if parsed:
context = hentry_to_entry(parsed, in_reply_to)
if context:
entry.reply_context.append(context)
session.commit()
def proxy_url(url):
if Config.TWITTER_AU_KEY and Config.TWITTER_AU_SECRET:
# swap out the a-u url for twitter urls
match = TWITTER_RE.match(url)
if match:
proxy_url = (
'https://twitter-activitystreams.appspot.com/@me/@all/@app/{}?'
.format(match.group(2)) + urllib.parse.urlencode({
'format': 'html',
'access_token_key': Config.TWITTER_AU_KEY,
'access_token_secret': Config.TWITTER_AU_SECRET,
}))
logger.debug('proxied twitter url %s', proxy_url)
return proxy_url
return url
logger.debug('built entry: %s', entry.permalink)
yield entry
def fallback_photo(url):

View file

@ -21,6 +21,31 @@
{% block body %}
{% for entry in entries %}
{% for context in entry.reply_context %}
<article class="reply-context">
<header>
{% if context.author_photo %}
<img src="{{context.author_photo}}"/>
{% endif %}
{% if context.author_name %}
{{ context.author_name }} -
{% endif %}
{{ context.feed.name }}
</header>
{% if context.title %}
<h1>{{ context.title }}</h1>
{% endif %}
{% if context.content %}
<div>
{{ context.content_cleaned() | add_preview }}
</div>
{% endif %}
<footer>
<a href="{{ context.permalink }}">{{ context.published }}</a>
</footer>
</article>
{% endfor %}
<article>
<header>
{% if entry.author_photo %}