diff --git a/requirements.txt b/requirements.txt index 0dbacb2..f502369 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,8 +17,8 @@ idna==2.1 itsdangerous==0.24 Jinja2==2.8 MarkupSafe==0.23 -mf2py==1.0.4 -mf2util==0.4.1 +mf2py==1.0.5 +mf2util==0.4.2 psycopg2==2.6.1 pyasn1==0.1.9 pycparser==2.14 diff --git a/woodwind/models.py b/woodwind/models.py index a4511af..0fa6d76 100644 --- a/woodwind/models.py +++ b/woodwind/models.py @@ -115,6 +115,7 @@ class Entry(db.Model): feed = db.relationship(Feed, backref='entries') published = db.Column(db.DateTime) updated = db.Column(db.DateTime) + deleted = db.Column(db.DateTime) retrieved = db.Column(db.DateTime, index=True) uid = db.Column(db.String(512)) permalink = db.Column(db.String(512), index=True) diff --git a/woodwind/tasks.py b/woodwind/tasks.py index 6a192ba..41d3a7d 100644 --- a/woodwind/tasks.py +++ b/woodwind/tasks.py @@ -471,6 +471,18 @@ def process_html_feed_for_new_entries(feed, content, backfill, now): def hentry_to_entry(hentry, feed, backfill, now): + def normalize_datetime(dt): + if (dt and hasattr(dt, 'year') and hasattr(dt, 'month') + and hasattr(dt, 'day')): + # make sure published is in UTC and strip the timezone + if hasattr(dt, 'tzinfo') and dt.tzinfo: + return dt.astimezone(datetime.timezone.utc).replace( + tzinfo=None) + # convert datetime.date to datetime.datetime + elif not hasattr(dt, 'hour'): + return datetime.datetime(year=dt.year, month=dt.month, + day=dt.day) + permalink = url = hentry.get('url') uid = hentry.get('uid') or url if not uid: @@ -494,20 +506,9 @@ def hentry_to_entry(hentry, feed, backfill, now): content = title title = None - published = hentry.get('published') - updated = hentry.get('updated') - - if published: - # make sure published is in UTC and strip the timezone - if hasattr(published, 'tzinfo') and published.tzinfo: - published = published.astimezone(datetime.timezone.utc)\ - .replace(tzinfo=None) - # convert datetime.date to datetime.datetime - elif not hasattr(published, 'hour'): - published = datetime.datetime( - year=published.year, - month=published.month, - day=published.day) + published = normalize_datetime(hentry.get('published')) + updated = normalize_datetime(hentry.get('updated')) + deleted = normalize_datetime(hentry.get('deleted')) # retrieved time is now unless we're backfilling old posts retrieved = now @@ -525,6 +526,7 @@ def hentry_to_entry(hentry, feed, backfill, now): permalink=permalink, published=published, updated=updated, + deleted=deleted, title=title, content=content, content_cleaned=util.clean(content), diff --git a/woodwind/views.py b/woodwind/views.py index b71bb07..457d63e 100644 --- a/woodwind/views.py +++ b/woodwind/views.py @@ -32,6 +32,7 @@ def index(): ws_topic = None solo = False all_tags = set() + now = datetime.datetime.now() if flask_login.current_user.is_authenticated: for subsc in flask_login.current_user.subscriptions: @@ -48,7 +49,9 @@ def index(): .join(Entry.feed)\ .join(Feed.subscriptions)\ .join(Subscription.user)\ - .filter(User.id == flask_login.current_user.id) + .filter(User.id == flask_login.current_user.id)\ + .filter(db.or_(Entry.deleted == None, + Entry.deleted >= now)) if 'entry' in flask.request.args: entry_url = flask.request.args.get('entry')