commit d0a9851ae626985f9d7079ddcbea10aff76a36ea Author: Kyle Mahan Date: Mon Jan 26 08:07:11 2015 -0800 initial check in diff --git a/app.py b/app.py new file mode 100644 index 0000000..05180cd --- /dev/null +++ b/app.py @@ -0,0 +1,394 @@ +from config import Config +from flask.ext.login import LoginManager +from flask.ext.micropub import MicropubClient +from flask.ext.sqlalchemy import SQLAlchemy +import bleach +import bs4 +import datetime +import feedparser +import flask +import flask.ext.login as flask_login +import itertools +import mf2py +import mf2util +import requests +import time +import urllib.parse + + +app = flask.Flask(__name__) +app.config.from_object(Config) +db = SQLAlchemy(app) +micropub = MicropubClient(app, client_id='redwind-reader') +login_mgr = LoginManager(app) +login_mgr.login_view = 'login' + + +bleach.ALLOWED_TAGS += ['a', 'img', 'p', 'br', 'marquee', 'blink'] +bleach.ALLOWED_ATTRIBUTES.update({ + 'img': ['src', 'alt', 'title'] +}) + + +class User(db.Model): + id = db.Column(db.Integer, primary_key=True) + domain = db.Column(db.String(256)) + micropub_endpoint = db.Column(db.String(512)) + access_token = db.Column(db.String(512)) + + # Flask-Login integration + def is_authenticated(self): + return True + + def is_active(self): + return True + + def is_anonymous(self): + return False + + def get_id(self): + return self.domain + + def __eq__(self, other): + if type(other) is type(self): + return self.domain == other.domain + return False + + def __repr__(self): + return ''.format(self.domain) + + +class Feed(db.Model): + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey(User.id)) + user = db.relationship(User, backref='feeds') + # the name of this feed + name = db.Column(db.String(256)) + # url that we subscribed to; periodically check if the feed url + # has changed + origin = db.Column(db.String(512)) + # url of the feed itself + feed = db.Column(db.String(512)) + # h-feed, xml, etc. + type = db.Column(db.String(64)) + # last time this feed returned new data + last_updated = db.Column(db.DateTime) + # last time we checked this feed + last_checked = db.Column(db.DateTime) + etag = db.Column(db.String(512)) + + def __repr__(self): + return ''.format(self.name, self.feed) + + +class Entry(db.Model): + id = db.Column(db.Integer, primary_key=True) + feed_id = db.Column(db.Integer, db.ForeignKey(Feed.id)) + feed = db.relationship(Feed, backref='entries') + published = db.Column(db.DateTime) + updated = db.Column(db.DateTime) + retrieved = db.Column(db.DateTime) + uid = db.Column(db.String(512)) + permalink = db.Column(db.String(512)) + author_name = db.Column(db.String(512)) + author_url = db.Column(db.String(512)) + author_photo = db.Column(db.String(512)) + title = db.Column(db.String(512)) + content = db.Column(db.Text) + + def content_cleaned(self): + if self.content: + return bleach.clean(self.content, strip=True) + + def __repr__(self): + return ''.format(self.title, (self.content or '')[:140]) + + +@app.route('/') +def index(): + if flask_login.current_user.is_authenticated(): + feed_ids = [f.id for f in flask_login.current_user.feeds] + entries = Entry.query.filter( + Entry.feed_id.in_(feed_ids)).order_by( + Entry.published.desc()).limit(100).all() + else: + entries = [] + return flask.render_template('feed.jinja2', entries=entries) + + +@app.route('/install') +def install(): + db.drop_all() + db.create_all() + + user = User(domain='kylewm.com',) + db.session.add(user) + db.session.commit() + + flask_login.login_user(user) + + return 'Success!' + + +def process_feed_for_new_entries(feed): + if feed.type == 'xml': + return process_xml_feed_for_new_entries(feed) + elif feed.type == 'html': + return process_html_feed_for_new_entries(feed) + + +def process_xml_feed_for_new_entries(feed): + app.logger.debug('updating feed: %s', feed) + + now = datetime.datetime.utcnow() + parsed = feedparser.parse(feed.feed) + + feed_props = parsed.get('feed', {}) + default_author_url = feed_props.get('author_detail', {}).get('href') + default_author_name = feed_props.get('author_detail', {}).get('name') + default_author_photo = feed_props.get('logo') + + all_uids = [e.id or e.link for e in parsed.entries] + preexisting = set(row[0] for row in db.session.query(Entry.uid) + .filter(Entry.uid.in_(all_uids)) + .filter(Entry.feed == feed)) + + for p_entry in parsed.entries: + permalink = p_entry.link + uid = p_entry.id or permalink + + if not uid or uid in preexisting: + continue + + updated = datetime.datetime.fromtimestamp( + time.mktime(p_entry.updated_parsed) + ) if p_entry.updated_parsed else None + published = datetime.datetime.fromtimestamp( + time.mktime(p_entry.published_parsed) + ) if p_entry.published_parsed else None + + title = p_entry.get('title') + + content = None + content_list = p_entry.get('content') + if content_list: + content = content_list[0].value + else: + content = p_entry.get('summary') + + if title and content: + title_trimmed = title.rstrip('...').rstrip('…') + if content.startswith(title_trimmed): + title = None + + entry = Entry( + feed=feed, + published=published, + updated=updated, + uid=uid, + permalink=permalink, + retrieved=now, + title=p_entry.get('title'), + content=content, + author_name=p_entry.get('author_detail', {}).get('name') + or default_author_name, + author_url=p_entry.get('author_detail', {}).get('href') + or default_author_url, + author_photo=default_author_photo) + + db.session.add(entry) + db.session.commit() + yield entry + + +def process_html_feed_for_new_entries(feed): + app.logger.debug('updating feed: %s', feed) + + now = datetime.datetime.utcnow() + parsed = mf2util.interpret_feed( + mf2py.parse(url=feed.feed), feed.feed) + hfeed = parsed.get('entries', []) + + all_uids = [e.get('uid') or e.get('url') for e in hfeed] + preexisting = set(row[0] for row in db.session.query(Entry.uid) + .filter(Entry.uid.in_(all_uids)) + .filter(Entry.feed == feed)) + + # app.logger.debug('preexisting urls: %r', preexisting) + + for hentry in hfeed: + permalink = url = hentry.get('url') + uid = hentry.get('uid') or url + + if not uid or uid in preexisting: + continue + + # hentry = mf2util.interpret(mf2py.parse(url=url), url) + # permalink = hentry.get('url') or url + # uid = hentry.get('uid') or uid + entry = Entry( + feed=feed, + published=hentry.get('published'), + updated=hentry.get('updated'), + uid=uid, + permalink=permalink, + retrieved=now, + title=hentry.get('name'), + content=hentry.get('content'), + author_name=hentry.get('author', {}).get('name'), + author_photo=hentry.get('author', {}).get('photo'), + author_url=hentry.get('author', {}).get('url')) + db.session.add(entry) + db.session.commit() + app.logger.debug('saved entry: %s', entry.permalink) + yield entry + + +@app.route('/update') +def update(): + new_urls = [] + for feed in Feed.query.all(): + new_entries = process_feed_for_new_entries(feed) + for entry in new_entries: + new_urls.append(entry.permalink) + return ('Success!
    ' + '\n'.join( + '
  • ' + url + '
  • ' for url in new_urls) + '
') + + +@app.route('/login') +def login(): + if True: + flask_login.login_user(User.query.all()[0], remember=True) + + me = flask.request.args.get('me') + if me: + return micropub.authorize( + me, flask.url_for('login_callback', _external=True), + next_url=flask.request.args.get('next'), + scope='write') + return flask.render_template('login.jinja2') + + +@app.route('/login-callback') +@micropub.authorized_handler +def login_callback(resp): + if not resp.me: + flask.flash('Login error: ' + resp.error) + return flask.redirect(flask.url_for('login')) + + domain = urllib.parse.urlparse(resp.me).netloc + user = load_user(domain) + if not user: + user = User() + user.domain = domain + db.session.add(user) + + user.micropub_endpoint = resp.micropub_endpoint + user.access_token = resp.access_token + db.session.commit() + + flask_login.login_user(user, remember=True) + return flask.redirect(resp.next_url or flask.url_for('index')) + + +@login_mgr.user_loader +def load_user(domain): + return User.query.filter_by(domain=domain).first() + + +@app.route('/subscribe', methods=['GET', 'POST']) +def subscribe(): + if flask.request.method == 'POST': + origin = flask.request.form.get('origin') + if origin: + type = None + feed = None + typed_feed = flask.request.form.get('feed') + if typed_feed: + type, feed = typed_feed.split('|', 1) + else: + feeds = find_possible_feeds(origin) + if not feeds: + flask.flash('No feeds found for: ' + origin) + return flask.redirect(flask.url_for('subscribe')) + if len(feeds) > 1: + return flask.render_template( + 'select-feed.jinja2', origin=origin, feeds=feeds) + feed = feeds[0]['feed'] + type = feeds[0]['type'] + new_feed = add_subscription(origin, feed, type) + flask.flash('Successfully subscribed to: {}'.format(new_feed.name)) + return flask.redirect(flask.url_for('index')) + else: + flask.abort(400) + + return flask.render_template('subscribe.jinja2') + + +def add_subscription(origin, feed, type): + if type == 'html': + parsed = mf2util.interpret_feed(mf2py.parse(url=feed), feed) + name = parsed.get('name') + if not name or len(name) > 140: + p = urllib.parse.urlparse(origin) + name = p.netloc + p.path + + feed = Feed(user=flask_login.current_user, name=name, + origin=origin, feed=feed, type=type) + + db.session.add(feed) + db.session.commit() + return feed + + elif type == 'xml': + parsed = feedparser.parse(feed) + feed = Feed(user=flask_login.current_user, + name=parsed.feed.title, origin=origin, feed=feed, + type=type) + + db.session.add(feed) + db.session.commit() + return feed + + +def find_possible_feeds(origin): + # scrape an origin source to find possible alternative feeds + resp = requests.get(origin) + + feeds = [] + xml_feed_types = [ + 'application/rss+xml', + 'application/atom+xml', + 'application/rdf+xml', + ] + + content_type = resp.headers['content-type'] + content_type = content_type.split(';', 1)[0].strip() + if content_type in xml_feed_types: + feeds.append({ + 'origin': origin, + 'feed': origin, + 'type': 'xml', + }) + + elif content_type == 'text/html': + # if text/html, then parse and look for rel="alternate" + soup = bs4.BeautifulSoup(resp.text) + for link in soup.find_all('link', {'rel': 'alternate'}): + if link.get('type') in xml_feed_types: + feeds.append({ + 'origin': origin, + 'feed': link.get('href'), + 'type': 'xml', + }) + feeds.append({ + 'origin': origin, + 'feed': origin, + 'type': 'html', + }) + + return feeds + + +if __name__ == '__main__': + app.run(debug=True, port=4000) diff --git a/static/normalize.scss b/static/normalize.scss new file mode 100644 index 0000000..458eea1 --- /dev/null +++ b/static/normalize.scss @@ -0,0 +1,427 @@ +/*! normalize.css v3.0.2 | MIT License | git.io/normalize */ + +/** + * 1. Set default font family to sans-serif. + * 2. Prevent iOS text size adjust after orientation change, without disabling + * user zoom. + */ + +html { + font-family: sans-serif; /* 1 */ + -ms-text-size-adjust: 100%; /* 2 */ + -webkit-text-size-adjust: 100%; /* 2 */ +} + +/** + * Remove default margin. + */ + +body { + margin: 0; +} + +/* HTML5 display definitions + ========================================================================== */ + +/** + * Correct `block` display not defined for any HTML5 element in IE 8/9. + * Correct `block` display not defined for `details` or `summary` in IE 10/11 + * and Firefox. + * Correct `block` display not defined for `main` in IE 11. + */ + +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +main, +menu, +nav, +section, +summary { + display: block; +} + +/** + * 1. Correct `inline-block` display not defined in IE 8/9. + * 2. Normalize vertical alignment of `progress` in Chrome, Firefox, and Opera. + */ + +audio, +canvas, +progress, +video { + display: inline-block; /* 1 */ + vertical-align: baseline; /* 2 */ +} + +/** + * Prevent modern browsers from displaying `audio` without controls. + * Remove excess height in iOS 5 devices. + */ + +audio:not([controls]) { + display: none; + height: 0; +} + +/** + * Address `[hidden]` styling not present in IE 8/9/10. + * Hide the `template` element in IE 8/9/11, Safari, and Firefox < 22. + */ + +[hidden], +template { + display: none; +} + +/* Links + ========================================================================== */ + +/** + * Remove the gray background color from active links in IE 10. + */ + +a { + background-color: transparent; +} + +/** + * Improve readability when focused and also mouse hovered in all browsers. + */ + +a:active, +a:hover { + outline: 0; +} + +/* Text-level semantics + ========================================================================== */ + +/** + * Address styling not present in IE 8/9/10/11, Safari, and Chrome. + */ + +abbr[title] { + border-bottom: 1px dotted; +} + +/** + * Address style set to `bolder` in Firefox 4+, Safari, and Chrome. + */ + +b, +strong { + font-weight: bold; +} + +/** + * Address styling not present in Safari and Chrome. + */ + +dfn { + font-style: italic; +} + +/** + * Address variable `h1` font-size and margin within `section` and `article` + * contexts in Firefox 4+, Safari, and Chrome. + */ + +h1 { + font-size: 2em; + margin: 0.67em 0; +} + +/** + * Address styling not present in IE 8/9. + */ + +mark { + background: #ff0; + color: #000; +} + +/** + * Address inconsistent and variable font size in all browsers. + */ + +small { + font-size: 80%; +} + +/** + * Prevent `sub` and `sup` affecting `line-height` in all browsers. + */ + +sub, +sup { + font-size: 75%; + line-height: 0; + position: relative; + vertical-align: baseline; +} + +sup { + top: -0.5em; +} + +sub { + bottom: -0.25em; +} + +/* Embedded content + ========================================================================== */ + +/** + * Remove border when inside `a` element in IE 8/9/10. + */ + +img { + border: 0; +} + +/** + * Correct overflow not hidden in IE 9/10/11. + */ + +svg:not(:root) { + overflow: hidden; +} + +/* Grouping content + ========================================================================== */ + +/** + * Address margin not present in IE 8/9 and Safari. + */ + +figure { + margin: 1em 40px; +} + +/** + * Address differences between Firefox and other browsers. + */ + +hr { + -moz-box-sizing: content-box; + box-sizing: content-box; + height: 0; +} + +/** + * Contain overflow in all browsers. + */ + +pre { + overflow: auto; +} + +/** + * Address odd `em`-unit font size rendering in all browsers. + */ + +code, +kbd, +pre, +samp { + font-family: monospace, monospace; + font-size: 1em; +} + +/* Forms + ========================================================================== */ + +/** + * Known limitation: by default, Chrome and Safari on OS X allow very limited + * styling of `select`, unless a `border` property is set. + */ + +/** + * 1. Correct color not being inherited. + * Known issue: affects color of disabled elements. + * 2. Correct font properties not being inherited. + * 3. Address margins set differently in Firefox 4+, Safari, and Chrome. + */ + +button, +input, +optgroup, +select, +textarea { + color: inherit; /* 1 */ + font: inherit; /* 2 */ + margin: 0; /* 3 */ +} + +/** + * Address `overflow` set to `hidden` in IE 8/9/10/11. + */ + +button { + overflow: visible; +} + +/** + * Address inconsistent `text-transform` inheritance for `button` and `select`. + * All other form control elements do not inherit `text-transform` values. + * Correct `button` style inheritance in Firefox, IE 8/9/10/11, and Opera. + * Correct `select` style inheritance in Firefox. + */ + +button, +select { + text-transform: none; +} + +/** + * 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` + * and `video` controls. + * 2. Correct inability to style clickable `input` types in iOS. + * 3. Improve usability and consistency of cursor style between image-type + * `input` and others. + */ + +button, +html input[type="button"], /* 1 */ +input[type="reset"], +input[type="submit"] { + -webkit-appearance: button; /* 2 */ + cursor: pointer; /* 3 */ +} + +/** + * Re-set default cursor for disabled elements. + */ + +button[disabled], +html input[disabled] { + cursor: default; +} + +/** + * Remove inner padding and border in Firefox 4+. + */ + +button::-moz-focus-inner, +input::-moz-focus-inner { + border: 0; + padding: 0; +} + +/** + * Address Firefox 4+ setting `line-height` on `input` using `!important` in + * the UA stylesheet. + */ + +input { + line-height: normal; +} + +/** + * It's recommended that you don't attempt to style these elements. + * Firefox's implementation doesn't respect box-sizing, padding, or width. + * + * 1. Address box sizing set to `content-box` in IE 8/9/10. + * 2. Remove excess padding in IE 8/9/10. + */ + +input[type="checkbox"], +input[type="radio"] { + box-sizing: border-box; /* 1 */ + padding: 0; /* 2 */ +} + +/** + * Fix the cursor style for Chrome's increment/decrement buttons. For certain + * `font-size` values of the `input`, it causes the cursor style of the + * decrement button to change from `default` to `text`. + */ + +input[type="number"]::-webkit-inner-spin-button, +input[type="number"]::-webkit-outer-spin-button { + height: auto; +} + +/** + * 1. Address `appearance` set to `searchfield` in Safari and Chrome. + * 2. Address `box-sizing` set to `border-box` in Safari and Chrome + * (include `-moz` to future-proof). + */ + +input[type="search"] { + -webkit-appearance: textfield; /* 1 */ + -moz-box-sizing: content-box; + -webkit-box-sizing: content-box; /* 2 */ + box-sizing: content-box; +} + +/** + * Remove inner padding and search cancel button in Safari and Chrome on OS X. + * Safari (but not Chrome) clips the cancel button when the search input has + * padding (and `textfield` appearance). + */ + +input[type="search"]::-webkit-search-cancel-button, +input[type="search"]::-webkit-search-decoration { + -webkit-appearance: none; +} + +/** + * Define consistent border, margin, and padding. + */ + +fieldset { + border: 1px solid #c0c0c0; + margin: 0 2px; + padding: 0.35em 0.625em 0.75em; +} + +/** + * 1. Correct `color` not being inherited in IE 8/9/10/11. + * 2. Remove padding so people aren't caught out if they zero out fieldsets. + */ + +legend { + border: 0; /* 1 */ + padding: 0; /* 2 */ +} + +/** + * Remove default vertical scrollbar in IE 8/9/10/11. + */ + +textarea { + overflow: auto; +} + +/** + * Don't inherit the `font-weight` (applied by a rule above). + * NOTE: the default cannot safely be changed in Chrome and Safari on OS X. + */ + +optgroup { + font-weight: bold; +} + +/* Tables + ========================================================================== */ + +/** + * Remove most spacing between table cells. + */ + +table { + border-collapse: collapse; + border-spacing: 0; +} + +td, +th { + padding: 0; +} diff --git a/static/style.scss b/static/style.scss new file mode 100644 index 0000000..14497a0 --- /dev/null +++ b/static/style.scss @@ -0,0 +1,62 @@ +@import "normalize"; + + +$title-font: Helvetica, Arial, sans-serif; +/*$body-font: Georgia, Times New Roman, serif;*/ +$body-font: $title-font; + +/* Subtlety of Hue */ +$lunar-green: #484A47; +$pine-glade: #C1CE96; +$athens-gray: #ECEBF0; +$sirocco: #687D77; +$armadillo: #353129; + +$box-shadow: 0 0 2px $sirocco; + + +body { + font: 12pt/1.5em $body-font; + background: $athens-gray; +} + + +main { + max-width: 600px; + margin: 0 auto; +} + +article { + margin-bottom: 2em; + box-shadow: $box-shadow; + background-color: white; + padding: 0.5em; + + img { + max-width: 100%; + } + + header { + img { + max-width: 64px; + max-height: 64px; + margin-left: -85px; + float: left; + border-radius: 4px; + } + } +} + +@media screen and (max-width: 640px) { + article { + header { + img { + vertical-align: text-middle; + margin: inherit; + display: inline; + max-width: 1.2em; + max-height: 1.2em; + } + } + } +} diff --git a/templates/base.jinja2 b/templates/base.jinja2 new file mode 100644 index 0000000..9c563b4 --- /dev/null +++ b/templates/base.jinja2 @@ -0,0 +1,18 @@ + + + + + + + {% block head %}{% endblock %} + + + {% if current_user.is_authenticated() %} +
{{ current_user.domain }}
+ {% endif %} + {% for message in get_flashed_messages() %} +
{{ message | safe }}
+ {% endfor %} + {% block body %}{% endblock %} + + diff --git a/templates/feed.jinja2 b/templates/feed.jinja2 new file mode 100644 index 0000000..bad80d2 --- /dev/null +++ b/templates/feed.jinja2 @@ -0,0 +1,26 @@ +{% extends "base.jinja2" %} +{% block body %} + +
+ {% for entry in entries %} +
+
+ + {{ entry.author_name }} - {{ entry.feed.name }} +
+ {% if entry.title %} +

{{ entry.title }}

+ {% endif %} + {% if entry.content %} +
+ {{ entry.content_cleaned() }} +
+ {% endif %} + +
+ {% endfor %} +
+ +{% endblock body %} diff --git a/templates/login.jinja2 b/templates/login.jinja2 new file mode 100644 index 0000000..9d83e59 --- /dev/null +++ b/templates/login.jinja2 @@ -0,0 +1,9 @@ +{% extends "base.jinja2" %} +{% block body %} +
+
+ + +
+
+{% endblock %} diff --git a/templates/select-feed.jinja2 b/templates/select-feed.jinja2 new file mode 100644 index 0000000..2a784ce --- /dev/null +++ b/templates/select-feed.jinja2 @@ -0,0 +1,23 @@ +{% extends "base.jinja2" %} +{% block body %} +
+
+ + + {% for feed in feeds %} +

+ + +

+ {% endfor %} + + + +
+ + +
+{% endblock body %} diff --git a/templates/subscribe.jinja2 b/templates/subscribe.jinja2 new file mode 100644 index 0000000..2d7dc87 --- /dev/null +++ b/templates/subscribe.jinja2 @@ -0,0 +1,10 @@ +{% extends "base.jinja2" %} +{% block body %} +
+
+ + +
+ +
+{% endblock body %}