remove one-module app
This commit is contained in:
Kyle Mahan
parent
9e1e3ad03f
commit
65667e6010
1 changed files with 0 additions and 394 deletions
394
woodwind.py
394
woodwind.py
|
|
@ -1,394 +0,0 @@
|
||||||
from config import Config
|
|
||||||
from flask.ext.login import LoginManager
|
|
||||||
from flask.ext.micropub import MicropubClient
|
|
||||||
from flask.ext.sqlalchemy import SQLAlchemy
|
|
||||||
import bleach
|
|
||||||
import bs4
|
|
||||||
import datetime
|
|
||||||
import feedparser
|
|
||||||
import flask
|
|
||||||
import flask.ext.login as flask_login
|
|
||||||
import itertools
|
|
||||||
import mf2py
|
|
||||||
import mf2util
|
|
||||||
import requests
|
|
||||||
import time
|
|
||||||
import urllib.parse
|
|
||||||
|
|
||||||
|
|
||||||
app = flask.Flask(__name__)
|
|
||||||
app.config.from_object(Config)
|
|
||||||
db = SQLAlchemy(app)
|
|
||||||
micropub = MicropubClient(app, client_id='redwind-reader')
|
|
||||||
login_mgr = LoginManager(app)
|
|
||||||
login_mgr.login_view = 'login'
|
|
||||||
|
|
||||||
|
|
||||||
bleach.ALLOWED_TAGS += ['a', 'img', 'p', 'br', 'marquee', 'blink']
|
|
||||||
bleach.ALLOWED_ATTRIBUTES.update({
|
|
||||||
'img': ['src', 'alt', 'title']
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
class User(db.Model):
|
|
||||||
id = db.Column(db.Integer, primary_key=True)
|
|
||||||
domain = db.Column(db.String(256))
|
|
||||||
micropub_endpoint = db.Column(db.String(512))
|
|
||||||
access_token = db.Column(db.String(512))
|
|
||||||
|
|
||||||
# Flask-Login integration
|
|
||||||
def is_authenticated(self):
|
|
||||||
return True
|
|
||||||
|
|
||||||
def is_active(self):
|
|
||||||
return True
|
|
||||||
|
|
||||||
def is_anonymous(self):
|
|
||||||
return False
|
|
||||||
|
|
||||||
def get_id(self):
|
|
||||||
return self.domain
|
|
||||||
|
|
||||||
def __eq__(self, other):
|
|
||||||
if type(other) is type(self):
|
|
||||||
return self.domain == other.domain
|
|
||||||
return False
|
|
||||||
|
|
||||||
def __repr__(self):
|
|
||||||
return '<User:{}>'.format(self.domain)
|
|
||||||
|
|
||||||
|
|
||||||
class Feed(db.Model):
|
|
||||||
id = db.Column(db.Integer, primary_key=True)
|
|
||||||
user_id = db.Column(db.Integer, db.ForeignKey(User.id))
|
|
||||||
user = db.relationship(User, backref='feeds')
|
|
||||||
# the name of this feed
|
|
||||||
name = db.Column(db.String(256))
|
|
||||||
# url that we subscribed to; periodically check if the feed url
|
|
||||||
# has changed
|
|
||||||
origin = db.Column(db.String(512))
|
|
||||||
# url of the feed itself
|
|
||||||
feed = db.Column(db.String(512))
|
|
||||||
# h-feed, xml, etc.
|
|
||||||
type = db.Column(db.String(64))
|
|
||||||
# last time this feed returned new data
|
|
||||||
last_updated = db.Column(db.DateTime)
|
|
||||||
# last time we checked this feed
|
|
||||||
last_checked = db.Column(db.DateTime)
|
|
||||||
etag = db.Column(db.String(512))
|
|
||||||
|
|
||||||
def __repr__(self):
|
|
||||||
return '<Feed:{},{}>'.format(self.name, self.feed)
|
|
||||||
|
|
||||||
|
|
||||||
class Entry(db.Model):
|
|
||||||
id = db.Column(db.Integer, primary_key=True)
|
|
||||||
feed_id = db.Column(db.Integer, db.ForeignKey(Feed.id))
|
|
||||||
feed = db.relationship(Feed, backref='entries')
|
|
||||||
published = db.Column(db.DateTime)
|
|
||||||
updated = db.Column(db.DateTime)
|
|
||||||
retrieved = db.Column(db.DateTime)
|
|
||||||
uid = db.Column(db.String(512))
|
|
||||||
permalink = db.Column(db.String(512))
|
|
||||||
author_name = db.Column(db.String(512))
|
|
||||||
author_url = db.Column(db.String(512))
|
|
||||||
author_photo = db.Column(db.String(512))
|
|
||||||
title = db.Column(db.String(512))
|
|
||||||
content = db.Column(db.Text)
|
|
||||||
|
|
||||||
def content_cleaned(self):
|
|
||||||
if self.content:
|
|
||||||
return bleach.clean(self.content, strip=True)
|
|
||||||
|
|
||||||
def __repr__(self):
|
|
||||||
return '<Entry:{},{}>'.format(self.title, (self.content or '')[:140])
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/')
|
|
||||||
def index():
|
|
||||||
if flask_login.current_user.is_authenticated():
|
|
||||||
feed_ids = [f.id for f in flask_login.current_user.feeds]
|
|
||||||
entries = Entry.query.filter(
|
|
||||||
Entry.feed_id.in_(feed_ids)).order_by(
|
|
||||||
Entry.published.desc()).limit(100).all()
|
|
||||||
else:
|
|
||||||
entries = []
|
|
||||||
return flask.render_template('feed.jinja2', entries=entries)
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/install')
|
|
||||||
def install():
|
|
||||||
db.drop_all()
|
|
||||||
db.create_all()
|
|
||||||
|
|
||||||
user = User(domain='kylewm.com',)
|
|
||||||
db.session.add(user)
|
|
||||||
db.session.commit()
|
|
||||||
|
|
||||||
flask_login.login_user(user)
|
|
||||||
|
|
||||||
return 'Success!'
|
|
||||||
|
|
||||||
|
|
||||||
def process_feed_for_new_entries(feed):
|
|
||||||
if feed.type == 'xml':
|
|
||||||
return process_xml_feed_for_new_entries(feed)
|
|
||||||
elif feed.type == 'html':
|
|
||||||
return process_html_feed_for_new_entries(feed)
|
|
||||||
|
|
||||||
|
|
||||||
def process_xml_feed_for_new_entries(feed):
|
|
||||||
app.logger.debug('updating feed: %s', feed)
|
|
||||||
|
|
||||||
now = datetime.datetime.utcnow()
|
|
||||||
parsed = feedparser.parse(feed.feed)
|
|
||||||
|
|
||||||
feed_props = parsed.get('feed', {})
|
|
||||||
default_author_url = feed_props.get('author_detail', {}).get('href')
|
|
||||||
default_author_name = feed_props.get('author_detail', {}).get('name')
|
|
||||||
default_author_photo = feed_props.get('logo')
|
|
||||||
|
|
||||||
all_uids = [e.id or e.link for e in parsed.entries]
|
|
||||||
preexisting = set(row[0] for row in db.session.query(Entry.uid)
|
|
||||||
.filter(Entry.uid.in_(all_uids))
|
|
||||||
.filter(Entry.feed == feed))
|
|
||||||
|
|
||||||
for p_entry in parsed.entries:
|
|
||||||
permalink = p_entry.link
|
|
||||||
uid = p_entry.id or permalink
|
|
||||||
|
|
||||||
if not uid or uid in preexisting:
|
|
||||||
continue
|
|
||||||
|
|
||||||
updated = datetime.datetime.fromtimestamp(
|
|
||||||
time.mktime(p_entry.updated_parsed)
|
|
||||||
) if p_entry.updated_parsed else None
|
|
||||||
published = datetime.datetime.fromtimestamp(
|
|
||||||
time.mktime(p_entry.published_parsed)
|
|
||||||
) if p_entry.published_parsed else None
|
|
||||||
|
|
||||||
title = p_entry.get('title')
|
|
||||||
|
|
||||||
content = None
|
|
||||||
content_list = p_entry.get('content')
|
|
||||||
if content_list:
|
|
||||||
content = content_list[0].value
|
|
||||||
else:
|
|
||||||
content = p_entry.get('summary')
|
|
||||||
|
|
||||||
if title and content:
|
|
||||||
title_trimmed = title.rstrip('...').rstrip('…')
|
|
||||||
if content.startswith(title_trimmed):
|
|
||||||
title = None
|
|
||||||
|
|
||||||
entry = Entry(
|
|
||||||
feed=feed,
|
|
||||||
published=published,
|
|
||||||
updated=updated,
|
|
||||||
uid=uid,
|
|
||||||
permalink=permalink,
|
|
||||||
retrieved=now,
|
|
||||||
title=p_entry.get('title'),
|
|
||||||
content=content,
|
|
||||||
author_name=p_entry.get('author_detail', {}).get('name')
|
|
||||||
or default_author_name,
|
|
||||||
author_url=p_entry.get('author_detail', {}).get('href')
|
|
||||||
or default_author_url,
|
|
||||||
author_photo=default_author_photo)
|
|
||||||
|
|
||||||
db.session.add(entry)
|
|
||||||
db.session.commit()
|
|
||||||
yield entry
|
|
||||||
|
|
||||||
|
|
||||||
def process_html_feed_for_new_entries(feed):
|
|
||||||
app.logger.debug('updating feed: %s', feed)
|
|
||||||
|
|
||||||
now = datetime.datetime.utcnow()
|
|
||||||
parsed = mf2util.interpret_feed(
|
|
||||||
mf2py.parse(url=feed.feed), feed.feed)
|
|
||||||
hfeed = parsed.get('entries', [])
|
|
||||||
|
|
||||||
all_uids = [e.get('uid') or e.get('url') for e in hfeed]
|
|
||||||
preexisting = set(row[0] for row in db.session.query(Entry.uid)
|
|
||||||
.filter(Entry.uid.in_(all_uids))
|
|
||||||
.filter(Entry.feed == feed))
|
|
||||||
|
|
||||||
# app.logger.debug('preexisting urls: %r', preexisting)
|
|
||||||
|
|
||||||
for hentry in hfeed:
|
|
||||||
permalink = url = hentry.get('url')
|
|
||||||
uid = hentry.get('uid') or url
|
|
||||||
|
|
||||||
if not uid or uid in preexisting:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# hentry = mf2util.interpret(mf2py.parse(url=url), url)
|
|
||||||
# permalink = hentry.get('url') or url
|
|
||||||
# uid = hentry.get('uid') or uid
|
|
||||||
entry = Entry(
|
|
||||||
feed=feed,
|
|
||||||
published=hentry.get('published'),
|
|
||||||
updated=hentry.get('updated'),
|
|
||||||
uid=uid,
|
|
||||||
permalink=permalink,
|
|
||||||
retrieved=now,
|
|
||||||
title=hentry.get('name'),
|
|
||||||
content=hentry.get('content'),
|
|
||||||
author_name=hentry.get('author', {}).get('name'),
|
|
||||||
author_photo=hentry.get('author', {}).get('photo'),
|
|
||||||
author_url=hentry.get('author', {}).get('url'))
|
|
||||||
db.session.add(entry)
|
|
||||||
db.session.commit()
|
|
||||||
app.logger.debug('saved entry: %s', entry.permalink)
|
|
||||||
yield entry
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/update')
|
|
||||||
def update():
|
|
||||||
new_urls = []
|
|
||||||
for feed in Feed.query.all():
|
|
||||||
new_entries = process_feed_for_new_entries(feed)
|
|
||||||
for entry in new_entries:
|
|
||||||
new_urls.append(entry.permalink)
|
|
||||||
return ('Success!<ul>' + '\n'.join(
|
|
||||||
'<li>' + url + '</li>' for url in new_urls) + '</ul>')
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/login')
|
|
||||||
def login():
|
|
||||||
if True:
|
|
||||||
flask_login.login_user(User.query.all()[0], remember=True)
|
|
||||||
|
|
||||||
me = flask.request.args.get('me')
|
|
||||||
if me:
|
|
||||||
return micropub.authorize(
|
|
||||||
me, flask.url_for('login_callback', _external=True),
|
|
||||||
next_url=flask.request.args.get('next'),
|
|
||||||
scope='write')
|
|
||||||
return flask.render_template('login.jinja2')
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/login-callback')
|
|
||||||
@micropub.authorized_handler
|
|
||||||
def login_callback(resp):
|
|
||||||
if not resp.me:
|
|
||||||
flask.flash('Login error: ' + resp.error)
|
|
||||||
return flask.redirect(flask.url_for('login'))
|
|
||||||
|
|
||||||
domain = urllib.parse.urlparse(resp.me).netloc
|
|
||||||
user = load_user(domain)
|
|
||||||
if not user:
|
|
||||||
user = User()
|
|
||||||
user.domain = domain
|
|
||||||
db.session.add(user)
|
|
||||||
|
|
||||||
user.micropub_endpoint = resp.micropub_endpoint
|
|
||||||
user.access_token = resp.access_token
|
|
||||||
db.session.commit()
|
|
||||||
|
|
||||||
flask_login.login_user(user, remember=True)
|
|
||||||
return flask.redirect(resp.next_url or flask.url_for('index'))
|
|
||||||
|
|
||||||
|
|
||||||
@login_mgr.user_loader
|
|
||||||
def load_user(domain):
|
|
||||||
return User.query.filter_by(domain=domain).first()
|
|
||||||
|
|
||||||
|
|
||||||
@app.route('/subscribe', methods=['GET', 'POST'])
|
|
||||||
def subscribe():
|
|
||||||
if flask.request.method == 'POST':
|
|
||||||
origin = flask.request.form.get('origin')
|
|
||||||
if origin:
|
|
||||||
type = None
|
|
||||||
feed = None
|
|
||||||
typed_feed = flask.request.form.get('feed')
|
|
||||||
if typed_feed:
|
|
||||||
type, feed = typed_feed.split('|', 1)
|
|
||||||
else:
|
|
||||||
feeds = find_possible_feeds(origin)
|
|
||||||
if not feeds:
|
|
||||||
flask.flash('No feeds found for: ' + origin)
|
|
||||||
return flask.redirect(flask.url_for('subscribe'))
|
|
||||||
if len(feeds) > 1:
|
|
||||||
return flask.render_template(
|
|
||||||
'select-feed.jinja2', origin=origin, feeds=feeds)
|
|
||||||
feed = feeds[0]['feed']
|
|
||||||
type = feeds[0]['type']
|
|
||||||
new_feed = add_subscription(origin, feed, type)
|
|
||||||
flask.flash('Successfully subscribed to: {}'.format(new_feed.name))
|
|
||||||
return flask.redirect(flask.url_for('index'))
|
|
||||||
else:
|
|
||||||
flask.abort(400)
|
|
||||||
|
|
||||||
return flask.render_template('subscribe.jinja2')
|
|
||||||
|
|
||||||
|
|
||||||
def add_subscription(origin, feed, type):
|
|
||||||
if type == 'html':
|
|
||||||
parsed = mf2util.interpret_feed(mf2py.parse(url=feed), feed)
|
|
||||||
name = parsed.get('name')
|
|
||||||
if not name or len(name) > 140:
|
|
||||||
p = urllib.parse.urlparse(origin)
|
|
||||||
name = p.netloc + p.path
|
|
||||||
|
|
||||||
feed = Feed(user=flask_login.current_user, name=name,
|
|
||||||
origin=origin, feed=feed, type=type)
|
|
||||||
|
|
||||||
db.session.add(feed)
|
|
||||||
db.session.commit()
|
|
||||||
return feed
|
|
||||||
|
|
||||||
elif type == 'xml':
|
|
||||||
parsed = feedparser.parse(feed)
|
|
||||||
feed = Feed(user=flask_login.current_user,
|
|
||||||
name=parsed.feed.title, origin=origin, feed=feed,
|
|
||||||
type=type)
|
|
||||||
|
|
||||||
db.session.add(feed)
|
|
||||||
db.session.commit()
|
|
||||||
return feed
|
|
||||||
|
|
||||||
|
|
||||||
def find_possible_feeds(origin):
|
|
||||||
# scrape an origin source to find possible alternative feeds
|
|
||||||
resp = requests.get(origin)
|
|
||||||
|
|
||||||
feeds = []
|
|
||||||
xml_feed_types = [
|
|
||||||
'application/rss+xml',
|
|
||||||
'application/atom+xml',
|
|
||||||
'application/rdf+xml',
|
|
||||||
]
|
|
||||||
|
|
||||||
content_type = resp.headers['content-type']
|
|
||||||
content_type = content_type.split(';', 1)[0].strip()
|
|
||||||
if content_type in xml_feed_types:
|
|
||||||
feeds.append({
|
|
||||||
'origin': origin,
|
|
||||||
'feed': origin,
|
|
||||||
'type': 'xml',
|
|
||||||
})
|
|
||||||
|
|
||||||
elif content_type == 'text/html':
|
|
||||||
# if text/html, then parse and look for rel="alternate"
|
|
||||||
soup = bs4.BeautifulSoup(resp.text)
|
|
||||||
for link in soup.find_all('link', {'rel': 'alternate'}):
|
|
||||||
if link.get('type') in xml_feed_types:
|
|
||||||
feeds.append({
|
|
||||||
'origin': origin,
|
|
||||||
'feed': link.get('href'),
|
|
||||||
'type': 'xml',
|
|
||||||
})
|
|
||||||
feeds.append({
|
|
||||||
'origin': origin,
|
|
||||||
'feed': origin,
|
|
||||||
'type': 'html',
|
|
||||||
})
|
|
||||||
|
|
||||||
return feeds
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
app.run(debug=True, port=4000)
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue