<commit_before>from pony import orm
from datetime import datetime
import feedparser
from flask import Flask, render_template, request, redirect, url_for, flash
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup, Comment, Doctype
from flask_bootstrap import Bootstrap
from flask_moment import Moment
from flask_login import LoginManager, UserMixin, login_user, logout_user, login_required
from werkzeug.security import generate_password_hash, check_password_hash

db = orm.Database()
db.bind('sqlite', 'fbdb.sqlite', create_db=True)
app = Flask(__name__)
app.config['SECRET_KEY'] = '<KEY>'
bootstrap = Bootstrap(app)
moment = Moment(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'

class Feed(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    articles = orm.Set('Article')
    categories = orm.Set('Category')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    etag = orm.Optional(str)
    modified = orm.Optional(str)

class Article(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    feed = orm.Required('Feed')
    author = orm.Optional('Author')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    read = orm.Optional(bool, default=False)
    published = orm.Required(datetime)
    summary = orm.Optional(str)

class Category(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    title = orm.Required(str, unique=True)
    feeds = orm.Set('Feed')

class Author(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    name = orm.Required(str, unique=True)
    articles = orm.Set('Article')

class User(UserMixin, db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    username = orm.Required(str, unique=True)
    password_hash = orm.Required(str, unique=True)

db.generate_mapping(create_tables=True)

@orm.db_session
def add_category(title):
    exists = Category.get(title=title)
    if exists:
        return exists.id
    else:
        new_category = Category(title=title)
        orm.commit()
        return new_category.id

@orm.db_session
def add_author(name):
    exists=Author.get(name=name)
    if exists:
        return exists.id
    else:
        new_author = Author(name=name)
        orm.commit()
        return new_author.id

@orm.db_session
def fetch_feed(id):
    feed = Feed[id]
    p = feedparser.parse(feed.url, etag=feed.etag, modified=feed.modified)
    feed.etag = p.etag if 'etag' in p else ''
    feed.modified = p.modified if 'modified' in p else ''
    for e in p.entries if 'entries' in p else []:
        author = Author[add_author(e.author.title())] if 'author' in e and e.author else Author[add_author('None')]
        title = e.title if 'title' in e else ''
        url = e.link if 'link' in e else feed.url
        published = datetime(*e.published_parsed[:6]) if 'published_parsed' in e else datetime.utcnow()
        summary = strip_summary(e.summary) if 'summary' in e else ''
        a = Article.get(url=url)
        if a:
            if a.published != published:
                a.feed = feed
                a.author = author
                a.title = title
                a.published = published
                a.summary = summary
        else:
            new_article = Article(feed=feed, author=author, title=title, url=url, published=published, summary=summary)

def strip_summary(summary):
    soup = BeautifulSoup(summary, 'html.parser')
    text = soup.find_all(text=True)
    stripped = ' '.join(list(filter(visible, text)))
    if len(stripped) > 300:
        return stripped[:300] + '...'
    elif stripped:
        return stripped
    else:
        return ''

def visible(element):
    if element.parent.name in ['style', 'script', 'head', 'title', 'meta']:
        return False
    elif isinstance(element, (Comment, Doctype)):
        return False
    elif element.string == '\n':
        return False
    return True

@login_manager.user_loader
@orm.db_session
def load_user(user_id):
    return User.get(id=user_id)

@orm.db_session
def add_user(username, password):
    exists=User.get()
    if exists:
        return False
    else:
        password_hash = generate_password_hash(password)
        new_user = User(username=username, password_hash=password_hash)
        orm.commit()
        return new_user

@app.context_processor
@orm.db_session
def nav_variables():
    feeds = list(Feed.select())
    uncategorized = list(Feed.select(lambda u: not u.categories))<commit_msg>Article Images - Backend Support<commit_after>from pony import orm
from datetime import datetime
import feedparser
from flask import Flask, render_template, request, redirect, url_for, flash
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup, Comment, Doctype
from flask_bootstrap import Bootstrap
from flask_moment import Moment
from flask_login import LoginManager, UserMixin, login_user, logout_user, login_required
from werkzeug.security import generate_password_hash, check_password_hash

db = orm.Database()
db.bind('sqlite', 'fbdb.sqlite', create_db=True)
app = Flask(__name__)
app.config['SECRET_KEY'] = '<KEY>'
bootstrap = Bootstrap(app)
moment = Moment(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'

class Feed(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    articles = orm.Set('Article')
    categories = orm.Set('Category')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    etag = orm.Optional(str)
    modified = orm.Optional(str)

class Article(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    feed = orm.Required('Feed')
    author = orm.Optional('Author')
    title = orm.Required(str)
    url = orm.Required(str, unique=True)
    read = orm.Optional(bool, default=False)
    published = orm.Required(datetime)
    summary = orm.Optional(str)
    image = orm.Optional(str)

class Category(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    title = orm.Required(str, unique=True)
    feeds = orm.Set('Feed')

class Author(db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    name = orm.Required(str, unique=True)
    articles = orm.Set('Article')

class User(UserMixin, db.Entity):
    id = orm.PrimaryKey(int, auto=True)
    username = orm.Required(str, unique=True)
    password_hash = orm.Required(str, unique=True)

db.generate_mapping(create_tables=True)

@orm.db_session
def add_category(title):
    exists = Category.get(title=title)
    if exists:
        return exists.id
    else:
        new_category = Category(title=title)
        orm.commit()
        return new_category.id

@orm.db_session
def add_author(name):
    exists=Author.get(name=name)
    if exists:
        return exists.id
    else:
        new_author = Author(name=name)
        orm.commit()
        return new_author.id

@orm.db_session
def fetch_feed(id):
    feed = Feed[id]
    p = feedparser.parse(feed.url, etag=feed.etag, modified=feed.modified)
    feed.etag = p.etag if 'etag' in p else ''
    feed.modified = p.modified if 'modified' in p else ''
    for e in p.entries if 'entries' in p else []:
        author = Author[add_author(e.author.title())] if 'author' in e and e.author else Author[add_author('None')]
        title = e.title if 'title' in e else ''
        url = e.link if 'link' in e else feed.url
        published = datetime(*e.published_parsed[:6]) if 'published_parsed' in e else datetime.utcnow()
        summary = strip_summary(e.summary) if 'summary' in e else ''
        image = find_image(e)

        a = Article.get(url=url)
        if a:
            if a.published != published:
                a.feed = feed
                a.author = author
                a.title = title
                a.published = published
                a.summary = summary
                a.image = image
        else:
            new_article = Article(feed=feed, author=author, title=title, url=url, published=published, summary=summary, image=image)

def strip_summary(summary):
    soup = BeautifulSoup(summary, 'html.parser')
    text = soup.find_all(text=True)
    stripped = ' '.join(list(filter(visible, text)))
    if len(stripped) > 300:
        return stripped[:300] + '...'
    elif stripped:
        return stripped
    else:
        return ''

def visible(element):
    if element.parent.name in ['style', 'script', 'head', 'title', 'meta']:
        return False
    elif isinstance(element, (Comment, Doctype)):
        return False
    elif element.string == '\n':
        return False
    return True

def find_image(entry):
    if 'media_content' in entry and 'url' in entry['media_content'][0]:
        return entry['media_content'][0]['url']
    else:
        image = ''
        if 'content' in entry and 'value' in entry['content'][0]:
            image = parse_image(entry['content'][0]['value'])
        if not image and 'summary' in entry:
            image = parse_image(entry['summary'])
        return image

def parse_image(html):
    soup = BeautifulSoup(html, 'html.parser')
    image = soup.find('img')
    return image.get('src') if image else ''

@login_manager.user_loader
@orm.db_session
def load_user(user_id):
    return User.get(id=user_id)

@orm.db_session
def add_user(username, password):
    exists=User.get()
    if exists:
        return False
    else:
        password_hash = generate_password_hash(password)
        new_user = User(username=username, password_hash=password_<PASSWORD>)
        orm.commit()
        return new_user

@app.context_processor
@orm.db_session
def nav_variables():
    feeds = list(Feed.select())
    uncategorized = list(Feed.select(lambda u: not u.categories))
{'fraction_non_alphanumeric': 0.11822711231056675, 'fraction_numerical': 0.002075980900975711, 'mean_word_length': 3.2074235807860263, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '41018', 'n_tokens_mistral': 3077, 'n_tokens_neox': 3032, 'n_words': 752}