<commit_before>from datetime import datetime

from bottle import abort, default_app, response, route, run
from rss2producer import RSS2Feed
import dataset

from settings import TABLE, PAGES


# copy pasted from scraper
LIMIT = 50


slug_to_name = {slug: name for slug, pk, name in PAGES}
slug_to_id = {slug: pk for slug, pk, name in PAGES}


@route('/')
def index():
    out = []
    for slug, pk, name in PAGES:
        out.append('<li><a href="{}/">{}</a></li>'.format(slug, name))
    return u''.join(out)


@route('/<slug>/')
def feed_detail(slug):
    if slug != 'all' and slug not in slug_to_name:
        abort(404, 'No results for that Board or Commission')
    db = dataset.connect()  # uses DATABASE_URL
    # table = db.load_table(TABLE)  # don't allow dataset to create the table
    if slug == 'all':
        feed_info = dict(
            title='Boards and Commissions',
            link='http://www.austintexas.gov/cityclerk/boards_commissions/',
            description='Feed of Boards and Commissions activity',
        )
    else:
        feed_info = dict(
            title=slug_to_name[slug],
            link=('http://www.austintexas.gov/cityclerk/boards_commissions/'
                'meetings/{}_1.htm'.format(slug_to_id[slug])),
            description='Feed of {} activity'.format(slug_to_name[slug]),
        )
    feed = RSS2Feed(**feed_info)
    filter_kwargs = {}
    if slug != 'all':
        filter_kwargs['bandc'] = slug
    # results = table.find(_limit=LIMIT, order_by='-date', **filter_kwargs)
    # HACK for 'order_by' not working
    sql = 'SELECT * from {} ORDER BY date DESC LIMIT {}'.format(
        TABLE,
        LIMIT,
    )
    results = db.query(sql)
    import re
    for row in results:
        title = row['title'] or row['type']
        if slug == 'all':
            title = '[{}] {}'.format(row['bandc'], title)
        text = row['text'].strip().encode('ascii', 'ignore')[:600].strip()
        text = re.sub(r'\s+', ' ', text)[:500]
        if text:
            print text
        feed.append_item(
            title=title,
            link=row['url'],
            description=text if text and not text.startswith('no text') else row['type'],
            # convert date into datetime
            pub_date=datetime.combine(row['date'], datetime.min.time()),
        )
    response.content_type = 'application/rss+xml'<commit_msg>more copy tweaks to feed page<commit_after>from datetime import datetime

from bottle import abort, default_app, response, route, run
from rss2producer import RSS2Feed
import dataset

from settings import TABLE, PAGES


# copy pasted from scraper
LIMIT = 50


slug_to_name = {slug: name for slug, pk, name in PAGES}
slug_to_id = {slug: pk for slug, pk, name in PAGES}


@route('/')
def index():
    out = []
    out.append('<li><a href="{}/">{}</a></li>'.format('all', 'All'))
    for slug, pk, name in PAGES:
        out.append('<li><a href="{}/">{}</a></li>'.format(slug, name))
    return u''.join(out)


@route('/<slug>/')
def feed_detail(slug):
    if slug != 'all' and slug not in slug_to_name:
        abort(404, 'No results for that Board or Commission')
    db = dataset.connect()  # uses DATABASE_URL
    # table = db.load_table(TABLE)  # don't allow dataset to create the table
    if slug == 'all':
        feed_info = dict(
            title='Boards and Commissions',
            link='http://www.austintexas.gov/cityclerk/boards_commissions/',
            description='Feed of Boards and Commissions activity',
        )
    else:
        feed_info = dict(
            title=slug_to_name[slug],
            link=('http://www.austintexas.gov/cityclerk/boards_commissions/'
                'meetings/{}_1.htm'.format(slug_to_id[slug])),
            description='Feed of {} activity'.format(slug_to_name[slug]),
        )
    feed = RSS2Feed(**feed_info)
    filter_kwargs = {}
    if slug != 'all':
        filter_kwargs['bandc'] = slug
    # results = table.find(_limit=LIMIT, order_by='-date', **filter_kwargs)
    # HACK for 'order_by' not working
    sql = 'SELECT * from {} ORDER BY date DESC LIMIT {}'.format(
        TABLE,
        LIMIT,
    )
    results = db.query(sql)
    import re
    for row in results:
        title = row['title'] or row['type']
        if slug == 'all':
            title = '[{}] {}'.format(row['bandc'], title)
        text = row['text'].strip().encode('ascii', 'ignore')[:600].strip()
        text = re.sub(r'\s+', ' ', text)[:500]
        if text.startswith('no text'):
            text = '{}: {}'.format(row['type'], text)
        feed.append_item(
            title=title,
            link=row['url'],
            description=text if text else row['type'],
            # convert date into datetime
            pub_date=datetime.combine(row['date'], datetime.min.time()),
        )
    response.content_type = 'application/rss+xml'
{'fraction_non_alphanumeric': 0.14961926322288538, 'fraction_numerical': 0.006174109899156205, 'mean_word_length': 2.5814296241709656, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5351006', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1546, 'n_words': 393}