<commit_before>#!/usr/bin/python2.6
"""This script will parse RSS feeds from the Dutch government in an attempt to
find publications relevant to the field of digitial civil rights and privacy.
Some of the feeds will be included completely, others will be filtered using a
list of keywords."""

from datetime import datetime
from time import mktime
from ConfigParser import SafeConfigParser
import cgi
import feedparser
import MySQLdb
import PyRSS2Gen
import sys

CONFIG = SafeConfigParser()
CONFIG.read('feeds.cfg')

# Not including "buma" because it matches the name of a senator.
#
# Removed, because matching too much.
#      '/ digita/i',
#
# There's a space prepending the string, to make sure the match
# won't be made in the middle of a word (e.g. "anpr" matching
# "methaanproductie"). There's no space at the end, as sometimes
# the words match only partial (e.g. "biometri" matching both
# "biometrie" as well as "biometrisch").

KEYWORDS = [' acta',
    ' aftap',
    ' aivd',
    ' anoniem',
    ' anonimiteit',
    ' anpr',
    ' auteursrecht',
    ' bewaarplicht',
    ' biometri',
    ' bits of freedom',
    ' bodyscanners',
    ' cameratoezicht',
    ' centraal informatiepunt onderzoek telecommunicatie',
    ' ciot',
    ' ctivd',
    ' computer',
    ' copyright',
    ' cyber',
    ' data',
    ' digid',
    ' embed',
    ' fileshar',
    ' filter',
    ' google',
    ' intellectu',
    ' internet',
    ' interoperabiliteit',
    ' ip adres',
    ' ip-adres',
    ' kinderporno',
    ' mensenrechten',
    ' mivd',
    ' netneutraliteit',
    ' ncss',
    ' octrooi',
    ' ov.chipkaart',
    ' patent',
    ' persoonsgegevens',
    ' piracy',
    ' piraterij',
    ' provider',
    ' retentie',
    ' retention',
    ' skimmen',
    ' software',
    ' swift',
    ' telecom',
    ' thuiskopie',
    ' vingerafdruk',
    ' voip',
    ' wob']

try:
    CONN = MySQLdb.connect(host = CONFIG.get('database', 'hostname'),
            user = CONFIG.get('database', 'username'),
            passwd = CONFIG.get('database', 'password'),
            db = CONFIG.get('database', 'database'),
            charset = CONFIG.get('database', 'charset'))

except MySQLdb.Error, e:
    print "Error %d: %s" % (e.args[0], e.args[1])
    sys.exit (1)

CURSOR = CONN.cursor()

def convert_unicode_to_html(string):
    """Converts unicode to HTML entities. For example '&' becomes '&amp;'."""
    string = cgi.escape(string).encode('ascii', 'xmlcharrefreplace')
    return string

def does_match_keyword(item):
    """Return TRUE if the string contains any of the given keywords."""
    for key in KEYWORDS:
        if key in item['title'].lower() or key in item['description'].lower():
            return True
    return False

def is_existing_item(link):
    """See if URI of item is already known in the database."""
    CURSOR.execute('''SELECT id FROM items WHERE link = %s;''', link)
    if CURSOR.rowcount > 0:
        return True
    return False

def insert_item_into_db(link, feed_id, title, description, updated_parsed):
    """Add a new item to the database."""
    CURSOR.execute('''INSERT INTO items (link, feed_id, title, 
            description, time_published) VALUES (%s, %s, %s, %s, 
            %s);''', (link, feed_id, convert_unicode_to_html(title), 
                convert_unicode_to_html(description), 
                datetime.fromtimestamp(mktime(updated_parsed))))

CURSOR.execute('''SELECT id, uri, type FROM feeds WHERE active = '1';''')
FEEDS = CURSOR.fetchall()

for feed in FEEDS:
    f = feedparser.parse('%s' % feed[1])
    if feed[2] == '1':
        for item in f.entries:
            if is_existing_item(item['link']) is not True:
                insert_item_into_db(item['link'], feed[0], item['title'],
                        item['description'], item['updated_parsed'])
    elif feed[2] == '2':
        for item in f.entries:
            if does_match_keyword(item) is True:
                if is_existing_item(item['link']) is not True:
                    insert_item_into_db(item['link'], feed[0], item['title'],
                            item['description'], item['updated_parsed'])

CURSOR.execute('''SELECT items.link, items.title, items.description,
        items.time_published, feeds.description FROM items, feeds
        WHERE items.feed_id = feeds.id
        AND feeds.active = '1'
        ORDER BY items.time_published DESC
        LIMIT 50;''')
PUBLISHED_ITEMS = CURSOR.fetchall()

FEED_ITEMS = [<commit_msg>Minor cosmetic changes.<commit_after>#!/usr/bin/python2.6
"""This script will parse RSS feeds from the Dutch government in an attempt to
find publications relevant to the field of digital civil rights and privacy.
Some of the feeds will be included completely, others will be filtered using a
list of keywords."""

from datetime import datetime
from time import mktime
from ConfigParser import SafeConfigParser
import cgi
import feedparser
import MySQLdb
import PyRSS2Gen
import sys

CONFIG = SafeConfigParser()
CONFIG.read('feeds.cfg')

# Not including "buma" because it matches the name of a senator.
#
# Removed, because matching too much.
#      '/ digita/i',
#
# There's a space prepending the string, to make sure the match
# won't be made in the middle of a word (e.g. "anpr" matching
# "methaanproductie"). There's no space at the end, as sometimes
# the words match only partial (e.g. "biometri" matching both
# "biometrie" as well as "biometrisch").

KEYWORDS = [' acta',
    ' aftap',
    ' aivd',
    ' anoniem',
    ' anonimiteit',
    ' anpr',
    ' auteursrecht',
    ' bewaarplicht',
    ' biometri',
    ' bits of freedom',
    ' bodyscanners',
    ' cameratoezicht',
    ' centraal informatiepunt onderzoek telecommunicatie',
    ' ciot',
    ' ctivd',
    ' computer',
    ' copyright',
    ' cyber',
    ' data',
    ' digid',
    ' embed',
    ' fileshar',
    ' filter',
    ' google',
    ' intellectu',
    ' internet',
    ' interoperabiliteit',
    ' ip adres',
    ' ip-adres',
    ' kinderporno',
    ' mensenrechten',
    ' mivd',
    ' netneutraliteit',
    ' ncss',
    ' octrooi',
    ' ov.chipkaart',
    ' patent',
    ' persoonsgegevens',
    ' piracy',
    ' piraterij',
    ' provider',
    ' retentie',
    ' retention',
    ' skimmen',
    ' software',
    ' swift',
    ' telecom',
    ' thuiskopie',
    ' vingerafdruk',
    ' voip',
    ' wob']

try:
    CONN = MySQLdb.connect(host = CONFIG.get('database', 'hostname'),
            user = CONFIG.get('database', 'username'),
            passwd = CONFIG.get('database', 'password'),
            db = CONFIG.get('database', 'database'),
            charset = CONFIG.get('database', 'charset'))

except MySQLdb.Error, e:
    print "Error %d: %s" % (e.args[0], e.args[1])
    sys.exit (1)

CURSOR = CONN.cursor()

def convert_unicode_to_html(string):
    """Converts unicode to HTML entities. For example '&' becomes '&amp;'."""
    string = cgi.escape(string).encode('ascii', 'xmlcharrefreplace')
    return string

def does_match_keyword(item):
    """Return TRUE if the string contains any of the given keywords."""
    for key in KEYWORDS:
        if key in item['title'].lower() or key in item['description'].lower():
            return True
    return False

def is_existing_item(link):
    """See if URI of item is already known in the database."""
    CURSOR.execute('''SELECT id FROM items WHERE link = %s''', link)
    return CURSOR.rowcount > 0

def insert_item_into_db(link, feed_id, title, description, updated_parsed):
    """Add a new item to the database."""
    CURSOR.execute('''INSERT INTO items (link, feed_id, title, 
            description, time_published) VALUES (%s, %s, %s, %s, 
            %s)''', (link, feed_id, convert_unicode_to_html(title), 
                convert_unicode_to_html(description), 
                datetime.fromtimestamp(mktime(updated_parsed))))

CURSOR.execute('''SELECT id, uri, type FROM feeds WHERE active = '1' ''')
FEEDS = CURSOR.fetchall()

for feed in FEEDS:
    f = feedparser.parse('%s' % feed[1])
    if feed[2] == '1':
        for item in f.entries:
            if is_existing_item(item['link']) is not True:
                insert_item_into_db(item['link'], feed[0], item['title'],
                        item['description'], item['updated_parsed'])
    elif feed[2] == '2':
        for item in f.entries:
            if does_match_keyword(item) is True:
                if is_existing_item(item['link']) is not True:
                    insert_item_into_db(item['link'], feed[0], item['title'],
                            item['description'], item['updated_parsed'])

CURSOR.execute('''SELECT items.link, items.title, items.description,
        items.time_published, feeds.description FROM items, feeds
        WHERE items.feed_id = feeds.id
        AND feeds.active = '1'
        ORDER BY items.time_published DESC
        LIMIT 50''')
PUBLISHED_ITEMS = CURSOR.fetchall()

FEED_ITEMS = [
{'fraction_non_alphanumeric': 0.1328843995510662, 'fraction_numerical': 0.00404040404040404, 'mean_word_length': 2.849244060475162, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1280801', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2782, 'n_words': 808}