<commit_before>import time
import urllib2
from urllib2 import urlopen
import re
import cookielib
from cookielib import CookieJar
import datetime
import psycopg2
import nltk

cj = CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]

visitedLinks = []

def striphtml(data):
    p = re.compile(r'<.*?>')
    return p.sub('', data)

# called from
def isLinkVisited(link):
    try:
        c.execute("""SELECT url FROM visitedLinks WHERE url = '%s';""" % (str(link)))
        article = c.fetchone()
        if article is None:
            return False
        else:
            return True
    except Exception, e:
        print 'failed in isLinkVisited'
        print str(e)

def addLinkAndGetId(link, sourceName):
    try:
        currentTime = time.time()
        datestamp = datetime.datetime.fromtimestamp(currentTime).strftime('%Y-%m-%d %H:%M:%S')
        c.execute("""SELECT sourceid FROM sources WHERE name = '%s';""" % (str(sourceName)))
        sourceid = c.fetchone()[0]
        c.execute("""INSERT INTO visitedlinks (url, created, sourceid) VALUES ('%s', now(), '%s') RETURNING linkid;""" %
                (str(link), str(sourceid)))
        linkid = c.fetchone()[0]
        conn.commit()
        return linkid
    except Exception, e:
        print 'failed in addLink'
        print str(e)

def processor(data, linkId):
    try:
        tokens = nltk.word_tokenize(data)
        tagged = nltk.pos_tag(tokens)

        tagsWeWant = ['JJ', 'JJR', 'JJS',
                'NN', 'NNS',
                'PRP', 'PRPS',
                'RB', 'RBR', 'RBS',
                'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'VBT']
        for tag in tagged:
            if tag[1] in tagsWeWant:
                currentTime = time.time()
                datestamp = datetime.datetime.fromtimestamp(currentTime).strftime('%Y-%m-%d %H:%M:%S')
                word = re.sub(r'\'','',tag[0].lower())
                pos = tag[1]
                c.execute("""INSERT INTO associations (created, word, linkid, pos) VALUES (now(), '%s', '%s', '%s')""" %
                        (str(word), str(linkId), str(pos)))
                conn.commit()
            else:
                pass

    except Exception, e:
        print 'failed in the first try of processor'
        print str(e)


def huffingtonRSSVisit():
    try:
        page = 'http://www.huffingtonpost.com/feeds/index.xml'
        sourceCode = opener.open(page).read()
        try:
            links = re.findall(r'<link>(.*?)</link>', sourceCode)
            for link in links[1:]:<commit_msg>Ignoring errors about symbol imcompatability<commit_after>import time
import urllib2
from urllib2 import urlopen
import re
import cookielib
from cookielib import CookieJar
import datetime
import psycopg2
import nltk

cj = CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
opener.addheaders = [('User-Agent', 'Mozilla/5.0')]

visitedLinks = []

def striphtml(data):
    p = re.compile(r'<.*?>')
    return p.sub('', data)

# called from
def isLinkVisited(link):
    try:
        c.execute("""SELECT url FROM visitedLinks WHERE url = '%s';""" % (str(link)))
        article = c.fetchone()
        if article is None:
            return False
        else:
            return True
    except Exception, e:
        print 'failed in isLinkVisited'
        print str(e)

def addLinkAndGetId(link, sourceName):
    try:
        currentTime = time.time()
        datestamp = datetime.datetime.fromtimestamp(currentTime).strftime('%Y-%m-%d %H:%M:%S')
        c.execute("""SELECT sourceid FROM sources WHERE name = '%s';""" % (str(sourceName)))
        sourceid = c.fetchone()[0]
        c.execute("""INSERT INTO visitedlinks (url, created, sourceid) VALUES ('%s', now(), '%s') RETURNING linkid;""" %
                (str(link), str(sourceid)))
        linkid = c.fetchone()[0]
        conn.commit()
        return linkid
    except Exception, e:
        print 'failed in addLink'
        print str(e)

def processor(data, linkId):
    try:
        tokens = nltk.word_tokenize(data)
        tagged = nltk.pos_tag(tokens)

        tagsWeWant = ['JJ', 'JJR', 'JJS',
                'NN', 'NNS',
                'PRP', 'PRPS',
                'RB', 'RBR', 'RBS',
                'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'VBT']
        for tag in tagged:
            if tag[1] in tagsWeWant:
                currentTime = time.time()
                datestamp = datetime.datetime.fromtimestamp(currentTime).strftime('%Y-%m-%d %H:%M:%S')
                word = re.sub(r'\'','',tag[0].lower())
                pos = tag[1]
                c.execute("""INSERT INTO associations (created, word, linkid, pos) VALUES (now(), '%s', '%s', '%s')""" %
                        (str(word), str(linkId), str(pos)))
                conn.commit()
            else:
                pass

    except Exception, e:
        if "'ascii' codec can't decode byte" not in e:
            print str(e)


def huffingtonRSSVisit():
    try:
        page = 'http://www.huffingtonpost.com/feeds/index.xml'
        sourceCode = opener.open(page).read()
        try:
            links = re.findall(r'<link>(.*?)</link>', sourceCode)
            for link in links[1:]:
{'fraction_non_alphanumeric': 0.15217804831653034, 'fraction_numerical': 0.0049457865702872365, 'mean_word_length': 2.21001221001221, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326032', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1657, 'n_words': 416}