<commit_before>#!/usr/bin/python
import sys
from sets import Set
import json
import helpers
from datetime import date
from datetime import datetime
from dateutil.rrule import rrule, DAILY
from multiprocessing import Pool


datadir = '/home/mingf/data/'
source = 'imdb'
homedir = '/home/mingf/Weiss/'
module = 'scrapers/'
historyfile = homedir + module + source + '/crawled.txt'    ## file that stores the movie id crawled
start = datetime.now().date()
#start = date(2015, 6, 3)
end = start
release_date = ''
cfile = ''
efile = ''
query = "groups=now-playing-us&title_type=feature"
#query = "release_date=2015-01-01,2015-06-03&title_type=feature"


def getToCrawl():
    with open(historyfile, 'r') as f:
        crawled = json.load(f)
    crawledIDs = Set(crawled.keys())
    IDs = helpers.get_movie_id_adv(query)
    print "Num of ID in the query", len(IDs)
    pool = Pool(processes=6)
    nums = pool.map(helpers.get_review_amount, IDs)
    pool.close()<commit_msg>imdb scraper fetch scraping history from db<commit_after>#!/usr/bin/python
import sys
from sets import Set
import json
import helpers
from datetime import date
from datetime import datetime
from dateutil.rrule import rrule, DAILY
from multiprocessing import Pool
import MySQLdb as mdb

datadir = '/home/mingf/data/'
source = 'imdb'
homedir = '/home/mingf/Weiss/'
module = 'scrapers/'
historyfile = homedir + module + source + '/crawled.txt'    ## file that stores the movie id crawled
dbsetting = homedir + module + source + '/dbsetting.json'

start = datetime.now().date()
#start = date(2015, 6, 3)
end = start
release_date = ''
cfile = ''
efile = ''
query = "groups=now-playing-us&title_type=feature"
#query = "release_date=2015-01-01,2015-06-03&title_type=feature"


def getHistory():
    with open(dbsetting, 'r') as f:
        setting = json.load(f)
    dbh = mdb.connect(host=setting['host'], user=setting['user'], passwd=setting['passed'], db=setting['db'])
    dbc = dbh.cursor()
    dbc.execute("select id, count(cid) from comment, entity where comment.eid = entity.eid and source='imdb' group by comment.eid")
    res = dbc.fetchall()
    return {ID: num for ID, num in res}

def getToCrawl():
    crawled = getHistory()
    crawledIDs = Set(crawled.keys())
    IDs = helpers.get_movie_id_adv(query)
    print "Num of ID in the query", len(IDs)
    pool = Pool(processes=6)
    nums = pool.map(helpers.get_review_amount, IDs)
    pool.close()
{'fraction_non_alphanumeric': 0.1318181818181818, 'fraction_numerical': 0.019008264462809916, 'mean_word_length': 4.195278969957082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5610980', 'n_tokens_mistral': 876, 'n_tokens_neox': 828, 'n_words': 229}