<commit_before># !/usr/bin/python
# -*- coding: utf-8 -*-
#
# Created on Oct 16, 2015
# @author:       <NAME>
# @email:        <EMAIL>
# @website:      http://yenching.org
# @organization: Harvard Kennedy School

import sys

sys.path.append("/home/bo/.local/lib/python2.7/site-packages")
# sys.path.append("/home/bo/Workspace/wbcrawler")
# sys.path.append("/home/bo/Workspace/wbcrawler/cga")
sys.path.append("/home/bo/wbcrawler")
sys.path.append("/home/bo/wbcrawler/cga")

from wbcrawler.parser import parse_discovery
from wbcrawler.robot import register, unregister, create_database, unlock_robots
from wbcrawler.log import *
from settings import SETTINGS2 as SETTINGS

# unlock the unreleased robots
unlock_robots(SETTINGS)

# start to crawl
start = datetime.datetime.now()
t = SETTINGS['robot_num']

robot = {}
accounts = []
for i in range(t):
    if robot == {}:
        robot = register(SETTINGS)
    else:
        break

try:
    db = create_database(SETTINGS)
    # for d_type in SETTINGS['d_types']:
    for d_type_num in range(len(SETTINGS['d_types'])):
        round_start = datetime.datetime.now()
        parse_discovery(SETTINGS['d_types'][d_type_num], robot, db)
        log(NOTICE, 'The completion of processing the keyword "%s". Time: %d sec(s)' % (SETTINGS['d_types'][d_type_num].keys()[0].decode('utf-8'), int((datetime.datetime.now() - round_start).seconds)))
except:
    log(ERROR, 'An error occurs.', 'crawler.py')
finally:
    # out of the stak
    unregister(robot)
    log(NOTICE, 'The completion of processing all keywords. Time: %d min(s)' % int((datetime.datetime.now() - start).seconds / 60))

if __name__ == '__main__':
    pass
<commit_msg>2.12 crawling d.weibo.com more categories<commit_after># !/usr/bin/python
# -*- coding: utf-8 -*-
#
# Created on Oct 16, 2015
# @author:       <NAME>
# @email:        <EMAIL>
# @website:      http://yenching.org
# @organization: Harvard Kennedy School

import sys

sys.path.append("/home/bo/.local/lib/python2.7/site-packages")
# sys.path.append("/home/bo/Workspace/wbcrawler")
# sys.path.append("/home/bo/Workspace/wbcrawler/cga")
sys.path.append("/home/bo/wbcrawler")
sys.path.append("/home/bo/wbcrawler/cga")

from wbcrawler.parser import parse_discovery
from wbcrawler.robot import register, unregister, create_database, unlock_robots
from wbcrawler.log import *
from settings import SETTINGS2 as SETTINGS

# unlock the unreleased robots
unlock_robots(SETTINGS)

# start to crawl
start = datetime.datetime.now()
t = SETTINGS['robot_num']

robot = {}
accounts = []
for i in range(t):
    if robot == {}:
        robot = register(SETTINGS)
    else:
        break

try:
    db = create_database(SETTINGS)
    # for d_type in SETTINGS['d_types']:
    for d_type_num in range(len(SETTINGS['d_types'])):
        round_start = datetime.datetime.now()
        parse_discovery(SETTINGS['d_types'][d_type_num], robot, db)
        log(NOTICE, 'The completion of processing the keyword "%s". Time: %d sec(s)' % (SETTINGS['d_types'][d_type_num].keys()[0].decode('utf-8'), int((datetime.datetime.now() - round_start).seconds)))
except:
    log(ERROR, 'An error occurs .', 'crawler.py')
finally:
    # out of the stak
    unregister(robot)
    log(NOTICE, 'The completion of processing all keywords. Time: %d min(s)' % int((datetime.datetime.now() - start).seconds / 60))

if __name__ == '__main__':
    pass

{'fraction_non_alphanumeric': 0.1528643514395963, 'fraction_numerical': 0.009201543484713566, 'mean_word_length': 3.9341142020497806, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '866693', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1178, 'n_words': 262}