<commit_before># NeteaseCrawler
a distributed crawler base on celery


#依赖版本 具体见requirements.txt

MySQL 5.7
Python 2.7
celery==3.1.25
redis==2.10.5


## 启动celery 
celery -A tasks worker --loglevel=info

##启动 redis
redis-server

##开始任务

Python start.py



<commit_msg>add informatiom about running<commit_after># netease crawler
a distributed crawler base on celery


#依赖
#具体见requirements.txt

MySQL 5.7

Python 2.7

celery==3.1.25

redis==2.10.5


## 启动celery 
celery -A tasks worker --loglevel=info

##启动 redis
redis-server

##开始任务

Python start.py




{'fraction_non_alphanumeric': 0.10307414104882459, 'fraction_numerical': 0.0433996383363472, 'mean_word_length': 3.8596491228070176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1610601', 'n_tokens_mistral': 261, 'n_tokens_neox': 250, 'n_words': 60}