<commit_before><commit_msg>Adds requirements (probably more than what we actually use, sorry).<commit_after>appnope==0.1.0
backports-abc==0.4
backports.ssl-match-hostname==3.4.0.2
beautifulsoup4==4.4.1
boto==2.39.0
breadability==0.1.20
bz2file==0.98
certifi==2015.11.20
cffi==1.5.0
characteristic==14.3.0
chardet==2.3.0
cryptography==1.2.2
cssselect==0.9.1
cycler==0.9.0
decorator==4.0.4
docopt==0.6.2
enum34==1.1.2
functools32==3.2.3.post2
gensim==0.12.4
gnureadline==6.3.3
httpretty==0.8.10
idna==2.0
ipaddress==1.0.16
ipykernel==4.1.1
ipython==4.0.0
ipython-genutils==0.1.0
Janome==0.2.7
Jinja2==2.8
jsonschema==2.5.1
jupyter-client==4.1.1
jupyter-core==4.0.6
lxml==3.5.0
MarkupSafe==0.23
matplotlib==1.5.0
mistune==0.7.1
mpld3==0.2
nbconvert==4.1.0
nbformat==4.0.1
networkx==1.11
nltk==3.1
notebook==4.0.6
numpy==1.10.4
pandas==0.17.1
path.py==8.1.2
pexpect==4.0.1
pickleshare==0.5
ptyprocess==0.5
pyasn1==0.1.9
pyasn1-modules==0.0.8
pycparser==2.14
pydot==1.0.2
Pygments==2.0.2
pyOpenSSL==0.15.1
pyparsing==1.5.7
python-dateutil==2.4.2
pytz==2015.7
pyzmq==15.0.0
qtconsole==4.1.1
queuelib==1.4.2
requests==2.8.1
scikit-learn==0.17
scipy==0.17.0
Scrapy==1.0.5
service-identity==14.0.0
simplegeneric==0.8.1
singledispatch==3.4.0.3
six==1.10.0
smart-open==1.3.2
sumy==0.4.0
terminado==0.5
textblob==0.11.1
textblob-aptagger==0.2.0
tornado==4.3
traitlets==4.0.0
Twisted==15.5.0
w3lib==1.13.0
yolk==0.4.3
zope.interface==4.1.3
{'fraction_non_alphanumeric': 0.22791023842917252, 'fraction_numerical': 0.18373071528751753, 'mean_word_length': 4.777327935222672, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6153087', 'n_tokens_mistral': 817, 'n_tokens_neox': 781, 'n_words': 87}