<commit_before>argh==0.26.2
brotlipy==0.6.0
cffi==1.9.1
chardet==2.3.0
enum34==1.1.6
idna==2.1
Jinja2==2.8
MarkupSafe==0.23
pathtools==0.1.2
pycparser==2.17
pywb==0.33.0
PyYAML==3.12
redis==2.10.5
requests==2.12.1
requests-file==1.4.1
simhash==1.7.1
six==1.10.0
surt==0.3.0
tldextract==2.0.2
watchdog==0.8.3
webencodings==0.5
<commit_msg>update - requirements<commit_after>appnope==0.1.0
argh==0.26.2
backports.shutil-get-terminal-size==1.0.0
brotlipy==0.6.0
cffi==1.9.1
chardet==2.3.0
decorator==4.0.11
enum34==1.1.6
idna==2.1
ipdb==0.10.1
ipython==5.1.0
ipython-genutils==0.1.0
Jinja2==2.8
MarkupSafe==0.23
pathlib2==2.2.1
pathtools==0.1.2
pexpect==4.2.1
pickleshare==0.7.4
prompt-toolkit==1.0.9
ptyprocess==0.5.1
pycparser==2.17
Pygments==2.2.0
pywb==0.33.0
PyYAML==3.12
redis==2.10.5
requests==2.12.1
requests-file==1.4.1
scandir==1.4
simhash==1.7.1
simplegeneric==0.8.1
six==1.10.0
surt==0.3.0
tldextract==2.0.2
traitlets==4.3.1
warc==0.2.1
watchdog==0.8.3
wcwidth==0.1.7
webencodings==0.5

{'fraction_non_alphanumeric': 0.23919597989949748, 'fraction_numerical': 0.1899497487437186, 'mean_word_length': 4.659090909090909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1727471', 'n_tokens_mistral': 596, 'n_tokens_neox': 566, 'n_words': 60}