<commit_before>pyliblzma>=0.5.3
python-dateutil>=1.5
requests>=1.1.0
lxml>=2.3.5
msgpack-python==0.1.13  # OpenStack compatibility
SQLAlchemy==0.8.2       # OpenStack compatibility
<commit_msg>install pyliblzma not from pip but from ubuntu package manager<commit_after>python-dateutil>=1.5
requests>=1.1.0
lxml>=2.3.5
msgpack-python==0.1.13  # Use this version for OpenStack(Grizzly) compatibility
SQLAlchemy==0.8.2       # Use this version for OpenStack(Grizzly) compatibility

{'fraction_non_alphanumeric': 0.12552301255230125, 'fraction_numerical': 0.06903765690376569, 'mean_word_length': 5.063291139240507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1498211', 'n_tokens_mistral': 183, 'n_tokens_neox': 176, 'n_words': 36}