<commit_before># Amazon S3
boto3==1.9.78

# HTML parsing and stripping
beautifulsoup4==4.7.1

# Job broker
Celery==4.3.1

# Detect encodings of web responses
chardet==3.0.4

# Language identification
cld2-cffi==0.1.4

# Colorful sentence diffs in unit tests
colorama==0.4.1

# Unit test coverage
coverage==4.5.2

# Some modules require latest version of this
cryptography==2.4.2

# Hindi stemming
CyHunspell==1.3.0

# Guessing publication dates of stories
date_guesser==2.1.4

# Parsing RSS / Atom feeds
feedparser==6.0.0b1

# Stemming Hausa words
hausastemmer==1.0

# Code conventions
flake8==3.6.0

# URL manipulation
furl==2.0.0

# Chinese language tokenizer, stemmer, etc.
jieba==0.39

# Parsing email templates
Jinja2==2.10.1

# One of Celery's dependencies (here just for PyCharm to stop complaining)
kombu==4.5.0

# XML manipulations, HTML parsing
lxml==4.6.2

# Japanese language tokenizer, stemmer, etc.
mecab-python3==0.996.1

# Natural language processing, e.g. tokenization
nltk==3.4.5

# PostgreSQL access
psycopg2-binary==2.8.6

# Process information
psutil==5.7.0

# Snowball stemmer (NLTK's implementation doesn't support Turkish)
PyStemmer==1.3.0

# Unit tests
pytest==3.8.0

# Test coverage
pytest-cov==2.6.0

# Timezone handling
pytz==2018.9

# Date parsing
python-dateutil==2.7.5

# Making HTTP requests
requests==2.25.0

# Asynchronous (parallell / concurrent) HTTP requests
requests-futures==0.9.9

# Splitting text into sentences using Lingua::Sentence's method
sentence_splitter==1.3

# Celery PostgreSQL result backend support
sqlalchemy==1.3.3

# Normalizing URLs
url_normalize==1.4.1

# Low level HTTP requests (with SSL certificate verification)
urllib3[secure]==1.24.2<commit_msg>Update most dependencies in common, remove test coverage libs<commit_after># Amazon S3
boto3==1.16.59

# HTML parsing and stripping
beautifulsoup4==4.9.3

# Job broker
Celery==5.0.5

# Detect encodings of web responses
chardet==4.0.0

# Language identification
cld2-cffi==0.1.4

# Colorful sentence diffs in unit tests
colorama==0.4.4

# Some modules require latest version of this
cryptography==3.3.1

# Hindi stemming
CyHunspell==2.0.1

# Guessing publication dates of stories
date_guesser==2.1.4

# Parsing RSS / Atom feeds
feedparser==6.0.2

# Stemming Hausa words
hausastemmer==1.0

# Code conventions
flake8==3.8.4

# URL manipulation
furl==2.1.0

# Chinese language tokenizer, stemmer, etc.
jieba==0.42.1

# Parsing email templates
Jinja2==2.11.2

# One of Celery's dependencies (here just for PyCharm to stop complaining)
kombu==5.0.2

# XML manipulations, HTML parsing
lxml==4.6.2

# Japanese language tokenizer, stemmer, etc.
mecab-python3==1.0.3

# Natural language processing, e.g. tokenization
nltk==3.5

# PostgreSQL access
psycopg2-binary==2.8.6

# Process information
psutil==5.8.0

# Snowball stemmer (NLTK's implementation doesn't support Turkish)
PyStemmer==2.0.1

# Unit tests
pytest==3.8.0

# Timezone handling
pytz==2020.5

# Date parsing
python-dateutil==2.8.1

# Making HTTP requests
requests==2.25.1

# Asynchronous (parallell / concurrent) HTTP requests
requests-futures==1.0.0

# Splitting text into sentences using Lingua::Sentence's method
sentence_splitter==1.4

# Celery PostgreSQL result backend support
sqlalchemy==1.3.22

# Normalizing URLs
url_normalize==1.4.3

# Low level HTTP requests (with SSL certificate verification)
urllib3[secure]==1.26.2
{'fraction_non_alphanumeric': 0.11383072839870245, 'fraction_numerical': 0.06576231200235919, 'mean_word_length': 4.643926788685524, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3929536', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1266, 'n_words': 330}