<commit_before># The pip resolver became more strict starting with version 20.3
# This constraints file is "a way to specify global (version) limits for packages"
# without specifying the install of these packages.
# See also: https://pip.pypa.io/en/stable/user_guide/#constraints-files
# We use it to place limits on certain packages to help the resolver come to a compatible configuration more quickly
# To install dev dependencies using the new pip resolver (recommended) please use the following syntax:
# `python -m pip install -r requirements-dev.txt -c constraints-dev.txt`

# NOTE - 20220511
# pypandoc is a dependency of pyspark and is only compatible up to 1.7.5 with pyspark<3.0.0
# constraining the version of pypandoc for now.
#
# ` AttributeError: module 'pypandoc' has no attribute 'convert'` was the error we were getting
pypandoc==1.7.5
# END NOTE

# NOTE - 20210114
# aiobotocore is a dependency of the s3fs package
# The latest version of aiobotocore (v1.4.0) is only compatible with botocore up to 1.20.106
# botocore==1.20.106 is compatible with boto3==1.17.106
botocore==1.20.106  # From aiobotocore v1.4.0 dependencies https://pypi.org/project/aiobotocore/
boto3==1.17.106  # from botocore==1.20.106 dependency
# END NOTE<commit_msg>Don't use jupyter-client 7.3.2 (#5252)<commit_after># The pip resolver became more strict starting with version 20.3
# This constraints file is "a way to specify global (version) limits for packages"
# without specifying the install of these packages.
# See also: https://pip.pypa.io/en/stable/user_guide/#constraints-files
# We use it to place limits on certain packages to help the resolver come to a compatible configuration more quickly
# To install dev dependencies using the new pip resolver (recommended) please use the following syntax:
# `python -m pip install -r requirements-dev.txt -c constraints-dev.txt`

# NOTE - 20220511
# pypandoc is a dependency of pyspark and is only compatible up to 1.7.5 with pyspark<3.0.0
# constraining the version of pypandoc for now.
#
# ` AttributeError: module 'pypandoc' has no attribute 'convert'` was the error we were getting
pypandoc==1.7.5
# END NOTE

# NOTE - 20210114
# aiobotocore is a dependency of the s3fs package
# The latest version of aiobotocore (v1.4.0) is only compatible with botocore up to 1.20.106
# botocore==1.20.106 is compatible with boto3==1.17.106
botocore==1.20.106  # From aiobotocore v1.4.0 dependencies https://pypi.org/project/aiobotocore/
boto3==1.17.106  # from botocore==1.20.106 dependency
# END NOTE

# NOTE - 20220606
# jupyter-client is used in nbconvert
# temporarily excluding the latest version due to failing tests
jupyter-client<7.3.2,>7.3.2
# END NOTE
{'fraction_non_alphanumeric': 0.0871338524286244, 'fraction_numerical': 0.06192065257693734, 'mean_word_length': 4.353174603174603, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7348399', 'n_tokens_mistral': 949, 'n_tokens_neox': 857, 'n_words': 338}