<commit_before>from datetime import datetime
from getpass import getuser
from itertools import groupby
import errno
from .key import key_factory
from .remote import cache_if_remote

import os
import socket
import stat
import sys
import time
from io import BytesIO
from . import xattr
from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython
if have_cython():
    from .platform import acl_get, acl_set
    from .chunker import Chunker
    from .hashindex import ChunkIndex
    import msgpack

ITEMS_BUFFER = 1024 * 1024

CHUNK_MIN_EXP = 10  # 2**10 == 1kiB
CHUNK_MAX_EXP = 23  # 2**23 == 8MiB
HASH_WINDOW_SIZE = 0xfff  # 4095B
HASH_MASK_BITS = 16  # results in ~64kiB chunks statistically

# defaults, use --chunker-params to override
CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)

utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {})
has_mtime_ns = sys.version >= '3.3'
has_lchmod = hasattr(os, 'lchmod')
has_lchflags = hasattr(os, 'lchflags')

# Python <= 3.2 raises OSError instead of PermissionError (See #164)<commit_msg>try to mock msgpack altogether to fix RTD again<commit_after>from datetime import datetime
from getpass import getuser
from itertools import groupby
import errno
from .key import key_factory
from .remote import cache_if_remote

import os
import socket
import stat
import sys
import time
from io import BytesIO
from . import xattr
from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
    Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython
if have_cython():
    from .platform import acl_get, acl_set
    from .chunker import Chunker
    from .hashindex import ChunkIndex
    import msgpack
else:
    import mock
    msgpack = mock.Mock()

ITEMS_BUFFER = 1024 * 1024

CHUNK_MIN_EXP = 10  # 2**10 == 1kiB
CHUNK_MAX_EXP = 23  # 2**23 == 8MiB
HASH_WINDOW_SIZE = 0xfff  # 4095B
HASH_MASK_BITS = 16  # results in ~64kiB chunks statistically

# defaults, use --chunker-params to override
CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)

utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {})
has_mtime_ns = sys.version >= '3.3'
has_lchmod = hasattr(os, 'lchmod')
has_lchflags = hasattr(os, 'lchflags')

# Python <= 3.2 raises OSError instead of PermissionError (See #164)
{'fraction_non_alphanumeric': 0.07927728613569321, 'fraction_numerical': 0.029498525073746312, 'mean_word_length': 5.280092592592593, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4176739', 'n_tokens_mistral': 1021, 'n_tokens_neox': 969, 'n_words': 278}