<commit_before>"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
import os
import sys

from mmap import mmap, ACCESS_READ
from mmap import ALLOCATIONGRANULARITY

__all__ = ["align_to_mmap", "is_64_bit", "buffer",
           "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]

#{ Utilities

try:
    # Python 2
    buffer = buffer
except NameError:
    # Python 3 has no `buffer`; only `memoryview`
    def buffer(obj, offset, size):
        # Actually, for gitpython this is fastest ... .
        return memoryview(obj)[offset:offset+size]
        # doing it directly is much faster !
        # return obj[offset:offset + size]


def string_types():
    return str

def align_to_mmap(num, round_up):
    """<commit_msg>Revert "Remove basestring reference, no py2 support"<commit_after>"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
import os
import sys

from mmap import mmap, ACCESS_READ
from mmap import ALLOCATIONGRANULARITY

__all__ = ["align_to_mmap", "is_64_bit", "buffer",
           "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]

#{ Utilities

try:
    # Python 2
    buffer = buffer
except NameError:
    # Python 3 has no `buffer`; only `memoryview`
    def buffer(obj, offset, size):
        # Actually, for gitpython this is fastest ... .
        return memoryview(obj)[offset:offset+size]
        # doing it directly is much faster !
        # return obj[offset:offset + size]


def string_types():
    if sys.version_info[0] >= 3:
        return str
    else:
        return basestring


def align_to_mmap(num, round_up):
    """
{'fraction_non_alphanumeric': 0.09795918367346938, 'fraction_numerical': 0.00641399416909621, 'mean_word_length': 3.5157894736842104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4174247', 'n_tokens_mistral': 521, 'n_tokens_neox': 512, 'n_words': 176}