<commit_before>from chainer.backends import cuda
from chainer import function_node
from chainer.utils import type_check

import numpy as np
from six.moves import range
from functools import reduce

index_dtype = {t().itemsize: t for t in np.sctypes['int']}


def _byte2step(iterable, itemsize):
    for i in iterable:
        assert i % itemsize == 0
    return tuple([i // itemsize for i in iterable])


def _step2byte(iterable, itemsize):
    return tuple([i * itemsize for i in iterable])


def _maybe_overlapping_memory(shape: tuple, strides: tuple):
    """Returns bool value indicating the array with such shape and strides
    might have overlapping memory.

    Args:
    shape (tuple of int): The shape of output.
    strides (tuple of int): The strides of output, given in the unit of steps.
    storage_offset (int):
        The offset between the head of allocated memory and the pointer of
        first element, given in the unit of steps.

    Returns:
        bool: Existence of the overlapping memory
    """
    max_ptr_in_slice = 0
    for stride, size in sorted(zip([abs(s) for s in strides], shape)):
        if stride <= max_ptr_in_slice:
            return True
        max_ptr_in_slice += stride * (size - 1)
    return False
<commit_msg>removed type annotation<commit_after>from chainer.backends import cuda
from chainer import function_node
from chainer.utils import type_check

import numpy as np
from six.moves import range
from functools import reduce

index_dtype = {t().itemsize: t for t in np.sctypes['int']}


def _byte2step(iterable, itemsize):
    for i in iterable:
        assert i % itemsize == 0
    return tuple([i // itemsize for i in iterable])


def _step2byte(iterable, itemsize):
    return tuple([i * itemsize for i in iterable])


def _maybe_overlapping_memory(shape, strides):
    """Returns bool value indicating the array with such shape and strides
    might have overlapping memory.

    Args:
    shape (tuple of int): The shape of output.
    strides (tuple of int): The strides of output, given in the unit of steps.
    storage_offset (int):
        The offset between the head of allocated memory and the pointer of
        first element, given in the unit of steps.

    Returns:
        bool: Existence of the overlapping memory
    """
    max_ptr_in_slice = 0
    for stride, size in sorted(zip([abs(s) for s in strides], shape)):
        if stride <= max_ptr_in_slice:
            return True
        max_ptr_in_slice += stride * (size - 1)
    return False

{'fraction_non_alphanumeric': 0.07222222222222222, 'fraction_numerical': 0.003968253968253968, 'mean_word_length': 3.1532125205930805, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6067671', 'n_tokens_mistral': 744, 'n_tokens_neox': 719, 'n_words': 308}