<commit_before>from typing import List

import numpy as np


def boolean_mask_by_value(mask: np.ndarray, value: object) -> np.ndarray:
    return boolean_mask_by_values(mask=mask, values=[value])


def boolean_mask_by_values(mask: np.ndarray, values: List[object]) -> np.ndarray:
    return np.isin(mask, values)


def replace_value(mask: np.ndarray, old_value: object, new_value: object) -> np.ndarray:
    return replace_values(mask=mask, old_values=[old_value], new_value=new_value)


def replace_values(mask: np.ndarray, old_values: List[object], new_value: object) -> np.ndarray:
    boolean_mask = boolean_mask_by_values(mask=mask, values=old_values)
    mask[boolean_mask] = new_value
    return mask


def encode_int32_as_rgb8(mask: np.ndarray) -> np.ndarray:
    return np.concatenate([mask & 0xFF, mask >> 8 & 0xFF, mask >> 16 & 0xFF], axis=-1).astype(np.uint8)


def encode_2int16_as_rgba8(mask: np.ndarray) -> np.ndarray:
    return np.concatenate(
        [mask[..., [0]] >> 8, mask[..., [0]] & 0xFF, mask[..., [1]] >> 8, mask[..., [1]] & 0xFF], axis=-1
    ).astype(np.uint8)
<commit_msg>Faster mass replacement (at the same time)<commit_after>from typing import Dict, List, Union

import numpy as np


def boolean_mask_by_value(mask: np.ndarray, value: int) -> np.ndarray:
    return boolean_mask_by_values(mask=mask, values=[value])


def boolean_mask_by_values(mask: np.ndarray, values: List[int]) -> np.ndarray:
    return np.isin(mask, values)


def replace_value(mask: np.ndarray, old_value: int, new_value: int) -> np.ndarray:
    return replace_values(mask=mask, value_map={old_value: new_value})


def replace_values(
    mask: np.ndarray, value_map: Dict[int, int], value_min: Union[int, None] = None, value_max: Union[int, None] = None
) -> np.ndarray:
    index_substitutes = np.array(
        [
            value_map.get(item, item)
            for item in range(
                value_min if value_min is not None else np.iinfo(mask.dtype).min,
                (value_max if value_max is not None else np.iinfo(mask.dtype).max) + 1,
            )
        ]
    )

    return index_substitutes[mask]


def encode_int32_as_rgb8(mask: np.ndarray) -> np.ndarray:
    return np.concatenate([mask & 0xFF, mask >> 8 & 0xFF, mask >> 16 & 0xFF], axis=-1).astype(np.uint8)


def encode_2int16_as_rgba8(mask: np.ndarray) -> np.ndarray:
    return np.concatenate(
        [mask[..., [0]] >> 8, mask[..., [0]] & 0xFF, mask[..., [1]] >> 8, mask[..., [1]] & 0xFF], axis=-1
    ).astype(np.uint8)

{'fraction_non_alphanumeric': 0.1490633718612993, 'fraction_numerical': 0.020326823435631726, 'mean_word_length': 3.6395563770794825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5154913', 'n_tokens_mistral': 941, 'n_tokens_neox': 897, 'n_words': 210}