<commit_before>import numpy as np
import json
import os
from os.path import join as pjoin

# Only for loading annotations
import h5py
from scipy.io import loadmat

from scipy.stats import multivariate_normal
from scipy import signal
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap

try:
    from functools import lru_cache  # Needs at least Python 3.2
except ImportError:
    def lru_cache(fn, *a, **kw): return fn


START_TIMES = [5543, 3607, 27244, 31182, 1, 22402, 18968, 46766]
TRAIN_START, TRAIN_END = 49700, 227540


###############################################################################
# Generic utilities


def scale_shape(shape, factors):
    try:
        len(factors)
    except TypeError:
        # It's a number
        factors = (factors, factors)

    return int(shape[0]*factors[0]), int(shape[1]*factors[1])


def argmax2d_xy(arr):
    idx = np.unravel_index(arr.argmax(), arr.shape)<commit_msg>Add an option to argmax2d_xy to smooth first.<commit_after>import numpy as np
import json
import os
from os.path import join as pjoin

# Only for loading annotations
import h5py
from scipy.io import loadmat

from scipy.stats import multivariate_normal
from scipy import signal
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap

try:
    from functools import lru_cache  # Needs at least Python 3.2
except ImportError:
    def lru_cache(fn, *a, **kw): return fn


START_TIMES = [5543, 3607, 27244, 31182, 1, 22402, 18968, 46766]
TRAIN_START, TRAIN_END = 49700, 227540


###############################################################################
# Generic utilities


def scale_shape(shape, factors):
    try:
        len(factors)
    except TypeError:
        # It's a number
        factors = (factors, factors)

    return int(shape[0]*factors[0]), int(shape[1]*factors[1])


def argmax2d_xy(arr, smooth=None):
    if smooth is not None:
        sy, sx = smooth
        arr = cv2.GaussianBlur(arr, (sx,sy), 0, borderType=cv2.BORDER_CONSTANT)

    idx = np.unravel_index(arr.argmax(), arr.shape)
{'fraction_non_alphanumeric': 0.15792007703418393, 'fraction_numerical': 0.05296100144439095, 'mean_word_length': 4.301020408163265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '456329', 'n_tokens_mistral': 771, 'n_tokens_neox': 659, 'n_words': 208}