<commit_before>import time
import logging
import numpy as np
import scipy.sparse as spp

log = logging.getLogger(__name__)

def rand64c(*shape, order='F'):
    """
    Constructs a `np.ndarray` of the requested shape and
    populates it with random np.complex64 values.
    """
    r = np.random.rand(*shape).astype(np.float32)
    i = np.random.rand(*shape).astype(np.float32)
    arr = (r + 1j*i).astype(np.complex64)
    if order == 'F':
        arr = np.asfortranarray(arr)
    return arr


def randM(M, N, density):
    """
    Constructs a `scipy.sparse.spmatrix'  of the requested shape and
    density and populates it with random np.complex64 values.
    """
    A_r = spp.random(M, N, density=density, format='csr', dtype=np.float32)
    A_i = spp.random(M, N, density=density, format='csr', dtype=np.float32)
    A = (A_r + 1j * A_i).astype(np.dtype('complex64'))
    return A


class profile(object):
    extra = dict()

    def __init__(self, event, **kwargs):
        self._event = event
        self._kwargs = kwargs

    def __enter__(self):
        self._start = time.time()

    def __exit__(self, exc_type, exc_value, traceback):
        data = dict(
            duration = time.time() - self._start,
            event    = self._event,
        )
        data.update(self.extra)
        data.update(self._kwargs)
        kvs = sorted(data.items(), key=lambda kv: kv[0])
        msg = "PROFILE(%s)" % ", ".join("%s=%s" % (k,repr(v)) for k,v in kvs)
        log.critical(msg)

<commit_msg>use debug log level for profiling statemens<commit_after>import time
import logging
import numpy as np
import scipy.sparse as spp

log = logging.getLogger(__name__)

def rand64c(*shape, order='F'):
    """
    Constructs a `np.ndarray` of the requested shape and
    populates it with random np.complex64 values.
    """
    r = np.random.rand(*shape).astype(np.float32)
    i = np.random.rand(*shape).astype(np.float32)
    arr = (r + 1j*i).astype(np.complex64)
    if order == 'F':
        arr = np.asfortranarray(arr)
    return arr


def randM(M, N, density):
    """
    Constructs a `scipy.sparse.spmatrix'  of the requested shape and
    density and populates it with random np.complex64 values.
    """
    A_r = spp.random(M, N, density=density, format='csr', dtype=np.float32)
    A_i = spp.random(M, N, density=density, format='csr', dtype=np.float32)
    A = (A_r + 1j * A_i).astype(np.dtype('complex64'))
    return A


class profile(object):
    extra = dict()

    def __init__(self, event, **kwargs):
        self._event = event
        self._kwargs = kwargs

    def __enter__(self):
        self._start = time.time()

    def __exit__(self, exc_type, exc_value, traceback):
        data = dict(
            duration = time.time() - self._start,
            event    = self._event,
        )
        data.update(self.extra)
        data.update(self._kwargs)
        kvs = sorted(data.items(), key=lambda kv: kv[0])
        msg = "PROFILE(%s)" % ", ".join("%s=%s" % (k,repr(v)) for k,v in kvs)
        log.debug(msg)


{'fraction_non_alphanumeric': 0.14074317658664912, 'fraction_numerical': 0.013811246300559027, 'mean_word_length': 2.696233292831106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6381829', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1033, 'n_words': 260}