<commit_before>import struct
import numpy as np


INT_SIZE = 4
FLOAT_SIZE = 4


def read(file, format, count, size):
    buffer = file.read(count * size)
    format = format.format(count)

    buffer = struct.unpack(format, buffer)

    if count == 1:
        return buffer[0]

    return buffer


def write(file, format, buffer):
    if isinstance(buffer, np.ndarray):
        buffer = buffer.flatten()
    elif not isinstance(buffer, list):
        buffer = [buffer]

    format = format.format(len(buffer))
    buffer = struct.pack(format, *buffer)

    file.write(buffer)


def readi(file, count=1):
    return read(file, '{0}i', count, INT_SIZE)


def writei(file, buffer):
    write(file, '{0}i', buffer)


def readf(file, count=1):
    return read(file, '{0}f', count, FLOAT_SIZE)


def writef(file, buffer):
    write(file, '{0}f', buffer)


def reads(file, length):
    return file.read(length)


def writes(file, buffer):
    file.write(buffer)


def dumpMatrix(path, matrix):
    rows, columns = matrix.shape
    values = np.asarray(matrix).flatten()

    with open(path, 'wb+') as file:
        writei(file, rows)
        writei(file, columns)
        writef(file, values)


def loadMatrix(path):
    with open(path, 'rb') as file:
        rows = readi(file)
        columns = readi(file)
        count = rows * count
        values = readf(file, count)
        matrix = np.reshape(values, (rows, columns))

        return matrix<commit_msg>Fixed matrix loading.<commit_after>import struct
import numpy as np


INT_SIZE = 4
FLOAT_SIZE = 4


def read(file, format, count, size):
    buffer = file.read(count * size)
    format = format.format(count)

    buffer = struct.unpack(format, buffer)

    if count == 1:
        return buffer[0]

    return buffer


def write(file, format, buffer):
    if isinstance(buffer, np.ndarray):
        buffer = buffer.flatten()
    elif not isinstance(buffer, list):
        buffer = [buffer]

    format = format.format(len(buffer))
    buffer = struct.pack(format, *buffer)

    file.write(buffer)


def readi(file, count=1):
    return read(file, '{0}i', count, INT_SIZE)


def writei(file, buffer):
    write(file, '{0}i', buffer)


def readf(file, count=1):
    return read(file, '{0}f', count, FLOAT_SIZE)


def writef(file, buffer):
    write(file, '{0}f', buffer)


def reads(file, length):
    return file.read(length)


def writes(file, buffer):
    file.write(buffer)


def dumpMatrix(path, matrix):
    rows, columns = matrix.shape
    values = np.asarray(matrix).flatten()

    with open(path, 'wb+') as file:
        writei(file, rows)
        writei(file, columns)
        writef(file, values)


def loadMatrix(path):
    with open(path, 'rb') as file:
        rows = readi(file)
        columns = readi(file)
        count = rows * columns
        values = readf(file, count)
        matrix = np.reshape(values, (rows, columns))

        return matrix
{'fraction_non_alphanumeric': 0.1313443072702332, 'fraction_numerical': 0.006858710562414266, 'mean_word_length': 2.9525745257452574, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7268419', 'n_tokens_mistral': 985, 'n_tokens_neox': 942, 'n_words': 261}