<commit_before>import binascii
import struct
import math
import os
import sys

def unpack_values(data, length):
    # unpack data stream

    values = []

    pixel = 0
    ptr = 0

    while len(values) < length:

        delta = struct.unpack('b', data[ptr])[0]
        ptr += 1

        if delta != -128:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<h', data[ptr:ptr + 2])[0]
        ptr += 2

        if delta != -32768:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<i', data[ptr:ptr + 4])[0]
        ptr += 4

        if delta != -2147483648:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<q', data[ptr:ptr + 8])[0]
        ptr += 8
        pixel += delta
        values.append(pixel)

    return values

def unpackbyteoffset(filename):

    hist = [0 for j in range(65536)]

    data = open(filename, 'r').read()

    start_tag = binascii.unhexlify('0c1a04d5')

    data_offset = data.find(start_tag) + 4

    header = data[:data.find(start_tag)].split('\n')

    fast = 0
    slow = 0
    length = 0

    for record in header:
        if 'X-Binary-Size-Fastest-Dimension' in record:
            fast = int(record.split()[-1])
        if 'X-Binary-Size-Second-Dimension' in record:
            slow = int(record.split()[-1])
        if 'X-Binary-Number-of-Elements' in record:
            length = int(record.split()[-1])

    assert(length == fast * slow)

    values = unpack_values(data[data_offset:], length)

    for i in range(slow):
        for j in range(fast):<commit_msg>More intelligent decision about the maximum<commit_after>import binascii
import struct
import math
import os
import sys

def unpack_values(data, length):
    # unpack data stream

    values = []

    pixel = 0
    ptr = 0

    while len(values) < length:

        delta = struct.unpack('b', data[ptr])[0]
        ptr += 1

        if delta != -128:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<h', data[ptr:ptr + 2])[0]
        ptr += 2

        if delta != -32768:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<i', data[ptr:ptr + 4])[0]
        ptr += 4

        if delta != -2147483648:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<q', data[ptr:ptr + 8])[0]
        ptr += 8
        pixel += delta
        values.append(pixel)

    return values

def unpackbyteoffset(filename):

    data = open(filename, 'r').read()

    start_tag = binascii.unhexlify('0c1a04d5')

    data_offset = data.find(start_tag) + 4

    header = data[:data.find(start_tag)].split('\n')

    fast = 0
    slow = 0
    length = 0

    for record in header:
        if 'X-Binary-Size-Fastest-Dimension' in record:
            fast = int(record.split()[-1])
        if 'X-Binary-Size-Second-Dimension' in record:
            slow = int(record.split()[-1])
        if 'X-Binary-Number-of-Elements' in record:
            length = int(record.split()[-1])

    assert(length == fast * slow)

    values = unpack_values(data[data_offset:], length)
    hist = [0 for j in range(max(values) + 1)]

    for i in range(slow):
        for j in range(fast):
{'fraction_non_alphanumeric': 0.13007892429114293, 'fraction_numerical': 0.02747734580532008, 'mean_word_length': 1.9272882805816938, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5143336', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1041, 'n_words': 271}