<commit_before>from struct import *
from common import *
import sys

GNAMELEN_OFFSET = 0x30
ENAMELEN_OFFSET = 0x14
FNAMELEN_OFFSET = 0x10

def read_group(group_dump):
    group = {}
    glen, group['flag'], group['domain'], group['event group offset'], \
        group['event group length'], group['schema index'], \
        group['event count'] = unpack_from(">HxxIBxHHBB", group_dump)
    group['event indexes'] = unpack_from(">16H", group_dump[0x10:])
    group['name'], nlen = read_string(group_dump[GNAMELEN_OFFSET:])
    group['description'], dlen = read_string(group_dump[(GNAMELEN_OFFSET + nlen):])

    return glen, group

def read_formula(formula_dump):
    formula = {}
    tlen, formula['flag'], formula['Group'] = unpack_from(">HxxIH6x",
                                                          formula_dump)
    if formula['flag'] == 0x4:
        formula['Grouped'] = 'y'
    else:
        formula['Grouped'] = 'n'

    formula['Formula Name'], nlen = read_string(formula_dump[FNAMELEN_OFFSET:])
    formula['Formula Description'], dlen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen):])
    formula['Formula'], flen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen + dlen):])
    formula['Unit'], ulen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen + dlen + flen):])

    return tlen, formula

def read_event(event_dump):
    event = {}
    elen, event['domain'], event['record byte offset'], event['record length'], \
        event['counter offset'], event['flag'], event['primary group index'], \
        event['group count'] = unpack_from(">HxxBxHHHIHH", event_dump)

    event['name'], nlen = read_string(event_dump[ENAMELEN_OFFSET:])
    event['description'], dlen = read_string(event_dump[(ENAMELEN_OFFSET + nlen):])
    event['detailed description'], ddlen = read_string(event_dump[(ENAMELEN_OFFSET + nlen + dlen):])

    return elen, event

def read_groups(dump_file, page_offset, num_pages, num_count, group_type):
    f = open(dump_file, 'r')
    f.seek(page_offset * PAGE_SIZE, 0)
    dump = f.read(PAGE_SIZE * num_pages)

    if group_type == 'groups':
        read_fn = read_group
    elif group_type == 'events':
        read_fn = read_event
    elif group_type == 'formulae':
        read_fn = read_formula
    else:<commit_msg>Dump formula index also from the lid to the csv<commit_after>from struct import *
from common import *
import sys

GNAMELEN_OFFSET = 0x30
ENAMELEN_OFFSET = 0x14
FNAMELEN_OFFSET = 0x10

def read_group(group_dump):
    group = {}
    glen, group['flag'], group['domain'], group['event group offset'], \
        group['event group length'], group['schema index'], \
        group['event count'] = unpack_from(">HxxIBxHHBB", group_dump)
    group['event indexes'] = unpack_from(">16H", group_dump[0x10:])
    group['name'], nlen = read_string(group_dump[GNAMELEN_OFFSET:])
    group['description'], dlen = read_string(group_dump[(GNAMELEN_OFFSET + nlen):])

    return glen, group

def read_formula(formula_dump):
    formula = {}
    tlen, formula['flag'], formula['Group'] = unpack_from(">HxxIH6x",
                                                          formula_dump)
    if formula['flag'] == 0x4:
        formula['Grouped'] = 'y'
    else:
        formula['Grouped'] = 'n'

    formula['Formula Name'], nlen = read_string(formula_dump[FNAMELEN_OFFSET:])
    formula['Formula Description'], dlen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen):])
    formula['Formula'], flen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen + dlen):])
    formula['Unit'], ulen = read_string(formula_dump[(FNAMELEN_OFFSET + nlen + dlen + flen):])

    return tlen, formula

def read_event(event_dump):
    event = {}
    elen, event['formala index'], event['domain'], event['record byte offset'], event['record length'], \
        event['counter offset'], event['flag'], event['primary group index'], \
        event['group count'] = unpack_from(">HHBxHHHIHH", event_dump)

    if event['formala index'] == 0xffff:
        event['formala index'] = '-1';

    event['name'], nlen = read_string(event_dump[ENAMELEN_OFFSET:])
    event['description'], dlen = read_string(event_dump[(ENAMELEN_OFFSET + nlen):])
    event['detailed description'], ddlen = read_string(event_dump[(ENAMELEN_OFFSET + nlen + dlen):])

    return elen, event

def read_groups(dump_file, page_offset, num_pages, num_count, group_type):
    f = open(dump_file, 'r')
    f.seek(page_offset * PAGE_SIZE, 0)
    dump = f.read(PAGE_SIZE * num_pages)

    if group_type == 'groups':
        read_fn = read_group
    elif group_type == 'events':
        read_fn = read_event
    elif group_type == 'formulae':
        read_fn = read_formula
    else:
{'fraction_non_alphanumeric': 0.14478186484174507, 'fraction_numerical': 0.008126603934987169, 'mean_word_length': 3.8871473354231973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4774559', 'n_tokens_mistral': 1550, 'n_tokens_neox': 1508, 'n_words': 321}