<commit_before>#!/usr/bin/python
# File:        section.py
# Description: section
# Created:     2017-06-27


from .common import format_bytes


SECTION_TYPE = 66666666
SECTION_UNK_3 = 33333333
SECTION_UNK_2 = 22222222


class Section(object):

    def __init__(self, ident, dbytes):
        self.ident = ident
        self.unknown = unknown = []
        if ident == SECTION_TYPE:
            unknown.append(dbytes.read_n(8))
            self.filetype = dbytes.read_string()
            unknown.append(dbytes.read_n(9))
        elif ident == SECTION_UNK_3:
            unknown.append(dbytes.read_n(20))
        elif ident == SECTION_UNK_2:
            unknown.append(dbytes.read_n(12))
            self.version = ver = dbytes.read_byte()
            unknown.append(dbytes.read_n(15))
        else:
            raise IOError(f"unknown section: {ident} (0x{ident:08x})")
        print(self.__dict__)

    @staticmethod
    def iter_to(dbytes, to):
        while True:
            ident = dbytes.read_fixed_number(4)
            yield Section(ident, dbytes)
            if ident == to:
                break

    @staticmethod
    def read_to_map(dbytes, to):
        return {s.ident: s for s in Section.iter_to(dbytes, to)}


# vim:set sw=4 ts=8 sts=4 et sr ft=python fdm=marker tw=0:<commit_msg>remove debug log<commit_after>#!/usr/bin/python
# File:        section.py
# Description: section
# Created:     2017-06-27


from .common import format_bytes


SECTION_TYPE = 66666666
SECTION_UNK_3 = 33333333
SECTION_UNK_2 = 22222222


class Section(object):

    def __init__(self, ident, dbytes):
        self.ident = ident
        self.unknown = unknown = []
        if ident == SECTION_TYPE:
            unknown.append(dbytes.read_n(8))
            self.filetype = dbytes.read_string()
            unknown.append(dbytes.read_n(9))
        elif ident == SECTION_UNK_3:
            unknown.append(dbytes.read_n(20))
        elif ident == SECTION_UNK_2:
            unknown.append(dbytes.read_n(12))
            self.version = ver = dbytes.read_byte()
            unknown.append(dbytes.read_n(15))
        else:
            raise IOError(f"unknown section: {ident} (0x{ident:08x})")

    @staticmethod
    def iter_to(dbytes, to):
        while True:
            ident = dbytes.read_fixed_number(4)
            yield Section(ident, dbytes)
            if ident == to:
                break

    @staticmethod
    def read_to_map(dbytes, to):
        return {s.ident: s for s in Section.iter_to(dbytes, to)}


# vim:set sw=4 ts=8 sts=4 et sr ft=python fdm=marker tw=0:
{'fraction_non_alphanumeric': 0.10811865729898516, 'fraction_numerical': 0.04059328649492584, 'mean_word_length': 2.264968152866242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5480977', 'n_tokens_mistral': 905, 'n_tokens_neox': 838, 'n_words': 178}