<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

from collections import defaultdict
from itertools import chain

from sc2reader.exceptions import ParseError, ReadError
from sc2reader.objects import *
from sc2reader.events import *
from sc2reader.utils import AttributeDict, BIG_ENDIAN, LITTLE_ENDIAN

class Reader(object):
    def __init__(self, **options):
        self.options = options

    def __call__(self, data, replay):
        raise NotImplementedError

class InitDataReader_Base(Reader):

    def get_player_names(self, data, replay):
        # The first block of the replay.initData file represents a list of
        # human player names; computers are no recorded. This list appears to
        # always be 16 long, with "" names filling in the balance. Each name
        # is followed by a 5 byte string that appears to be always all zeros.
        player_names = list()
        for player in range(data.read_byte()):
            name = data.read_string()
            data.skip(5)
            if name:
                player_names.append(name)
        return player_names

    def __call__(self, data, replay):
        player_names = self.get_player_names(data, replay)

        # The next block contains information about the structure of the MPQ
        # archive. We don't read this information because we've got mpyq for
        # that. Its split into 3 sections because of the variable length segment
        # in the middle that prevents bulk skipping. The last section also
        # appears to be variable length, hack it to do a find for the section
        # we are looking for.
        data.skip(24)
        sc_account_id = data.read_string()
        distance = data.read_range(data.tell(), data.length).find('s2ma')
        data.skip(distance)

        # The final block of this file that we concern ourselves with is a list
        # of what appears to be map data with the s2ma header on each element.
        # Each element consists of two unknown bytes, a realm id (e.g EU or US)
        # and a map hash which probably ties back to the sc2map files.
        #
        # Some replays don't seem to have a maps section at all, now we can't
        # know what gateway its from? Very strange...
        #
        # TODO: Figure out how we could be missing a maps section.
        map_data = list()
        while data.peek(4) == 's2ma':
            map_data.append(DepotFile(data.read(40)))

        # Return the extracted information inside an AttributeDict.
        return AttributeDict(
            map_data=map_data,
            player_names=player_names,
            sc_account_id=sc_account_id,
        )

class InitDataReader_24764(InitDataReader_Base):
    def get_player_names(self, data, replay):
        player_names = list()
        for player in range(data.read_byte()):
            name_length = data.read_byte()
            data.byte_align() # Strings seem to be always byte aligned
            name = data.read_string(length=name_length)
            data.read_bits(1) # Not sure why we have this offset, it could be a flag?
            clan_length = data.read_byte()
            data.byte_align()
            clan_name = data.read_string(length=clan_length)
            unknown = data.read_bits(42)
            data.read_bytes(5)
            if name:
                player_names.append(name)
        return player_names



class AttributesEventsReader_Base(Reader):
    header_length = 4
    offset=False

    def __call__(self, data, replay):
        # The replay.attribute.events file is comprised of a small header and
        # single long list of attributes with the 0x00 00 03 E7 header on each
        # element. Each element holds a four byte attribute id code, a one byte
        # player id, and a four byte value code. Unlike the other files, this
        # file is stored in little endian format.
        #
        # See: ``objects.Attribute`` for attribute id and value lookup logic
        #
        attribute_events = list()
        data.skip(self.header_length)
        for attribute in range(data.read_int(LITTLE_ENDIAN)):
            info = [
                    data.read_int(LITTLE_ENDIAN),
                    data.read_int(LITTLE_ENDIAN),
                    data.read_byte(),
                    data.read(4).strip("\x00 ")[::-1]
                ]<commit_msg>Use the 1 bit clan info flag in InitData.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

from collections import defaultdict
from itertools import chain

from sc2reader.exceptions import ParseError, ReadError
from sc2reader.objects import *
from sc2reader.events import *
from sc2reader.utils import AttributeDict, BIG_ENDIAN, LITTLE_ENDIAN

class Reader(object):
    def __init__(self, **options):
        self.options = options

    def __call__(self, data, replay):
        raise NotImplementedError

class InitDataReader_Base(Reader):

    def get_player_names(self, data, replay):
        # The first block of the replay.initData file represents a list of
        # human player names; computers are no recorded. This list appears to
        # always be 16 long, with "" names filling in the balance. Each name
        # is followed by a 5 byte string that appears to be always all zeros.
        player_names = list()
        for player in range(data.read_byte()):
            name = data.read_string()
            data.skip(5)
            if name:
                player_names.append(name)
        return player_names

    def __call__(self, data, replay):
        player_names = self.get_player_names(data, replay)

        # The next block contains information about the structure of the MPQ
        # archive. We don't read this information because we've got mpyq for
        # that. Its split into 3 sections because of the variable length segment
        # in the middle that prevents bulk skipping. The last section also
        # appears to be variable length, hack it to do a find for the section
        # we are looking for.
        data.skip(24)
        sc_account_id = data.read_string()
        distance = data.read_range(data.tell(), data.length).find('s2ma')
        data.skip(distance)

        # The final block of this file that we concern ourselves with is a list
        # of what appears to be map data with the s2ma header on each element.
        # Each element consists of two unknown bytes, a realm id (e.g EU or US)
        # and a map hash which probably ties back to the sc2map files.
        #
        # Some replays don't seem to have a maps section at all, now we can't
        # know what gateway its from? Very strange...
        #
        # TODO: Figure out how we could be missing a maps section.
        map_data = list()
        while data.peek(4) == 's2ma':
            map_data.append(DepotFile(data.read(40)))

        # Return the extracted information inside an AttributeDict.
        return AttributeDict(
            map_data=map_data,
            player_names=player_names,
            sc_account_id=sc_account_id,
        )

class InitDataReader_24764(InitDataReader_Base):
    def get_player_names(self, data, replay):
        player_names = list()
        for player in range(data.read_byte()):
            name_length = data.read_byte()
            data.byte_align() # Strings seem to be always byte aligned
            name = data.read_string(length=name_length)

            # Flag is 1 for multiplayer, 0 for single player
            if data.read_bits(1):
                clan_length = data.read_byte()
                data.byte_align()
                clan_name = data.read_string(length=clan_length)
                unknown = data.read_bits(42)

            data.read_bytes(5)

            if name:
                player_names.append(name)
        return player_names



class AttributesEventsReader_Base(Reader):
    header_length = 4
    offset=False

    def __call__(self, data, replay):
        # The replay.attribute.events file is comprised of a small header and
        # single long list of attributes with the 0x00 00 03 E7 header on each
        # element. Each element holds a four byte attribute id code, a one byte
        # player id, and a four byte value code. Unlike the other files, this
        # file is stored in little endian format.
        #
        # See: ``objects.Attribute`` for attribute id and value lookup logic
        #
        attribute_events = list()
        data.skip(self.header_length)
        for attribute in range(data.read_int(LITTLE_ENDIAN)):
            info = [
                    data.read_int(LITTLE_ENDIAN),
                    data.read_int(LITTLE_ENDIAN),
                    data.read_byte(),
                    data.read(4).strip("\x00 ")[::-1]
                ]
{'fraction_non_alphanumeric': 0.07084188911704312, 'fraction_numerical': 0.009696554871092859, 'mean_word_length': 2.25064886911383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701620', 'n_tokens_mistral': 2446, 'n_tokens_neox': 2315, 'n_words': 870}