<commit_before>from time import ctime
from collections import defaultdict

from objects import Attribute, Message, Player, Event
from eventparsers import *
from utils import ByteStream
from exceptions import ParseError

from pprint import PrettyPrinter
pprint = PrettyPrinter(indent=2).pprint

#################################################
# Parser Dispatch Functions
#################################################
def get_detail_parser(build):
    #This file format appears to have never changed
    return DetailParser()
    
def get_attribute_parser(build):
    if build >= 17326:
        return AttributeParser_17326()
    
    #All versions prior to 17326 have been the same
    return AttributeParser()
    
def get_event_parser(build):
    if build >= 17326:
        return EventParser_17326()
    if build >= 16561:
        return EventParser_16561()
    
    #All versions prior to 16561 appear to be the same
    return EventParser()
    
def get_message_parser(build):
    #format appears to have not changed
    return MessageParser()

def get_initdata_parser(build):
    return InitdataParser()
    
class InitdataParser(object):
    def load(self,replay,filecontents):
        bytes = ByteStream(filecontents)
        num_players = bytes.get_big_int(1)
        for p in range(0,num_players):
            name_length = bytes.get_big_int(1)
            name = bytes.get_string(name_length)
            bytes.skip(5)
        
        bytes.skip(5) # Unknown
        bytes.get_string(4) # Always Dflt
        bytes.skip(15) #Unknown
        id_length = bytes.get_big_int(1)
        sc_account_id = bytes.get_string(id_length)
        bytes.skip(684) # Fixed Length data for unknown purpose
        while( bytes.get_string(4).lower() == 's2ma' ):
            bytes.skip(2)
            replay.realm = bytes.get_string(2).lower()
            unknown_map_hash = bytes.get_big(32)
            
#################################################
# replay.attributes.events Parsing classes
#################################################
class AttributeParser(object):
    def load_header(self, replay, bytes):
        bytes.skip(4, byte_code=True)              #Always start with 4 nulls
        self.count = bytes.get_little_int(4)       #get total attribute count
    
    def load_attribute(self, replay, bytes):
        #Get the attribute data elements
        attr_data = [
                bytes.get_big(4),         #Header
                bytes.get_little_int(4),   #Attr Id
                bytes.get_big_int(1),       #Player
                bytes.get_little(4)      #Value
            ]

        #Complete the decoding in the attribute object
        return Attribute(attr_data)
        
    def load(self, replay, filecontents):
        bytes = ByteStream(filecontents)
		
        self.load_header(replay, bytes)
        
        replay.attributes = list()
        data = defaultdict(dict)
        for i in range(0, self.count):
            attr = self.load_attribute(replay, bytes)
            replay.attributes.append(attr)

            # Uknown attributes get named as such and are not stored
            # because we don't know what they are
            # Index by player,  then name for ease of access later on
            if attr.name != "Unknown":
                data[attr.player][attr.name] = attr.value
            
        #Get global settings first
        replay.speed = data[16]['Game Speed']
        replay.category = data[16]['Category']
        replay.type = data[16]['Game Type']

        #Set player attributes as available,  requires already populated player list
        for pid, attributes in data.iteritems():
            if pid == 16: continue
            player = replay.player[pid]
            player.color = attributes['Color']<commit_msg>Adds back support for replay.is_ladder and replay.is_private<commit_after>from time import ctime
from collections import defaultdict

from objects import Attribute, Message, Player, Event
from eventparsers import *
from utils import ByteStream
from exceptions import ParseError

from pprint import PrettyPrinter
pprint = PrettyPrinter(indent=2).pprint

#################################################
# Parser Dispatch Functions
#################################################
def get_detail_parser(build):
    #This file format appears to have never changed
    return DetailParser()
    
def get_attribute_parser(build):
    if build >= 17326:
        return AttributeParser_17326()
    
    #All versions prior to 17326 have been the same
    return AttributeParser()
    
def get_event_parser(build):
    if build >= 17326:
        return EventParser_17326()
    if build >= 16561:
        return EventParser_16561()
    
    #All versions prior to 16561 appear to be the same
    return EventParser()
    
def get_message_parser(build):
    #format appears to have not changed
    return MessageParser()

def get_initdata_parser(build):
    return InitdataParser()
    
class InitdataParser(object):
    def load(self,replay,filecontents):
        bytes = ByteStream(filecontents)
        num_players = bytes.get_big_int(1)
        for p in range(0,num_players):
            name_length = bytes.get_big_int(1)
            name = bytes.get_string(name_length)
            bytes.skip(5)
        
        bytes.skip(5) # Unknown
        bytes.get_string(4) # Always Dflt
        bytes.skip(15) #Unknown
        id_length = bytes.get_big_int(1)
        sc_account_id = bytes.get_string(id_length)
        bytes.skip(684) # Fixed Length data for unknown purpose
        while( bytes.get_string(4).lower() == 's2ma' ):
            bytes.skip(2)
            replay.realm = bytes.get_string(2).lower()
            unknown_map_hash = bytes.get_big(32)
            
#################################################
# replay.attributes.events Parsing classes
#################################################
class AttributeParser(object):
    def load_header(self, replay, bytes):
        bytes.skip(4, byte_code=True)              #Always start with 4 nulls
        self.count = bytes.get_little_int(4)       #get total attribute count
    
    def load_attribute(self, replay, bytes):
        #Get the attribute data elements
        attr_data = [
                bytes.get_big(4),         #Header
                bytes.get_little_int(4),   #Attr Id
                bytes.get_big_int(1),       #Player
                bytes.get_little(4)      #Value
            ]

        #Complete the decoding in the attribute object
        return Attribute(attr_data)
        
    def load(self, replay, filecontents):
        bytes = ByteStream(filecontents)
		
        self.load_header(replay, bytes)
        
        replay.attributes = list()
        data = defaultdict(dict)
        for i in range(0, self.count):
            attr = self.load_attribute(replay, bytes)
            replay.attributes.append(attr)

            # Uknown attributes get named as such and are not stored
            # because we don't know what they are
            # Index by player,  then name for ease of access later on
            if attr.name != "Unknown":
                data[attr.player][attr.name] = attr.value
            
        #Get global settings first
        replay.speed = data[16]['Game Speed']
        
        #TODO: Do we need the category variable at this point?
        replay.category = data[16]['Category']
        replay.is_ladder = (replay.category == "Ladder")
        replay.is_private = (replay.category == "Private")
        
        replay.type = data[16]['Game Type']

        #Set player attributes as available,  requires already populated player list
        for pid, attributes in data.iteritems():
            if pid == 16: continue
            player = replay.player[pid]
            player.color = attributes['Color']
{'fraction_non_alphanumeric': 0.12727040163724737, 'fraction_numerical': 0.01918649270913277, 'mean_word_length': 2.479750778816199, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2056880', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2068, 'n_words': 591}