<commit_before>import struct
from spock import utils

class MapBlock:
    def __init__(self, base_id = 0, add_id = 0):
        self.base_id = base_id
        self.add_id = add_id
        self.id = self.calc_id()
        self.meta = 0
        self.light = 0
        self.sky_light = 0
        self.block_light = 0
        self.biome = 0

    def calc_id(self):
        self.id = (self.add_id<<8)+self.base_id
        return self.id

    #Needs to do proper light calc based on time_of_day
    def calc_light(self, time):
        self.light = max(self.block_light, self.sky_light)

emptyBlock = MapBlock()

class EmptyChunk:
    def __init__(self):
        self.length = 16*16*16
        self.time = 0

    def get(self, x, y, z):
        return emptyBlock

    def unpack_biome(self, x, z, biome_id):
        # unpack_biome gets called and needs to be present, but we're not really very
        # interested in biome data for empty chunks
        pass

class Chunk:
    def __init__(self):
        self.length = 16*16*16
        self.time = 0
        self.blocks = [MapBlock() for i in range(self.length)]

    def get(self, x, y, z):
        return self.blocks[x+((y*16)+z)*16]

    def put(self, x, y, z, data):
        block = self.blocks[x+((y*16)+z)*16]
        block.id = data['block_id']
        block.base_id = data['block_id']&0xFF
        block.add_id = data['block_id']>>8
        block.meta = data['metadata']
        return block

    def unpack_data(self, buff):
        for idx, i in enumerate(buff.recv(self.length)):
            self.blocks[idx].id = i
            self.blocks[idx].base_id = i
            self.blocks[idx].add_id = 0

    def unpack_meta(self, buff):
        for idx, i in enumerate(buff.recv(self.length>>1)):
            self.blocks[idx*2].meta = i>>4
            self.blocks[idx*2+1].meta = i&0x0F

    def unpack_add(self, buff):
        for idx, i in enumerate(buff.recv(self.length>>1)):
            self.blocks[idx*2].add_id = i>>4
            self.blocks[idx*2].calc_id()
            self.blocks[idx*2+1].add_id = i&0x0F
            self.blocks[idx*2+1].calc_id()

    def unpack_blight(self, buff):
        for idx, i in enumerate(buff.recv(self.length>>1)):
            self.blocks[idx*2].block_light = i>>4
            self.blocks[idx*2+1].block_light = i&0x0F
        self.update_light()

    def unpack_slight(self, buff):
        for idx, i in enumerate(buff.recv(self.length>>1)):
            self.blocks[idx*2].sky_light = i>>4
            self.blocks[idx*2+1].sky_light = i&0x0F
        self.update_light()

    def unpack_biome(self, x, z, biome_id):
        for y in range(16):
            self.blocks[x+((y*16)+z)*16].biome = biome_id

    def update_light(self, time = None):
        if time: self.time = time
        for block in self.blocks:
            block.calc_light(self.time)

class ChunkColumn:
    def __init__(self):
        self.chunks = [None]*16
        self.biome = [None]*256

    def unpack(self, buff, primary_bitmap, add_bitmap, skylight, continuous):
        primary_mask = []
        add_mask = []
        for i in range(16):<commit_msg>Lowering the byte count for MapBlock and Chunk by  removing obvious redundancies<commit_after>import struct
from spock import utils

CHUNKLENGTH = 16*16*16

class MapBlock:
    def __init__(self, base_id = 0, add_id = 0):
        self.base_id = base_id
        self.add_id = add_id
        self.meta = 0
        self.sky_light = 0
        self.block_light = 0
        self.biome = 0

    @property
    def id(self):
        return (self.add_id<<8)+self.base_id

    @property
    def light(self):
        return max(self.block_light, self.sky_light)

    #Needs to do proper light calc based on time_of_day
    def calc_light(self, time):
        pass

emptyBlock = MapBlock()

class EmptyChunk:
    def __init__(self):
        self.time = 0

    def get(self, x, y, z):
        return emptyBlock

    def unpack_biome(self, x, z, biome_id):
        # unpack_biome gets called and needs to be present, but we're not really very
        # interested in biome data for empty chunks
        pass

class Chunk:
    def __init__(self):
        self.time = 0
        self.blocks = [MapBlock() for i in range(CHUNKLENGTH)]

    def get(self, x, y, z):
        return self.blocks[x+((y*16)+z)*16]

    def put(self, x, y, z, data):
        block = self.blocks[x+((y*16)+z)*16]
        block.base_id = data['block_id']&0xFF
        block.add_id = data['block_id']>>8
        block.meta = data['metadata']
        return block

    def unpack_data(self, buff):
        for idx, i in enumerate(buff.recv(CHUNKLENGTH)):
            self.blocks[idx].base_id = i
            self.blocks[idx].add_id = 0

    def unpack_meta(self, buff):
        for idx, i in enumerate(buff.recv(CHUNKLENGTH>>1)):
            self.blocks[idx*2].meta = i>>4
            self.blocks[idx*2+1].meta = i&0x0F

    def unpack_add(self, buff):
        for idx, i in enumerate(buff.recv(CHUNKLENGTH>>1)):
            self.blocks[idx*2].add_id = i>>4
            self.blocks[idx*2+1].add_id = i&0x0F

    def unpack_blight(self, buff):
        for idx, i in enumerate(buff.recv(CHUNKLENGTH>>1)):
            self.blocks[idx*2].block_light = i>>4
            self.blocks[idx*2+1].block_light = i&0x0F
        self.update_light()

    def unpack_slight(self, buff):
        for idx, i in enumerate(buff.recv(CHUNKLENGTH>>1)):
            self.blocks[idx*2].sky_light = i>>4
            self.blocks[idx*2+1].sky_light = i&0x0F
        self.update_light()

    def unpack_biome(self, x, z, biome_id):
        for y in range(16):
            self.blocks[x+((y*16)+z)*16].biome = biome_id

    def update_light(self, time = None):
        if time: self.time = time
        for block in self.blocks:
            block.calc_light(self.time)

class ChunkColumn:
    def __init__(self):
        self.chunks = [None]*16
        self.biome = [None]*256

    def unpack(self, buff, primary_bitmap, add_bitmap, skylight, continuous):
        primary_mask = []
        add_mask = []
        for i in range(16):
{'fraction_non_alphanumeric': 0.12493819020932916, 'fraction_numerical': 0.023734959617603427, 'mean_word_length': 2.2623655913978493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7449993', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2051, 'n_words': 489}