<commit_before># -*- coding: utf-8 -*-
#pylint: skip-file
'''
packeting module provides classes for Raet packets

'''

# Import python libs
from collections import Mapping
try:
    import simplejson as json
except ImportError:
    import json

# Import ioflo libs
from ioflo.base.odicting import odict
from ioflo.base.aiding import packByte, unpackByte

from ioflo.base.consoling import getConsole
console = getConsole()

from . import raeting

class Part(object):
    '''
    Base class for parts of a RAET packet
    Should be subclassed
    '''

    def __init__(self, packet=None, **kwa):
        '''
        Setup Part instance
        '''
        self.packet = packet  # Packet this Part belongs too
        self.packed = ''

    def __len__(self):
        '''
        Returns the length of .packed
        '''
        return len(self.packed)

    @property
    def size(self):
        '''
        Property is the length of this Part
        '''
        return self.__len__()

class Head(Part):
    '''
    RAET protocol packet header class
    Manages the header portion of a packet
    '''
    def __init__(self, **kwa):
        '''
        Setup Head instance
        '''
        super(Head, self).__init__(**kwa)

class TxHead(Head):
    '''
    RAET protocl transmit packet header class
    '''
    def pack(self):
        '''
        Composes .packed, which is the packed form of this part
        '''
        self.packed = ''
        data = self.packet.data  # for speed
        data['fl'] = self.packet.foot.size
        data['fg'] = "{:02x}".format(self.packFlags())

        # kit always includes header kind and length fields
        kit = odict([('ri', 'RAET'), ('pl', 0), ('hl', 0)])
        for k, v in raeting.PACKET_DEFAULTS.items():  # include if not equal to default
            if ((k in raeting.HEAD_FIELDS) and
                (k not in raeting.PACKET_FLAGS) and
                (data[k] != v)):
                kit[k] = data[k]

        if data['hk'] == raeting.headKinds.json:
            kit['pl'] = '0000000'  # need hex string so fixed length and jsonable
            kit['hl'] = '00'  # need hex string so fixed length and jsonable
            packed = json.dumps(kit, separators=(',', ':'), encoding='ascii',)
            packed = '{0}{1}'.format(packed, raeting.JSON_END)
            hl = len(packed)
            if hl > raeting.MAX_HEAD_SIZE:
                emsg = "Head length of {0}, exceeds max of {1}".format(hl, MAX_HEAD_SIZE)
                raise raeting.PacketError(emsg)
            data['hl'] = hl

            if self.packet.coat.size > raeting.MAX_MESSAGE_SIZE:
                emsg = "Packet length of {0}, exceeds max of {1}".format(
                         self.packet.coat.size, raeting.MAX_MESSAGE_SIZE)
                raise raeting.PacketError(emsg)
            pl = hl + self.packet.coat.size + data['fl']
            data['pl'] = pl
            #subsitute true length converted to 2 byte hex string
            packed = packed.replace('"pl":"0000000"', '"pl":"{0}"'.format("{0:07x}".format(pl)[-7:]), 1)
            self.packed = packed.replace('"hl":"00"', '"hl":"{0}"'.format("{0:02x}".format(hl)[-2:]), 1)


    def packFlags(self):
        '''
        Packs all the flag fields into a single two char hex string
        '''
        values = []
        for field in raeting.PACKET_FLAG_FIELDS:
            values.append(1 if self.packet.data.get(field, 0) else 0)
        return packByte(format='11111111', fields=values)

class RxHead(Head):
    '''
    RAET protocl receive packet header class
    '''
    def parse(self):
        '''
        From .packed.packed, Detects head kind. Unpacks head. Parses head and updates
        .packet.data
        Raises PacketError if failure occurs
        '''
        self.packed = ''
        data = self.packet.data  # for speed
        packed = self.packet.packed  # for speed
        if packed.startswith('{"ri":"RAET",') and raeting.JSON_END in packed:  # json header
            hk = raeting.headKinds.json
            front, sep, back = packed.partition(raeting.JSON_END)
            self.packed = "{0}{1}".format(front, sep)
            kit = json.loads(front,
                             encoding='ascii',
                             object_pairs_hook=odict)
            data.update(kit)
            if 'fg' in data:
                self.unpackFlags(data['fg'])

            if data['hk'] != hk:
                emsg = 'Recognized head kind does not match head field value.'
                raise raeting.PacketError(emsg)

            hl = int(data['hl'], 16)
            if hl != self.size:
                emsg = 'Actual head length does not match head field value.'
                raise raeting.PacketError(emsg)
            data['hl'] = hl
            pl = int(data['pl'], 16)
            if pl != self.packet.size:
                emsg = 'Actual packet length does not match head field value.'
                raise raeting.PacketError(emsg)
            data['pl'] = pl

        else:  # notify unrecognizible packet head
            data['hk'] = raeting.headKinds.unknown
            emsg = "Unrecognizible packet head."
            raise raeting.PacketError(emsg)

    def unpackFlags(self, flags):
        '''
        Unpacks all the flag fields from a single two char hex string
        '''
        values = unpackByte(format='11111111', byte=int(flags, 16), boolean=True)
        for i, field in enumerate(raeting.PACKET_FLAG_FIELDS):
            if field in self.packet.data:
                self.packet.data[field] = values[i]

class Body(Part):
    '''
    RAET protocol packet body class
    Manages the messsage  portion of the packet
    '''
    def __init__(self, data=None, **kwa):
        '''
        Setup Body instance
        '''
        super(Body, self).__init__(**kwa)
        if data is None:
            data = odict()
        self.data = data

class TxBody(Body):
    '''
    RAET protocol tx packet body class
    '''
    def pack(self):
        '''
        Composes .packed, which is the packed form of this part
        '''
        self.packed = ''
        bk = self.packet.data['bk']
        if bk == raeting.bodyKinds.json:
            if self.data:
                self.packed = json.dumps(self.data, separators=(',', ':'))
<commit_msg>Add support for the msgpack bodykind to packeting<commit_after># -*- coding: utf-8 -*-
#pylint: skip-file
'''
packeting module provides classes for Raet packets

'''

# Import python libs
from collections import Mapping
try:
    import simplejson as json
except ImportError:
    import json
import msgpack

# Import ioflo libs
from ioflo.base.odicting import odict
from ioflo.base.aiding import packByte, unpackByte

from ioflo.base.consoling import getConsole
console = getConsole()

from . import raeting

class Part(object):
    '''
    Base class for parts of a RAET packet
    Should be subclassed
    '''

    def __init__(self, packet=None, **kwa):
        '''
        Setup Part instance
        '''
        self.packet = packet  # Packet this Part belongs too
        self.packed = ''

    def __len__(self):
        '''
        Returns the length of .packed
        '''
        return len(self.packed)

    @property
    def size(self):
        '''
        Property is the length of this Part
        '''
        return self.__len__()

class Head(Part):
    '''
    RAET protocol packet header class
    Manages the header portion of a packet
    '''
    def __init__(self, **kwa):
        '''
        Setup Head instance
        '''
        super(Head, self).__init__(**kwa)

class TxHead(Head):
    '''
    RAET protocl transmit packet header class
    '''
    def pack(self):
        '''
        Composes .packed, which is the packed form of this part
        '''
        self.packed = ''
        data = self.packet.data  # for speed
        data['fl'] = self.packet.foot.size
        data['fg'] = "{:02x}".format(self.packFlags())

        # kit always includes header kind and length fields
        kit = odict([('ri', 'RAET'), ('pl', 0), ('hl', 0)])
        for k, v in raeting.PACKET_DEFAULTS.items():  # include if not equal to default
            if ((k in raeting.HEAD_FIELDS) and
                (k not in raeting.PACKET_FLAGS) and
                (data[k] != v)):
                kit[k] = data[k]

        if data['hk'] == raeting.headKinds.json:
            kit['pl'] = '0000000'  # need hex string so fixed length and jsonable
            kit['hl'] = '00'  # need hex string so fixed length and jsonable
            packed = json.dumps(kit, separators=(',', ':'), encoding='ascii',)
            packed = '{0}{1}'.format(packed, raeting.JSON_END)
            hl = len(packed)
            if hl > raeting.MAX_HEAD_SIZE:
                emsg = "Head length of {0}, exceeds max of {1}".format(hl, MAX_HEAD_SIZE)
                raise raeting.PacketError(emsg)
            data['hl'] = hl

            if self.packet.coat.size > raeting.MAX_MESSAGE_SIZE:
                emsg = "Packet length of {0}, exceeds max of {1}".format(
                         self.packet.coat.size, raeting.MAX_MESSAGE_SIZE)
                raise raeting.PacketError(emsg)
            pl = hl + self.packet.coat.size + data['fl']
            data['pl'] = pl
            #subsitute true length converted to 2 byte hex string
            packed = packed.replace('"pl":"0000000"', '"pl":"{0}"'.format("{0:07x}".format(pl)[-7:]), 1)
            self.packed = packed.replace('"hl":"00"', '"hl":"{0}"'.format("{0:02x}".format(hl)[-2:]), 1)


    def packFlags(self):
        '''
        Packs all the flag fields into a single two char hex string
        '''
        values = []
        for field in raeting.PACKET_FLAG_FIELDS:
            values.append(1 if self.packet.data.get(field, 0) else 0)
        return packByte(format='11111111', fields=values)

class RxHead(Head):
    '''
    RAET protocl receive packet header class
    '''
    def parse(self):
        '''
        From .packed.packed, Detects head kind. Unpacks head. Parses head and updates
        .packet.data
        Raises PacketError if failure occurs
        '''
        self.packed = ''
        data = self.packet.data  # for speed
        packed = self.packet.packed  # for speed
        if packed.startswith('{"ri":"RAET",') and raeting.JSON_END in packed:  # json header
            hk = raeting.headKinds.json
            front, sep, back = packed.partition(raeting.JSON_END)
            self.packed = "{0}{1}".format(front, sep)
            kit = json.loads(front,
                             encoding='ascii',
                             object_pairs_hook=odict)
            data.update(kit)
            if 'fg' in data:
                self.unpackFlags(data['fg'])

            if data['hk'] != hk:
                emsg = 'Recognized head kind does not match head field value.'
                raise raeting.PacketError(emsg)

            hl = int(data['hl'], 16)
            if hl != self.size:
                emsg = 'Actual head length does not match head field value.'
                raise raeting.PacketError(emsg)
            data['hl'] = hl
            pl = int(data['pl'], 16)
            if pl != self.packet.size:
                emsg = 'Actual packet length does not match head field value.'
                raise raeting.PacketError(emsg)
            data['pl'] = pl

        else:  # notify unrecognizible packet head
            data['hk'] = raeting.headKinds.unknown
            emsg = "Unrecognizible packet head."
            raise raeting.PacketError(emsg)

    def unpackFlags(self, flags):
        '''
        Unpacks all the flag fields from a single two char hex string
        '''
        values = unpackByte(format='11111111', byte=int(flags, 16), boolean=True)
        for i, field in enumerate(raeting.PACKET_FLAG_FIELDS):
            if field in self.packet.data:
                self.packet.data[field] = values[i]

class Body(Part):
    '''
    RAET protocol packet body class
    Manages the messsage  portion of the packet
    '''
    def __init__(self, data=None, **kwa):
        '''
        Setup Body instance
        '''
        super(Body, self).__init__(**kwa)
        if data is None:
            data = odict()
        self.data = data

class TxBody(Body):
    '''
    RAET protocol tx packet body class
    '''
    def pack(self):
        '''
        Composes .packed, which is the packed form of this part
        '''
        self.packed = ''
        bk = self.packet.data['bk']
        if bk == raeting.bodyKinds.json:
            if self.data:
                self.packed = json.dumps(self.data, separators=(',', ':'))
        elif bk == raeting.bodykinds.msgpack:
            if self.data:
                self.packed = msgpack.dumps(self.data)

{'fraction_non_alphanumeric': 0.1202541575149043, 'fraction_numerical': 0.010825227486664575, 'mean_word_length': 1.906086163665375, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5270447', 'n_tokens_mistral': 3877, 'n_tokens_neox': 3667, 'n_words': 1074}