<commit_before># Copyright (c) 2005, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its 
# contributors may be used to endorse or promote products derived from 
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: __init__.py,v 1.9 2006/09/05 07:30:56 gnn Exp $
#
# Author: <NAME>
#
# Description: A packet module for Python.  We will package the
# individual protocols separately.

"""PCS aka Packet Construction Set

PCS is a set of Python modules and objects that make building network
protocol testing tools easier for the protocol developer.  The core of
the system is the pcs module itself which provides the necessary
functionality to create classes that implement packets.

In PCS every packet is a class and the layout of the packet is defined
by a Layout class which contains a set of Fields.  Fields can be from
1 to many bits, so it is possible to build packets with arbitrary
width bit fields.  Fields know about the widths and will throw
exceptions when they are overloaded.

Every Packet object, that is an object instantiated from a specific
PCS packet class, has a field named bytes which shows the
representation of the data in the packet at that point in time.  It is
the bytes field that is used when transmitting the packet on the wire.

For more information please see the manual, called pcs, and available in various
formats after installation.
"""

__revision__ = "$Id: __init__.py,v 1.9 2006/09/05 07:30:56 gnn Exp $"

# We need the struct module to pack our class into a byte string.
import struct

# We need the socket module for to implement some of the Connector classes.
from socket import *

import pcs.pcap as pcap

import itertools

def attribreprlist(obj, attrs):
    return map(lambda x, y = obj: '%s: %s' % (x.name, repr(getattr(y, x.name))), itertools.ifilter(lambda x, y = obj: hasattr(y, x.name), attrs))

class FieldBoundsError(Exception):
    """When a programmer tries to set a field with an inappropriately
    sized piece of data this exception is raised."""

    def __init__(self, message):
        self.message = message
    def __str__(self):
        return repr(self.message)
    
class Field(object):
    """A field is a name, a type, a width in bits, possibly a default
value and can be marked as a dicriminator for higher level packet
demultiplexing .  These classes are used by the packet to define the
layout of the data and how it is addressed."""
    
    def __init__(self, name = "", width = 1, default = None,
                 discriminator = False):
        """initialize a field

        name - a string name
        width - a width in bits
        default - a default value
        discriminator - is this field used to demultiplex packets
        """
        ## the name of the Field
        self.name = name
        ## the width, in bites, of the field's data
        self.width = width
        ## the default value of the field, must fit into bits
        self.default = default
        ## Is this field used to demultiplex higher layer packets?
        self.discriminator = discriminator
        ## Fields store the values
        if default == None:
            self.value = 0
        else:
            self.value = default
        
    def __repr__(self):
        """return an appropriate representation for the Field object"""
        return "<pcs.Field  name %s, %d bits, default %s, discriminator %d>" % \
               (self.name, self.width, self.default, self.discriminator)

    def decode(self, bytes, curr, byteBR):
        """Decode a field and return the value and the updated current
        pointer into the bytes array

        bytes - the byte array for the packet
        curr - the current byte position in the bytes array
        byteBR - the number of Bits Remaining in the current byte
        """
        real_value = 0
        fieldBR = self.width
        while fieldBR > 0 and curr < len(bytes):
            if fieldBR < byteBR:
                shift = byteBR - fieldBR
                value = ord(bytes[curr]) >> shift
                mask = 2 ** fieldBR -1
                value = (value & mask)
                byteBR -= fieldBR
                fieldBR = 0 # next field
            elif fieldBR > byteBR:
                shift = fieldBR - byteBR
                mask = 2 ** byteBR - 1
                value = (ord(bytes[curr]) & mask)
                fieldBR -= byteBR
                byteBR = 8
                curr += 1 # next byte
            elif fieldBR == byteBR:
                mask = 2 ** byteBR - 1
                value = ord(bytes[curr]) & mask
                fieldBR -= byteBR
                byteBR = 8
                curr += 1 # next byte
            real_value += value << fieldBR
        self.value = real_value
        return [real_value, curr, byteBR]

    def encode(self, bytearray, value, byte, byteBR):
        """encode the a field into the bytes necessary to transmit it
        as part of a packet

        bytearray - the array of bytes that will be returned
        value - the value to encode
        byte - the byte we are encoding, we can encode partial bytes
        byteBR - the bits remaining in the current byte being encoded.
        """
        # The algorithm below hurts my head, as I'm not very
        # smart so it is heavily commented.
        #
        # fieldBR is the bits remaining in the field to be encoded
        # byteBR is the bits remaining in the current byte being encoded
        
        fieldBR = self.width
        while byteBR > 0:
            if fieldBR < byteBR:
                shift = byteBR - fieldBR
                byteBR -= fieldBR
                mask = ((2 ** fieldBR) - 1) << shift
                byte = (byte | ((value << shift) & mask))
                # Done with the field, not with the byte get new field
                break
            elif fieldBR > byteBR:
                shift = fieldBR - byteBR
                fieldBR -= byteBR
                mask = ((2 ** byteBR) - 1)
                byte = (byte | ((value >> shift) & mask))
                bytearray.append(struct.pack('B', byte))
                byteBR = 8
                byte = 0
                # Done with this byte, but not the field, get a new byte
            elif fieldBR == byteBR:
                mask = ((2 ** byteBR) - 1)
                byte = (byte | (value & mask))
                bytearray.append(struct.pack('B', byte))
                byte = 0
                byteBR = 8
                # Done with the byte and the field have a nice day
                break

        return [byte, byteBR]

    def set_value(self, value):
        """Set the value of a field."""
        self.value = value

    def get_value(self):
        return self.value
    
    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return 0

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (value < 0) or
            (value > (2 ** self.width) - 1)):
            raise FieldBoundsError, "Value must be between 0 and %d but is %d" % ((2 ** self.width - 1), value)
        
class FieldAlignmentError(Exception):
    """When a programmer tries to decode a field that is not
    on a byte boundary this exception is raised."""
    
    def __init__(self, message):
        """set the FieldAlignmentError message"""
        ## the message that will be output when this error is raised
        self.message = message

class StringField(Field):
    """A string field is a name, a width in bits, and possibly a
default value.  The data is to be interpreted as a string, but does
not encode the length into the packet.  Length encoded values are
handled by the LengthValueField."""
    
    def __init__(self, name = "", width = 1, default = None):
        """initialtize a StringField"""
        ## the name of the StringField
        self.name = name
        ## the width, in bits, of the StringField
        self.width = width
        ## the default value, if any, of the StringField
        self.default = default
        ## Fields store the values
        if default == None:
            self.value = ""
        else:
            self.value = default
        
    def __repr__(self):
        """return a human readable form of a StringFeild object"""
        return "<pcs.StringField  name %s, %d bits, default %s>" % \
               (self.name, self.width, self.default) # 

    def decode(self, bytes, curr, byteBR):
        """Decode the field and return the value as well as the new
        current position in the bytes array."""
        # byteBR == 8 is the neutral state
        if (byteBR != None and byteBR != 8):
            raise FieldAlignmentError, "Strings must start on a byte boundary"
        packarg = "%ds" % (self.width / 8)
        end = curr + self.width / 8
        value = struct.unpack(packarg, bytes[curr:end])[0]
        curr += self.width / 8
        self.value = value
        return [value, curr, byteBR]

    def encode(self, bytearray, value, byte, byteBR):
        """Encode a string field, make sure the bytes are aligned."""
        if (byteBR != None and byteBR != 8):
            raise FieldAlignmentError, "Strings must start on a byte boundary"
        packarg = "%ds" % (self.width / 8)
        bytearray.append(struct.pack(packarg, value))
        return [byte, byteBR]

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if (value == None) or (len (value) > (self.width / 8)):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (self.width / 8)

class LengthValueFieldError(Exception):
    """LengthValue fields only allow access to two internal pieces of data."""
    
    def __init__(self, message):
        """set the error message"""
        ## the message that will be output when this error is raised
        self.message = message

# TODO: Add a means of packing fields according to the length actually
# encoded in them, where a field has variable length.
class LengthValueField(object):
    """A length value field handles parts of packets where a length
    and value are encoded together, usually used to shove strings into
    packets.
    """

    def __init__(self, name, length, value):
        self.packet = None
        self.name = name
        if not isinstance(length, Field):
            raise LengthValueFieldError, "Length must be of type Field but is %s" % type(length)
        object.__setattr__(self, 'length', length)
        if isinstance(value, Field) or isinstance(value, StringField):
            object.__setattr__(self, 'value', value)
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = length.width + value.width
        #self.packed = packed

    def __repr__(self):
        return "<pcs.LengthValueField name %s, length %s, value %s" \
               % (self.name, self.length, self.value)

    def __len__(self):
        return self.length.width + self.value.width
    
    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        """Decode a LengthValue field."""
        [self.length.value, curr, byteBR] = self.length.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [self.value.value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        """Encode a LengthValue field."""
        if not isinstance(self.value, StringField):
            self.length.value = self.value.width
        else:
            self.length.value = len(self.value.value)
            #self.value.width = len(self.value.value) * 8	# XXX packed
        [byte, byteBR] = self.length.encode(bytearray, self.length.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]

    def set_value(self, value):
        """Set the value of a LengthValueField."""
	self.length.value = len(value)
        self.value.value = value
        if self.packet != None:
            self.packet.__needencode = True

    #def get_value(self):
    #    return self.value.value

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        self.value.bounds(value)

class TypeValueField(object):
    """A type-value field handles parts of packets where a type
    is encoded before a value.  """

    def __init__(self, name, type, value):
        self.packet = None
        self.name = name
        if not isinstance(type, Field):
            raise LengthValueFieldError, "Type must be of type Field but is %s" % type(type)
        self.type = type
        if isinstance(value, Field) or isinstance(value, StringField):
            self.value = value
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = type.width + value.width

    def __repr__(self):
        return "<pcs.TypeValueField name %s, type %s, value %s," % (self.name, self.type, self.value)

    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        [self.type.value, curr, byteBR] = self.type.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        [byte, byteBR] = self.type.encode(bytearray, self.type.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]
        
    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (len (value) > (((2 ** self.valuewidth) - 1) / 8))):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (((2 ** self.width) - 1) / 8)


class TypeLengthValueField(object):
    """A type-length-value field handles parts of packets where a type
    is encoded before a length and value.  """

    def __init__(self, name, type, length, value,
                 inclusive = False):
        self.packet = None
        self.name = name
        if not isinstance(type, Field):
            raise LengthValueFieldError, "Type must be of type Field but is %s" % type(type)
        self.type = type
        if not isinstance(length, Field):
            raise LengthValueFieldError, "Length must be of type Field but is %s" % type(type)
        self.length = length
        if isinstance(value, Field) or isinstance(value, StringField):
            self.value = value
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = type.width + length.width + value.width
        self.inclusive = inclusive

    def __repr__(self):
        return "<pcs.TypeLengthValueField type %s, length %s, value %s>" \
                % (self.type, self.length, self.value)

    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        [self.type.value, curr, byteBR] = self.type.decode(bytes, curr, byteBR)
        [self.length.value, curr, byteBR] = self.length.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        """Encode a TypeLengthValue field."""
	# XXX Break existing assumptions and assume the TLV encodes
	# the length in bytes, not bits, and that this length INCLUDES
	# the type and length fields.
	#  In PCS all field widths are currently defined in bits.
	# This, uh, fixes the TCP option stuff...
        if isinstance(self.value, Field):
            self.length.value = (self.type.width +
				 self.length.width +
				 self.value.width) / 8
        else:
            self.length.value = len(self.value)
        [byte, byteBR] = self.type.encode(bytearray, self.type.value, byte, byteBR)
        [byte, byteBR] = self.length.encode(bytearray, self.length.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (len (value) > (((2 ** self.lengthwidth) - 1) / 8))):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (((2 ** self.width) - 1) / 8)<commit_msg>Break out the default behaviour of TLV padding so that clients can control it on instantiation.<commit_after># Copyright (c) 2005, Neville-Neil Consulting
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# Neither the name of Neville-Neil Consulting nor the names of its 
# contributors may be used to endorse or promote products derived from 
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# File: $Id: __init__.py,v 1.9 2006/09/05 07:30:56 gnn Exp $
#
# Author: <NAME>
#
# Description: A packet module for Python.  We will package the
# individual protocols separately.

"""PCS aka Packet Construction Set

PCS is a set of Python modules and objects that make building network
protocol testing tools easier for the protocol developer.  The core of
the system is the pcs module itself which provides the necessary
functionality to create classes that implement packets.

In PCS every packet is a class and the layout of the packet is defined
by a Layout class which contains a set of Fields.  Fields can be from
1 to many bits, so it is possible to build packets with arbitrary
width bit fields.  Fields know about the widths and will throw
exceptions when they are overloaded.

Every Packet object, that is an object instantiated from a specific
PCS packet class, has a field named bytes which shows the
representation of the data in the packet at that point in time.  It is
the bytes field that is used when transmitting the packet on the wire.

For more information please see the manual, called pcs, and available in various
formats after installation.
"""

__revision__ = "$Id: __init__.py,v 1.9 2006/09/05 07:30:56 gnn Exp $"

# We need the struct module to pack our class into a byte string.
import struct

# We need the socket module for to implement some of the Connector classes.
from socket import *

import pcs.pcap as pcap

import itertools

def attribreprlist(obj, attrs):
    return map(lambda x, y = obj: '%s: %s' % (x.name, repr(getattr(y, x.name))), itertools.ifilter(lambda x, y = obj: hasattr(y, x.name), attrs))

class FieldBoundsError(Exception):
    """When a programmer tries to set a field with an inappropriately
    sized piece of data this exception is raised."""

    def __init__(self, message):
        self.message = message
    def __str__(self):
        return repr(self.message)
    
class Field(object):
    """A field is a name, a type, a width in bits, possibly a default
value and can be marked as a dicriminator for higher level packet
demultiplexing .  These classes are used by the packet to define the
layout of the data and how it is addressed."""
    
    def __init__(self, name = "", width = 1, default = None,
                 discriminator = False):
        """initialize a field

        name - a string name
        width - a width in bits
        default - a default value
        discriminator - is this field used to demultiplex packets
        """
        ## the name of the Field
        self.name = name
        ## the width, in bites, of the field's data
        self.width = width
        ## the default value of the field, must fit into bits
        self.default = default
        ## Is this field used to demultiplex higher layer packets?
        self.discriminator = discriminator
        ## Fields store the values
        if default == None:
            self.value = 0
        else:
            self.value = default
        
    def __repr__(self):
        """return an appropriate representation for the Field object"""
        return "<pcs.Field  name %s, %d bits, default %s, discriminator %d>" % \
               (self.name, self.width, self.default, self.discriminator)

    def decode(self, bytes, curr, byteBR):
        """Decode a field and return the value and the updated current
        pointer into the bytes array

        bytes - the byte array for the packet
        curr - the current byte position in the bytes array
        byteBR - the number of Bits Remaining in the current byte
        """
        real_value = 0
        fieldBR = self.width
        while fieldBR > 0 and curr < len(bytes):
            if fieldBR < byteBR:
                shift = byteBR - fieldBR
                value = ord(bytes[curr]) >> shift
                mask = 2 ** fieldBR -1
                value = (value & mask)
                byteBR -= fieldBR
                fieldBR = 0 # next field
            elif fieldBR > byteBR:
                shift = fieldBR - byteBR
                mask = 2 ** byteBR - 1
                value = (ord(bytes[curr]) & mask)
                fieldBR -= byteBR
                byteBR = 8
                curr += 1 # next byte
            elif fieldBR == byteBR:
                mask = 2 ** byteBR - 1
                value = ord(bytes[curr]) & mask
                fieldBR -= byteBR
                byteBR = 8
                curr += 1 # next byte
            real_value += value << fieldBR
        self.value = real_value
        return [real_value, curr, byteBR]

    def encode(self, bytearray, value, byte, byteBR):
        """encode the a field into the bytes necessary to transmit it
        as part of a packet

        bytearray - the array of bytes that will be returned
        value - the value to encode
        byte - the byte we are encoding, we can encode partial bytes
        byteBR - the bits remaining in the current byte being encoded.
        """
        # The algorithm below hurts my head, as I'm not very
        # smart so it is heavily commented.
        #
        # fieldBR is the bits remaining in the field to be encoded
        # byteBR is the bits remaining in the current byte being encoded
        
        fieldBR = self.width
        while byteBR > 0:
            if fieldBR < byteBR:
                shift = byteBR - fieldBR
                byteBR -= fieldBR
                mask = ((2 ** fieldBR) - 1) << shift
                byte = (byte | ((value << shift) & mask))
                # Done with the field, not with the byte get new field
                break
            elif fieldBR > byteBR:
                shift = fieldBR - byteBR
                fieldBR -= byteBR
                mask = ((2 ** byteBR) - 1)
                byte = (byte | ((value >> shift) & mask))
                bytearray.append(struct.pack('B', byte))
                byteBR = 8
                byte = 0
                # Done with this byte, but not the field, get a new byte
            elif fieldBR == byteBR:
                mask = ((2 ** byteBR) - 1)
                byte = (byte | (value & mask))
                bytearray.append(struct.pack('B', byte))
                byte = 0
                byteBR = 8
                # Done with the byte and the field have a nice day
                break

        return [byte, byteBR]

    def set_value(self, value):
        """Set the value of a field."""
        self.value = value

    def get_value(self):
        return self.value
    
    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return 0

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (value < 0) or
            (value > (2 ** self.width) - 1)):
            raise FieldBoundsError, "Value must be between 0 and %d but is %d" % ((2 ** self.width - 1), value)
        
class FieldAlignmentError(Exception):
    """When a programmer tries to decode a field that is not
    on a byte boundary this exception is raised."""
    
    def __init__(self, message):
        """set the FieldAlignmentError message"""
        ## the message that will be output when this error is raised
        self.message = message

class StringField(Field):
    """A string field is a name, a width in bits, and possibly a
default value.  The data is to be interpreted as a string, but does
not encode the length into the packet.  Length encoded values are
handled by the LengthValueField."""
    
    def __init__(self, name = "", width = 1, default = None):
        """initialtize a StringField"""
        ## the name of the StringField
        self.name = name
        ## the width, in bits, of the StringField
        self.width = width
        ## the default value, if any, of the StringField
        self.default = default
        ## Fields store the values
        if default == None:
            self.value = ""
        else:
            self.value = default
        
    def __repr__(self):
        """return a human readable form of a StringFeild object"""
        return "<pcs.StringField  name %s, %d bits, default %s>" % \
               (self.name, self.width, self.default) # 

    def decode(self, bytes, curr, byteBR):
        """Decode the field and return the value as well as the new
        current position in the bytes array."""
        # byteBR == 8 is the neutral state
        if (byteBR != None and byteBR != 8):
            raise FieldAlignmentError, "Strings must start on a byte boundary"
        packarg = "%ds" % (self.width / 8)
        end = curr + self.width / 8
        value = struct.unpack(packarg, bytes[curr:end])[0]
        curr += self.width / 8
        self.value = value
        return [value, curr, byteBR]

    def encode(self, bytearray, value, byte, byteBR):
        """Encode a string field, make sure the bytes are aligned."""
        if (byteBR != None and byteBR != 8):
            raise FieldAlignmentError, "Strings must start on a byte boundary"
        packarg = "%ds" % (self.width / 8)
        bytearray.append(struct.pack(packarg, value))
        return [byte, byteBR]

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if (value == None) or (len (value) > (self.width / 8)):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (self.width / 8)

class LengthValueFieldError(Exception):
    """LengthValue fields only allow access to two internal pieces of data."""
    
    def __init__(self, message):
        """set the error message"""
        ## the message that will be output when this error is raised
        self.message = message

# TODO: Add a means of packing fields according to the length actually
# encoded in them, where a field has variable length.
class LengthValueField(object):
    """A length value field handles parts of packets where a length
    and value are encoded together, usually used to shove strings into
    packets.
    """

    def __init__(self, name, length, value):
        self.packet = None
        self.name = name
        if not isinstance(length, Field):
            raise LengthValueFieldError, "Length must be of type Field but is %s" % type(length)
        object.__setattr__(self, 'length', length)
        if isinstance(value, Field) or isinstance(value, StringField):
            object.__setattr__(self, 'value', value)
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = length.width + value.width
        #self.packed = packed

    def __repr__(self):
        return "<pcs.LengthValueField name %s, length %s, value %s" \
               % (self.name, self.length, self.value)

    def __len__(self):
        return self.length.width + self.value.width
    
    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        """Decode a LengthValue field."""
        [self.length.value, curr, byteBR] = self.length.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [self.value.value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        """Encode a LengthValue field."""
        if not isinstance(self.value, StringField):
            self.length.value = self.value.width
        else:
            self.length.value = len(self.value.value)
            #self.value.width = len(self.value.value) * 8	# XXX packed
        [byte, byteBR] = self.length.encode(bytearray, self.length.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]

    def set_value(self, value):
        """Set the value of a LengthValueField."""
	self.length.value = len(value)
        self.value.value = value
        if self.packet != None:
            self.packet.__needencode = True

    #def get_value(self):
    #    return self.value.value

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        self.value.bounds(value)

class TypeValueField(object):
    """A type-value field handles parts of packets where a type
    is encoded before a value.  """

    def __init__(self, name, type, value):
        self.packet = None
        self.name = name
        if not isinstance(type, Field):
            raise LengthValueFieldError, "Type must be of type Field but is %s" % type(type)
        self.type = type
        if isinstance(value, Field) or isinstance(value, StringField):
            self.value = value
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = type.width + value.width

    def __repr__(self):
        return "<pcs.TypeValueField name %s, type %s, value %s," % (self.name, self.type, self.value)

    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        [self.type.value, curr, byteBR] = self.type.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        [byte, byteBR] = self.type.encode(bytearray, self.type.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]
        
    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (len (value) > (((2 ** self.valuewidth) - 1) / 8))):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (((2 ** self.width) - 1) / 8)


class TypeLengthValueField(object):
    """A type-length-value field handles parts of packets where a type
    is encoded before a length and value.  """

    def __init__(self, name, type, length, value,
                 inclusive = True, bytewise = True):
        self.packet = None
        self.name = name
        if not isinstance(type, Field):
            raise LengthValueFieldError, "Type must be of type Field but is %s" % type(type)
        self.type = type
        if not isinstance(length, Field):
            raise LengthValueFieldError, "Length must be of type Field but is %s" % type(type)
        self.length = length
        if isinstance(value, Field) or isinstance(value, StringField):
            self.value = value
        else:
            raise LengthValueFieldError, "Value must be of type Field or StringField but is %s" % type(value)
        self.width = type.width + length.width + value.width
        self.inclusive = inclusive
        self.bytewise = bytewise

    def __repr__(self):
        return "<pcs.TypeLengthValueField type %s, length %s, value %s>" \
                % (self.type, self.length, self.value)

    def __setattr__(self, name, value):
        object.__setattr__(self, name, value)
        if self.packet != None:
            self.packet.__needencode = True

    def decode(self, bytes, curr, byteBR):
        [self.type.value, curr, byteBR] = self.type.decode(bytes, curr, byteBR)
        [self.length.value, curr, byteBR] = self.length.decode(bytes, curr, byteBR)
        [self.value.value, curr, byteBR] = self.value.decode(bytes, curr, byteBR)
        return [value, curr, byteBR]
        
    def encode(self, bytearray, value, byte, byteBR):
        """Encode a TypeLengthValue field."""

        if isinstance(self.value, Field):
	    # Value is a field. Take its width in bits.
            self.length.value = self.value.width
        else:
	    # Value is any other Python type. Take its actual length in bits. 
            self.length.value = len(self.value) * 8

	if self.inclusive is True:
	    # Length field includes the size of the type and length fields.
            self.length.value += (self.type.width + self.length.width)

	if self.bytewise is True:
	    # Length should be encoded as a measure of bytes, not bits.
            self.length.value /= 8

        [byte, byteBR] = self.type.encode(bytearray, self.type.value, byte, byteBR)
        [byte, byteBR] = self.length.encode(bytearray, self.length.value, byte, byteBR)
        [byte, byteBR] = self.value.encode(bytearray, self.value.value, byte, byteBR)
        return [byte, byteBR]

    def reset(self):
        """Return a resonable value to use in resetting a field of this type."""
        return ""

    def bounds(self, value):
        """Check the bounds of this field."""
        if ((value == None) or
            (len (value) > (((2 ** self.lengthwidth) - 1) / 8))):
            raise FieldBoundsError, "Value must be between 0 and %d bytes long" % (((2 ** self.width) - 1) / 8)
{'fraction_non_alphanumeric': 0.0916924378188354, 'fraction_numerical': 0.005207094335632913, 'mean_word_length': 2.2939230367403796, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1725097', 'n_tokens_mistral': 10465, 'n_tokens_neox': 10048, 'n_words': 4246}