<commit_before># pypcapfile.savefile.py

import binascii
import ctypes
import struct
import sys

import linklayer

VERBOSE = False


def __TRACE__(msg, args=None):
    if VERBOSE:
        if args:
            print msg % args
        else:
            print msg


class __pcap_header__(ctypes.Structure):
    """
C-struct representation of a savefile header. See __validate_header__
for validation.
    """
    _fields_ = [('magic', ctypes.c_uint),       # file magic number
                ('major', ctypes.c_ushort),     # major version number
                ('minor', ctypes.c_ushort),     # minor version number
                ('tz_off', ctypes.c_uint),      # timezone offset
                ('ts_acc', ctypes.c_uint),      # timestamp accuracy
                ('snaplen', ctypes.c_uint),     # snapshot length
                ('ll_type', ctypes.c_uint)]     # link layer header type


class pcap_packet(ctypes.Structure):
    """
    ctypes Structure representation of a packet. The header field is a pointer
    to the header of the savefile the packet came from to provide context. It
    can be accessed with header[0]. By default, the raw packet is stored in
    a string of hexadecimal-encoded bytes as the packet field. The raw()
    method will return the raw binary packet.
    """
    _fields_ = [('header', ctypes.POINTER(__pcap_header__)),
                ('timestamp', ctypes.c_uint),
                ('timestamp_ms', ctypes.c_uint),
                ('capture_len', ctypes.c_uint),
                ('packet_len', ctypes.c_uint),
                ('packet', ctypes.c_char_p)]

    def raw(self):
        """
        Return the raw binary data from the packet.
        """
        return binascii.unhexlify(self.packet)

    def __repr__(self):
        return self.raw()


class pcap_savefile(object):
    """
    Represents a libpcap savefile. The packets member is a list of pcap_packet
    instances. The 'valid' member will be None for an uninitialised instance,
    False if the initial validation fails, or True if the instance has been
    successfully set up and the file has been parsed.
    """
    def __init__(self, header, packets = []):
        self.header = header
        self.packets = packets
        self.valid = None
        self.byteorder = sys.byteorder
        
        if not self.__validate__():
            self.valid = False
        else:
            self.valid = True

        assert self.valid, 'Invalid savefile.'

        if header.magic == 0xa1b2c3d4:
            self.byteorder = 'big'
        elif header.magic == 0xd4c3b2a1:
            self.byteorder = 'little'
        else:
            self.byteorder = 'unknown'

        assert self.byteorder in [ 'little', 'big' ], 'Invalid byte order.'


    def __validate__(self):
        assert __validate_header__(self.header),  "Invalid header."
        if not __validate_header__(self.header):
            return False
        
        valid_packet = lambda pkt: type(pkt) == pcap_packet or pkt == None
        if not 0 == len(self.packets):
            valid_packet = [valid_packet(pkt) for pkt in self.packets]
            assert False not in valid_packet, 'Invalid packets in savefile.'
            if False in valid_packet:
                return False
        
        return True

    def __repr__(self):
        string = '%s-endian capture file version %d.%d\n'
        string += 'snapshot length: %d\n'
        string += 'linklayer type: %s\nnumber of packets: %d\n'
        string = string % (self.byteorder, self.header.major, 
                           self.header.minor, self.header.snaplen,
                           linklayer.lookup(self.header.ll_type),
                           len(self.packets))
        return string



def _load_savefile_header(file_h):
    """
Load and validate the header of a pcap file.
    """
    raw_savefile_header = file_h.read(24)
    unpacked = struct.unpack('=IhhIIII', raw_savefile_header)
    (magic, major, minor, tz_off, ts_acc, snaplen, ll_type) = unpacked
    header = __pcap_header__(magic, major, minor, tz_off, ts_acc, snaplen, 
                             ll_type)
    if not __validate_header__(header):
        raise Exception('invalid savefile header!')
    else:
        return header


def load_savefile(filename, verbose = False):
    """
    Load and parse a savefile as a pcap_savefile instance. Returns the savefile
    on success and None on failure. Verbose mode prints additional information
    about the file's processing.
    """
    global VERBOSE
    old_verbose = VERBOSE
    VERBOSE = verbose

    file_h = open(filename)
    __TRACE__('[+] attempting to load %s', (filename, ))

    header = _load_savefile_header(file_h)
    if __validate_header__(header):
        __TRACE__('[+] found valid header')
        packets = _load_packets(file_h, header)
        __TRACE__('[+] loaded %d packets', (len(packets), ))
        sfile = pcap_savefile(header, packets)
        __TRACE__('[+] finished loading savefile.')

    else:
        __TRACE__('[!] invalid savefile')
        return None

    return sfile

<commit_msg>formatting fix<commit_after># pypcapfile.savefile.py

import binascii
import ctypes
import struct
import sys

import linklayer

VERBOSE = False


def __TRACE__(msg, args=None):
    if VERBOSE:
        if args:
            print msg % args
        else:
            print msg


class __pcap_header__(ctypes.Structure):
    """
C-struct representation of a savefile header. See __validate_header__
for validation.
    """
    _fields_ = [('magic', ctypes.c_uint),       # file magic number
                ('major', ctypes.c_ushort),     # major version number
                ('minor', ctypes.c_ushort),     # minor version number
                ('tz_off', ctypes.c_uint),      # timezone offset
                ('ts_acc', ctypes.c_uint),      # timestamp accuracy
                ('snaplen', ctypes.c_uint),     # snapshot length
                ('ll_type', ctypes.c_uint)]     # link layer header type


class pcap_packet(ctypes.Structure):
    """
    ctypes Structure representation of a packet. The header field is a pointer
    to the header of the savefile the packet came from to provide context. It
    can be accessed with header[0]. By default, the raw packet is stored in
    a string of hexadecimal-encoded bytes as the packet field. The raw()
    method will return the raw binary packet.
    """
    _fields_ = [('header', ctypes.POINTER(__pcap_header__)),
                ('timestamp', ctypes.c_uint),
                ('timestamp_ms', ctypes.c_uint),
                ('capture_len', ctypes.c_uint),
                ('packet_len', ctypes.c_uint),
                ('packet', ctypes.c_char_p)]

    def raw(self):
        """
        Return the raw binary data from the packet.
        """
        return binascii.unhexlify(self.packet)

    def __repr__(self):
        return self.raw()


class pcap_savefile(object):
    """
    Represents a libpcap savefile. The packets member is a list of pcap_packet
    instances. The 'valid' member will be None for an uninitialised instance,
    False if the initial validation fails, or True if the instance has been
    successfully set up and the file has been parsed.
    """
    def __init__(self, header, packets = []):
        self.header = header
        self.packets = packets
        self.valid = None
        self.byteorder = sys.byteorder
        
        if not self.__validate__():
            self.valid = False
        else:
            self.valid = True

        assert self.valid, 'Invalid savefile.'

        if header.magic == 0xa1b2c3d4:
            self.byteorder = 'big'
        elif header.magic == 0xd4c3b2a1:
            self.byteorder = 'little'
        else:
            self.byteorder = 'unknown'

        assert self.byteorder in [ 'little', 'big' ], 'Invalid byte order.'


    def __validate__(self):
        assert __validate_header__(self.header),  "Invalid header."
        if not __validate_header__(self.header):
            return False
        
        valid_packet = lambda pkt: type(pkt) == pcap_packet or pkt == None
        if not 0 == len(self.packets):
            valid_packet = [valid_packet(pkt) for pkt in self.packets]
            assert False not in valid_packet, 'Invalid packets in savefile.'
            if False in valid_packet:
                return False
        
        return True

    def __repr__(self):
        string = '%s-endian capture file version %d.%d\n'
        string += 'snapshot length: %d\n'
        string += 'linklayer type: %s\nnumber of packets: %d\n'
        string = string % (self.byteorder, self.header.major, 
                           self.header.minor, self.header.snaplen,
                           linklayer.lookup(self.header.ll_type),
                           len(self.packets))
        return string



def _load_savefile_header(file_h):
    """
Load and validate the header of a pcap file.
    """
    raw_savefile_header = file_h.read(24)
    unpacked = struct.unpack('=IhhIIII', raw_savefile_header)
    (magic, major, minor, tz_off, ts_acc, snaplen, ll_type) = unpacked
    header = __pcap_header__(magic, major, minor, tz_off, ts_acc, snaplen, 
                             ll_type)
    if not __validate_header__(header):
        raise Exception('invalid savefile header!')
    else:
        return header


def load_savefile(filename, verbose = False):
    """
    Load and parse a savefile as a pcap_savefile instance. Returns the savefile
    on success and None on failure. Verbose mode prints additional information
    about the file's processing.
    """
    global VERBOSE
    old_verbose = VERBOSE
    VERBOSE = verbose

    file_h = open(filename)
    __TRACE__('[+] attempting to load %s', (filename, ))

    header = _load_savefile_header(file_h)
    if __validate_header__(header):
        __TRACE__('[+] found valid header')
        packets = _load_packets(file_h, header)
        __TRACE__('[+] loaded %d packets', (len(packets), ))
        sfile = pcap_savefile(header, packets)
        __TRACE__('[+] finished loading savefile.')
    else:
        __TRACE__('[!] invalid savefile')
        return None

    return sfile


{'fraction_non_alphanumeric': 0.09383588535531999, 'fraction_numerical': 0.0027483313702394976, 'mean_word_length': 2.2071136292099465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852206', 'n_tokens_mistral': 2998, 'n_tokens_neox': 2927, 'n_words': 893}