<commit_before>import struct
from gevent.socket import htonl, inet_ntop, AF_INET

class Packet(object):
    def __init__(self, data, header=None):
        self.data = data
        self.header = header

    def dst_asstring(self):
        return inet_ntop(AF_INET, struct.pack("i", self.header["dst"]))

    def src_asstring(self):
        return inet_ntop(AF_INET, struct.pack("i", self.header["src"]))

    def __repr__(self):
        return repr(self.header)

    @staticmethod
    def read_from_socket(socket, header_safe=False, tun=False):
        ip_header_format = "!HHHHHHii"
        if tun:
            ip_header_format = "!iHHHHHHii"
        ip_header_size = struct.calcsize(ip_header_format)

        raw_header = socket.recv(ip_header_size)

        if tun:
            trash, c1, c2, c3, c4, c5, c6, src, dst = struct.unpack(ip_header_format, raw_header)
        else:
            c1, c2, c3, c4, c5, c6, src, dst = struct.unpack(ip_header_format, raw_header)


        header = {
            "ver" : c1 >> 12,
            "ihl" : (c1 >> 8) & 0b00001111,
            "dscp" : (c1 & 255) >> 3,
            "ecn" : c1 & 3,
            "length" : c2, # length of packet with header in bytes
            "id" : c3,
            "flags" : c4 >> 13,
            "offset" : c4 & 0b1111111111111,
            "ttl" : c5 >> 8,
            "protocol" : c5 & 255,
            "checksum" : c6,
            "src": htonl(src),
            "dst" : htonl(dst),
        }

        if header["ihl"] > 5:
            socket.recv((header["ihl"]-5)*4)

        data_size = header["length"]-header["ihl"]*4

        data = ""
        if header_safe:
            data += raw_header
        data += socket.recv(data_size)

        return Packet(data, header=header)

<commit_msg>dirty hack<commit_after>import struct
from gevent.socket import htonl, inet_ntop, AF_INET

class Packet(object):
    def __init__(self, data, header=None):
        self.data = data
        self.header = header

    def dst_asstring(self):
        return inet_ntop(AF_INET, struct.pack("i", self.header["dst"]))

    def src_asstring(self):
        return inet_ntop(AF_INET, struct.pack("i", self.header["src"]))

    def __repr__(self):
        return repr(self.header)

    @staticmethod
    def read_from_socket(socket, header_safe=False, tun=False):
        ip_header_format = "!HHHHHHii"
        if tun:
            ip_header_format = "!iHHHHHHii"
        ip_header_size = struct.calcsize(ip_header_format)

        raw_header = socket.recv(ip_header_size)

        if tun:
            trash, c1, c2, c3, c4, c5, c6, src, dst = struct.unpack(ip_header_format, raw_header)
        else:
            c1, c2, c3, c4, c5, c6, src, dst = struct.unpack(ip_header_format, raw_header)


        header = {
            "ver" : c1 >> 12,
            "ihl" : (c1 >> 8) & 0b00001111,
            "dscp" : (c1 & 255) >> 3,
            "ecn" : c1 & 3,
            "length" : c2, # length of packet with header in bytes
            "id" : c3,
            "flags" : c4 >> 13,
            "offset" : c4 & 0b1111111111111,
            "ttl" : c5 >> 8,
            "protocol" : c5 & 255,
            "checksum" : c6,
            "src": htonl(src),
            "dst" : htonl(dst),
        }

        if header["ihl"] > 5:
            socket.recv((header["ihl"]-5)*4)

        data_size = header["length"]-header["ihl"]*4

        data = ""
        if header_safe:
            if tun:
                data += raw_header[4:]
            else:
                data += raw_header
        data += socket.recv(data_size)

        return Packet(data, header=header)


{'fraction_non_alphanumeric': 0.13102493074792243, 'fraction_numerical': 0.03573407202216067, 'mean_word_length': 1.9074074074074074, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2588900', 'n_tokens_mistral': 1336, 'n_tokens_neox': 1237, 'n_words': 272}