<commit_before>from datetime import datetime
import re

from .utils import get_int2, get_int4, get_int8, get_mpi

NEW_TAG_FLAG    = 0x40
TAG_MASK        = 0x3f
PARTIAL_MASK    = 0x1f

OLD_LEN_MASK    = 0x03
OLD_TAG_SHIFT   = 2
TAG_COMPRESSED  = 8

CRITICAL_BIT    = 0x80
CRITICAL_MASK   = 0x7f


class Packet(object):
    '''The base packet object containing various fields pulled from the packet
    header as well as a slice of the packet data.'''
    def __init__(self, raw, name, new, partial, data):
        self.raw = raw
        self.name = name
        self.new = new
        self.partial = partial
        self.length = len(data)
        self.data = data

        # now let subclasses work their magic
        self.parse()

    def parse(self):
        '''Perform any parsing necessary to populate fields on this packet.
        This method is called as the last step in __init__(). The base class
        method is a no-op; subclasses should use this as required.'''
        pass

    def __repr__(self):
        new = "old"
        if self.new:
            new = "new"
        return "<%s: %s (%d), %s, length %d>" % (
                self.__class__.__name__, self.name, self.raw, new, self.length)


class AlgoLookup(object):
    '''Mixin class containing algorithm lookup methods.'''
    @staticmethod
    def lookup_pub_algorithm(alg):
        algorithms = {
            1:  "RSA Encrypt or Sign",
            2:  "RSA Encrypt-Only",
            3:  "RSA Sign-Only",
            16: "ElGamal Encrypt-Only",
            17: "DSA Digital Signature Algorithm",
            18: "Elliptic Curve",
            19: "ECDSA",
            20: "Formerly ElGamal Encrypt or Sign",
            21: "Diffie-Hellman",
        }
        return algorithms.get(alg, "Unknown")

    @staticmethod
    def lookup_hash_algorithm(alg):
        reserved_values = (4, 5, 6, 7)
        algorithms = {
            1:  "MD5",
            2:  "SHA1",
            3:  "RIPEMD160",
            8:  "SHA256",
            9:  "SHA384",
            10: "SHA512",
            11: "SHA224",
        }
        if alg in reserved_values:
            return "Reserved"
        return algorithms.get(alg, "Unknown")


class SignatureSubpacket(object):
    '''A signature subpacket containing a type, type name, some flags, and the
    contained data.'''
    def __init__(self, raw, name, hashed, critical, data):
        self.raw = raw
        self.name = name
        self.hashed = hashed
        self.critical = critical
        self.length = len(data)
        self.data = data

    def __repr__(self):
        hashed = ""
        if self.hashed:
            hashed = "hashed, "
        return "<%s: %slength %d>" % (
                self.__class__.__name__, hashed, self.length)

class SignaturePacket(Packet, AlgoLookup):
    def __init__(self, *args, **kwargs):
        self.hash_material = None
        self.sig_version = None
        self.raw_sig_type = None
        self.sig_type = None
        self.raw_pub_algorithm = None
        self.pub_algorithm = None
        self.raw_hash_algorithm = None
        self.hash_algorithm = None
        self.creation_time = None
        self.datetime = None
        self.key_id = None
        self.hash2 = None
        self.subpackets = []
        super(SignaturePacket, self).__init__(*args, **kwargs)

    def parse(self):
        self.sig_version = self.data[0]
        offset = 1
        if self.sig_version == 3:
            # 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
            # |  |  [  ctime  ] [ key_id                 ] |
            # |  |-type                           pub_algo-|
            # |-hash material
            # 10 11 12
            # |  [hash2]
            # |-hash_algo

            self.hash_material = self.data[offset]
            offset += 1

            self.raw_sig_type = self.data[offset]
            self.sig_type = self.lookup_signature_type(self.data[offset])
            offset += 1

            self.creation_time = get_int4(self.data, offset)
            self.datetime = datetime.utcfromtimestamp(self.creation_time)
            offset += 4

            self.key_id = get_int8(self.data, offset)
            offset += 8

            self.raw_pub_algorithm = self.data[offset]
            self.pub_algorithm = self.lookup_pub_algorithm(self.data[offset])
            offset += 1

            self.raw_hash_algorithm = self.data[offset]
            self.hash_algorithm = self.lookup_hash_algorithm(self.data[offset])
            offset += 1

            self.hash2 = self.data[offset:offset + 2]
            offset += 2

        elif self.sig_version == 4:
            # 00 01 02 03 ... <hashedsubpackets..> <subpackets..> [hash2]
            # |  |  |-hash_algo
            # |  |-pub_algo
            # |-type

            self.raw_sig_type = self.data[offset]
            self.sig_type = self.lookup_signature_type(self.data[offset])
            offset += 1

            self.raw_pub_algorithm = self.data[offset]
            self.pub_algorithm = self.lookup_pub_algorithm(self.data[offset])
            offset += 1

            self.raw_hash_algorithm = self.data[offset]
            self.hash_algorithm = self.lookup_hash_algorithm(self.data[offset])
            offset += 1

            # next is hashed subpackets
            length = get_int2(self.data, offset)
            offset += 2
            self.parse_subpackets(offset, length, True)
            offset += length

            # followed by subpackets
            length = get_int2(self.data, offset)
            offset += 2
            self.parse_subpackets(offset, length, False)
            offset += length

            self.hash2 = self.data[offset:offset + 2]
            offset += 2

    def parse_subpackets(self, outer_offset, outer_length, hashed=False):
        offset = outer_offset
        while offset < outer_offset + outer_length:
            # each subpacket is [variable length] [subtype] [data]
            sub_offset, sub_length = new_tag_length(self.data[offset:])
            # sub_length includes the subtype single byte, knock that off
            sub_length -= 1
            # initial length bytes
            offset += 1 + sub_offset

            subtype = self.data[offset]
            offset += 1

            critical = bool(subtype & CRITICAL_BIT)
            subtype &= CRITICAL_MASK
            name = self.lookup_signature_subtype(subtype)

            sub_data = self.data[offset:offset + sub_length]
            subpacket = SignatureSubpacket(subtype, name,
                    hashed, critical, sub_data)
            if subpacket.raw == 2:
                self.creation_time = get_int4(subpacket.data, 0)
                self.datetime = datetime.utcfromtimestamp(self.creation_time)
            elif subpacket.raw == 16:<commit_msg>Improve scope on some constants<commit_after>from datetime import datetime
import re

from .utils import get_int2, get_int4, get_int8, get_mpi

NEW_TAG_FLAG    = 0x40
TAG_MASK        = 0x3f
PARTIAL_MASK    = 0x1f

OLD_LEN_MASK    = 0x03
OLD_TAG_SHIFT   = 2
TAG_COMPRESSED  = 8


class Packet(object):
    '''The base packet object containing various fields pulled from the packet
    header as well as a slice of the packet data.'''
    def __init__(self, raw, name, new, partial, data):
        self.raw = raw
        self.name = name
        self.new = new
        self.partial = partial
        self.length = len(data)
        self.data = data

        # now let subclasses work their magic
        self.parse()

    def parse(self):
        '''Perform any parsing necessary to populate fields on this packet.
        This method is called as the last step in __init__(). The base class
        method is a no-op; subclasses should use this as required.'''
        pass

    def __repr__(self):
        new = "old"
        if self.new:
            new = "new"
        return "<%s: %s (%d), %s, length %d>" % (
                self.__class__.__name__, self.name, self.raw, new, self.length)


class AlgoLookup(object):
    '''Mixin class containing algorithm lookup methods.'''
    @staticmethod
    def lookup_pub_algorithm(alg):
        algorithms = {
            1:  "RSA Encrypt or Sign",
            2:  "RSA Encrypt-Only",
            3:  "RSA Sign-Only",
            16: "ElGamal Encrypt-Only",
            17: "DSA Digital Signature Algorithm",
            18: "Elliptic Curve",
            19: "ECDSA",
            20: "Formerly ElGamal Encrypt or Sign",
            21: "Diffie-Hellman",
        }
        return algorithms.get(alg, "Unknown")

    @staticmethod
    def lookup_hash_algorithm(alg):
        reserved_values = (4, 5, 6, 7)
        algorithms = {
            1:  "MD5",
            2:  "SHA1",
            3:  "RIPEMD160",
            8:  "SHA256",
            9:  "SHA384",
            10: "SHA512",
            11: "SHA224",
        }
        if alg in reserved_values:
            return "Reserved"
        return algorithms.get(alg, "Unknown")


class SignatureSubpacket(object):
    '''A signature subpacket containing a type, type name, some flags, and the
    contained data.'''
    def __init__(self, raw, name, hashed, critical, data):
        self.raw = raw
        self.name = name
        self.hashed = hashed
        self.critical = critical
        self.length = len(data)
        self.data = data

    def __repr__(self):
        hashed = ""
        if self.hashed:
            hashed = "hashed, "
        return "<%s: %slength %d>" % (
                self.__class__.__name__, hashed, self.length)

class SignaturePacket(Packet, AlgoLookup):
    def __init__(self, *args, **kwargs):
        self.hash_material = None
        self.sig_version = None
        self.raw_sig_type = None
        self.sig_type = None
        self.raw_pub_algorithm = None
        self.pub_algorithm = None
        self.raw_hash_algorithm = None
        self.hash_algorithm = None
        self.creation_time = None
        self.datetime = None
        self.key_id = None
        self.hash2 = None
        self.subpackets = []
        super(SignaturePacket, self).__init__(*args, **kwargs)

    def parse(self):
        self.sig_version = self.data[0]
        offset = 1
        if self.sig_version == 3:
            # 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
            # |  |  [  ctime  ] [ key_id                 ] |
            # |  |-type                           pub_algo-|
            # |-hash material
            # 10 11 12
            # |  [hash2]
            # |-hash_algo

            self.hash_material = self.data[offset]
            offset += 1

            self.raw_sig_type = self.data[offset]
            self.sig_type = self.lookup_signature_type(self.data[offset])
            offset += 1

            self.creation_time = get_int4(self.data, offset)
            self.datetime = datetime.utcfromtimestamp(self.creation_time)
            offset += 4

            self.key_id = get_int8(self.data, offset)
            offset += 8

            self.raw_pub_algorithm = self.data[offset]
            self.pub_algorithm = self.lookup_pub_algorithm(self.data[offset])
            offset += 1

            self.raw_hash_algorithm = self.data[offset]
            self.hash_algorithm = self.lookup_hash_algorithm(self.data[offset])
            offset += 1

            self.hash2 = self.data[offset:offset + 2]
            offset += 2

        elif self.sig_version == 4:
            # 00 01 02 03 ... <hashedsubpackets..> <subpackets..> [hash2]
            # |  |  |-hash_algo
            # |  |-pub_algo
            # |-type

            self.raw_sig_type = self.data[offset]
            self.sig_type = self.lookup_signature_type(self.data[offset])
            offset += 1

            self.raw_pub_algorithm = self.data[offset]
            self.pub_algorithm = self.lookup_pub_algorithm(self.data[offset])
            offset += 1

            self.raw_hash_algorithm = self.data[offset]
            self.hash_algorithm = self.lookup_hash_algorithm(self.data[offset])
            offset += 1

            # next is hashed subpackets
            length = get_int2(self.data, offset)
            offset += 2
            self.parse_subpackets(offset, length, True)
            offset += length

            # followed by subpackets
            length = get_int2(self.data, offset)
            offset += 2
            self.parse_subpackets(offset, length, False)
            offset += length

            self.hash2 = self.data[offset:offset + 2]
            offset += 2

    CRITICAL_BIT = 0x80
    CRITICAL_MASK = 0x7f

    def parse_subpackets(self, outer_offset, outer_length, hashed=False):
        offset = outer_offset
        while offset < outer_offset + outer_length:
            # each subpacket is [variable length] [subtype] [data]
            sub_offset, sub_length = new_tag_length(self.data[offset:])
            # sub_length includes the subtype single byte, knock that off
            sub_length -= 1
            # initial length bytes
            offset += 1 + sub_offset

            subtype = self.data[offset]
            offset += 1

            critical = bool(subtype & self.CRITICAL_BIT)
            subtype &= self.CRITICAL_MASK
            name = self.lookup_signature_subtype(subtype)

            sub_data = self.data[offset:offset + sub_length]
            subpacket = SignatureSubpacket(subtype, name,
                    hashed, critical, sub_data)
            if subpacket.raw == 2:
                self.creation_time = get_int4(subpacket.data, 0)
                self.datetime = datetime.utcfromtimestamp(self.creation_time)
            elif subpacket.raw == 16:
{'fraction_non_alphanumeric': 0.09534286762009535, 'fraction_numerical': 0.020682068206820683, 'mean_word_length': 1.7580906148867315, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1154496', 'n_tokens_mistral': 4239, 'n_tokens_neox': 3930, 'n_words': 1011}