<commit_before>import ptypes, datetime, time
from ptypes import pstruct,parray,ptype,dyn,pstr,pint,pfloat,pbinary

## primitives
byte = dyn.clone(pint.uint8_t)
word = dyn.clone(pint.uint16_t)
dword = dyn.clone(pint.uint32_t)
float = dyn.clone(pfloat.single)
double = dyn.clone(pfloat.double)

uint0 = dyn.clone(pint.uint_t)
int0 = dyn.clone(pint.int_t)
uint8 = dyn.clone(pint.uint8_t)
int8 = dyn.clone(pint.int8_t)
int16 = dyn.clone(pint.int16_t)
uint16 = dyn.clone(pint.uint16_t)
int32 = dyn.clone(pint.int32_t)
uint32 = dyn.clone(pint.uint32_t)
uint64 = dyn.clone(pint.uint64_t)

class off_t(pint.uint32_t): pass
class addr_t(pint.uint32_t): pass

### locating particular parts of the executable
def LocateBase(self):
    """Return the base object of the executable. This is used to find the base address."""
    try:
        nth = self.getparent(ptype.boundary)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

def LocateHeader(self):
    """Return the executable sub-header. This will return the executable main header."""
    try:
        nth = self.getparent(Header)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

## types of relative pointers in the executable
def CalculateRelativeAddress(self, address):
    """given a va, returns offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

def CalculateRelativeOffset(self, offset):
    """return an offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        return base + offset

    # memory
    pe = LocateHeader(self)
    section = pe['Sections'].getsectionbyoffset(offset)
    o = offset - section['PointerToRawData'].int()
    return base + section['VirtualAddress'].int() + o

def CalculateRealAddress(self, address):
    """given an rva, return offset relative to the baseaddress"""
    header = LocateHeader(self)
    base = LocateBase(header)
    address -= header['OptionalHeader']['ImageBase'].int()
    base = base.getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

## ptypes closures
def realaddress(target, **kwds):
    """Returns a pointer to /target/ where value is an rva"""
    kwds.setdefault('__name__', 'realaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRealAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRealAddress, **kwds)

def fileoffset(target, **kwds):
    """Returns a pointer to /target/ where value is a fileoffset"""
    kwds.setdefault('__name__', 'fileoffset')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeOffset, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeOffset, **kwds)
def virtualaddress(target, **kwds):
    """Returns a pointer to /target/ where value is a va"""
    kwds.setdefault('__name__', 'virtualaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeAddress, **kwds)

## core header types
class Header(object): pass

class Machine(pint.enum, pint.uint16_t):
    _values_ = [
        ('UNKNOWN', 0x0000), ('AM33', 0x01d3), ('AMD64', 0x8664), ('ARM', 0x01c0),
        ('EBC', 0x0ebc), ('I386', 0x014c), ('IA64', 0x0200), ('M32R', 0x9041),
        ('MIPS16', 0x0266), ('MIPSFPU', 0x0366), ('MIPSFPU16', 0x0466), ('POWERPC', 0x01f0),
        ('POWERPCFP', 0x01f1), ('R4000', 0x0166), ('SH3', 0x01a2), ('SH3DSP', 0x01a3),
        ('SH4', 0x01a6), ('SH5', 0x01a8), ('THUMB', 0x01c2), ('WCEMIPSV2', 0x0169),
    ]

class TimeDateStamp(uint32):
    def datetime(self):
        res = self.int()
        epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
        delta = datetime.timedelta(seconds=res)
        return epoch + delta
    def details(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return res.isoformat()
    def summary(self):<commit_msg>Added some microsoft-y primitive types to the `pecoff.headers` library module.<commit_after>import ptypes, datetime, time
from ptypes import pstruct,parray,ptype,dyn,pstr,pint,pfloat,pbinary

## primitives
byte = dyn.clone(pint.uint8_t)
word = dyn.clone(pint.uint16_t)
dword = dyn.clone(pint.uint32_t)
float = dyn.clone(pfloat.single)
double = dyn.clone(pfloat.double)

uint0 = dyn.clone(pint.uint_t)
int0 = dyn.clone(pint.int_t)
uint8 = dyn.clone(pint.uint8_t)
int8 = dyn.clone(pint.int8_t)
int16 = dyn.clone(pint.int16_t)
uint16 = dyn.clone(pint.uint16_t)
int32 = dyn.clone(pint.int32_t)
uint32 = dyn.clone(pint.uint32_t)
uint64 = dyn.clone(pint.uint64_t)

class off_t(pint.uint32_t): pass
class addr_t(pint.uint32_t): pass

class VOID(ptype.undefined): pass
class BYTE(pint.uint8_t): pass
class WORD(pint.uint16_t): pass
class DWORD(pint.uint32_t): pass
class ULONGLONG(pint.uint64_t): pass

### locating particular parts of the executable
def LocateBase(self):
    """Return the base object of the executable. This is used to find the base address."""
    try:
        nth = self.getparent(ptype.boundary)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

def LocateHeader(self):
    """Return the executable sub-header. This will return the executable main header."""
    try:
        nth = self.getparent(Header)
    except ValueError as msg:
        nth = list(self.backtrace(fn=lambda x:x))[-1]
    return nth

## types of relative pointers in the executable
def CalculateRelativeAddress(self, address):
    """given a va, returns offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

def CalculateRelativeOffset(self, offset):
    """return an offset relative to the baseaddress"""
    base = LocateBase(self).getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        return base + offset

    # memory
    pe = LocateHeader(self)
    section = pe['Sections'].getsectionbyoffset(offset)
    o = offset - section['PointerToRawData'].int()
    return base + section['VirtualAddress'].int() + o

def CalculateRealAddress(self, address):
    """given an rva, return offset relative to the baseaddress"""
    header = LocateHeader(self)
    base = LocateBase(header)
    address -= header['OptionalHeader']['ImageBase'].int()
    base = base.getoffset()

    # file
    if issubclass(self.source.__class__, ptypes.provider.fileobj):
        pe = LocateHeader(self)
        section = pe['Sections'].getsectionbyaddress(address)
        return base + section.getoffsetbyaddress(address)

    # memory
    return base + address

## ptypes closures
def realaddress(target, **kwds):
    """Returns a pointer to /target/ where value is an rva"""
    kwds.setdefault('__name__', 'realaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRealAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRealAddress, **kwds)

def fileoffset(target, **kwds):
    """Returns a pointer to /target/ where value is a fileoffset"""
    kwds.setdefault('__name__', 'fileoffset')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeOffset, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeOffset, **kwds)
def virtualaddress(target, **kwds):
    """Returns a pointer to /target/ where value is a va"""
    kwds.setdefault('__name__', 'virtualaddress')
    if 'type' in kwds:
        return dyn.opointer(target, CalculateRelativeAddress, kwds.pop('type'), **kwds)
    return dyn.opointer(target, CalculateRelativeAddress, **kwds)

## core header types
class Header(object): pass

class Machine(pint.enum, WORD):
    _values_ = [
        ('UNKNOWN', 0x0000), ('AM33', 0x01d3), ('AMD64', 0x8664), ('ARM', 0x01c0),
        ('EBC', 0x0ebc), ('I386', 0x014c), ('IA64', 0x0200), ('M32R', 0x9041),
        ('MIPS16', 0x0266), ('MIPSFPU', 0x0366), ('MIPSFPU16', 0x0466), ('POWERPC', 0x01f0),
        ('POWERPCFP', 0x01f1), ('R4000', 0x0166), ('SH3', 0x01a2), ('SH3DSP', 0x01a3),
        ('SH4', 0x01a6), ('SH5', 0x01a8), ('THUMB', 0x01c2), ('WCEMIPSV2', 0x0169),
    ]

class TimeDateStamp(DWORD):
    def datetime(self):
        res = self.int()
        epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
        delta = datetime.timedelta(seconds=res)
        return epoch + delta
    def details(self):
        tzinfo = datetime.timezone(datetime.timedelta(seconds=-(time.altzone if time.daylight else time.timezone)))
        res = self.datetime().astimezone(tzinfo)
        return res.isoformat()
    def summary(self):
{'fraction_non_alphanumeric': 0.13651092296435663, 'fraction_numerical': 0.0333437859308038, 'mean_word_length': 3.9345023207839094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620604', 'n_tokens_mistral': 3409, 'n_tokens_neox': 3176, 'n_words': 765}