<commit_before>#!/usr/bin/env python
import six,argparse
import functools,operator,itertools,types
import ptypes,pecoff,ber
#ptypes.setbyteorder(ptypes.config.byteorder.littleendian)
ptypes.config.defaults.ptype.clone_name = '{}'
ptypes.config.defaults.pint.bigendian_name = 'be({})'
ptypes.config.defaults.pint.littleendian_name = '{}'

def Locate(path, obj):
    return Resolve(obj, path.split(':'))

def Extract(obj, out):
    if out == 'print':
        res = obj.repr()
    elif out == 'hex':
        res = obj.hexdump()
    elif out == 'raw':
        res = obj.serialize()
    elif out == 'list':
        res = '\n'.join((n.summary() if hasattr(n, 'summary') else str(n) for n in obj))

    # default
    elif not obj:
        res = obj.details()
    elif isinstance(obj, (ptypes.ptype.pointer_t, ptypes.pbinary.partial)):
        res = repr(obj)
    elif hasattr(obj, 'num'):
        res = obj.num()
    elif hasattr(obj, 'int'):
        res = obj.int()
    elif hasattr(obj, 'str'):
        res = obj.str()
    else:
        res = repr(obj)
    six.print_(res, file=sys.stdout)

def Resolve(result, path):
    for p in path:
        if p == '':
            break
        elif p == '.':
            pass
        elif p == '..':
            result = result.p
        elif p == '!' and hasattr(result, 'd'):
            result = result.d.li
        elif p == '@' and hasattr(result, 'o'):
            result = result.o.li
        elif hasattr(result, '__field__'):
            result = result.__field__(p) if hasattr(result, '__keys__') else result.__field__(int(p))
        else:
            raise AttributeError(p, result)
        continue
    return result

## commands
def dump_exe(t, output, F=None):
    global result; result = t
    if F:
        return Extract(F(result), output)
    if not output or output in {'print'}:
        six.print_(result['Dos'].repr(), file=sys.stdout)
        six.print_(result['Stub'].hexdump(), file=sys.stdout)
        return
    if output in {'hex','raw'}:
        t = ptypes.dyn.block(result['Dos'].size() + result['Stub'].size())
        return Extract(t(offset=0).li, output)
    return Extract(result, output)

def dump_header(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H
    if F:
        return Extract(F(result), output)
    if not output:
        six.print_(result.p['Signature'], repr(result.p['Signature'].serialize()+result['SignaturePadding'].serialize()), file=sys.stdout)
        six.print_(result['FileHeader'].repr(), file=sys.stdout)
        six.print_(result['OptionalHeader'].repr(), file=sys.stdout)
        return
    return Extract(result, output)

def list_sections(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H['Sections']
    if F:
        return Extract(F(result), output)
    if not output:
        summary = lambda s: '{!r} {:#x}:{:+#x} Raw:{:#x}:{:+#x}'.format(s['Name'].str(), s['VirtualAddress'].int(), s['VirtualSize'].int(), s['PointerToRawData'].int(), s['SizeOfRawData'].int())
        for i, S in enumerate(result):
            six.print_('[{:d}] {:s}'.format(i, summary(S)), file=sys.stdout)
        return
    if output in {'list'}:
        return Extract((S['Name'].str() for S in result), output)
    return Extract(result, output)

def extract_section(t, index, output, F=None):
    H = t['Next']['Header']
    if not (0 <= index < len(H['Sections'])):
        raise IndexError("Invalid section index was specified ({:d} <= {:d} < {:d}).".format(0, index, len(H['Sections'])))
    S = H['Sections'][index]
    global result; result = S
    #if 'load' in kwds:
    #    return Extract(t['VirtualAddress'].d.li, output or 'raw')
    if F:
        return Extract(F(result), output)
    return Extract(result['PointerToRawData'].d.li, output or 'hex')

def list_entries(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H['DataDirectory']
    if F:
        return Extract(F(result), output)
    summary = lambda n: '{:s} {:#x}:{:+#x}'.format(n.classname(), n['Address'].int(), n['Size'].int())
    if not output:
        for i, n in enumerate(result):
            six.print_('[{:d}] {}'.format(i, summary(n)), file=sys.stdout)
        return
    if output in {'list'}:
        return Extract((n.classname() for n in result if n['Address'].int()), output)
    return Extract(result, output)

def extract_entry(t, index, output, F=None):
    d = t['Next']['Header']['DataDirectory']
    if not(0 <= index < len(d)):
        raise IndexError("Invalid DataDirectory entry number was specified ({:d} <= {:d} < {:d}).".format(0, index, len(d)))
    E = d[index]
    global result; result = E
    T, cb = E['Address'], E['Size']
    if F:
        return Extract(F(result), output)
    if output in {'hex', 'raw'}:
        res = ptypes.dyn.block(cb.int())(offset=T.d.getoffset())
        return Extract(res.li, output)
    return Extract(result, output or 'print')

def list_exports(t, output, F=None):
    E = t['Next']['Header']['DataDirectory']['Export']
    if E['Address'].int() == 0:
        raise ValueError("No Exports directory entry was found.")<commit_msg>Modified the ability to extract a data directory entry in tools/pe.py so that it dereferences the entry instead of just emitting it.<commit_after>#!/usr/bin/env python
import six,argparse
import functools,operator,itertools,types
import ptypes,pecoff,ber
#ptypes.setbyteorder(ptypes.config.byteorder.littleendian)
ptypes.config.defaults.ptype.clone_name = '{}'
ptypes.config.defaults.pint.bigendian_name = 'be({})'
ptypes.config.defaults.pint.littleendian_name = '{}'

def Locate(path, obj):
    return Resolve(obj, path.split(':'))

def Extract(obj, out):
    if out == 'print':
        res = obj.repr()
    elif out == 'hex':
        res = obj.hexdump()
    elif out == 'raw':
        res = obj.serialize()
    elif out == 'list':
        res = '\n'.join((n.summary() if hasattr(n, 'summary') else str(n) for n in obj))

    # default
    elif not obj:
        res = obj.details()
    elif isinstance(obj, (ptypes.ptype.pointer_t, ptypes.pbinary.partial)):
        res = repr(obj)
    elif hasattr(obj, 'num'):
        res = obj.num()
    elif hasattr(obj, 'int'):
        res = obj.int()
    elif hasattr(obj, 'str'):
        res = obj.str()
    else:
        res = repr(obj)
    six.print_(res, file=sys.stdout)

def Resolve(result, path):
    for p in path:
        if p == '':
            break
        elif p == '.':
            pass
        elif p == '..':
            result = result.p
        elif p == '!' and hasattr(result, 'd'):
            result = result.d.li
        elif p == '@' and hasattr(result, 'o'):
            result = result.o.li
        elif hasattr(result, '__field__'):
            result = result.__field__(p) if hasattr(result, '__keys__') else result.__field__(int(p))
        else:
            raise AttributeError(p, result)
        continue
    return result

## commands
def dump_exe(t, output, F=None):
    global result; result = t
    if F:
        return Extract(F(result), output)
    if not output or output in {'print'}:
        six.print_(result['Dos'].repr(), file=sys.stdout)
        six.print_(result['Stub'].hexdump(), file=sys.stdout)
        return
    if output in {'hex','raw'}:
        t = ptypes.dyn.block(result['Dos'].size() + result['Stub'].size())
        return Extract(t(offset=0).li, output)
    return Extract(result, output)

def dump_header(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H
    if F:
        return Extract(F(result), output)
    if not output:
        six.print_(result.p['Signature'], repr(result.p['Signature'].serialize()+result['SignaturePadding'].serialize()), file=sys.stdout)
        six.print_(result['FileHeader'].repr(), file=sys.stdout)
        six.print_(result['OptionalHeader'].repr(), file=sys.stdout)
        return
    return Extract(result, output)

def list_sections(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H['Sections']
    if F:
        return Extract(F(result), output)
    if not output:
        summary = lambda s: '{!r} {:#x}:{:+#x} Raw:{:#x}:{:+#x}'.format(s['Name'].str(), s['VirtualAddress'].int(), s['VirtualSize'].int(), s['PointerToRawData'].int(), s['SizeOfRawData'].int())
        for i, S in enumerate(result):
            six.print_('[{:d}] {:s}'.format(i, summary(S)), file=sys.stdout)
        return
    if output in {'list'}:
        return Extract((S['Name'].str() for S in result), output)
    return Extract(result, output)

def extract_section(t, index, output, F=None):
    H = t['Next']['Header']
    if not (0 <= index < len(H['Sections'])):
        raise IndexError("Invalid section index was specified ({:d} <= {:d} < {:d}).".format(0, index, len(H['Sections'])))
    S = H['Sections'][index]
    global result; result = S
    #if 'load' in kwds:
    #    return Extract(t['VirtualAddress'].d.li, output or 'raw')
    if F:
        return Extract(F(result), output)
    return Extract(result['PointerToRawData'].d.li, output or 'hex')

def list_entries(t, output, F=None):
    H = t['Next']['Header']
    global result; result = H['DataDirectory']
    if F:
        return Extract(F(result), output)
    summary = lambda n: '{:s} {:#x}:{:+#x}'.format(n.classname(), n['Address'].int(), n['Size'].int())
    if not output:
        for i, n in enumerate(result):
            six.print_('[{:d}] {}'.format(i, summary(n)), file=sys.stdout)
        return
    if output in {'list'}:
        return Extract((n.classname() for n in result if n['Address'].int()), output)
    return Extract(result, output)

def extract_entry(t, index, output, F=None):
    d = t['Next']['Header']['DataDirectory']
    if not(0 <= index < len(d)):
        raise IndexError("Invalid DataDirectory entry number was specified ({:d} <= {:d} < {:d}).".format(0, index, len(d)))
    E = d[index]
    global result; result = E['Address'].d.li
    T, cb = E['Address'], E['Size']
    if F:
        return Extract(F(result), output)
    if output in {'hex', 'raw'}:
        res = ptypes.dyn.block(cb.int())(offset=T.d.getoffset())
        return Extract(res.li, output)
    return Extract(result, output or 'print')

def list_exports(t, output, F=None):
    E = t['Next']['Header']['DataDirectory']['Export']
    if E['Address'].int() == 0:
        raise ValueError("No Exports directory entry was found.")
{'fraction_non_alphanumeric': 0.1793786669231509, 'fraction_numerical': 0.0011541790901221506, 'mean_word_length': 2.979334098737084, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2582006', 'n_tokens_mistral': 3418, 'n_tokens_neox': 3360, 'n_words': 858}