<commit_before>import ctypes, struct, re
from collections import OrderedDict, namedtuple

class Registers(OrderedDict):
    __register_names = ['rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'eflags']

    def __init__(self, *args):
        super(Registers, self).__init__(zip(self.__register_names, args))
        self.__initialized = True

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        if name in self.__register_names:
            if not isinstance(value, int):
                raise ValueError(type(value))
            else:
                self[name] = value
        super(Registers, self).__setattr__(name, value)

    def __str__(self):
        s='('
        for k,v in self.items():
            s += k+'='+hex(v)+', '
        return s[:-2]+')'
    __repr__ = __str__

    def values(self):
        return list(super(Registers, self).values())

class Memory:
    def __init__(self):
        Mapping = namedtuple('Mapping', ['start', 'end', 'perms', 'offset', 'dev', 'inode', 'pathname'])
        self.mappings = []
        with open('/proc/self/maps') as f:
            for l in f.readlines():
                data = re.split('\s+', l)
                #print(data)
                data += [''] * (6-len(data))
                addr = data[0].split('-')
                self.mappings.append(Mapping(
                    int(addr[0], 16),
                    int(addr[1], 16),
                    data[1],
                    int(data[2], 16),
                    data[3],
                    int(data[4]),
                    data[5]))

    def __getitem__(self, idx):
        if isinstance(idx, slice):
            if idx.step:
                raise IndexError('step unsupported')
            return ctypes.string_at(idx.start, idx.stop-idx.start)
        else:
            return ctypes.string_at(idx, 1)
    def __setitem__(self, idx, bytes):
        if isinstance(idx, slice):
            if idx.step:
                raise IndexError('step unsupported')
            if idx.stop-idx.start != len(bytes):
                raise ValueError('slice length not equal value')
            start = idx.start
        else:
            start = idx
            if len(bytes) != 1:
                raise ValueError('got more than one byte')
        cbytes = ctypes.create_string_buffer(bytes)
        ctypes.memmove(start, ctypes.addressof(cbytes), len(bytes))


libc = ctypes.cdll.LoadLibrary('libc.so.6')
memory = Memory()

def get_long(addr):
    return struct.unpack('<Q', ctypes.string_at(addr, 8))[0]

def set_long(addr, value):
    new_address_raw_bytes = struct.pack('<Q', value)
    new_address_buffer = ctypes.create_string_buffer(new_address_raw_bytes)
    ctypes.memmove(addr, ctypes.addressof(new_address_buffer), 8)

def new_buffer(bytes):
    string_buffer = ctypes.create_string_buffer(bytes)
    addr = libc.malloc(len(bytes)+1)
    ctypes.memmove(addr, ctypes.addressof(string_buffer), len(bytes)+1)
    return addr

def free(addr):
    return libc.free(addr)<commit_msg>Moved, renamed and extended memory manipulation functions<commit_after>import ctypes, struct, re
from collections import OrderedDict, namedtuple

class Registers(OrderedDict):
    __register_names = ['rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15', 'eflags']

    def __init__(self, *args):
        super(Registers, self).__init__(zip(self.__register_names, args))
        self.__initialized = True

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        if name in self.__register_names:
            if not isinstance(value, int):
                raise ValueError(type(value))
            else:
                self[name] = value
        super(Registers, self).__setattr__(name, value)

    def __str__(self):
        s='('
        for k,v in self.items():
            s += k+'='+hex(v)+', '
        return s[:-2]+')'
    __repr__ = __str__

    def values(self):
        return list(super(Registers, self).values())

class Memory:
    def __init__(self):
        Mapping = namedtuple('Mapping', ['start', 'end', 'perms', 'offset', 'dev', 'inode', 'pathname'])
        self.mappings = []
        with open('/proc/self/maps') as f:
            for l in f.readlines():
                data = re.split('\s+', l)
                #print(data)
                data += [''] * (6-len(data))
                addr = data[0].split('-')
                self.mappings.append(Mapping(
                    int(addr[0], 16),
                    int(addr[1], 16),
                    data[1],
                    int(data[2], 16),
                    data[3],
                    int(data[4]),
                    data[5]))

    def __getitem__(self, idx):
        if isinstance(idx, slice):
            if idx.step:
                raise IndexError('step unsupported')
            return ctypes.string_at(idx.start, idx.stop-idx.start)
        else:
            return ctypes.string_at(idx, 1)
    def __setitem__(self, idx, bytes):
        if isinstance(idx, slice):
            if idx.step:
                raise IndexError('step unsupported')
            if idx.stop-idx.start != len(bytes):
                raise ValueError('slice length not equal value')
            start = idx.start
        else:
            start = idx
            if len(bytes) != 1:
                raise ValueError('got more than one byte')
        cbytes = ctypes.create_string_buffer(bytes)
        ctypes.memmove(start, ctypes.addressof(cbytes), len(bytes))

    def get(self, addr, fmt):
        size = struct.calcsize(fmt)
        return struct.unpack(fmt, ctypes.string_at(addr, size))

    def get_single(self, addr, fmt):
        return self.get(addr, fmt)[0]

    def set(self, addr, values, fmt):
        value_raw = struct.pack(fmt, *values)
        value_buffer = ctypes.create_string_buffer(value_raw)
        ctypes.memmove(addr, ctypes.addressof(value_buffer), len(value_raw))

    def set_single(self, addr, value, fmt):
        self.set(addr, (value,), fmt)

    def set_ptr(self, addr, value):
        self.set_single(addr, value, '@P')

    def get_ptr(self, addr):
        return self.get_single(addr, '@P')

    def new_buffer(self, bytes, fmt=None):
        if fmt:
            bytes = struct.pack(fmt, *bytes)
        string_buffer = ctypes.create_string_buffer(bytes)
        addr = libc.malloc(len(bytes)+1)
        ctypes.memmove(addr, ctypes.addressof(string_buffer), len(bytes)+1)
        return addr

    def free(self, addr):
        return libc.free(addr)


libc = ctypes.cdll.LoadLibrary('libc.so.6')
memory = Memory()


{'fraction_non_alphanumeric': 0.1373713954499348, 'fraction_numerical': 0.01072308361107086, 'mean_word_length': 2.0936799641416406, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3990294', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2093, 'n_words': 487}