<commit_before># -*- coding: utf-8 -*-

import collections
import json
import logging
import signal
import subprocess
import time

import ptrace.debugger as pdbg
import ptrace.signames

crash_signals = (signal.SIGILL, signal.SIGABRT, signal.SIGFPE, signal.SIGBUS, signal.SIGSEGV, signal.SIGSYS)


def get_pids(name):
    pgrep = ("pgrep", name)
    pid = subprocess.Popen(pgrep, stdout=subprocess.PIPE).communicate()[0]
    return list(map(int, pid.split()))


def get_pid_command(pid):
    proc_path = "/proc/%s/cmdline" % pid
    try:
        with open(proc_path, "r") as f:
            cmdline = f.read().replace("\x00", " ").strip()
    except IOError:
        cmdline = None
    return cmdline


class PtraceDbg(pdbg.Application):

    def __init__(self, options):
        self.options = options
        self.program = self.options.program
        self.processes = []
        self.processOptions()
        self.debugger = pdbg.debugger.PtraceDebugger()
        self.setupDebugger()
        self.is_running = False
        super(PtraceDbg, self).__init__()

    def spawn_traced_process(self):
        try:
            process = self.createProcess()
        except pdbg.child.ChildError as ce:
            raise IOError("Failed to create traced process: %s => %s" % (" ".join(self.program), ce))
        logging.info("Successfully attached to process: %d" % process.pid)
        process.cont()
        logging.info("Moving process to running state: %d" % process.pid)
        self.processes.append(process)
        return process

    def stop(self):
        for process in self.processes:
            process.detach()
            self.processes.remove(process)
            logging.warn("Detached from process: %d" % process.pid)
        self.is_running = False

    def watch(self, on_signal, on_event, on_exit):
        # Spawning of tracee MUST be done in same thread as event waitProcessEvent() on Linux
        self.spawn_traced_process()
        self.is_running = True
        logging.info("Debugger entered event monitoring loop")
        while self.is_running and self.processes != []:
            event = self.debugger.waitProcessEvent()
            process = event.process
            logging.info("Caught event on process: %d => \"%s\". Dispatching to callback" % (process.pid, event))
            if event.__class__ == pdbg.ProcessSignal:
                on_signal(event)
            elif event.__class__ == pdbg.ProcessEvent:
                on_event(event)
            elif event.__class__ == pdbg.ProcessExit:
                on_exit(event)
            else:
                raise RuntimeError("Unexpected process event: %s" % event)
            if not process.is_attached:
                try:
                    self.processes.remove(process)
                    logging.info("Detected process as dead: %d" % process.pid)
                except ValueError:
                    pass
        logging.info("Debugger exiting event monitoring loop")
        self.is_running = False


class CrashReport(object):
    def __init__(self, sessid, pid, signum, stream_id):
        self.pid = pid
        self.sessid = sessid
        self.signal = ptrace.signames.signalName(signum)
        self.stream_id = stream_id
        self.time = time.time()
        self.registers = {}
        self.stack = collections.OrderedDict()
        self.backtrace = collections.OrderedDict()
        self.disassembly = collections.OrderedDict()
        self.maps = []
        self.stream = []
        self.history = []

    def to_json(self, f):
        json.dump({"session_id": self.sessid,
                   "stream_count": self.stream_id,
                   "pid": self.pid,
                   "signal": self.signal,
                   "time": self.time,
                   "registers": self.registers,
                   "backtrace": self.backtrace,
                   "disassembly": self.disassembly,
                   "maps": self.maps,
                   "stack": self.stack,
                   "stream": self.stream,
                   "history": self.history},
                  f,
                  indent=4)

    def dump_regs(self, str_):
        try:
            reg, val = list(map(lambda x: x.strip(), str_.split("=")))
            self.registers[reg] = val
        except ValueError:
            pass

    def dump_maps(self, str_):
        pass

    def dump_stack(self, str_):
        try:
            addr, val = list(map(lambda x: x.strip(), str_.split(":")))
            self.stack[addr] = val
        except ValueError:
            pass

    def dump_backtrace(self, frame):
        self.backtrace[hex(frame.ip)] = (frame.name, frame.arguments)

    def dump_code(self, str_):
        self.disassembly[hex(str_.address)] = str_.text<commit_msg>Added memory mappings parsing support<commit_after># -*- coding: utf-8 -*-

import collections
import json
import logging
import re
import signal
import subprocess
import time

import ptrace.debugger as pdbg
import ptrace.signames

crash_signals = (signal.SIGILL, signal.SIGABRT, signal.SIGFPE, signal.SIGBUS, signal.SIGSEGV, signal.SIGSYS)


def get_pids(name):
    pgrep = ("pgrep", name)
    pid = subprocess.Popen(pgrep, stdout=subprocess.PIPE).communicate()[0]
    return list(map(int, pid.split()))


def get_pid_command(pid):
    proc_path = "/proc/%s/cmdline" % pid
    try:
        with open(proc_path, "r") as f:
            cmdline = f.read().replace("\x00", " ").strip()
    except IOError:
        cmdline = None
    return cmdline


class PtraceDbg(pdbg.Application):

    def __init__(self, options):
        self.options = options
        self.program = self.options.program
        self.processes = []
        self.processOptions()
        self.debugger = pdbg.debugger.PtraceDebugger()
        self.setupDebugger()
        self.is_running = False
        super(PtraceDbg, self).__init__()

    def spawn_traced_process(self):
        try:
            process = self.createProcess()
        except pdbg.child.ChildError as ce:
            raise IOError("Failed to create traced process: %s => %s" % (" ".join(self.program), ce))
        logging.info("Successfully attached to process: %d" % process.pid)
        process.cont()
        logging.info("Moving process to running state: %d" % process.pid)
        self.processes.append(process)
        return process

    def stop(self):
        for process in self.processes:
            process.detach()
            self.processes.remove(process)
            logging.warn("Detached from process: %d" % process.pid)
        self.is_running = False

    def watch(self, on_signal, on_event, on_exit):
        # Spawning of tracee MUST be done in same thread as event waitProcessEvent() on Linux
        self.spawn_traced_process()
        self.is_running = True
        logging.info("Debugger entered event monitoring loop")
        while self.is_running and self.processes != []:
            event = self.debugger.waitProcessEvent()
            process = event.process
            logging.info("Caught event on process: %d => \"%s\". Dispatching to callback" % (process.pid, event))
            if event.__class__ == pdbg.ProcessSignal:
                on_signal(event)
            elif event.__class__ == pdbg.ProcessEvent:
                on_event(event)
            elif event.__class__ == pdbg.ProcessExit:
                on_exit(event)
            else:
                raise RuntimeError("Unexpected process event: %s" % event)
            if not process.is_attached:
                try:
                    self.processes.remove(process)
                    logging.info("Detected process as dead: %d" % process.pid)
                except ValueError:
                    pass
        logging.info("Debugger exiting event monitoring loop")
        self.is_running = False


class CrashReport(object):

    # Matches
    # 'MAPS: 0x00007fb7b25ae000-0x00007fb7b2730000 => /lib/x86_64-linux-gnu/libc-2.13.so (r-xp)'
    # 'MAPS: 0x0000000000df5000-0x0000000000e16000 => [heap] (rwxp)'
    # 'MAPS: 0x00007fb7b2b56000-0x00007fb7b2b59000 (rwxp)'
    # Into start/stop address, binary, permissions
    MAPS_REGEXP = "MAPS:\s(0x[0-9a-fA-F]+)-(0x[0-9a-fA-F]+)\s(?:=>\s)?(.*?)\s?\((.+)\)"

    def __init__(self, sessid, pid, signum, stream_id):
        self.pid = pid
        self.sessid = sessid
        self.signal = ptrace.signames.signalName(signum)
        self.stream_id = stream_id
        self.time = time.time()
        self.registers = {}
        self.stack = collections.OrderedDict()
        self.backtrace = collections.OrderedDict()
        self.disassembly = collections.OrderedDict()
        self.maps = []
        self.stream = []
        self.history = []
        self.regexp = re.compile(CrashReport.MAPS_REGEXP)

    def to_json(self, f):
        json.dump({"session_id": self.sessid,
                   "stream_count": self.stream_id,
                   "pid": self.pid,
                   "signal": self.signal,
                   "time": self.time,
                   "registers": self.registers,
                   "backtrace": self.backtrace,
                   "disassembly": self.disassembly,
                   "maps": self.maps,
                   "stack": self.stack,
                   "stream": self.stream,
                   "history": self.history},
                  f,
                  indent=4)

    def dump_regs(self, str_):
        try:
            reg, val = list(map(lambda x: x.strip(), str_.split("=")))
            self.registers[reg] = val
        except ValueError:
            pass

    def dump_maps(self, str_):
        try:
            start_addr, stop_addr, binary, perms = self.regexp.match(str_).groups()
            self.maps.append(((start_addr, stop_addr), binary, perms))
        except ValueError:
            pass

    def dump_stack(self, str_):
        try:
            addr, val = list(map(lambda x: x.strip(), str_.split(":")))
            self.stack[addr] = val
        except ValueError:
            pass

    def dump_backtrace(self, frame):
        self.backtrace[hex(frame.ip)] = (frame.name, frame.arguments)

    def dump_code(self, str_):
        self.disassembly[hex(str_.address)] = str_.text
{'fraction_non_alphanumeric': 0.11544874938695439, 'fraction_numerical': 0.01039725355566454, 'mean_word_length': 2.025519287833828, 'pattern_counts': {'":': 26, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6161607', 'n_tokens_mistral': 3097, 'n_tokens_neox': 2957, 'n_words': 670}