<commit_before>''' provides polling method/class to monitor gdb output '''
import os
import time
import re
import logging
import pprint

POLL_INTERVAL = 0.05

class LogHandler(object):
    ''' monitors gdb output (from "set logging on") and outputs 2 files for location/breakpoints
        
        one could use the watchdog module but the latency between a modified
        event and the "on_modified" call is too long to be of use
    '''

    FNAME_GDB = 'gdb.txt'
    FNAME_LOC = '.debug_location'
    FNAME_BRK = '.debug_breakpoint'

    def __init__(self, debug):

        logging.basicConfig(filename="gdb_monitor.log")
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO)
        self.logger.debug('in __init__')

        self.active_fname = ""
        self.info_loc = [None, None]
        self.info_brk = []
        self.last_mod = -1

        if os.path.isfile(self.FNAME_GDB):
            self.on_modified()
        self.logger.debug('done with init')

    def _add_breakpoints(self, bp_num, ln_num, fname):

        found = False

        for i in range(len(self.info_brk)):

            if self.info_brk[i][0] == fname:

                if not ln_num in self.info_brk[i][1]:

                    self.info_brk[i][1].append(ln_num)
                    self.info_brk[i][2].append(bp_num)

                found = True
                break

        if not found:
            self.info_brk.append([fname, [ln_num], [bp_num]])

    def _delete_breakpoints(self, ln):

        # convert comma separated string to list
        bp_nums = []
        old_pos = len("Deleted breakpoint ")

        while True:

            pos_comma = ln.find(",", old_pos+1)
            pos_space = ln.find(" ", old_pos+1)

            if pos_space == -1 and pos_comma == -1:
                break
            elif pos_space >= 0 and pos_comma >= 0:
                pos = min(pos_comma, pos_space)
            elif pos_space == -1:
                pos = pos_comma
            elif pos_comma == -1:
                pos = pos_space

            bp_nums.append(int(ln[old_pos:pos]))
            old_pos = pos

        # delete breakpoints
        for num in bp_nums:

            for i in range(len(self.info_brk)):
                try:
                    pos = self.info_brk[i][2].index(num)
                    if len(self.info_brk[i][2]) == 1:
                        del self.info_brk[i]
                    else:
                        del self.info_brk[i][2][pos]
                        del self.info_brk[i][1][pos]
                    break
                except ValueError:
                    pass

    def on_modified(self):
        ''' parses the output of gdb and if there changes, update FNAME_BRK and FNAME_LOC '''

        if not os.path.isfile(self.FNAME_GDB) \
           or abs(self.last_mod - os.path.getmtime(self.FNAME_GDB)) < POLL_INTERVAL/10.0:
            return

        try:
            # read the file then clear it
            with open(self.FNAME_GDB, 'r') as f:
                lines = f.read().splitlines()

            self.logger.debug('processing file: {}'.format(self.FNAME_GDB))
            self.logger.debug('lines = {}'.format(pprint.pformat(lines)))

            if not lines:
                self.logger.debug('exiting'.format(pprint.pformat(lines)))
                return

            with open(self.FNAME_GDB, 'w') as f:
                pass

        except FileNotFoundError:
            return

        self.last_mod = os.path.getmtime(self.FNAME_GDB)
        info_loc_updated = False
        info_brk_updated = False

        for ln in lines:

            self.logger.debug('processing line: {}'.format(ln))

            if ln == '' or ln[0] == '$':

                self.logger.debug('empty/printed variable, doing nothing')
                continue

            elif ln[0:len("delete all breakpoints")] == "Delete all breakpoints":

                self.logger.debug('clearing info_brk')
                info_brk_updated = True
                self.info_brk = []

            elif ln[0:len("deleted")] == "Deleted":

                self._delete_breakpoints(ln)
                info_brk_updated = True
                self.logger.debug('info_brk = {}'.format(pprint.pformat(self.info_brk)))
                self.logger.debug('calling _delete_breakpoints({},{}).format(ln, info_brk)')
                self.logger.debug('info_brk = {}'.format(pprint.pformat(self.info_brk)))<commit_msg>handle case of non-ascii characters in gdb output<commit_after>''' provides polling method/class to monitor gdb output '''
import os
import time
import re
import logging
import pprint
import subprocess

POLL_INTERVAL = 0.05

class LogHandler(object):
    ''' monitors gdb output (from "set logging on") and outputs 2 files for location/breakpoints

        one could use the watchdog module but the latency between a modified
        event and the "on_modified" call is too long to be of use
    '''

    FNAME_GDB = 'gdb.txt'
    FNAME_LOC = '.debug_location'
    FNAME_BRK = '.debug_breakpoint'

    def __init__(self, debug):

        logging.basicConfig(filename="gdb_monitor.log")
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO)
        self.logger.debug('in __init__')

        self.active_fname = ""
        self.info_loc = [None, None]
        self.info_brk = []
        self.last_mod = -1

        if os.path.isfile(self.FNAME_GDB):
            self.on_modified()
        self.logger.debug('done with init')

    def _add_breakpoints(self, bp_num, ln_num, fname):

        found = False

        for i in range(len(self.info_brk)):

            if self.info_brk[i][0] == fname:

                if not ln_num in self.info_brk[i][1]:

                    self.info_brk[i][1].append(ln_num)
                    self.info_brk[i][2].append(bp_num)

                found = True
                break

        if not found:
            self.info_brk.append([fname, [ln_num], [bp_num]])

    def _delete_breakpoints(self, ln):

        # convert comma separated string to list
        bp_nums = []
        old_pos = len("Deleted breakpoint ")

        while True:

            pos_comma = ln.find(",", old_pos+1)
            pos_space = ln.find(" ", old_pos+1)

            if pos_space == -1 and pos_comma == -1:
                break
            elif pos_space >= 0 and pos_comma >= 0:
                pos = min(pos_comma, pos_space)
            elif pos_space == -1:
                pos = pos_comma
            elif pos_comma == -1:
                pos = pos_space

            bp_nums.append(int(ln[old_pos:pos]))
            old_pos = pos

        # delete breakpoints
        for num in bp_nums:

            for i in range(len(self.info_brk)):
                try:
                    pos = self.info_brk[i][2].index(num)
                    if len(self.info_brk[i][2]) == 1:
                        del self.info_brk[i]
                    else:
                        del self.info_brk[i][2][pos]
                        del self.info_brk[i][1][pos]
                    break
                except ValueError:
                    pass

    def on_modified(self):
        ''' parses the output of gdb and if there changes, update FNAME_BRK and FNAME_LOC '''

        if not os.path.isfile(self.FNAME_GDB) \
           or abs(self.last_mod - os.path.getmtime(self.FNAME_GDB)) < POLL_INTERVAL/10.0:
            return

        try:
            # read the file then clear it
            with open(self.FNAME_GDB, 'r') as f:
                lines = f.read().splitlines()
        except FileNotFoundError:
            return
        except UnicodeError:
            subprocess.call("tr -cd '\11\12\15\40-\176' <{} > {}" \
                            .format(self.FNAME_GDB, self.FNAME_GDB),
                            shell=True)
            with open(self.FNAME_GDB, 'r') as f:
                lines = f.read().splitlines()

        self.logger.debug('processing file: {}'.format(self.FNAME_GDB))
        self.logger.debug('lines = {}'.format(pprint.pformat(lines)))

        if not lines:
            self.logger.debug('exiting'.format(pprint.pformat(lines)))
            return

        with open(self.FNAME_GDB, 'w') as f:
            pass

        self.last_mod = os.path.getmtime(self.FNAME_GDB)
        info_loc_updated = False
        info_brk_updated = False

        for ln in lines:

            self.logger.debug('processing line: {}'.format(ln))

            if ln == '' or ln[0] == '$':

                self.logger.debug('empty/printed variable, doing nothing')
                continue

            elif ln[0:len("delete all breakpoints")] == "Delete all breakpoints":

                self.logger.debug('clearing info_brk')
                info_brk_updated = True
                self.info_brk = []

            elif ln[0:len("deleted")] == "Deleted":

                self._delete_breakpoints(ln)
                info_brk_updated = True
                self.logger.debug('info_brk = {}'.format(pprint.pformat(self.info_brk)))
                self.logger.debug('calling _delete_breakpoints({},{}).format(ln, info_brk)')
                self.logger.debug('info_brk = {}'.format(pprint.pformat(self.info_brk)))
{'fraction_non_alphanumeric': 0.110907917383821, 'fraction_numerical': 0.007207401032702237, 'mean_word_length': 1.7081270026216138, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599660', 'n_tokens_mistral': 2955, 'n_tokens_neox': 2766, 'n_words': 625}