<commit_before>import xml.sax as sax
import re

class GangliaContentHandler(sax.ContentHandler):
    """
    Base Ganglia content handler. Does not do anything with the data, but it will
    execute logic to filter out data that doesn't match ``grid_expr``, 
    ``cluster_expr``, or ``host_expr``. 
    """
    def __init__(self, grid_expr=".*", cluster_expr=".*", host_expr=".*"):

        self.exprs = {
            'grid': re.compile(grid_expr),
            'cluster': re.compile(cluster_expr),
            'host': re.compile(host_expr),
        }

        self.current = {
            'grid': None, 'cluster': None, 'host': None 
        }

        self.skip = {}

        sax.ContentHandler.__init__(self)

    def _eval_re(self, type, text):
        pat = self.exprs.get(type, None)

        if pat is None:
            raise ValueError("Invalid regular expression type '%s" % type)

        return bool(pat.search(text))

    def _check_skip(self, type):
        skip = self.skip.get(type, None)
        current = self.current.get(type, None)

        if skip is None or current is None:
            ret = False
        else:
            ret = skip == current

        return ret

    def _set_skip(self, type, value):
        if not self._eval_re(type, value):
            self.skip[type] = value
            return True
        return False

    def _handle_grid(self, attrs):
        name = attrs['NAME']
        self.current['grid'] = name

        # if grid doesn't match regexp, set skip
        if self._set_skip('grid', name): return

        self.handle_grid(attrs)

    def handle_grid(self, attrs):
        self.data.setdefault(attrs['NAME'], {})

    def _handle_cluster(self, attrs):
        name = attrs['NAME']

        self.current['cluster'] = name

        # if cluster doesn't match regexp, set skip
        if self._set_skip('cluster', name): return

        # skip node if GRID set to skip
        if self._check_skip('grid'): return

        self.handle_cluster(attrs)

    def handle_cluster(self, attrs):
        self.data[self.current['grid']].setdefault(attrs['NAME'], {})

    def _handle_host(self, attrs):
        name = attrs['NAME']
        self.current['host'] = name

        # if host doesn't match regexp, set skip
        if self._set_skip('host', name): return

        # skip node if GRID or CLUSTER set to skip
        if self._check_skip('grid'): return 
        if self._check_skip('cluster'): return

        self.handle_host(attrs)

    def handle_host(self, attrs):
        self.data[self.current['grid']][self.current['cluster']].setdefault(attrs['NAME'], {})

    def _handle_metric(self, attrs):
        name = attrs['NAME']

        # skip node if GRID, CLUSTER, or HOST set to skip
        if self._check_skip('grid'): return 
        if self._check_skip('cluster'): return
        if self._check_skip('host'): return

        self.handle_metric(attrs)

    def handle_metric(self, attrs):

        metric_data = { 
            'name': attrs['VAL'], 
            'units': attrs['UNITS'], 
            'type': attrs['TYPE']
        } 

        self.data[self.current['grid']][self.current['cluster']]\
                 [self.current['host']][name] = metric_data

    def _handle_grid_end(self):
        self.current['grid'] = None
        self.skip['grid'] = None
        self.handle_grid_end()

    def handle_grid_end(self):
        pass

    def _handle_cluster_end(self):
        self.current['cluster'] = None
        self.skip['cluster'] = None
        self.handle_cluster_end()

    def handle_cluster_end(self):
        pass

    def _handle_host_end(self):
        self.current['host'] = None
        self.skip['host'] = None
        self.handle_host_end()

    def handle_host_end(self):
        pass<commit_msg>Removed 'loading' handlers that were extracted to LodingGangliaContentHandler<commit_after>import xml.sax as sax
import re

class GangliaContentHandler(sax.ContentHandler):
    """
    Base Ganglia content handler. Does not do anything with the data, but it will
    execute logic to filter out data that doesn't match ``grid_expr``, 
    ``cluster_expr``, or ``host_expr``. 
    """
    def __init__(self, grid_expr=".*", cluster_expr=".*", host_expr=".*"):

        self.exprs = {
            'grid': re.compile(grid_expr),
            'cluster': re.compile(cluster_expr),
            'host': re.compile(host_expr),
        }

        self.current = {
            'grid': None, 'cluster': None, 'host': None 
        }

        self.skip = {}

        sax.ContentHandler.__init__(self)

    def _eval_re(self, type, text):
        pat = self.exprs.get(type, None)

        if pat is None:
            raise ValueError("Invalid regular expression type '%s" % type)

        return bool(pat.search(text))

    def _check_skip(self, type):
        skip = self.skip.get(type, None)
        current = self.current.get(type, None)

        if skip is None or current is None:
            ret = False
        else:
            ret = skip == current

        return ret

    def _set_skip(self, type, value):
        if not self._eval_re(type, value):
            self.skip[type] = value
            return True
        return False

    def _handle_grid(self, attrs):
        name = attrs['NAME']
        self.current['grid'] = name

        # if grid doesn't match regexp, set skip
        if self._set_skip('grid', name): return

        self.handle_grid(attrs)

    def handle_grid(self, attrs):
        pass

    def _handle_cluster(self, attrs):
        name = attrs['NAME']

        self.current['cluster'] = name

        # if cluster doesn't match regexp, set skip
        if self._set_skip('cluster', name): return

        # skip node if GRID set to skip
        if self._check_skip('grid'): return

        self.handle_cluster(attrs)

    def handle_cluster(self, attrs):
        pass

    def _handle_host(self, attrs):
        name = attrs['NAME']
        self.current['host'] = name

        # if host doesn't match regexp, set skip
        if self._set_skip('host', name): return

        # skip node if GRID or CLUSTER set to skip
        if self._check_skip('grid'): return 
        if self._check_skip('cluster'): return

        self.handle_host(attrs)

    def handle_host(self, attrs):
        pass

    def _handle_metric(self, attrs):
        name = attrs['NAME']

        # skip node if GRID, CLUSTER, or HOST set to skip
        if self._check_skip('grid'): return 
        if self._check_skip('cluster'): return
        if self._check_skip('host'): return

        self.handle_metric(attrs)

    def handle_metric(self, attrs):
        pass

    def _handle_grid_end(self):
        self.current['grid'] = None
        self.skip['grid'] = None
        self.handle_grid_end()

    def handle_grid_end(self):
        pass

    def _handle_cluster_end(self):
        self.current['cluster'] = None
        self.skip['cluster'] = None
        self.handle_cluster_end()

    def handle_cluster_end(self):
        pass

    def _handle_host_end(self):
        self.current['host'] = None
        self.skip['host'] = None
        self.handle_host_end()

    def handle_host_end(self):
        pass
{'fraction_non_alphanumeric': 0.11992209237618252, 'fraction_numerical': 0.0, 'mean_word_length': 2.2223218287763333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2683550', 'n_tokens_mistral': 2169, 'n_tokens_neox': 2099, 'n_words': 551}