<commit_before>#!/usr/bin/env python

import logging
from sys_helper import is_num

from tabula.table import Table
from tabula.section import Section

from libcarbon.carbon_key import CarbonKey
from bls_helper import BLSHelper
from server import Server
from mc_source import MemcachedSource
from mc_collector import MemcachedCollector
from carbon_handler import CarbonHandler
from json_handler import JsonHandler
from data_helper import DataHelper

class VisitorCallback(object):
    """
    Callbacks for metadata visitor
    """
    def __init__(self, c_feeder, tbl):
        self.c_feeder = c_feeder
        self.tbl = tbl
        self.cur_row = {}      # {sec_nam: row name}

    def _show_stats(self, key, val, meta_inf):
        """
        Show stats on the ascii table
        """
        if not self.tbl or not isinstance(self.tbl, Table):
            return False

        if not meta_inf or not "section" in meta_inf:
            logging.error(
                "unable to show slow data: key %s, val %s, invalid meta info"
                % (key, val))
            return False

        # ok, not deal with unicode for now
        sec_nam = str(meta_inf["section"])
        val = str(val)

        section = self.tbl.get_section(sec_nam)

        if not section:
            section = Section(sec_nam)
            self.tbl.add_section(section)

        if "new_row" in meta_inf:
            # create a new row using the name
            section.add_cell(val, sec_nam, val, "S50")
            self.cur_row[sec_nam] = val
            return True

        if not sec_nam in self.cur_row:
            logging.debug("stats %s is not associated with a section" % key)
            return True

        if "col" in meta_inf:
            col = str(meta_inf["col"])
        else:
            col = str(key)

        row = self.cur_row[sec_nam]
        section.add_cell(row, col, val, type="S50")

        return True

    def store_fast(self, root, parents, data, meta, coll,
                   key, val, meta_val, meta_inf, level):
        """Store time-series data into fast-changing database"""
        if not self.c_feeder:
            logging.error(
                "unable to store fast changing data : invalid CarbonFeeder")
            return False<commit_msg>added section display configurations<commit_after>#!/usr/bin/env python

import logging
from sys_helper import is_num

from tabula.table import Table
from tabula.section import Section

from libcarbon.carbon_key import CarbonKey
from bls_helper import BLSHelper
from server import Server
from mc_source import MemcachedSource
from mc_collector import MemcachedCollector
from carbon_handler import CarbonHandler
from json_handler import JsonHandler
from data_helper import DataHelper

# configuration for each tabula.section
SECTION_CONFIG = {"settings": {"show_row_hdrs": False,
                               "show_col_hdrs": False,
                               "show_col_hdr_in_cell": True},
                  "nodes": {"show_row_hdrs": True,
                            "show_col_hdrs": True,
                            "show_col_hdr_in_cell": False}}

class VisitorCallback(object):
    """
    Callbacks for metadata visitor
    """
    def __init__(self, c_feeder, tbl):
        self.c_feeder = c_feeder
        self.tbl = tbl
        self.cur_row = {}      # {sec_nam: row name}

    def _show_stats(self, key, val, meta_inf):
        """
        Show stats on the ascii table
        """
        if not self.tbl or not isinstance(self.tbl, Table):
            return False

        if not meta_inf or not "section" in meta_inf:
            logging.error(
                "unable to show slow data: key %s, val %s, invalid meta info"
                % (key, val))
            return False

        # ok, not deal with unicode for now
        sec_nam = str(meta_inf["section"])
        val = str(val)

        section = self.tbl.get_section(sec_nam)

        if not section:
            section = Section(sec_nam)
            if sec_nam in SECTION_CONFIG:
                config = SECTION_CONFIG[sec_nam]
                section.config(config["show_row_hdrs"],
                               config["show_col_hdrs"],
                               config["show_col_hdr_in_cell"])
            self.tbl.add_section(section)

        if "new_row" in meta_inf:
            # create a new row using the name
            section.add_cell(val, sec_nam, val, "S50")
            self.cur_row[sec_nam] = val
            return True

        if not sec_nam in self.cur_row:
            logging.debug("stats %s is not associated with a section" % key)
            return True

        if "col" in meta_inf:
            col = str(meta_inf["col"])
        else:
            col = str(key)

        row = self.cur_row[sec_nam]
        section.add_cell(row, col, val, type="S50")

        return True

    def store_fast(self, root, parents, data, meta, coll,
                   key, val, meta_val, meta_inf, level):
        """Store time-series data into fast-changing database"""
        if not self.c_feeder:
            logging.error(
                "unable to store fast changing data : invalid CarbonFeeder")
            return False
{'fraction_non_alphanumeric': 0.07977700884275278, 'fraction_numerical': 0.0015378700499807767, 'mean_word_length': 1.9971198156682028, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5965926', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1452, 'n_words': 458}