<commit_before># system
import os
import re
import shutil
import logging
import logging.handlers
# scientific
import pandas as pd
# custom
from data import PdfSlider
from emailer import Emailer


class ExpCommon(object):
  def __init__(self, root, name):
    """
    `root`: file root
    `name`: project name
    """
    self.root = root
    self.name = name
    self.comp = 6
    self.upass = None
    self._init_logger()

  def _init_logger(self):
    cn = self._underscore(self.__class__.__name__)
    fmt = logging. \
        Formatter(fmt='%(asctime)s,%(levelname)s,' +
                  '%(name)s,%(funcName)s,%(lineno)d, %(message)s',
                  datefmt='%m/%d/%Y %H:%M:%S')
    logger = logging.getLogger("{}.{}.{}.{}".
                               format(__name__, cn, self.root, self.name))
    logger.setLevel(logging.INFO)
    logger.propagate = 0
    # FileHandler
    fn = self.make_path('log', 'log', True, False)
    fnh = fn + "_fh"
    cnh = fn + "_ch"
    if fnh not in [lh.name for lh in logger.handlers]:
      fh = logging.handlers. \
          RotatingFileHandler(fn, maxBytes=10485760, backupCount=5)
      fh.name = fnh
      fh.setFormatter(fmt)
      logger.addHandler(fh)
    # StreamHandler
    if 0 and cnh not in [lh.name for lh in logger.handlers]:
      ch = logging.StreamHandler()
      ch.name = cnh
      ch.setFormatter(fmt)
      logger.addHandler(ch)
    logger.info(">>============== {} inited ================= <<".format(cn))
    self.log = logger

  def _save_key(self, fpath, key):
    kf = self.load('keys')
    if kf is None:
      kf = pd.DataFrame([key], columns=['key'])
    else:
      kf = kf.append(pd.DataFrame([key], columns=['key']))
      kf = kf.reset_index()
      for rc in ['index', 'level']:
        if rc in kf.columns:
          del kf[rc]
    kf.to_hdf(fpath, 'keys', mode='a', data_columns=True,
              format='t', complib='blosc', complevel=self.comp)

  def _underscore(self, string=''):
    # move pre-compile out the loop to improve performance
    first_cap_re = re.compile('(.)([A-Z][a-z]+)')
    all_cap_re = re.compile('([a-z0-9])([A-Z])')
    s1 = first_cap_re.sub(r'\1_\2', string)
    return all_cap_re.sub(r'\1_\2', s1).lower()

  def _ustr_dict(self, di):
    """
    Make dict keys underscore for saving to hdfs
    """
    s = ""
    ks = di.keys()
    ks.sort()
    for k in ks:
      s += "/{}_{}".format(k, di[k])
    return s

  def _asure_path(self, path):
    if not os.path.exists(path):
      os.makedirs(path)

  def slide_pages(self):
    ps = PdfSlider(self.name, self.root)
    return ps.pages()

  def slides_path(self, size='mid'):
    ps = PdfSlider(self.name, self.root)
    return ps.slides_path(size)
<commit_msg>Rename PdfSlider to Slider in exp.exp_common<commit_after># system
import os
import re
import shutil
import logging
import logging.handlers
# scientific
import pandas as pd
# custom
from summary import Slider
from emailer import Emailer


class ExpCommon(object):
  def __init__(self, root, name):
    """
    `root`: file root
    `name`: project name
    """
    self.root = root
    self.name = name
    self.comp = 6
    self.upass = None
    self._init_logger()

  def _init_logger(self):
    cn = self._underscore(self.__class__.__name__)
    fmt = logging. \
        Formatter(fmt='%(asctime)s,%(levelname)s,' +
                  '%(name)s,%(funcName)s,%(lineno)d, %(message)s',
                  datefmt='%m/%d/%Y %H:%M:%S')
    logger = logging.getLogger("{}.{}.{}.{}".
                               format(__name__, cn, self.root, self.name))
    logger.setLevel(logging.INFO)
    logger.propagate = 0
    # FileHandler
    fn = self.make_path('log', 'log', True, False)
    fnh = fn + "_fh"
    cnh = fn + "_ch"
    if fnh not in [lh.name for lh in logger.handlers]:
      fh = logging.handlers. \
          RotatingFileHandler(fn, maxBytes=10485760, backupCount=5)
      fh.name = fnh
      fh.setFormatter(fmt)
      logger.addHandler(fh)
    # StreamHandler
    if 0 and cnh not in [lh.name for lh in logger.handlers]:
      ch = logging.StreamHandler()
      ch.name = cnh
      ch.setFormatter(fmt)
      logger.addHandler(ch)
    logger.info(">>============== {} inited ================= <<".format(cn))
    self.log = logger

  def _save_key(self, fpath, key):
    kf = self.load('keys')
    if kf is None:
      kf = pd.DataFrame([key], columns=['key'])
    else:
      kf = kf.append(pd.DataFrame([key], columns=['key']))
      kf = kf.reset_index()
      for rc in ['index', 'level']:
        if rc in kf.columns:
          del kf[rc]
    kf.to_hdf(fpath, 'keys', mode='a', data_columns=True,
              format='t', complib='blosc', complevel=self.comp)

  def _underscore(self, string=''):
    # move pre-compile out the loop to improve performance
    first_cap_re = re.compile('(.)([A-Z][a-z]+)')
    all_cap_re = re.compile('([a-z0-9])([A-Z])')
    s1 = first_cap_re.sub(r'\1_\2', string)
    return all_cap_re.sub(r'\1_\2', s1).lower()

  def _ustr_dict(self, di):
    """
    Make dict keys underscore for saving to hdfs
    """
    s = ""
    ks = di.keys()
    ks.sort()
    for k in ks:
      s += "/{}_{}".format(k, di[k])
    return s

  def _asure_path(self, path):
    if not os.path.exists(path):
      os.makedirs(path)

  def slide_pages(self):
    ps = Slider(self.root, self.name)
    return ps.pages()

  def slides_path(self, size='mid'):
    ps = Slider(self.root, self.name)
    return ps.slides_path(size)

{'fraction_non_alphanumeric': 0.1654203902972825, 'fraction_numerical': 0.007295276308590188, 'mean_word_length': 2.624586913417052, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179632', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1944, 'n_words': 453}