<commit_before>import logging
import logging.config
import os
import ConfigParser
import time
import sys

from biomaj.bmajindex import BmajIndex

class BiomajConfig:
  '''
  Manage Biomaj configuration
  '''

  DEFAULTS = {
  'http.parse.dir.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})',
  'http.parse.file.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})',
  'http.group.dir.name': 1,
  'http.group.dir.date': 2,
  'http.group.file.name': 1,
  'http.group.file.date': 2,
  'http.group.file.size': 3,
  'visibility.default': 'public',
  'historic.logfile.level': 'INFO',
  'bank.num.threads': 2,
  'files.num.threads': 4
  }

  # Old biomaj level compatibility
  LOGLEVEL = {
    'DEBUG': logging.DEBUG,
    'VERBOSE': logging.INFO,
    'INFO': logging.INFO,
    'WARN': logging.WARNING,
    'ERR': logging.ERROR
  }

  '''
  Global configuration file
  '''
  global_config = None

  '''
  Per use global configuration file, overriding global_config
  '''
  user_config = None

  @staticmethod
  def load_config(config_file='global.properties'):
    '''
    Loads general config

    :param config_file: global.properties file path
    :type config_file: str
    '''
    if not os.path.exists(config_file) and not os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
      raise Exception('Missing global configuration file')

    BiomajConfig.config_file = config_file

    BiomajConfig.global_config = ConfigParser.ConfigParser()

    if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
      BiomajConfig.user_config = ConfigParser.ConfigParser()
      BiomajConfig.user_config.read([os.path.expanduser('~/.biomaj.cfg')])

    BiomajConfig.global_config.read([config_file])

    # ElasticSearch indexation support
    if BiomajConfig.global_config.get('GENERAL','use_elastic'):
      if BiomajConfig.global_config.get('GENERAL','elastic_nodes'):
        elastic_hosts = BiomajConfig.global_config.get('GENERAL','elastic_nodes').split(',')
      else:
        elastic_hosts = ['localhost']
      elastic_index = BiomajConfig.global_config.get('GENERAL','elastic_index')
      if elastic_index is None:
        elastic_index = 'biomaj'
      BmajIndex.load(index=elastic_index, hosts=elastic_hosts)


<commit_msg>fix use or not of elasticsearch<commit_after>import logging
import logging.config
import os
import ConfigParser
import time
import sys

from biomaj.bmajindex import BmajIndex

class BiomajConfig:
  '''
  Manage Biomaj configuration
  '''

  DEFAULTS = {
  'http.parse.dir.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})',
  'http.parse.file.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})',
  'http.group.dir.name': 1,
  'http.group.dir.date': 2,
  'http.group.file.name': 1,
  'http.group.file.date': 2,
  'http.group.file.size': 3,
  'visibility.default': 'public',
  'historic.logfile.level': 'INFO',
  'bank.num.threads': 2,
  'files.num.threads': 4
  }

  # Old biomaj level compatibility
  LOGLEVEL = {
    'DEBUG': logging.DEBUG,
    'VERBOSE': logging.INFO,
    'INFO': logging.INFO,
    'WARN': logging.WARNING,
    'ERR': logging.ERROR
  }

  '''
  Global configuration file
  '''
  global_config = None

  '''
  Per use global configuration file, overriding global_config
  '''
  user_config = None

  @staticmethod
  def load_config(config_file='global.properties'):
    '''
    Loads general config

    :param config_file: global.properties file path
    :type config_file: str
    '''
    if not os.path.exists(config_file) and not os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
      raise Exception('Missing global configuration file')

    BiomajConfig.config_file = config_file

    BiomajConfig.global_config = ConfigParser.ConfigParser()

    if os.path.exists(os.path.expanduser('~/.biomaj.cfg')):
      BiomajConfig.user_config = ConfigParser.ConfigParser()
      BiomajConfig.user_config.read([os.path.expanduser('~/.biomaj.cfg')])

    BiomajConfig.global_config.read([config_file])

    # ElasticSearch indexation support
    do_index = False
    if BiomajConfig.global_config.get('GENERAL','use_elastic') and \
      BiomajConfig.global_config.get('GENERAL','use_elastic') == 1:
      do_index = True
    if do_index:
      if BiomajConfig.global_config.get('GENERAL','elastic_nodes'):
        elastic_hosts = BiomajConfig.global_config.get('GENERAL','elastic_nodes').split(',')
      else:
        elastic_hosts = ['localhost']
      elastic_index = BiomajConfig.global_config.get('GENERAL','elastic_index')
      if elastic_index is None:
        elastic_index = 'biomaj'
      BmajIndex.load(index=elastic_index, hosts=elastic_hosts, do_index=do_index)



{'fraction_non_alphanumeric': 0.19644284572342127, 'fraction_numerical': 0.00859312549960032, 'mean_word_length': 3.8734177215189876, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5783667', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1779, 'n_words': 275}