<commit_before>"""The backbone of pax

"""
import glob
import itertools
import logging
import inspect
from configparser import ConfigParser, ExtendedInterpolation
from collections import OrderedDict  # DO NOT REMOVE, used in eval(configuration)

import re
import os
from io import StringIO
import importlib
from tqdm import tqdm     # Progress bar
from prettytable import PrettyTable     # Timing report

import pax
from pax import units, simulation

# Store the directory of pax (i.e. this file's directory) as PAX_DIR
PAX_DIR = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))


def data_file_name(filename):
    """Returns filename if a file exists there, else returns PAX_DIR/data/filename"""
    if os.path.exists(filename):
        return filename
    new_filename = os.path.join(PAX_DIR, 'data', filename)
    if os.path.exists(new_filename):
        return new_filename
    else:
        raise ValueError('File name or path %s not found!' % filename)


def get_named_configuration_options():
    """ Return the names of all working named configurations
    """
    config_files = []
    for filename in glob.glob(os.path.join(PAX_DIR, 'config', '*.ini')):
        filename = os.path.basename(filename)
        m = re.match(r'(\w+)\.ini', filename)
        if m is None:
            print("Weird file in config dir: %s" % filename)
        filename = m.group(1)
        # Config files starting with '_' won't appear in the usage list (they won't work by themselves)
        if filename[0] == '_':
            continue
        config_files.append(filename)
    return config_files


##
# Processor class
##
class Processor:
    fallback_configuration = 'XENON100'    # Configuration to use when none is specified

    def __init__(self, config_names=(), config_paths=(), config_string=None, config_dict=None, just_testing=False):
        """Setup pax using configuration data from three sources:
          - config_names: List of named configurations to load (sequentially)
          - config_paths: List of config file paths to load (sequentially)
          - config_string: A config string (overrides all config from files)
          - config_dict: A final config ovveride dict: {'section' : {'setting' : 17, ...

        Files from config_paths will be loaded after config_names, and can thus override their settings.
        Configuration files can inherit from others using parent_configuration and parent_configuration_files.
        Each value in the ini files (and the config_string) is eval()'ed in a context with physical unit variables:
            4 * 2 -> 8
            4 * cm**(2)/s -> correctly interpreted as a physical value with units of cm^2/s

        The config_dict's settings are not evaluated.

        Setting just_testing disables some warnings about not specifying any plugins or plugin groups in the config.
        Use only if, for some reason, you don't want to load a full configuration file.
        
        .. note:: 
          Although the log level can be specified in the configuration, it is an application wide
          setting that will not be modified once set. New instances of the Processor class will have
          the same log level as the first, regardless of their configuration.  See #78.
        """
        self.config = self.load_configuration(config_names, config_paths, config_string, config_dict)
        self.log = self.setup_logging()
        pc = self.config['pax']

        self.log.info("This is PAX version %s, running with configuration for %s." % (
            pax.__version__, self.config['DEFAULT'].get('tpc_name', 'UNSPECIFIED TPC NAME')))

        # Start up the simulator
        # Must be done explicitly here, as plugins can rely on its presence in startup
        self.simulator = simulation.Simulator(self.config['WaveformSimulator'])

        # Get the list of plugins from the configuration file
        # plugin_names[group] is a list of all plugins we have to initialize in the group 'group'
        plugin_names = {}
        if not 'plugin_group_names' in pc:
            if not just_testing:
                self.log.warning('You did not specify any plugin groups to load: are you testing me?')
            pc['plugin_group_names'] = []

        for plugin_group_name in pc['plugin_group_names']:

            if not plugin_group_name in pc:
                raise ValueError('Invalid configuration: plugin group list %s missing' % plugin_group_name)

            plugin_names[plugin_group_name] = pc[plugin_group_name]

            if not isinstance(plugin_names[plugin_group_name], (str, list)):
                raise ValueError("Invalid configuration: plugin group list %s should be a string, not %s" %
                                 (plugin_group_name, type(plugin_names)))

            if not isinstance(plugin_names[plugin_group_name], list):
                plugin_names[plugin_group_name] = [plugin_names[plugin_group_name]]

        # Separate input and actions (which for now includes output).
        # For the plugin groups which are action plugins, get all names, flatten them
        action_plugin_names = list(itertools.chain(*[plugin_names[g]
                                                   for g in pc['plugin_group_names']
                                                   if g != 'input']))<commit_msg>Fix tests after Simulation class move<commit_after>"""The backbone of pax

"""
import glob
import itertools
import logging
import inspect
from configparser import ConfigParser, ExtendedInterpolation
from collections import OrderedDict  # DO NOT REMOVE, used in eval(configuration)

import re
import os
from io import StringIO
import importlib
from tqdm import tqdm     # Progress bar
from prettytable import PrettyTable     # Timing report

import pax
from pax import units, simulation

# Store the directory of pax (i.e. this file's directory) as PAX_DIR
PAX_DIR = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))))


def data_file_name(filename):
    """Returns filename if a file exists there, else returns PAX_DIR/data/filename"""
    if os.path.exists(filename):
        return filename
    new_filename = os.path.join(PAX_DIR, 'data', filename)
    if os.path.exists(new_filename):
        return new_filename
    else:
        raise ValueError('File name or path %s not found!' % filename)


def get_named_configuration_options():
    """ Return the names of all working named configurations
    """
    config_files = []
    for filename in glob.glob(os.path.join(PAX_DIR, 'config', '*.ini')):
        filename = os.path.basename(filename)
        m = re.match(r'(\w+)\.ini', filename)
        if m is None:
            print("Weird file in config dir: %s" % filename)
        filename = m.group(1)
        # Config files starting with '_' won't appear in the usage list (they won't work by themselves)
        if filename[0] == '_':
            continue
        config_files.append(filename)
    return config_files


##
# Processor class
##
class Processor:
    fallback_configuration = 'XENON100'    # Configuration to use when none is specified

    def __init__(self, config_names=(), config_paths=(), config_string=None, config_dict=None, just_testing=False):
        """Setup pax using configuration data from three sources:
          - config_names: List of named configurations to load (sequentially)
          - config_paths: List of config file paths to load (sequentially)
          - config_string: A config string (overrides all config from files)
          - config_dict: A final config ovveride dict: {'section' : {'setting' : 17, ...

        Files from config_paths will be loaded after config_names, and can thus override their settings.
        Configuration files can inherit from others using parent_configuration and parent_configuration_files.
        Each value in the ini files (and the config_string) is eval()'ed in a context with physical unit variables:
            4 * 2 -> 8
            4 * cm**(2)/s -> correctly interpreted as a physical value with units of cm^2/s

        The config_dict's settings are not evaluated.

        Setting just_testing disables some warnings about not specifying any plugins or plugin groups in the config.
        Use only if, for some reason, you don't want to load a full configuration file.
        
        .. note:: 
          Although the log level can be specified in the configuration, it is an application wide
          setting that will not be modified once set. New instances of the Processor class will have
          the same log level as the first, regardless of their configuration.  See #78.
        """
        self.config = self.load_configuration(config_names, config_paths, config_string, config_dict)
        self.log = self.setup_logging()
        pc = self.config['pax']

        self.log.info("This is PAX version %s, running with configuration for %s." % (
            pax.__version__, self.config['DEFAULT'].get('tpc_name', 'UNSPECIFIED TPC NAME')))

        # Start up the simulator
        # Must be done explicitly here, as plugins can rely on its presence in startup
        if 'WaveformSimulator' in self.config:
            self.simulator = simulation.Simulator(self.config['WaveformSimulator'])
        else:
            if not just_testing:
                self.log.warning('You did not specify any configuration for the waveform simulator!\n'+
                                 'If you attempt to load the waveform simulator, pax will crash!')

        # Get the list of plugins from the configuration file
        # plugin_names[group] is a list of all plugins we have to initialize in the group 'group'
        plugin_names = {}
        if not 'plugin_group_names' in pc:
            if not just_testing:
                self.log.warning('You did not specify any plugin groups to load: are you testing me?')
            pc['plugin_group_names'] = []

        for plugin_group_name in pc['plugin_group_names']:

            if not plugin_group_name in pc:
                raise ValueError('Invalid configuration: plugin group list %s missing' % plugin_group_name)

            plugin_names[plugin_group_name] = pc[plugin_group_name]

            if not isinstance(plugin_names[plugin_group_name], (str, list)):
                raise ValueError("Invalid configuration: plugin group list %s should be a string, not %s" %
                                 (plugin_group_name, type(plugin_names)))

            if not isinstance(plugin_names[plugin_group_name], list):
                plugin_names[plugin_group_name] = [plugin_names[plugin_group_name]]

        # Separate input and actions (which for now includes output).
        # For the plugin groups which are action plugins, get all names, flatten them
        action_plugin_names = list(itertools.chain(*[plugin_names[g]
                                                   for g in pc['plugin_group_names']
                                                   if g != 'input']))
{'fraction_non_alphanumeric': 0.07863945578231292, 'fraction_numerical': 0.0027210884353741495, 'mean_word_length': 2.714959568733154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '558856', 'n_tokens_mistral': 2885, 'n_tokens_neox': 2804, 'n_words': 1091}