<commit_before>from configparser import ConfigParser, ExtendedInterpolation
from datetime import datetime
from functools import partial
import sys
import importlib.util
from pathlib import Path
import shutil
from .rtfits import Fits


class RT1_configparser(object):
    '''
    A configparser that can be used to fully specify a RT1-processing routine

    Extended interpolation is used -> variables can be addressed within the
    config-file by using:

        >>> [class1]
        >>> var1 = asdf
        >>> var2 = ${var1} bsdf
        >>>
        >>> [class2]
        >>> var3 = asdf
        >>> var4 = ${class1:var1} bsdf

    Methods
    --------
    get_config():
        get a dict of the following structure:

        >>> {fits_kwargs : parsed args from "[fits_kwargs]",
        >>>  lsq_kwargs  : parsed args from "[lsq_kwargs]",
        >>>  defdict     : parsed args from "[defdict]",
        >>>  set_V_SRF   : parsed args from "[set_V_SRF]"}

    get_fitobject():
        get the rt1.rtfits.Fits() object

    get_modules():
        get a dict of the following structure:

        >>> {module_name1 : reference to imported module1 from "[CONFIGFILES]",
        >>>  module_name2 : reference to imported module2 from "[CONFIGFILES]"}

    get_process_specs():
        get a dict of the following structure:

        >>> {name1 : parsed value1 from "[PROCESS_SPECS]",
        >>>  name2 : parsed value2 from "[PROCESS_SPECS]",
             ...}

    Notes
    -----
    The sections are interpreted as follows:

    - [fits_kwargs]
        keyword-arguments passed to the `rt1.rtfits.Fits()` object
    - [defdict]
        the defdict passed to the `rt1.rtfits.Fits()` object
    - [RT1_V]
        the specifications of the `rt1.volume` object passed to the
        `rt1.rtfits.Fits()` object
    - [RT1_SRF]
        the specifications of the `rt1.surface` object passed to the
        `rt1.rtfits.Fits()` object
    - [least_squares_kwargs]
        keyword-arguments passed to the call of `scipy.stats.least_squares()`
    - [CONFIGFILES]

        - any passed argument starting with `module__NAME` will be interpreted as the
          location of a python-module that is intended to be imported.

          the `.get_modules()` function returns a dict:  {NAME : imported module}

        - if `copy = path to a folder` is provided, both the .ini file as well
          as any file imported via `module__` arguments will be copied to the
          specified folder. All modules will be imported from the files within
          the specified folder.

    - [PROCESS_SPECS]
        additional properties needed to specify the process

        - any passed argument starting with `int__NAME` will be passed as
          `integer` with the name `NAME`
        - any passed argument starting with `float__NAME` will be passed as
          `float` with the name `NAME`
        - any passed argument starting with `bool__NAME` will be passed as
          `bool` with the name `NAME`
        - any passed argument starting with `path__NAME` will be passed as
          `pathlib.Path` with the name `NAME`
        - any passed argument starting with `datetime__NAME` will be passed as
          `datetime.datetime` with the name `NAME` as follows:

              >>> datetime__NAME = datetime-string, datetime-format

              for example:

              >>> datetime_d1 = 1.1.2018, %d%m%Y
    '''


    def __init__(self, configpath):
        self.configpath = Path(configpath)
        # setup config (allow empty values -> will result in None)
        self.config = ConfigParser(allow_no_value=True,
                                   interpolation=ExtendedInterpolation())
        # avoid converting uppercase characters
        # (see https://stackoverflow.com/a/19359720/9703451)
        self.config.optionxform = str

        # read config file
        self.cfg = self.config.read(self.configpath)

        # keys that will be converted to int, float or bool
        self.lsq_parse_props = dict(section = 'least_squares_kwargs',
                                    int_keys = ['verbose', '<KEY>'],
                                    float_keys = ['ftol', 'gtol', 'xtol'])

        self.fitargs_parse_props = dict(section = 'fits_kwargs',
                                        bool_keys = ['sig0', 'dB', 'int_Q'],
                                        int_keys= ['verbose'],
                                        list_keys = [])


    def _parse_dict(self, section, int_keys=[], float_keys=[], bool_keys=[],
                    list_keys=[]):<commit_msg>update doc of rtparse to include list parsing<commit_after>from configparser import ConfigParser, ExtendedInterpolation
from datetime import datetime
from functools import partial
import sys
import importlib.util
from pathlib import Path
import shutil
from .rtfits import Fits


class RT1_configparser(object):
    '''
    A configparser that can be used to fully specify a RT1-processing routine

    Extended interpolation is used -> variables can be addressed within the
    config-file by using:

        >>> [class1]
        >>> var1 = asdf
        >>> var2 = ${var1} bsdf
        >>>
        >>> [class2]
        >>> var3 = asdf
        >>> var4 = ${class1:var1} bsdf

    Methods
    --------
    get_config():
        get a dict of the following structure:

        >>> {fits_kwargs : parsed args from "[fits_kwargs]",
        >>>  lsq_kwargs  : parsed args from "[lsq_kwargs]",
        >>>  defdict     : parsed args from "[defdict]",
        >>>  set_V_SRF   : parsed args from "[set_V_SRF]"}

    get_fitobject():
        get the rt1.rtfits.Fits() object

    get_modules():
        get a dict of the following structure:

        >>> {module_name1 : reference to imported module1 from "[CONFIGFILES]",
        >>>  module_name2 : reference to imported module2 from "[CONFIGFILES]"}

    get_process_specs():
        get a dict of the following structure:

        >>> {name1 : parsed value1 from "[PROCESS_SPECS]",
        >>>  name2 : parsed value2 from "[PROCESS_SPECS]",
             ...}

    Notes
    -----
    The sections are interpreted as follows:

    - [fits_kwargs]
        keyword-arguments passed to the `rt1.rtfits.Fits()` object
    - [defdict]
        the defdict passed to the `rt1.rtfits.Fits()` object
    - [RT1_V]
        the specifications of the `rt1.volume` object passed to the
        `rt1.rtfits.Fits()` object
    - [RT1_SRF]
        the specifications of the `rt1.surface` object passed to the
        `rt1.rtfits.Fits()` object
    - [least_squares_kwargs]
        keyword-arguments passed to the call of `scipy.stats.least_squares()`
    - [CONFIGFILES]

        - any passed argument starting with `module__NAME` will be interpreted as the
          location of a python-module that is intended to be imported.

          the `.get_modules()` function returns a dict:  {NAME : imported module}

        - if `copy = path to a folder` is provided, both the .ini file as well
          as any file imported via `module__` arguments will be copied to the
          specified folder. All modules will be imported from the files within
          the specified folder.

    - [PROCESS_SPECS]
        additional properties needed to specify the process

        - any argument starting with `int__NAME` will be parsed as
          `integer` with the name `NAME`
        - any argument starting with `float__NAME` will be parsed as
          `float` with the name `NAME`
        - any argument starting with `bool__NAME` will be parsed as
          `bool` with the name `NAME`
        - any argument starting with `path__NAME` will be parsed as
          `pathlib.Path` with the name `NAME`
        - any argument starting with `datetime__NAME` will be parsed as
          `datetime.datetime` with the name `NAME` as follows:

              >>> datetime__NAME = datetime-string  fmt= datetime-format

              for example:

              >>> datetime_d1 = 1.1.2018 fmt=%d%m%Y

              if fmt is not provided, `"%Y-%m-%d %H:%M:%S.%f"` is used.
              Note that the exact string "fmt=" must be used to separate
              the datetime-string from the format-string!
        - any argument starting with list__NAME will be parsed as a `list`
            - list__NAME will be parsed as a list of strings with the
              name NAME
            - list__float__NAME will be parsed as a list of floats with the
              name NAME
            - list__int__NAME will be parsed as a list of integers with the
              name NAME
            - list__bool__NAME will be parsed as a list of bools with the
              name NAME
            - list__datetime__NAME will be parsed as a list of datetimes with
              the name NAME as follows:

              >>> list__datetime__NAME = [df1, dt2, ...]  fmt= datetime-format

              for example:

              >>> list__datetime__dts = [1.1.2018, 1.10.2018] fmt= %d%m%Y

    '''


    def __init__(self, configpath):
        self.configpath = Path(configpath)
        # setup config (allow empty values -> will result in None)
        self.config = ConfigParser(allow_no_value=True,
                                   interpolation=ExtendedInterpolation())
        # avoid converting uppercase characters
        # (see https://stackoverflow.com/a/19359720/9703451)
        self.config.optionxform = str

        # read config file
        self.cfg = self.config.read(self.configpath)

        # keys that will be converted to int, float or bool
        self.lsq_parse_props = dict(section = 'least_squares_kwargs',
                                    int_keys = ['verbose', '<KEY>'],
                                    float_keys = ['ftol', 'gtol', 'xtol'])

        self.fitargs_parse_props = dict(section = 'fits_kwargs',
                                        bool_keys = ['sig0', 'dB', 'int_Q'],
                                        int_keys= ['verbose'],
                                        list_keys = [])


    def _parse_dict(self, section, int_keys=[], float_keys=[], bool_keys=[],
                    list_keys=[]):
{'fraction_non_alphanumeric': 0.09137556739688178, 'fraction_numerical': 0.011545293072824156, 'mean_word_length': 2.0517916290274014, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 117, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301574', 'n_tokens_mistral': 2822, 'n_tokens_neox': 2645, 'n_words': 967}