<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

from collections import OrderedDict
from distutils.util import strtobool
from io import StringIO
from sys import stderr
from typing import Any, Optional, Union

from supervisor.datatypes import list_of_strings

from supvisors.application import ApplicationStatus, ApplicationRules
from supvisors.process import ProcessStatus, ProcessRules
from supvisors.ttypes import StartingFailureStrategies, RunningFailureStrategies
from supvisors.utils import supvisors_shortcuts

# XSD contents for XML validation
XSDContents = StringIO('''\
<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified"
xmlns:xs="http://www.w3.org/2001/XMLSchema">
    <xs:simpleType name="Loading">
        <xs:restriction base="xs:int">
            <xs:minInclusive value="0"/>
            <xs:maxInclusive value="100"/>
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="StartingFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="ABORT" />
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="STOP" />
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="RunningFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="RESTART_PROCESS" />
            <xs:enumeration value="STOP_APPLICATION" />
            <xs:enumeration value="RESTART_APPLICATION" />
        </xs:restriction>
    </xs:simpleType>
    <xs:complexType name="ProgramModel">
        <xs:sequence>
            <xs:element type="xs:string" name="reference" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:string" name="addresses" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:boolean" name="required" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:boolean" name="wait_exit" minOccurs="0" maxOccurs="1"/>
            <xs:element type="Loading" name="expected_loading" minOccurs="0" maxOccurs="1"/>
            <xs:element type="RunningFailureStrategy" name="running_failure_strategy" minOccurs="0" maxOccurs="1"/>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:complexType name="ApplicationModel">
        <xs:sequence>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="StartingFailureStrategy" name="starting_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:element type="RunningFailureStrategy" name="running_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:choice minOccurs="0" maxOccurs="unbounded">
                <xs:element type="ProgramModel" name="program"/>
                <xs:element type="ProgramModel" name="pattern"/>
            </xs:choice>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:element name="root">
        <xs:complexType>
            <xs:sequence>
                <xs:choice minOccurs="0" maxOccurs="unbounded">
                    <xs:element type="ProgramModel" name="model"/>
                    <xs:element type="ApplicationModel" name="application"/>
                </xs:choice>
            </xs:sequence>
        </xs:complexType>
    </xs:element>
</xs:schema>
''')


class Parser(object):
    """ The Parser class used to get application and program rules from an XML file. """

    # for recursive references to program models, depth is limited to 3
    LOOP_CHECK = 3

    def __init__(self, supvisors: Any) -> None:
        """ The constructor parses the XML file and stores references to models and patterns found.

        :param supvisors: the global Supvisors structure.
        """
        self.supvisors = supvisors
        supvisors_shortcuts(self, ['logger'])
        self.tree = self.parse(supvisors.options.rules_file)
        self.root = self.tree.getroot()
        # get models
        elements = self.root.findall("./model[@name]")
        self.models = {element.get('name'): element for element in elements}
        self.logger.debug(self.models)
        # get patterns
        elements = self.root.findall(".//pattern[@name]")
        self.patterns = {element.get('name'): element for element in elements}
        self.logger.debug(self.patterns)

    def load_application_rules(self, application: ApplicationStatus) -> None:
        """ Find an entry corresponding to the application in the rules file, then load the parameters found.

        :param application: the application for which to find rules in the XML rules file
        :return: None
        """
        # find application element using an xpath
        self.logger.trace('searching application element for {}'.format(application.application_name))
        application_elt = self.root.find('./application[@name="{}"]'.format(application.application_name))
        if application_elt is not None:
            rules = application.rules
            self.load_sequence(application_elt, 'start_sequence', rules)
            self.load_sequence(application_elt, 'stop_sequence', rules)
            self.load_enum(application_elt, 'starting_failure_strategy', StartingFailureStrategies, rules)
            self.load_enum(application_elt, 'running_failure_strategy', RunningFailureStrategies, rules)
            self.logger.debug('application {} - rules {}'.format(application.application_name, rules))

    def load_process_rules(self, process: ProcessStatus) -> None:
        """ Find an entry corresponding to the process in the rules, then load the parameters found.
        A final check is performed to detect inconsistencies.

        :param process: the process for which to find rules in the XML rules file
        :return: None
        """
        self.logger.trace('searching program element for {}'.format(process.namespec()))
        program_elt = self.get_program_element(process)
        if program_elt is not None:
            # load element parameters into rules
            self.load_model_rules(program_elt, process.rules, Parser.LOOP_CHECK)
            # check that rules are compliant with dependencies
            process.rules.check_dependencies(process.namespec())
            self.logger.debug('process {} - rules {}'.format(process.namespec(), process.rules))

    def load_model_rules(self, program_elt: Any, rules: ProcessRules, loop_check: int) -> None:
        """ Load the parameters found whatever it is given by a program or a model section.
        If the section includes a reference to a model, it is loaded first and then eventually superseded by other attributes.

        :param program_elt: the XML element containing rules definition
        :param rules: the process structure used to store the rules found
        :param loop_check: the counter used to check the maximum number of recursive model references
        :return: None
        """
        # check if recursive depth has been reached
        if loop_check == 0:
            self.logger.warn('Maximum number of model referencing reached: {} levels', Parser.LOOP_CHECK)
            return
        # find check if a model is referenced in the program rules
        model_elt = self.get_model_element(program_elt)
        if model_elt is not None:
            self.logger.trace('{} - model search ({}) program element {}'
                              .format(program_elt.get('name'), model_elt, program_elt))
            # a model can reference another model
            # WARN: recursive call, counter decreased
            self.load_model_rules(model_elt, rules, loop_check - 1)
        # other attributes found may be used to complete or supersede the possible model
        self.load_program_addresses(program_elt, rules)
        self.load_sequence(program_elt, 'start_sequence', rules)
        self.load_sequence(program_elt, 'stop_sequence', rules)
        self.load_boolean(program_elt, 'required', rules)
        self.load_boolean(program_elt, 'wait_exit', rules)
        self.load_loading(program_elt, 'expected_loading', rules)
        self.load_enum(program_elt, 'running_failure_strategy', RunningFailureStrategies, rules)

    def get_program_element(self, process: ProcessStatus) -> Optional[Any]:
        """ Try to find the definition of a program in rules files.
        First try to to find the definition based on the exact program name.
        If not found, second try to find a corresponding pattern.

        :param process: the process for which to find rules in the XML rules file
        :return: the XML element containing rules definition for a program
        """
        # try to find program name in file
        program_elt = self.root.find('./application[@name="{}"]/program[@name="{}"]'
                                     .format(process.application_name, process.process_name))
        self.logger.trace('{} - direct search for program element {}'
                          .format(process.namespec(), program_elt))
        if program_elt is None:
            # if not found as it is, try to find a corresponding pattern
            patterns = [name for name, element in self.patterns.items() if name in process.namespec()]
            self.supvisors.logger.trace('{} - found {} patterns'
                                        .format(process.namespec(), len(patterns)))
            if patterns:
                pattern = max(patterns, key=len)
                program_elt = self.patterns[pattern]
                self.logger.trace('{} - pattern search for program element {}'
                                  .format(process.namespec(), program_elt))
        return program_elt

    def get_model_element(self, elt: Any) -> Optional[Any]:
        """ Find if element references a model

        :param elt: the XML element containing rules definition for a program
        :return: the XML element containing rules definition for a model
        """
        model = elt.findtext('reference')
        return self.models.get(model, None)

    def load_program_addresses(self, elt: Any, rules: ProcessRules) -> None:
        """ Get the addresses where the program is authorized to run.

        :param elt: the XML element containing rules definition for a program
        :param rules: the process structure used to store the rules found
        :return: None
        """
        value = elt.findtext('addresses')
        if value:
            # sort and trim
            addresses = list(OrderedDict.fromkeys(filter(None, list_of_strings(value))))
            if '*' in addresses:
                rules.addresses = ['*']
            elif '#' in addresses:
                rules.addresses = ['#']
            else:
                rules.addresses = self.supvisors.address_mapper.filter(addresses)

    def load_sequence(self, elt: Any, attr_string: str, rules: Union[ApplicationRules, ProcessRules]) -> None:
        """ Return the sequence value found from the XML element.
        The value must be greater than or equal to 0.

        :param elt: the XML element containing rules definition for an application or a program
        :param attr_string: the XML tag searched and the name of the rule attribute
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext(attr_string)
        if str_value:
            try:
                value = int(str_value)
                if value >= 0:
                    setattr(rules, attr_string, value)
                else:
                    self.logger.warn('Invalid value for {} {}: {} (expected integer >= 0)'
                                     .format(elt.get('name'), attr_string, value))
            except (TypeError, ValueError):
                self.logger.warn('Not an integer for {} {}: {}'
                                 .format(elt.get('name'), attr_string, str_value))

    def load_loading(self, elt: Any, attr_string: str, rules: ProcessRules) -> None:
        """ Return the loading value found from the XML element.
        The value must be in [0 ; 100].

        :param elt: the XML element containing rules definition for an application or a program
        :param attr_string: the XML tag searched and the name of the rule attribute
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext(attr_string)
        try:
            value = int(str_value)
            if 0 <= value <= 100:
                setattr(rules, attr_string, value)
            else:
                self.logger.warn('Invalid value for {} {}: {} (expected integer >= 0)'
                                 .format(elt.get('name'), attr_string, value))
        except (TypeError, ValueError):
            self.logger.warn('Not an integer for {} {}: {}'
                             .format(elt.get('name'), attr_string, str_value))

    def load_boolean(self, elt: Any, attr_string: str, rules: ProcessRules) -> None:
        """ Return the boolean value found from XML element.

        :param elt: the XML element containing rules definition for an application or a program
        :param attr_string: the XML tag searched and the name of the rule attribute
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext(attr_string)
        if str_value:
            try:
                value = strtobool(str_value)
                setattr(rules, attr_string, value)
            except ValueError:
                self.logger.warn('Not a boolean-like for {} {}: {}'
                                 .format(elt.get('name'), attr_string, str_value))
<commit_msg>Closes #84<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# ======================================================================
# Copyright 2016 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ======================================================================

from collections import OrderedDict
from distutils.util import strtobool
from io import StringIO
from sys import stderr
from typing import Any, Optional, Union

from supervisor.datatypes import list_of_strings

from supvisors.application import ApplicationStatus, ApplicationRules
from supvisors.process import ProcessStatus, ProcessRules
from supvisors.ttypes import StartingFailureStrategies, RunningFailureStrategies
from supvisors.utils import supvisors_shortcuts

# XSD contents for XML validation
XSDContents = StringIO('''\
<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified"
xmlns:xs="http://www.w3.org/2001/XMLSchema">
    <xs:simpleType name="Loading">
        <xs:restriction base="xs:int">
            <xs:minInclusive value="0"/>
            <xs:maxInclusive value="100"/>
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="StartingFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="ABORT" />
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="STOP" />
        </xs:restriction>
    </xs:simpleType>
    <xs:simpleType name="RunningFailureStrategy" final="restriction" >
        <xs:restriction base="xs:string">
            <xs:enumeration value="CONTINUE" />
            <xs:enumeration value="RESTART_PROCESS" />
            <xs:enumeration value="STOP_APPLICATION" />
            <xs:enumeration value="RESTART_APPLICATION" />
        </xs:restriction>
    </xs:simpleType>
    <xs:complexType name="ProgramModel">
        <xs:sequence>
            <xs:element type="xs:string" name="reference" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:string" name="addresses" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:boolean" name="required" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:boolean" name="wait_exit" minOccurs="0" maxOccurs="1"/>
            <xs:element type="Loading" name="expected_loading" minOccurs="0" maxOccurs="1"/>
            <xs:element type="RunningFailureStrategy" name="running_failure_strategy" minOccurs="0" maxOccurs="1"/>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:complexType name="ApplicationModel">
        <xs:sequence>
            <xs:element type="xs:byte" name="start_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="xs:byte" name="stop_sequence" minOccurs="0" maxOccurs="1"/>
            <xs:element type="StartingFailureStrategy" name="starting_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:element type="RunningFailureStrategy" name="running_failure_strategy" minOccurs="0" maxOccurs="1"/>
            <xs:choice minOccurs="0" maxOccurs="unbounded">
                <xs:element type="ProgramModel" name="program"/>
                <xs:element type="ProgramModel" name="pattern"/>
            </xs:choice>
        </xs:sequence>
        <xs:attribute type="xs:string" name="name" use="required"/>
    </xs:complexType>
    <xs:element name="root">
        <xs:complexType>
            <xs:sequence>
                <xs:choice minOccurs="0" maxOccurs="unbounded">
                    <xs:element type="ProgramModel" name="model"/>
                    <xs:element type="ApplicationModel" name="application"/>
                </xs:choice>
            </xs:sequence>
        </xs:complexType>
    </xs:element>
</xs:schema>
''')


class Parser(object):
    """ The Parser class used to get application and program rules from an XML file. """

    # for recursive references to program models, depth is limited to 3
    LOOP_CHECK = 3

    def __init__(self, supvisors: Any) -> None:
        """ The constructor parses the XML file and stores references to models and patterns found.

        :param supvisors: the global Supvisors structure.
        """
        self.supvisors = supvisors
        supvisors_shortcuts(self, ['logger'])
        self.tree = self.parse(supvisors.options.rules_file)
        self.root = self.tree.getroot()
        # get models
        elements = self.root.findall("./model[@name]")
        self.models = {element.get('name'): element for element in elements}
        self.logger.debug(self.models)
        # get patterns
        elements = self.root.findall(".//pattern[@name]")
        self.patterns = {element.get('name'): element for element in elements}
        self.logger.debug(self.patterns)

    def load_application_rules(self, application: ApplicationStatus) -> None:
        """ Find an entry corresponding to the application in the rules file, then load the parameters found.

        :param application: the application for which to find rules in the XML rules file
        :return: None
        """
        # find application element using an xpath
        self.logger.trace('searching application element for {}'.format(application.application_name))
        application_elt = self.root.find('./application[@name="{}"]'.format(application.application_name))
        if application_elt is not None:
            rules = application.rules
            self.load_sequence(application_elt, 'start_sequence', rules)
            self.load_sequence(application_elt, 'stop_sequence', rules)
            self.load_enum(application_elt, 'starting_failure_strategy', StartingFailureStrategies, rules)
            self.load_enum(application_elt, 'running_failure_strategy', RunningFailureStrategies, rules)
            self.logger.debug('application {} - rules {}'.format(application.application_name, rules))

    def load_process_rules(self, process: ProcessStatus) -> None:
        """ Find an entry corresponding to the process in the rules, then load the parameters found.
        A final check is performed to detect inconsistencies.

        :param process: the process for which to find rules in the XML rules file
        :return: None
        """
        self.logger.trace('searching program element for {}'.format(process.namespec()))
        program_elt = self.get_program_element(process)
        if program_elt is not None:
            # load element parameters into rules
            self.load_model_rules(program_elt, process.rules, Parser.LOOP_CHECK)
            # check that rules are compliant with dependencies
            process.rules.check_dependencies(process.namespec())
            self.logger.debug('process {} - rules {}'.format(process.namespec(), process.rules))

    def load_model_rules(self, program_elt: Any, rules: ProcessRules, loop_check: int) -> None:
        """ Load the parameters found whatever it is given by a program or a model section.
        If the section includes a reference to a model, it is loaded first and then eventually superseded by other attributes.

        :param program_elt: the XML element containing rules definition
        :param rules: the process structure used to store the rules found
        :param loop_check: the counter used to check the maximum number of recursive model references
        :return: None
        """
        # check if recursive depth has been reached
        if loop_check == 0:
            self.logger.warn('Maximum number of model referencing reached: {} levels', Parser.LOOP_CHECK)
            return
        # find check if a model is referenced in the program rules
        model_elt = self.get_model_element(program_elt)
        if model_elt is not None:
            self.logger.trace('{} - model search ({}) program element {}'
                              .format(program_elt.get('name'), model_elt, program_elt))
            # a model can reference another model
            # WARN: recursive call, counter decreased
            self.load_model_rules(model_elt, rules, loop_check - 1)
        # other attributes found may be used to complete or supersede the possible model
        self.load_program_addresses(program_elt, rules)
        self.load_sequence(program_elt, 'start_sequence', rules)
        self.load_sequence(program_elt, 'stop_sequence', rules)
        self.load_boolean(program_elt, 'required', rules)
        self.load_boolean(program_elt, 'wait_exit', rules)
        self.load_loading(program_elt, rules)
        self.load_enum(program_elt, 'running_failure_strategy', RunningFailureStrategies, rules)

    def get_program_element(self, process: ProcessStatus) -> Optional[Any]:
        """ Try to find the definition of a program in rules files.
        First try to to find the definition based on the exact program name.
        If not found, second try to find a corresponding pattern.

        :param process: the process for which to find rules in the XML rules file
        :return: the XML element containing rules definition for a program
        """
        # try to find program name in file
        program_elt = self.root.find('./application[@name="{}"]/program[@name="{}"]'
                                     .format(process.application_name, process.process_name))
        self.logger.trace('{} - direct search for program element {}'
                          .format(process.namespec(), program_elt))
        if program_elt is None:
            # if not found as it is, try to find a corresponding pattern
            patterns = [name for name, element in self.patterns.items() if name in process.namespec()]
            self.supvisors.logger.trace('{} - found {} patterns'
                                        .format(process.namespec(), len(patterns)))
            if patterns:
                pattern = max(patterns, key=len)
                program_elt = self.patterns[pattern]
                self.logger.trace('{} - pattern search for program element {}'
                                  .format(process.namespec(), program_elt))
        return program_elt

    def get_model_element(self, elt: Any) -> Optional[Any]:
        """ Find if element references a model

        :param elt: the XML element containing rules definition for a program
        :return: the XML element containing rules definition for a model
        """
        model = elt.findtext('reference')
        return self.models.get(model, None)

    def load_program_addresses(self, elt: Any, rules: ProcessRules) -> None:
        """ Get the addresses where the program is authorized to run.

        :param elt: the XML element containing rules definition for a program
        :param rules: the process structure used to store the rules found
        :return: None
        """
        value = elt.findtext('addresses')
        if value:
            # sort and trim
            addresses = list(OrderedDict.fromkeys(filter(None, list_of_strings(value))))
            if '*' in addresses:
                rules.addresses = ['*']
            elif '#' in addresses:
                rules.addresses = ['#']
            else:
                rules.addresses = self.supvisors.address_mapper.filter(addresses)

    def load_sequence(self, elt: Any, attr_string: str, rules: Union[ApplicationRules, ProcessRules]) -> None:
        """ Return the sequence value found from the XML element.
        The value must be greater than or equal to 0.

        :param elt: the XML element containing rules definition for an application or a program
        :param attr_string: the XML tag searched and the name of the rule attribute
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext(attr_string)
        if str_value:
            try:
                value = int(str_value)
                if value >= 0:
                    setattr(rules, attr_string, value)
                else:
                    self.logger.warn('Invalid value for {} {}: {} (expected integer >= 0)'
                                     .format(elt.get('name'), attr_string, value))
            except (TypeError, ValueError):
                self.logger.warn('Not an integer for {} {}: {}'
                                 .format(elt.get('name'), attr_string, str_value))

    def load_loading(self, elt: Any, rules: ProcessRules) -> None:
        """ Return the loading value found from the XML element.
        The value must be in [0 ; 100].

        :param elt: the XML element containing rules definition for an application or a program
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext('expected_loading')
        if str_value:
            try:
                value = int(str_value)
                if 0 <= value <= 100:
                    setattr(rules, 'expected_loading', value)
                else:
                    self.logger.warn('Invalid value for {} expected_loading: {} (expected integer in [0;100])'
                                     .format(elt.get('name'), value))
            except (TypeError, ValueError):
                self.logger.warn('Not an integer for {} expected_loading: {}'
                                 .format(elt.get('name'), str_value))

    def load_boolean(self, elt: Any, attr_string: str, rules: ProcessRules) -> None:
        """ Return the boolean value found from XML element.

        :param elt: the XML element containing rules definition for an application or a program
        :param attr_string: the XML tag searched and the name of the rule attribute
        :param rules: the structure used to store the rules found
        :return: None
        """
        str_value = elt.findtext(attr_string)
        if str_value:
            try:
                value = strtobool(str_value)
                setattr(rules, attr_string, value)
            except ValueError:
                self.logger.warn('Not a boolean-like for {} {}: {}'
                                 .format(elt.get('name'), attr_string, str_value))

{'fraction_non_alphanumeric': 0.111310107119927, 'fraction_numerical': 0.004223971885243132, 'mean_word_length': 2.619618395303327, 'pattern_counts': {'":': 0, '<': 127, '<?xml version=': 0, '>': 148, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1030845', 'n_tokens_mistral': 7799, 'n_tokens_neox': 7527, 'n_words': 2380}