<commit_before>import logging
import itertools

from pysb import (Model, Monomer, Parameter, Rule, Annotation,
        ComponentDuplicateNameError, ComplexPattern, ReactionPattern, ANY)
from pysb.core import SelfExporter
import pysb.export

from indra import statements as ist
from indra.databases import context_client

logger = logging.getLogger('pysb_assembler')

SelfExporter.do_export = False

# Here we define the types of INDRA statements that are meant to be
# assembled using the PySB assembler. If a type of statement appears
# in this list then we require that there is at least one default
# policy implemented to assemble that type of statement.
statement_whitelist = [ist.Phosphorylation, ist.Dephosphorylation,
                       ist.SelfModification, ist.Complex,
                       ist.Activation, ist.ActiveForm,
                       ist.RasGef, ist.RasGap, ist.Translocation]

def _is_whitelisted(stmt):
    """Return True if the statement type is in the whitelist."""
    for s in statement_whitelist:
        if isinstance(stmt, s):
            return True
    return False

# BaseAgent classes ####################################################

class _BaseAgentSet(object):
    """Container for a dict of BaseAgents with their names as keys."""
    def __init__(self):
        self.agents = {}

    def get_create_base_agent(self, agent):
        """Return base agent with given name, creating it if needed."""
        try:
            base_agent = self.agents[agent.name]
        except KeyError:
            base_agent = _BaseAgent(agent.name)
            self.agents[agent.name] = base_agent

        # Handle bound conditions
        for bc in agent.bound_conditions:
            bound_base_agent = self.get_create_base_agent(bc.agent)
            bound_base_agent.create_site(get_binding_site_name(agent.name))
            base_agent.create_site(get_binding_site_name(bc.agent.name))

        # Handle modification conditions
        for mc in agent.mods:
            mod_site_name =\
                get_mod_site_name(mc.mod_type, mc.residue, mc.position)
            site_states = states[mc.mod_type]
            base_agent.create_site(mod_site_name, site_states)

        # Handle mutation conditions
        for mc in agent.mutations:
            if mc.residue_from is None:
                res_from = 'X'
            else:
                res_from = mc.residue_from
            mut_site_name = res_from + mc.position
            base_agent.create_site(mut_site_name, states=['WT'])
            if mc.residue_to is not None:
                base_agent.add_site_states(mut_site_name, [mc.residue_to])

        # Handle location condition
        if agent.location is not None:
            base_agent.create_site('loc', [agent.location])

        # There might be overwrites here
        for db_name, db_ref in agent.db_refs.iteritems():
            base_agent.db_refs[db_name] = db_ref

        return base_agent

    def iteritems(self):
        """Return iteritems for the set of BaseAgents that this class wraps.
        """
        return self.agents.iteritems()

    def __getitem__(self, name):
        return self.agents[name]


class _BaseAgent(object):
    """A BaseAgent aggregates the global properties of an Agent.

    The BaseAgent class aggregates the name, sites, site states, active forms,
    inactive forms and database references of Agents from individual INDRA
    Statements. This allows the PySB Assembler to correctly assemble the
    Monomer signatures in the model.
    """

    def __init__(self, name):
        self.name = name
        self.sites = []
        self.site_states = {}
        # The list of site/state configurations that lead to this agent
        # being active (where the agent is currently assumed to have only
        # one type of activity)
        self.active_forms = []
        self.inactive_forms = []
        self.db_refs = {}

    def create_site(self, site, states=None):
        """Create a new site on an agent if it doesn't already exist."""
        if site not in self.sites:
            self.sites.append(site)
        if states is not None:
            self.site_states.setdefault(site, [])
            try:
                states = list(states)
            except TypeError:
                return
            self.add_site_states(site, states)

    def add_site_states(self, site, states):
        """Create new states on an agent site if the state doesn't exist."""
        for state in states:
            if state not in self.site_states[site]:
                self.site_states[site].append(state)

    def add_activity_form(self, activity_pattern, is_active):
        """Adds the pattern as an active or inactive form to an Agent.

        Parameters
        ----------
        activity_pattern : dict
            A dictionary of site names and their states.
        is_active : bool
            Is True if the given pattern corresponds to an active state.
        """
        if is_active:
            self.active_forms.append(activity_pattern)
        else:
            self.inactive_forms.append(activity_pattern)

# Site/state information ###############################################

abbrevs = {
    'phosphorylation': 'phospho',
    'ubiquitination': 'ub',
    'farnesylation': 'farnesyl',
    'hydroxylation': 'hydroxyl',
    'acetylation': 'acetyl',
    'sumoylation': 'sumo',
    'glycosylation': 'glycosyl',
    'methylation': 'methyl',
    'modification': 'mod',
}

active_site_names = {
    'kinase': 'kin_site',
    'phosphatase': 'phos_site',
    'gtpbound': 'switch',
    'catalytic': 'cat_site',
    'transcription': 'trans_act',
    # For general molecular activity
    'activity': 'act'
}

states = {
    'phosphorylation': ['u', 'p'],
    'ubiquitination': ['n', 'y'],
    'farnesylation': ['n', 'y'],
    'hydroxylation': ['n', 'y'],
    'acetylation': ['n', 'y'],
    'sumoylation': ['n', 'y'],
    'glycosylation': ['n', 'y'],
    'methylation': ['n', 'y'],
    'modification': ['n', 'y'],
}

# The following dict specifies the default modification/binding site names for
# modifications resulting from a particular type of activity. For example, a
# protein with Kinase activity makes a modification of type "phospho" on its
# substrate, and a RasGTPase (with GtpBound activity) binds to a site of type
# "RBD" (Ras binding domain). This comes in handy for specifying
# Activation rules, where the modification site mediating the activation
# is not specified.
default_mod_site_names = {
    'kinase': 'phospho',
    'gtpbound': 'rbd',
    'phosphatase': 'phospho',
    'activity': 'act'
}


def get_binding_site_name(name):
    """Return a binding site name from a given agent name."""
    binding_site = name.lower()
    return binding_site

def get_mod_site_name(mod_type, residue, position):
    """Return site names for a modification."""
    names = []
    if residue is None:
        mod_str = abbrevs[mod_type]
    else:
        mod_str = residue
    mod_pos = position if position is not None else ''
    name = ('%s%s' % (mod_str, mod_pos))
    return name

def get_active_forms(agent, agent_set):
    '''Returns all the patterns (dicts of site states) of an Agent
    that are known to be active.'''
    act_forms = agent_set[agent.name].active_forms
    if not act_forms:
        act_forms = [{}]
    return act_forms

def get_inactive_forms(agent, agent_set):
    '''Returns all the patterns (dicts of site states) of an Agent
    that are known to be inactive.'''
    inact_forms = agent_set[agent.name].inactive_forms
    if not inact_forms:
        inact_forms = [{}]
    return inact_forms

# PySB model elements ##################################################

def get_agent_rule_str(agent):
    """Construct a string from an Agent as part of a PySB rule name."""
    rule_str_list = [agent.name]
    for mod in agent.mods:
        mstr = abbrevs[mod.mod_type]
        if mod.residue is not None:
            mstr += mod.residue
        if mod.position is not None:
            mstr += mod.position
        rule_str_list.append('%s' % mstr)
    for mut in agent.mutations:
        if mut.residue_from is None:
            res_from = 'X'
        else:
            res_from = mut.residue_from
        mstr = res_from + mut.position
        if mut.residue_to is not None:
            mstr += mut.residue_to
        rule_str_list.append(mstr)
    if agent.bound_conditions:
        for b in agent.bound_conditions:
            if b.is_bound:
                rule_str_list.append(b.agent.name)
            else:
                rule_str_list.append('n' + b.agent.name)
    if agent.location is not None:
        rule_str_list.append(agent.location.replace(' ', '_'))
    rule_str = '_'.join(rule_str_list)
    return rule_str

def add_rule_to_model(model, rule):
    """Add a Rule to a PySB model and handle duplicate component errors."""
    try:
        model.add_component(rule)
    # If this rule is already in the model, issue a warning and continue
    except ComponentDuplicateNameError:
        msg = "Rule %s already in model! Skipping." % rule.name
        logger.warning(msg)


def get_create_parameter(model, name, value, unique=True):
    """Return parameter with given name, creating it if needed.

    If unique is false and the parameter exists, the value is not changed; if
    it does not exist, it will be created. If unique is true then upon conflict
    a number is added to the end of the parameter name.
    """
    parameter = model.parameters.get(name)

    if not unique and parameter is not None:
        return parameter

    if unique:
        pnum = 1
        while True:
            pname = name + '_%d' % pnum
            if model.parameters.get(pname) is None:
                break
            pnum += 1
    else:
        pname = name

    parameter = Parameter(pname, value)
    model.add_component(parameter)
    return parameter

def get_uncond_agent(agent):
    """Construct the unconditional state of an Agent.

    The unconditional Agent is a copy of the original agent but
    without any bound conditions and modification conditions.
    Mutation conditions, however, are preserved since they are static.
    """
    agent_uncond = ist.Agent(agent.name, mutations=agent.mutations)
    return agent_uncond

def get_monomer_pattern(model, agent, extra_fields=None):
    """Construct a PySB MonomerPattern from an Agent."""
    pattern = get_site_pattern(agent)
    if extra_fields is not None:
        for k, v in extra_fields.iteritems():
            pattern[k] = v

    # If a model is given, return the Monomer with the generated pattern,
    # otherwise just return the pattern
    monomer = model.monomers[agent.name]
    monomer_pattern = monomer(**pattern)
    return monomer_pattern

def get_site_pattern(agent):
    """Construct a dictionary of Monomer site states from an Agent.

    This crates the mapping to the associated PySB monomer from an
    INDRA Agent object."""
    pattern = {}
    # Handle bound conditions
    for bc in agent.bound_conditions:
        # Here we make the assumption that the binding site
        # is simply named after the binding partner
        if bc.is_bound:
            pattern[get_binding_site_name(bc.agent.name)] = ANY
        else:
            pattern[get_binding_site_name(bc.agent.name)] = None

    # Handle modifications
    for mod in agent.mods:
        mod_site_str = abbrevs[mod.mod_type]
        if mod.residue is not None:
            mod_site_str = mod.residue
        mod_pos_str = mod.position if mod.position is not None else ''
        mod_site = ('%s%s' % (mod_site_str, mod_pos_str))
        site_states = states[mod.mod_type]
        if mod.is_modified:
            pattern[mod_site] = site_states[1]
        else:
            pattern[mod_site] = site_states[0]

    # Handle mutations
    for mc in agent.mutations:
        if mc.residue_from is None:
            res_from = 'X'
        else:
            res_from = mc.residue_from
        mut_site_name = res_from + mc.position
        mut_site_state = mc.residue_to
        pattern[mut_site_name] = mut_site_state

    # Handle location
    if agent.location is not None:
        pattern['loc'] = agent.location

    return pattern


def set_base_initial_condition(model, monomer, value):
    """Set an initial condition for a monomer in its 'default' state."""
    # Build up monomer pattern dict
    sites_dict = {}
    for site in monomer.sites:
        if site in monomer.site_states:
            sites_dict[site] = monomer.site_states[site][0]
        else:
            sites_dict[site] = None
    mp = monomer(**sites_dict)
    pname = monomer.name + '_0'
    try:
        p = model.parameters[pname]
        p.value = value
    except KeyError:
        p = Parameter(pname, value)
        model.add_component(p)
        model.initial(mp, p)

def set_extended_initial_condition(model, monomer, value=0):
    """Set an initial condition for monomers in "modified" state.

    This is useful when using downstream analysis that relies on reactions
    being active in the model. One example is BioNetGen-based reaction network
    diagram generation.
    """
    # Build up monomer pattern dict for default state
    sites_dict = {}
    for site in monomer.sites:
        if site in monomer.site_states:
            sites_dict[site] = monomer.site_states[site][-1]
        else:
            mod_sites_dict[site] = ANY
    mp = monomer(**sites_dict)
    pname = monomer.name + '_0_mod'
    try:
        p = model.parameters[pname]
        p.value = value
    except KeyError:
        p = Parameter(pname, value)
        model.add_component(p)
        model.initial(mp, p)

def get_annotation(component, db_name, db_ref):
    """Construct model Annotations for each component.

    Annotation formats follow guidelines at http://identifiers.org/.
    """
    url = 'http://identifiers.org/'
    subj = component
    if db_name == 'UP':
        obj = url + 'uniprot/%s' % db_ref
        pred = 'is'
    elif db_name == 'HGNC':<commit_msg>Fix redifining extended initial condition in PySB assembler<commit_after>import logging
import itertools

from pysb import (Model, Monomer, Parameter, Rule, Annotation,
        ComponentDuplicateNameError, ComplexPattern, ReactionPattern, ANY,
        InvalidInitialConditionError)
from pysb.core import SelfExporter
import pysb.export

from indra import statements as ist
from indra.databases import context_client

logger = logging.getLogger('pysb_assembler')

SelfExporter.do_export = False

# Here we define the types of INDRA statements that are meant to be
# assembled using the PySB assembler. If a type of statement appears
# in this list then we require that there is at least one default
# policy implemented to assemble that type of statement.
statement_whitelist = [ist.Phosphorylation, ist.Dephosphorylation,
                       ist.SelfModification, ist.Complex,
                       ist.Activation, ist.ActiveForm,
                       ist.RasGef, ist.RasGap, ist.Translocation]

def _is_whitelisted(stmt):
    """Return True if the statement type is in the whitelist."""
    for s in statement_whitelist:
        if isinstance(stmt, s):
            return True
    return False

# BaseAgent classes ####################################################

class _BaseAgentSet(object):
    """Container for a dict of BaseAgents with their names as keys."""
    def __init__(self):
        self.agents = {}

    def get_create_base_agent(self, agent):
        """Return base agent with given name, creating it if needed."""
        try:
            base_agent = self.agents[agent.name]
        except KeyError:
            base_agent = _BaseAgent(agent.name)
            self.agents[agent.name] = base_agent

        # Handle bound conditions
        for bc in agent.bound_conditions:
            bound_base_agent = self.get_create_base_agent(bc.agent)
            bound_base_agent.create_site(get_binding_site_name(agent.name))
            base_agent.create_site(get_binding_site_name(bc.agent.name))

        # Handle modification conditions
        for mc in agent.mods:
            mod_site_name =\
                get_mod_site_name(mc.mod_type, mc.residue, mc.position)
            site_states = states[mc.mod_type]
            base_agent.create_site(mod_site_name, site_states)

        # Handle mutation conditions
        for mc in agent.mutations:
            if mc.residue_from is None:
                res_from = 'X'
            else:
                res_from = mc.residue_from
            mut_site_name = res_from + mc.position
            base_agent.create_site(mut_site_name, states=['WT'])
            if mc.residue_to is not None:
                base_agent.add_site_states(mut_site_name, [mc.residue_to])

        # Handle location condition
        if agent.location is not None:
            base_agent.create_site('loc', [agent.location])

        # There might be overwrites here
        for db_name, db_ref in agent.db_refs.iteritems():
            base_agent.db_refs[db_name] = db_ref

        return base_agent

    def iteritems(self):
        """Return iteritems for the set of BaseAgents that this class wraps.
        """
        return self.agents.iteritems()

    def __getitem__(self, name):
        return self.agents[name]


class _BaseAgent(object):
    """A BaseAgent aggregates the global properties of an Agent.

    The BaseAgent class aggregates the name, sites, site states, active forms,
    inactive forms and database references of Agents from individual INDRA
    Statements. This allows the PySB Assembler to correctly assemble the
    Monomer signatures in the model.
    """

    def __init__(self, name):
        self.name = name
        self.sites = []
        self.site_states = {}
        # The list of site/state configurations that lead to this agent
        # being active (where the agent is currently assumed to have only
        # one type of activity)
        self.active_forms = []
        self.inactive_forms = []
        self.db_refs = {}

    def create_site(self, site, states=None):
        """Create a new site on an agent if it doesn't already exist."""
        if site not in self.sites:
            self.sites.append(site)
        if states is not None:
            self.site_states.setdefault(site, [])
            try:
                states = list(states)
            except TypeError:
                return
            self.add_site_states(site, states)

    def add_site_states(self, site, states):
        """Create new states on an agent site if the state doesn't exist."""
        for state in states:
            if state not in self.site_states[site]:
                self.site_states[site].append(state)

    def add_activity_form(self, activity_pattern, is_active):
        """Adds the pattern as an active or inactive form to an Agent.

        Parameters
        ----------
        activity_pattern : dict
            A dictionary of site names and their states.
        is_active : bool
            Is True if the given pattern corresponds to an active state.
        """
        if is_active:
            self.active_forms.append(activity_pattern)
        else:
            self.inactive_forms.append(activity_pattern)

# Site/state information ###############################################

abbrevs = {
    'phosphorylation': 'phospho',
    'ubiquitination': 'ub',
    'farnesylation': 'farnesyl',
    'hydroxylation': 'hydroxyl',
    'acetylation': 'acetyl',
    'sumoylation': 'sumo',
    'glycosylation': 'glycosyl',
    'methylation': 'methyl',
    'modification': 'mod',
}

active_site_names = {
    'kinase': 'kin_site',
    'phosphatase': 'phos_site',
    'gtpbound': 'switch',
    'catalytic': 'cat_site',
    'transcription': 'trans_act',
    # For general molecular activity
    'activity': 'act'
}

states = {
    'phosphorylation': ['u', 'p'],
    'ubiquitination': ['n', 'y'],
    'farnesylation': ['n', 'y'],
    'hydroxylation': ['n', 'y'],
    'acetylation': ['n', 'y'],
    'sumoylation': ['n', 'y'],
    'glycosylation': ['n', 'y'],
    'methylation': ['n', 'y'],
    'modification': ['n', 'y'],
}

# The following dict specifies the default modification/binding site names for
# modifications resulting from a particular type of activity. For example, a
# protein with Kinase activity makes a modification of type "phospho" on its
# substrate, and a RasGTPase (with GtpBound activity) binds to a site of type
# "RBD" (Ras binding domain). This comes in handy for specifying
# Activation rules, where the modification site mediating the activation
# is not specified.
default_mod_site_names = {
    'kinase': 'phospho',
    'gtpbound': 'rbd',
    'phosphatase': 'phospho',
    'activity': 'act'
}


def get_binding_site_name(name):
    """Return a binding site name from a given agent name."""
    binding_site = name.lower()
    return binding_site

def get_mod_site_name(mod_type, residue, position):
    """Return site names for a modification."""
    names = []
    if residue is None:
        mod_str = abbrevs[mod_type]
    else:
        mod_str = residue
    mod_pos = position if position is not None else ''
    name = ('%s%s' % (mod_str, mod_pos))
    return name

def get_active_forms(agent, agent_set):
    '''Returns all the patterns (dicts of site states) of an Agent
    that are known to be active.'''
    act_forms = agent_set[agent.name].active_forms
    if not act_forms:
        act_forms = [{}]
    return act_forms

def get_inactive_forms(agent, agent_set):
    '''Returns all the patterns (dicts of site states) of an Agent
    that are known to be inactive.'''
    inact_forms = agent_set[agent.name].inactive_forms
    if not inact_forms:
        inact_forms = [{}]
    return inact_forms

# PySB model elements ##################################################

def get_agent_rule_str(agent):
    """Construct a string from an Agent as part of a PySB rule name."""
    rule_str_list = [agent.name]
    for mod in agent.mods:
        mstr = abbrevs[mod.mod_type]
        if mod.residue is not None:
            mstr += mod.residue
        if mod.position is not None:
            mstr += mod.position
        rule_str_list.append('%s' % mstr)
    for mut in agent.mutations:
        if mut.residue_from is None:
            res_from = 'X'
        else:
            res_from = mut.residue_from
        mstr = res_from + mut.position
        if mut.residue_to is not None:
            mstr += mut.residue_to
        rule_str_list.append(mstr)
    if agent.bound_conditions:
        for b in agent.bound_conditions:
            if b.is_bound:
                rule_str_list.append(b.agent.name)
            else:
                rule_str_list.append('n' + b.agent.name)
    if agent.location is not None:
        rule_str_list.append(agent.location.replace(' ', '_'))
    rule_str = '_'.join(rule_str_list)
    return rule_str

def add_rule_to_model(model, rule):
    """Add a Rule to a PySB model and handle duplicate component errors."""
    try:
        model.add_component(rule)
    # If this rule is already in the model, issue a warning and continue
    except ComponentDuplicateNameError:
        msg = "Rule %s already in model! Skipping." % rule.name
        logger.warning(msg)


def get_create_parameter(model, name, value, unique=True):
    """Return parameter with given name, creating it if needed.

    If unique is false and the parameter exists, the value is not changed; if
    it does not exist, it will be created. If unique is true then upon conflict
    a number is added to the end of the parameter name.
    """
    parameter = model.parameters.get(name)

    if not unique and parameter is not None:
        return parameter

    if unique:
        pnum = 1
        while True:
            pname = name + '_%d' % pnum
            if model.parameters.get(pname) is None:
                break
            pnum += 1
    else:
        pname = name

    parameter = Parameter(pname, value)
    model.add_component(parameter)
    return parameter

def get_uncond_agent(agent):
    """Construct the unconditional state of an Agent.

    The unconditional Agent is a copy of the original agent but
    without any bound conditions and modification conditions.
    Mutation conditions, however, are preserved since they are static.
    """
    agent_uncond = ist.Agent(agent.name, mutations=agent.mutations)
    return agent_uncond

def get_monomer_pattern(model, agent, extra_fields=None):
    """Construct a PySB MonomerPattern from an Agent."""
    pattern = get_site_pattern(agent)
    if extra_fields is not None:
        for k, v in extra_fields.iteritems():
            pattern[k] = v

    # If a model is given, return the Monomer with the generated pattern,
    # otherwise just return the pattern
    monomer = model.monomers[agent.name]
    monomer_pattern = monomer(**pattern)
    return monomer_pattern

def get_site_pattern(agent):
    """Construct a dictionary of Monomer site states from an Agent.

    This crates the mapping to the associated PySB monomer from an
    INDRA Agent object."""
    pattern = {}
    # Handle bound conditions
    for bc in agent.bound_conditions:
        # Here we make the assumption that the binding site
        # is simply named after the binding partner
        if bc.is_bound:
            pattern[get_binding_site_name(bc.agent.name)] = ANY
        else:
            pattern[get_binding_site_name(bc.agent.name)] = None

    # Handle modifications
    for mod in agent.mods:
        mod_site_str = abbrevs[mod.mod_type]
        if mod.residue is not None:
            mod_site_str = mod.residue
        mod_pos_str = mod.position if mod.position is not None else ''
        mod_site = ('%s%s' % (mod_site_str, mod_pos_str))
        site_states = states[mod.mod_type]
        if mod.is_modified:
            pattern[mod_site] = site_states[1]
        else:
            pattern[mod_site] = site_states[0]

    # Handle mutations
    for mc in agent.mutations:
        if mc.residue_from is None:
            res_from = 'X'
        else:
            res_from = mc.residue_from
        mut_site_name = res_from + mc.position
        mut_site_state = mc.residue_to
        pattern[mut_site_name] = mut_site_state

    # Handle location
    if agent.location is not None:
        pattern['loc'] = agent.location

    return pattern


def set_base_initial_condition(model, monomer, value):
    """Set an initial condition for a monomer in its 'default' state."""
    # Build up monomer pattern dict
    sites_dict = {}
    for site in monomer.sites:
        if site in monomer.site_states:
            sites_dict[site] = monomer.site_states[site][0]
        else:
            sites_dict[site] = None
    mp = monomer(**sites_dict)
    pname = monomer.name + '_0'
    try:
        p = model.parameters[pname]
        p.value = value
    except KeyError:
        p = Parameter(pname, value)
        model.add_component(p)
        model.initial(mp, p)

def set_extended_initial_condition(model, monomer, value=0):
    """Set an initial condition for monomers in "modified" state.

    This is useful when using downstream analysis that relies on reactions
    being active in the model. One example is BioNetGen-based reaction network
    diagram generation.
    """
    # Build up monomer pattern dict for default state
    sites_dict = {}
    for site in monomer.sites:
        if site in monomer.site_states:
            sites_dict[site] = monomer.site_states[site][-1]
        else:
            mod_sites_dict[site] = ANY
    mp = monomer(**sites_dict)
    pname = monomer.name + '_0_mod'
    try:
        p = model.parameters[pname]
        p.value = value
    except KeyError:
        p = Parameter(pname, value)
        model.add_component(p)
        try:
            model.initial(mp, p)
        except InvalidInitialConditionError:
            pass

def get_annotation(component, db_name, db_ref):
    """Construct model Annotations for each component.

    Annotation formats follow guidelines at http://identifiers.org/.
    """
    url = 'http://identifiers.org/'
    subj = component
    if db_name == 'UP':
        obj = url + 'uniprot/%s' % db_ref
        pred = 'is'
    elif db_name == 'HGNC':
{'fraction_non_alphanumeric': 0.09866628931262603, 'fraction_numerical': 0.0006367849435737786, 'mean_word_length': 2.565140623029386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228081', 'n_tokens_mistral': 8383, 'n_tokens_neox': 8023, 'n_words': 2585}