<commit_before># coding=utf-8
#
# BSD 3-Clause License
#
# Copyright (c) 2016-18, University of Liverpool
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""ContactMap container used throughout ConKit"""

from __future__ import division
from __future__ import print_function

__author__ = "<NAME>"
__date__ = "03 Aug 2016"
__version__ = "1.0"

import collections
import numpy as np
import os
import sys

if sys.version_info.major < 3:
    from itertools import izip as zip

from conkit.core._entity import _Entity
from conkit.core._struct import _Gap, _Residue
from conkit.core.mappings import AminoAcidMapping, ContactMatchState
from conkit.core.sequence import Sequence
from conkit.misc import deprecate, normalize


class ContactMap(_Entity):
    """A contact map object representing a single prediction

    The :obj:`ContactMap <conkit.core.contactmap.ContactMap>` class represents a data structure to hold a single
    contact map prediction in one place. It contains functions to store,
    manipulate and organise :obj:`Contact <conkit.core.contact.Contact>` instances.

    Attributes
    ----------
    coverage : float
       The sequence coverage score
    id : str
       A unique identifier
    ncontacts : int
       The number of :obj:`Contact <conkit.core.contact.Contact>` instances in the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    precision : float
       The precision (Positive Predictive Value) score
    repr_sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The representative :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
       <conkit.core.contactmap.ContactMap>`
    repr_sequence_altloc : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The representative altloc :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    top_contact : :obj:`Contact <conkit.core.contact.Contact>`
       The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`

    Examples
    --------
    >>> from conkit.core import Contact, ContactMap
    >>> contact_map = ContactMap("example")
    >>> contact_map.add(Contact(1, 10, 0.333))
    >>> contact_map.add(Contact(5, 30, 0.667))
    >>> print(contact_map)
    ContactMap(id="example" ncontacts=2)

    """
    __slots__ = ['_sequence']

    def __init__(self, id):
        """Initialise a new contact map"""
        self._sequence = None
        super(ContactMap, self).__init__(id)

    def __repr__(self):
        return "{0}(id=\"{1}\", ncontacts={2})".format(self.__class__.__name__, self.id, self.ncontacts)

    @property
    def coverage(self):
        """The sequence coverage score

        The coverage score is calculated by dividing the number of residues
        covered by the predicted contact pairs :math:`x_{cov}` by the number 
        of residues in the sequence :math:`L`.

        .. math::

           Coverage=\\frac{x_{cov}}{L}

        Returns
        -------
        float
           The calculated coverage score

        See Also
        --------
        precision

        """
        seq = np.array(self.repr_sequence.seq_encoded, dtype=np.int64)
        cov = seq != AminoAcidMapping["X"].value
        return np.sum(cov) / float(seq.shape[0])

    @property
    def empty(self):
        """Empty contact map"""
        return len(self) < 1

    @property
    def ncontacts(self):
        """The number of :obj:`Contact <conkit.core.contact.Contact>` instances in the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        int
           The number of contacts in the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        return len(self)

    @property
    @deprecate('0.11', msg='Use short_range instead.')
    def short_range_contacts(self):
        """The short range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.short_range

    @property
    def short_range(self):
        """The short range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Short range contacts are defined as 6 <= x <= 11 residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with short-range contacts only

        See Also
        --------
        medium_range, long_range

        """
        return self.remove_neighbors(min_distance=6, max_distance=11)

    @property
    @deprecate('0.11', msg='Use medium_range instead.')
    def medium_range_contacts(self):
        """The medium range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.medium_range

    @property
    def medium_range(self):
        """The medium range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Medium range contacts are defined as 12 <= x <= 23 residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with medium-range contacts only

        See Also
        --------
        short_range, long_range

        """
        return self.remove_neighbors(min_distance=12, max_distance=23)

    @property
    @deprecate('0.11', msg='Use long_range instead.')
    def long_range_contacts(self):
        """The long range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.long_range

    @property
    def long_range(self):
        """The long range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Long range contacts are defined as 24 <= x residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with long-range contacts only

        See Also
        --------
        short_range, medium_range

        """
        return self.remove_neighbors(min_distance=24)

    @property
    def precision(self):
        """The precision (Positive Predictive Value) score

        The precision value is calculated by analysing the true and false
        postive contacts.

        .. math::

           Precision=\\frac{TruePositives}{TruePositives - FalsePositives}

        The status of each contact, i.e true or false positive status, can be
        determined by running the :func:`match` function providing a reference
        structure.

        Returns
        -------
        float
           The calculated precision score

        See Also
        --------
        coverage

        """
        if self.empty:
            return 0.0

        import warnings

        s = np.array([c.status for c in self])
        cdict = dict(zip(np.unique(s), np.array([s[s == i].shape[0] for i in np.unique(s)])))
        fp_count = cdict[ContactMatchState.mismatched.value] if ContactMatchState.mismatched.value in cdict else 0.0
        uk_count = cdict[ContactMatchState.unknown.value] if ContactMatchState.unknown.value in cdict else 0.0
        tp_count = cdict[ContactMatchState.matched.value] if ContactMatchState.matched.value in cdict else 0.0

        if fp_count == 0.0 and tp_count == 0.0:
            warnings.warn("No matches or mismatches found in your contact map. Match two ContactMaps first.")
            return 0.0
        elif uk_count > 0:
            warnings.warn("Some contacts between the ContactMaps are unmatched due to non-identical sequences. "
                          "The precision value might be inaccurate.")

        return tp_count / (tp_count + fp_count)

    @property
    def repr_sequence(self):
        """The representative :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        The peptide sequence constructed from the available
        contacts using the normal res_seq positions

        Returns
        -------
        obj
           A :obj:`conkit.coreSequence` object

        Raises
        ------
        TypeError
           Sequence undefined

        See Also
        --------
        repr_sequence_altloc, sequence

        """
        if isinstance(self.sequence, Sequence):
            res_seqs = np.unique(np.array(self.as_list()).flatten()).tolist()
            return self._construct_repr_sequence(res_seqs)
        else:
            raise TypeError('Define the sequence as Sequence() instance')

    @property
    def repr_sequence_altloc(self):
        """The representative altloc :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        The peptide sequence constructed from the available
        contacts using the altloc res_seq positions

        Returns
        -------
        obj
           A :obj:`Sequence <conkit.core.sequence.Sequence>` object

        Raises
        ------
        ValueError
           Sequence undefined

        See Also
        --------
        repr_sequence, sequence

        """
        if isinstance(self.sequence, Sequence):
            res_seqs = np.unique(np.array(self.as_list(altloc=True)).flatten()).tolist()
            return self._construct_repr_sequence(res_seqs)
        else:
            raise TypeError('Define the sequence as Sequence() instance')

    @property
    def singletons(self):
        """Singleton contact pairs in the current :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
        
        Contacts are identified by a distance-based grouping analysis. A :obj:`Contact <conkit.core.contact.Contact>` is
        classified as singleton if not other contacts are found within 2 residues.

        Returns
        -------
        :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        from conkit.core.ext._contactmap import nb_singletons
        X = np.array(self.as_list(), dtype=np.int64)
        throwables = np.zeros(X.shape[0], dtype=np.int8)
        nb_singletons(X, 2, throwables)
        singletons = self.deepcopy()
        for i, contact in enumerate(self):
            if throwables[i]:
                singletons.remove(contact.id)
        return singletons

    @property
    def sequence(self):
        """The :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        obj
           A :obj:`Sequence <conkit.core.sequence.Sequence>` object

        See Also
        --------
        repr_sequence, repr_sequence_altloc

        """
        return self._sequence

    @sequence.setter
    def sequence(self, sequence):
        """Associate a :obj:`Sequence <conkit.core.sequence.Sequence>` instance with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        Parameters
        ----------
        sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`

        Raises
        ------
        ValueError
           Incorrect hierarchy instance provided

        """
        if isinstance(sequence, Sequence):
            self._sequence = sequence
        else:
            msg = "Instance of Sequence() required: {}".format(sequence)
            raise TypeError(msg)

    @property
    def top_contact(self):
        """The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        obj, None
           The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactFile <conkit.core.contactfile.ContactFile>`

        """
        return self.top

    def _construct_repr_sequence(self, res_seqs):
        """Construct the representative sequence"""
        representative_sequence = ''
        for i in np.arange(1, self.sequence.seq_len + 1):
            if i in res_seqs:
                representative_sequence += self.sequence.seq[i - 1]
            else:
                representative_sequence += '-'
        return Sequence(self.sequence.id + '_repr', representative_sequence)

    def as_list(self, altloc=False):
        """The :obj:`ContactMap <conkit.core.contactmap.ContactMap>` as a 2D-list containing contact-pair residue indexes

        Parameters
        ----------
        altloc : bool
           Use the res_altloc positions [default: False]

        """
        if altloc:
            return [[c.res1_altseq, c.res2_altseq] for c in self]
        else:
            return [[c.res1_seq, c.res2_seq] for c in self]

    @deprecate('0.11', msg='Use set_sequence_register instead.')
    def assign_sequence_register(self, altloc=False):
        """Assign the amino acids from :obj:`Sequence <conkit.core.sequence.Sequence>` to all :obj:`Contact <conkit.core.contact.Contact>` instances
        """
        return self.set_sequence_register(altloc=altloc)

    def set_sequence_register(self, altloc=False):
        """Assign the amino acids from :obj:`Sequence <conkit.core.sequence.Sequence>` to all :obj:`Contact <conkit.core.contact.Contact>` instances

        Parameters
        ----------
        altloc : bool
           Use the res_altloc positions [default: False]

        """
        for c in self:
            if altloc:
                res1_index = c.res1_altseq
                res2_index = c.res2_altseq
            else:
                res1_index = c.res1_seq
                res2_index = c.res2_seq
            c.res1 = self.sequence.seq[res1_index - 1]
            c.res2 = self.sequence.seq[res2_index - 1]

    @deprecate('0.11', msg='Use get_jaccard_index instead.')
    def calculate_jaccard_index(self, other):
        """Calculate the Jaccard index between two :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instances"""
        return self.get_jaccard_index(other)

    def get_jaccard_index(self, other):
        """Calculate the Jaccard index between two :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instances

        This score analyzes the difference of the predicted contacts from two maps,

        .. math::

           J_{x,y}=\\frac{\\left|x \\cap y\\right|}{\\left|x \\cup y\\right|}

        where :math:`x` and :math:`y` are the sets of predicted contacts from two
        different predictors, :math:`\\left|x \\cap y\\right|` is the number of
        elements in the intersection of :math:`x` and :math:`y`, and the
        :math:`\\left|x \\cup y\\right|` represents the number of elements in the
        union of :math:`x` and :math:`y`.

        The J-score has values in the range of :math:`[0, 1]`, with a value of :math:`1`
        corresponding to identical contact maps and :math:`0` to dissimilar ones.

        Parameters
        ----------
        other : :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
           A ConKit :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        float
           The Jaccard index

        See Also
        --------
        match, precision

        Warnings
        --------
        The Jaccard distance ranges from :math:`[0, 1]`, where :math:`1` means
        the maps contain identical contacts pairs.

        Notes
        -----
        The Jaccard index is different from the Jaccard distance mentioned in [#]_. The
        Jaccard distance corresponds to :math:`1-Jaccard_{index}`.

        .. [#] <NAME>, <NAME>, <NAME>, <NAME> (2016). A large-scale comparative assessment
           of methods for residue-residue contact prediction. *Briefings in Bioinformatics*,
           [doi: 10.1093/bib/bbw106].

        """
        union = len(self) + np.sum([1 for contact in other if contact.id not in self])
        if union == 0:
            return 1.0
        intersection = np.sum([1 for contact in self if contact.id in other])
        return float(intersection) / union

    @deprecate('0.11', msg='Use get_contact_density instead.')
    def calculate_kernel_density(self, *args, **kwargs):
        """Calculate the contact density in the contact map using Gaussian kernels"""
        return self.get_contact_density(*args, **kwargs)

    def get_contact_density(self, bw_method="amise"):
        """Calculate the contact density in the contact map using Gaussian kernels

        Various algorithms can be used to estimate the bandwidth. To calculate the
        bandwidth for an 1D data array ``X`` with ``n`` data points and ``d`` dimensions,
        the listed algorithms have been implemented. Please note, in rules 2 and 3, the
        value of :math:`\\sigma` is the smaller of the standard deviation of ``X`` or
        the normalized interquartile range.

        Parameters
        ----------
        bw_method : str, optional
           The bandwidth estimator to use [default: amise]

        Returns
        -------
        list
           The list of per-residue density estimates

        Raises
        ------
        RuntimeError
           Cannot find SciKit package
        ValueError
           Undefined bandwidth method

        """
        try:
            import sklearn.neighbors
        except ImportError:
            raise RuntimeError("Cannot find SKlearn package")

        if self.empty:
            raise ValueError("ContactMap is empty")

        x = np.array([i for c in self for i in np.arange(c.res1_seq, c.res2_seq + 1)])[:, np.newaxis]
        x_fit = np.arange(x.min(), x.max() + 1)[:, np.newaxis]
        from conkit.misc.bandwidth import bandwidth_factory
        bandwidth = bandwidth_factory(bw_method)(x).bw
        kde = sklearn.neighbors.KernelDensity(bandwidth=bandwidth).fit(x)
        return np.exp(kde.score_samples(x_fit)).tolist()

    @deprecate('0.11', msg='Use set_scalar_score instead.')
    def calculate_scalar_score(self):
        """Calculate a scaled score for the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.set_scalar_score()

    def set_scalar_score(self):
        """Calculate a scaled score for the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        This score is a scaled score for all raw scores in a contact
        map. It is defined by the formula

        .. math::

           {x}'=\\frac{x}{\\overline{d}}

        where :math:`x` corresponds to the raw score of each predicted
        contact and :math:`\\overline{d}` to the mean of all raw scores.

        The score is saved in a separate :obj:`Contact <conkit.core.contact.Contact>` attribute called ``scalar_score``

        This score is described in more detail in [#]_.

        .. [#] <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>,
           <NAME>, <NAME>, <NAME> (2015). Large-scale determination
           of previously unsolved protein structures using evolutionary information.
           *Elife* **4**, e09248.

        """
        raw_scores = np.array([c.raw_score for c in self])
        sca_scores = raw_scores / np.mean(raw_scores)
        for contact, sca_score in zip(self, sca_scores):
            contact.scalar_score = sca_score

    def find(self, register, altloc=False, strict=False):
        """Find all contacts with one or both residues in ``register``

        Parameters
        ----------
        register : int, list, tuple
           A list of residue register to find
        altloc : bool
           Use the res_altloc positions [default: False]
        strict : bool
           Both residues of :obj:`Contact <conkit.core.contact.Contact>` in register [default: False]

        Returns
        -------
        obj
           A modified version of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` containing
           the found contacts

        """
        if isinstance(register, int):
            register = [register]
        register = set(register)
        comparison_operator = _AND if strict else _OR
        contact_map = self.deepcopy()
        for contactid in self.as_list(altloc=altloc):
            if not comparison_operator(contactid[0] in register, contactid[1] in register):
                contact_map.remove(tuple(contactid))
        return contact_map

    def match(self, other, match_other=False, remove_unmatched=False, renumber=False, inplace=False):
        """Modify both hierarchies so residue numbers match one another.

        This function is key when plotting contact maps or visualising
        contact maps in 3-dimensional space. In particular, when residue
        numbers in the structure do not start at count 0 or when peptide
        chain breaks are present.

        Parameters
        ----------
        other : :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
           A ConKit :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
        match_other: bool, optional
           Match `other` to `self` [default: False]
        remove_unmatched : bool, optional
           Remove all unmatched contacts [default: False]
        renumber : bool, optional
           Renumber the res_seq entries [default: False]

           If ``True``, ``res1_seq`` and ``res2_seq`` changes
           but ``id`` remains the same
        inplace : bool, optional
           Replace the saved order of contacts [default: False]

        Returns
        -------
        obj
            :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instance, regardless of inplace

        Raises
        ------
        ValueError
           Error creating reliable keymap matching the sequence in :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        contact_map1 = self._inplace(inplace)
        if match_other:
            contact_map2 = other._inplace(inplace)
        else:
            contact_map2 = other._inplace(False)

        # ================================================================
        # 1. Align all sequences
        # ================================================================

        aligned_sequences_full = contact_map1.sequence.align_local(
            contact_map2.sequence, id_chars=2, nonid_chars=1, gap_open_pen=-0.5, gap_ext_pen=-0.1)
        contact_map1_full_sequence, contact_map2_full_sequence = aligned_sequences_full

        aligned_sequences_map1 = contact_map1_full_sequence.align_local(
            contact_map1.repr_sequence, id_chars=2, nonid_chars=1, gap_open_pen=-0.5, gap_ext_pen=-0.2, inplace=True)
        contact_map1_repr_sequence = aligned_sequences_map1[-1]

        aligned_sequences_map2 = contact_map2_full_sequence.align_local(
            contact_map2.repr_sequence_altloc,
            id_chars=2,
            nonid_chars=1,
            gap_open_pen=-0.5,
            gap_ext_pen=-0.2,
            inplace=True)
        contact_map2_repr_sequence = aligned_sequences_map2[-1]

        aligned_sequences_repr = contact_map1_repr_sequence.align_local(
            contact_map2_repr_sequence, id_chars=2, nonid_chars=1, gap_open_pen=-1.0, gap_ext_pen=-0.5, inplace=True)
        contact_map1_repr_sequence, contact_map2_repr_sequence = aligned_sequences_repr

        # ================================================================
        # 2. Identify TPs in other, map them, and match them to self
        # ================================================================

        encoded_repr = np.array(
            [list(contact_map1_repr_sequence.seq_ascii),
             list(contact_map2_repr_sequence.seq_ascii)])

        contact_map1_keymap = ContactMap._create_keymap(contact_map1)
        contact_map2_keymap = ContactMap._create_keymap(contact_map2, altloc=True)

        msg = "Error creating reliable keymap matching the sequence in ContactMap: "
        if len(contact_map1_keymap) != np.where(encoded_repr[0] != ord('-'))[0].shape[0]:
            raise ValueError(msg + contact_map1.id)
        elif len(contact_map2_keymap) != np.where(encoded_repr[1] != ord('-'))[0].shape[0]:
            raise ValueError(msg + contact_map2.id)

        contact_map1_keymap = ContactMap._insert_states(encoded_repr[0], contact_map1_keymap)
        contact_map2_keymap = ContactMap._insert_states(encoded_repr[1], contact_map2_keymap)

        contact_map1_keymap = ContactMap._reindex_by_keymap(contact_map1_keymap)
        contact_map2_keymap = ContactMap._reindex_by_keymap(contact_map2_keymap)

        contact_map2 = ContactMap._adjust(contact_map2, contact_map2_keymap)

        residues_map2 = tuple(i + 1 for i, a in enumerate(aligned_sequences_full[1].seq) if a != '-')

        for contact in contact_map1:
            _id = (contact.res1_seq, contact.res2_seq)
            _id_alt = tuple(r.res_seq for r in contact_map2_keymap for i in _id if i == r.res_altseq)

            if any(i == _Gap.IDENTIFIER for i in _id_alt) and any(j not in residues_map2 for j in _id):
                contact_map1[_id].define_unknown()
            elif all(i in residues_map2 for i in _id):
                if _id_alt in contact_map2:
                    contact_map1[_id].define_match()
                else:
                    contact_map1[_id].define_mismatch()
            else:
                raise RuntimeError("Error matching two contact maps - this should never happen")
<commit_msg>tidied match function a little<commit_after># coding=utf-8
#
# BSD 3-Clause License
#
# Copyright (c) 2016-18, University of Liverpool
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""ContactMap container used throughout ConKit"""

from __future__ import division
from __future__ import print_function

__author__ = "<NAME>"
__date__ = "03 Aug 2016"
__version__ = "1.0"

import collections
import numpy as np
import os
import sys

if sys.version_info.major < 3:
    from itertools import izip as zip

from conkit.core._entity import _Entity
from conkit.core._struct import _Gap, _Residue
from conkit.core.mappings import AminoAcidMapping, ContactMatchState
from conkit.core.sequence import Sequence
from conkit.misc import deprecate, normalize


class ContactMap(_Entity):
    """A contact map object representing a single prediction

    The :obj:`ContactMap <conkit.core.contactmap.ContactMap>` class represents a data structure to hold a single
    contact map prediction in one place. It contains functions to store,
    manipulate and organise :obj:`Contact <conkit.core.contact.Contact>` instances.

    Attributes
    ----------
    coverage : float
       The sequence coverage score
    id : str
       A unique identifier
    ncontacts : int
       The number of :obj:`Contact <conkit.core.contact.Contact>` instances in the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    precision : float
       The precision (Positive Predictive Value) score
    repr_sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The representative :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
       <conkit.core.contactmap.ContactMap>`
    repr_sequence_altloc : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The representative altloc :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`
       The :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`
    top_contact : :obj:`Contact <conkit.core.contact.Contact>`
       The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactMap 
       <conkit.core.contactmap.ContactMap>`

    Examples
    --------
    >>> from conkit.core import Contact, ContactMap
    >>> contact_map = ContactMap("example")
    >>> contact_map.add(Contact(1, 10, 0.333))
    >>> contact_map.add(Contact(5, 30, 0.667))
    >>> print(contact_map)
    ContactMap(id="example" ncontacts=2)

    """
    __slots__ = ['_sequence']

    def __init__(self, id):
        """Initialise a new contact map"""
        self._sequence = None
        super(ContactMap, self).__init__(id)

    def __repr__(self):
        return "{0}(id=\"{1}\", ncontacts={2})".format(self.__class__.__name__, self.id, self.ncontacts)

    @property
    def coverage(self):
        """The sequence coverage score

        The coverage score is calculated by dividing the number of residues
        covered by the predicted contact pairs :math:`x_{cov}` by the number 
        of residues in the sequence :math:`L`.

        .. math::

           Coverage=\\frac{x_{cov}}{L}

        Returns
        -------
        float
           The calculated coverage score

        See Also
        --------
        precision

        """
        seq = np.array(self.repr_sequence.seq_encoded, dtype=np.int64)
        cov = seq != AminoAcidMapping["X"].value
        return np.sum(cov) / float(seq.shape[0])

    @property
    def empty(self):
        """Empty contact map"""
        return len(self) < 1

    @property
    def ncontacts(self):
        """The number of :obj:`Contact <conkit.core.contact.Contact>` instances in the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        int
           The number of contacts in the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        return len(self)

    @property
    @deprecate('0.11', msg='Use short_range instead.')
    def short_range_contacts(self):
        """The short range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.short_range

    @property
    def short_range(self):
        """The short range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Short range contacts are defined as 6 <= x <= 11 residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with short-range contacts only

        See Also
        --------
        medium_range, long_range

        """
        return self.remove_neighbors(min_distance=6, max_distance=11)

    @property
    @deprecate('0.11', msg='Use medium_range instead.')
    def medium_range_contacts(self):
        """The medium range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.medium_range

    @property
    def medium_range(self):
        """The medium range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Medium range contacts are defined as 12 <= x <= 23 residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with medium-range contacts only

        See Also
        --------
        short_range, long_range

        """
        return self.remove_neighbors(min_distance=12, max_distance=23)

    @property
    @deprecate('0.11', msg='Use long_range instead.')
    def long_range_contacts(self):
        """The long range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.long_range

    @property
    def long_range(self):
        """The long range contacts found :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Long range contacts are defined as 24 <= x residues apart

        Returns
        -------
        obj
           A copy of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` with long-range contacts only

        See Also
        --------
        short_range, medium_range

        """
        return self.remove_neighbors(min_distance=24)

    @property
    def precision(self):
        """The precision (Positive Predictive Value) score

        The precision value is calculated by analysing the true and false
        postive contacts.

        .. math::

           Precision=\\frac{TruePositives}{TruePositives - FalsePositives}

        The status of each contact, i.e true or false positive status, can be
        determined by running the :func:`match` function providing a reference
        structure.

        Returns
        -------
        float
           The calculated precision score

        See Also
        --------
        coverage

        """
        if self.empty:
            return 0.0

        import warnings

        s = np.array([c.status for c in self])
        cdict = dict(zip(np.unique(s), np.array([s[s == i].shape[0] for i in np.unique(s)])))
        fp_count = cdict[ContactMatchState.mismatched.value] if ContactMatchState.mismatched.value in cdict else 0.0
        uk_count = cdict[ContactMatchState.unknown.value] if ContactMatchState.unknown.value in cdict else 0.0
        tp_count = cdict[ContactMatchState.matched.value] if ContactMatchState.matched.value in cdict else 0.0

        if fp_count == 0.0 and tp_count == 0.0:
            warnings.warn("No matches or mismatches found in your contact map. Match two ContactMaps first.")
            return 0.0
        elif uk_count > 0:
            warnings.warn("Some contacts between the ContactMaps are unmatched due to non-identical sequences. "
                          "The precision value might be inaccurate.")

        return tp_count / (tp_count + fp_count)

    @property
    def repr_sequence(self):
        """The representative :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        The peptide sequence constructed from the available
        contacts using the normal res_seq positions

        Returns
        -------
        obj
           A :obj:`conkit.coreSequence` object

        Raises
        ------
        TypeError
           Sequence undefined

        See Also
        --------
        repr_sequence_altloc, sequence

        """
        if isinstance(self.sequence, Sequence):
            res_seqs = np.unique(np.array(self.as_list()).flatten()).tolist()
            return self._construct_repr_sequence(res_seqs)
        else:
            raise TypeError('Define the sequence as Sequence() instance')

    @property
    def repr_sequence_altloc(self):
        """The representative altloc :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        The peptide sequence constructed from the available
        contacts using the altloc res_seq positions

        Returns
        -------
        obj
           A :obj:`Sequence <conkit.core.sequence.Sequence>` object

        Raises
        ------
        ValueError
           Sequence undefined

        See Also
        --------
        repr_sequence, sequence

        """
        if isinstance(self.sequence, Sequence):
            res_seqs = np.unique(np.array(self.as_list(altloc=True)).flatten()).tolist()
            return self._construct_repr_sequence(res_seqs)
        else:
            raise TypeError('Define the sequence as Sequence() instance')

    @property
    def singletons(self):
        """Singleton contact pairs in the current :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
        
        Contacts are identified by a distance-based grouping analysis. A :obj:`Contact <conkit.core.contact.Contact>` is
        classified as singleton if not other contacts are found within 2 residues.

        Returns
        -------
        :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        from conkit.core.ext._contactmap import nb_singletons
        X = np.array(self.as_list(), dtype=np.int64)
        throwables = np.zeros(X.shape[0], dtype=np.int8)
        nb_singletons(X, 2, throwables)
        singletons = self.deepcopy()
        for i, contact in enumerate(self):
            if throwables[i]:
                singletons.remove(contact.id)
        return singletons

    @property
    def sequence(self):
        """The :obj:`Sequence <conkit.core.sequence.Sequence>` associated with the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        obj
           A :obj:`Sequence <conkit.core.sequence.Sequence>` object

        See Also
        --------
        repr_sequence, repr_sequence_altloc

        """
        return self._sequence

    @sequence.setter
    def sequence(self, sequence):
        """Associate a :obj:`Sequence <conkit.core.sequence.Sequence>` instance with the :obj:`ContactMap
        <conkit.core.contactmap.ContactMap>`

        Parameters
        ----------
        sequence : :obj:`Sequence <conkit.core.sequence.Sequence>`

        Raises
        ------
        ValueError
           Incorrect hierarchy instance provided

        """
        if isinstance(sequence, Sequence):
            self._sequence = sequence
        else:
            msg = "Instance of Sequence() required: {}".format(sequence)
            raise TypeError(msg)

    @property
    def top_contact(self):
        """The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        obj, None
           The first :obj:`Contact <conkit.core.contact.Contact>` entry in :obj:`ContactFile <conkit.core.contactfile.ContactFile>`

        """
        return self.top

    def _construct_repr_sequence(self, res_seqs):
        """Construct the representative sequence"""
        representative_sequence = ''
        for i in np.arange(1, self.sequence.seq_len + 1):
            if i in res_seqs:
                representative_sequence += self.sequence.seq[i - 1]
            else:
                representative_sequence += '-'
        return Sequence(self.sequence.id + '_repr', representative_sequence)

    def as_list(self, altloc=False):
        """The :obj:`ContactMap <conkit.core.contactmap.ContactMap>` as a 2D-list containing contact-pair residue indexes

        Parameters
        ----------
        altloc : bool
           Use the res_altloc positions [default: False]

        """
        if altloc:
            return [[c.res1_altseq, c.res2_altseq] for c in self]
        else:
            return [[c.res1_seq, c.res2_seq] for c in self]

    @deprecate('0.11', msg='Use set_sequence_register instead.')
    def assign_sequence_register(self, altloc=False):
        """Assign the amino acids from :obj:`Sequence <conkit.core.sequence.Sequence>` to all :obj:`Contact <conkit.core.contact.Contact>` instances
        """
        return self.set_sequence_register(altloc=altloc)

    def set_sequence_register(self, altloc=False):
        """Assign the amino acids from :obj:`Sequence <conkit.core.sequence.Sequence>` to all :obj:`Contact <conkit.core.contact.Contact>` instances

        Parameters
        ----------
        altloc : bool
           Use the res_altloc positions [default: False]

        """
        for c in self:
            if altloc:
                res1_index = c.res1_altseq
                res2_index = c.res2_altseq
            else:
                res1_index = c.res1_seq
                res2_index = c.res2_seq
            c.res1 = self.sequence.seq[res1_index - 1]
            c.res2 = self.sequence.seq[res2_index - 1]

    @deprecate('0.11', msg='Use get_jaccard_index instead.')
    def calculate_jaccard_index(self, other):
        """Calculate the Jaccard index between two :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instances"""
        return self.get_jaccard_index(other)

    def get_jaccard_index(self, other):
        """Calculate the Jaccard index between two :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instances

        This score analyzes the difference of the predicted contacts from two maps,

        .. math::

           J_{x,y}=\\frac{\\left|x \\cap y\\right|}{\\left|x \\cup y\\right|}

        where :math:`x` and :math:`y` are the sets of predicted contacts from two
        different predictors, :math:`\\left|x \\cap y\\right|` is the number of
        elements in the intersection of :math:`x` and :math:`y`, and the
        :math:`\\left|x \\cup y\\right|` represents the number of elements in the
        union of :math:`x` and :math:`y`.

        The J-score has values in the range of :math:`[0, 1]`, with a value of :math:`1`
        corresponding to identical contact maps and :math:`0` to dissimilar ones.

        Parameters
        ----------
        other : :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
           A ConKit :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        Returns
        -------
        float
           The Jaccard index

        See Also
        --------
        match, precision

        Warnings
        --------
        The Jaccard distance ranges from :math:`[0, 1]`, where :math:`1` means
        the maps contain identical contacts pairs.

        Notes
        -----
        The Jaccard index is different from the Jaccard distance mentioned in [#]_. The
        Jaccard distance corresponds to :math:`1-Jaccard_{index}`.

        .. [#] <NAME>, <NAME>, <NAME>, <NAME> (2016). A large-scale comparative assessment
           of methods for residue-residue contact prediction. *Briefings in Bioinformatics*,
           [doi: 10.1093/bib/bbw106].

        """
        union = len(self) + np.sum([1 for contact in other if contact.id not in self])
        if union == 0:
            return 1.0
        intersection = np.sum([1 for contact in self if contact.id in other])
        return float(intersection) / union

    @deprecate('0.11', msg='Use get_contact_density instead.')
    def calculate_kernel_density(self, *args, **kwargs):
        """Calculate the contact density in the contact map using Gaussian kernels"""
        return self.get_contact_density(*args, **kwargs)

    def get_contact_density(self, bw_method="amise"):
        """Calculate the contact density in the contact map using Gaussian kernels

        Various algorithms can be used to estimate the bandwidth. To calculate the
        bandwidth for an 1D data array ``X`` with ``n`` data points and ``d`` dimensions,
        the listed algorithms have been implemented. Please note, in rules 2 and 3, the
        value of :math:`\\sigma` is the smaller of the standard deviation of ``X`` or
        the normalized interquartile range.

        Parameters
        ----------
        bw_method : str, optional
           The bandwidth estimator to use [default: amise]

        Returns
        -------
        list
           The list of per-residue density estimates

        Raises
        ------
        RuntimeError
           Cannot find SciKit package
        ValueError
           Undefined bandwidth method

        """
        try:
            import sklearn.neighbors
        except ImportError:
            raise RuntimeError("Cannot find SKlearn package")

        if self.empty:
            raise ValueError("ContactMap is empty")

        x = np.array([i for c in self for i in np.arange(c.res1_seq, c.res2_seq + 1)])[:, np.newaxis]
        x_fit = np.arange(x.min(), x.max() + 1)[:, np.newaxis]
        from conkit.misc.bandwidth import bandwidth_factory
        bandwidth = bandwidth_factory(bw_method)(x).bw
        kde = sklearn.neighbors.KernelDensity(bandwidth=bandwidth).fit(x)
        return np.exp(kde.score_samples(x_fit)).tolist()

    @deprecate('0.11', msg='Use set_scalar_score instead.')
    def calculate_scalar_score(self):
        """Calculate a scaled score for the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`"""
        return self.set_scalar_score()

    def set_scalar_score(self):
        """Calculate a scaled score for the :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        This score is a scaled score for all raw scores in a contact
        map. It is defined by the formula

        .. math::

           {x}'=\\frac{x}{\\overline{d}}

        where :math:`x` corresponds to the raw score of each predicted
        contact and :math:`\\overline{d}` to the mean of all raw scores.

        The score is saved in a separate :obj:`Contact <conkit.core.contact.Contact>` attribute called ``scalar_score``

        This score is described in more detail in [#]_.

        .. [#] <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>,
           <NAME>, <NAME>, <NAME> (2015). Large-scale determination
           of previously unsolved protein structures using evolutionary information.
           *Elife* **4**, e09248.

        """
        raw_scores = np.array([c.raw_score for c in self])
        sca_scores = raw_scores / np.mean(raw_scores)
        for contact, sca_score in zip(self, sca_scores):
            contact.scalar_score = sca_score

    def find(self, register, altloc=False, strict=False):
        """Find all contacts with one or both residues in ``register``

        Parameters
        ----------
        register : int, list, tuple
           A list of residue register to find
        altloc : bool
           Use the res_altloc positions [default: False]
        strict : bool
           Both residues of :obj:`Contact <conkit.core.contact.Contact>` in register [default: False]

        Returns
        -------
        obj
           A modified version of the :obj:`ContactMap <conkit.core.contactmap.ContactMap>` containing
           the found contacts

        """
        if isinstance(register, int):
            register = [register]
        register = set(register)
        comparison_operator = _AND if strict else _OR
        contact_map = self.deepcopy()
        for contactid in self.as_list(altloc=altloc):
            if not comparison_operator(contactid[0] in register, contactid[1] in register):
                contact_map.remove(tuple(contactid))
        return contact_map

    def match(self, other, match_other=False, remove_unmatched=False, renumber=False, inplace=False):
        """Modify both hierarchies so residue numbers match one another.

        This function is key when plotting contact maps or visualising
        contact maps in 3-dimensional space. In particular, when residue
        numbers in the structure do not start at count 0 or when peptide
        chain breaks are present.

        Parameters
        ----------
        other : :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
           A ConKit :obj:`ContactMap <conkit.core.contactmap.ContactMap>`
        match_other: bool, optional
           Match `other` to `self` [default: False]
        remove_unmatched : bool, optional
           Remove all unmatched contacts [default: False]
        renumber : bool, optional
           Renumber the res_seq entries [default: False]

           If ``True``, ``res1_seq`` and ``res2_seq`` changes
           but ``id`` remains the same
        inplace : bool, optional
           Replace the saved order of contacts [default: False]

        Returns
        -------
        obj
            :obj:`ContactMap <conkit.core.contactmap.ContactMap>` instance, regardless of inplace

        Raises
        ------
        ValueError
           Error creating reliable keymap matching the sequence in :obj:`ContactMap <conkit.core.contactmap.ContactMap>`

        """
        contact_map1 = self._inplace(inplace)
        if match_other:
            contact_map2 = other._inplace(inplace)
        else:
            contact_map2 = other._inplace(False)

        # ================================================================
        # 1. Align all sequences
        # ================================================================

        config = dict(id_chars=2, nonid_chars=1, gap_open_pen=-0.5, gap_ext_pen=-0.1, inplace=False)

        contact_map1_full_sequence, contact_map2_full_sequence = contact_map1.sequence.align_local(
            contact_map2.sequence, **config)

        config['inplace'] = True
        config['gap_ext_pen'] = -0.2
        _, contact_map1_repr_sequence = contact_map1_full_sequence.align_local(contact_map1.repr_sequence, **config)
        _, contact_map2_repr_sequence = contact_map2_full_sequence.align_local(contact_map2.repr_sequence_altloc,
                                                                               **config)

        config['gap_open_pen'] = -1.0
        config['gap_ext_pen'] = -0.5
        contact_map1_repr_sequence, contact_map2_repr_sequence = contact_map1_repr_sequence.align_local(
            contact_map2_repr_sequence, **config)

        # ================================================================
        # 2. Identify TPs in other, map them, and match them to self
        # ================================================================

        encoded_repr = np.asarray([contact_map1_repr_sequence.seq_ascii, contact_map2_repr_sequence.seq_ascii])

        contact_map1_keymap = ContactMap._create_keymap(contact_map1)
        contact_map2_keymap = ContactMap._create_keymap(contact_map2, altloc=True)

        msg = "Error creating reliable keymap matching the sequence in ContactMap: "
        if len(contact_map1_keymap) != (encoded_repr[0] != ord('-')).sum():
            raise ValueError(msg + contact_map1.id)
        elif len(contact_map2_keymap) != (encoded_repr[1] != ord('-')).sum():
            raise ValueError(msg + contact_map2.id)

        contact_map1_keymap = ContactMap._insert_states(encoded_repr[0], contact_map1_keymap)
        contact_map2_keymap = ContactMap._insert_states(encoded_repr[1], contact_map2_keymap)

        contact_map1_keymap = ContactMap._reindex_by_keymap(contact_map1_keymap)
        contact_map2_keymap = ContactMap._reindex_by_keymap(contact_map2_keymap)

        contact_map2 = ContactMap._adjust(contact_map2, contact_map2_keymap)

        residues_map2 = tuple(i + 1 for i, a in enumerate(contact_map2_full_sequence.seq) if a != '-')

        for contact in contact_map1:
            _id = (contact.res1_seq, contact.res2_seq)
            _id_alt = tuple(r.res_seq for r in contact_map2_keymap for i in _id if i == r.res_altseq)

            if any(i == _Gap.IDENTIFIER for i in _id_alt) and any(j not in residues_map2 for j in _id):
                contact_map1[_id].define_unknown()
            elif all(i in residues_map2 for i in _id):
                if _id_alt in contact_map2:
                    contact_map1[_id].define_match()
                else:
                    contact_map1[_id].define_mismatch()
            else:
                raise RuntimeError("Error matching two contact maps - this should never happen")

{'fraction_non_alphanumeric': 0.11075089481475929, 'fraction_numerical': 0.009407267207615764, 'mean_word_length': 2.4607003891050585, 'pattern_counts': {'":': 0, '<': 173, '<?xml version=': 0, '>': 191, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3068464', 'n_tokens_mistral': 15266, 'n_tokens_neox': 14297, 'n_words': 4296}