<commit_before>"""
Subclassing of the xigt package to add a few convenience methods.
"""

#===============================================================================
# Logging
#===============================================================================

import logging
import re
import copy
import string

from intent.utils.logconsts import XIGT_PARSE_LOG_STR
from intent.utils.logconsts import GIZA_LOG_STR

from xigt.model import XigtCorpus, Igt, Item, Tier
from xigt.metadata import Metadata, Meta
from xigt.consts import ALIGNMENT, SEGMENTATION, CONTENT
from intent.igt.metadata import add_meta, find_meta_attr, del_meta_attr, set_intent_method, get_intent_method, \
    set_intent_proj_data
from xigt import ref





# Set up logging ---------------------------------------------------------------
from xigt.ref import ids

PARSELOG = logging.getLogger(XIGT_PARSE_LOG_STR)
GIZA_LOG = logging.getLogger(GIZA_LOG_STR)

# XIGT imports -----------------------------------------------------------------
from xigt.codecs import xigtxml

# INTERNAL imports -------------------------------------------------------------
from .igtutils import merge_lines, clean_lang_string, clean_gloss_string,\
    clean_trans_string, remove_hyphens, surrounding_quotes_and_parens, punc_re, rgencode, resolve_objects, rgp

from .consts import *

import intent.utils.token
from intent.utils.env import c
from intent.alignment.Alignment import Alignment, heur_alignments
from intent.utils.token import Token, POSToken
from intent.interfaces.giza import GizaAligner
from intent.utils.dicts import DefaultOrderedDict

# Other imports ----------------------------------------------------------------
from collections import defaultdict





#===============================================================================
# Exceptions
#===============================================================================

class RGXigtException(Exception): pass

# • Format Exceptions ------------------------------------------------------------

class XigtFormatException(RGXigtException): pass
class NoNormLineException(XigtFormatException): pass
class MultipleNormLineException(XigtFormatException): pass

class NoTransLineException(XigtFormatException): pass
class NoLangLineException(XigtFormatException):	pass
class NoGlossLineException(XigtFormatException): pass

class NoODINRawException(XigtFormatException):	pass

# • Alignment and Projection Exceptions ------------------------------------------

class GlossLangAlignException(RGXigtException):	pass
class ProjectionException(RGXigtException): pass
class ProjectionTransGlossException(ProjectionException): pass
class PhraseStructureProjectionException(RGXigtException): pass

class AlignerError(Exception): pass
class GizaAlignerError(Exception): pass

def project_creator_except(msg_start, msg_end, created_by):

    if created_by:
        msg_start += ' by the creator "%s".' % created_by
    else:
        msg_start += '.'
    raise ProjectionException(msg_start + ' ' + msg_end)

#===============================================================================
# Mixins
#===============================================================================



def ref_match(o, target_ref, ref_type):
    if hasattr(o, ref_type):
        my_ref = getattr(o, ref_type)
        if my_ref and target_ref in ref.ids(my_ref):
            return True
    return False

def seg_match(seg): return lambda o: ref_match(o, seg, SEGMENTATION)
def cnt_match(cnt): return lambda o: ref_match(o, cnt, CONTENT)
def aln_match(aln): return lambda o: ref_match(o, aln, ALIGNMENT)

def type_match(type): return lambda o: o.type == type
def id_match(id): return lambda o: o.id == id
def id_base_match(id_base): return lambda o: get_id_base(o.id) == id_base
def attr_match(attr): return lambda o: set(attr.items()).issubset(set(o.attributes.items()))

class FindMixin():
    """
    Extension of the recursive search for non-iterable elements.
    """

    # FindMixin objects should have an index.
    index = None

    def find_self(self, filters=list):
        """
        Check to see if this object matches all of the filter functions in filters.

        :param filters: List of functions to apply to this object. All filters have a logical and
                        applied to them.
        :type filters: list
        """

        assert len(filters) > 0, "Must have selected some attribute to filter."

        # Iterate through the filters...
        for filter in filters:
            if not filter(self): # If one evaluates to false...
                return None      # ..we're done. Exit with "None"

        # If we make it through all the iteration, we're a match. Return.
        return self

    def _build_filterlist(self, **kwargs):
        filters = []
        for kw, val in kwargs.items():
            if kw == 'id':
                filters += [id_match(val)]
            elif kw == 'content':
                filters += [cnt_match(val)]
            elif kw == 'segmentation':
                filters += [seg_match(val)]
            elif kw == 'id_base':
                filters += [id_base_match(val)]
            elif kw == 'attributes':
                filters += [attr_match(val)]
            elif kw == 'type':
                filters += [type_match(val)]

            elif kw == 'others': # Append any other filters...
                filters += val
            else:
                raise ValueError('Invalid keyword argument "%s"' % kw)
        return filters

    def find(self, **kwargs):
        return self.find_self(self._build_filterlist(**kwargs))

    def findall(self, **kwargs):
        found = self.find_self(self._build_filterlist(**kwargs))
        if found:
            return [found]
        else:
            return []

class RecursiveFindMixin(FindMixin):

    # Define the FindMixin as iterable,
    # but don't implement the class.
    def __iter__(self): pass

    def find(self, **kwargs):
        """
        Generic find function for non-iterable elements. NOTE: This version stops on the first match.

        :param id: id of an element to find, or None to search by attribute.
        :type id: str
        :param attributes: key:value pairs that are an inclusive subset of those found in the desired item.
        :type attributes: dict
        """
        if super().find(**kwargs) is not None:
            return self
        else:
            found = None
            for child in self:
                found = child.find(**kwargs)
                if found is not None:
                    break
            return found

    def findall(self, **kwargs):
        """
        Find function that does not terminate on the first match.
        """
        found = []
        if super().find(**kwargs) is not None:
            found = [self]

        for child in self:
            found += child.findall(**kwargs)

        return found


#===============================================================================
# • Parse Tree Functions ---
#===============================================================================
def read_pt(tier):

    # Assume that we are reading from a phrase structure tier.
    assert tier.type == PS_TIER_TYPE

    # Provide a way to look up the nodes by their ID so we can
    # pair them directly later...
    node_dict = {}

    # Also, keep track of the child-parent relationships that need
    # to be constructed.
    children_dict = defaultdict(list)

    for node in tier:

        # 1) If the node has an alignment, that means it's a terminal ----------
        aln = node.attributes.get(ALIGNMENT)
        if aln:
            w = tier.igt.find(id=aln)
            n = IdTree(node.value(), [Terminal(w.value(), w.index)])

            # If this is a preterminal, it shouldn't have children.
            assert not node.attributes.get(PS_CHILD_ATTRIBUTE)


        else:
            n = IdTree(node.value(), [])

            # 2) If there is a "children" attribute, split it on whitespace and store ---
            #    those IDs to revisit, with the current node as the parent.
            childids = node.attributes.get(PS_CHILD_ATTRIBUTE, '').split()

            for childid in childids:
                children_dict[node.id].append(childid)


        node_dict[node.id] = n


    # 3) Revisit the children and make the linkages.
    child_n = None
    for parent_id in children_dict.keys():
        parent_n = node_dict[parent_id]
        for child_id in children_dict[parent_id]:
            child_n = node_dict[child_id]
            parent_n.append(child_n)


    # Finally, pick an arbitrary node, and try to find the root.
    assert child_n, "There should have been at least one child found..."

    return child_n.root()


def gen_item_id(id_base, num):
    return '{}{}'.format(id_base, num+1)

def gen_tier_id(inst, id_base, tier_type=None, alignment=None, no_hyphenate=False):
    """
    Unified method to generate a tier ID string. (See: https://github.com/goodmami/xigt/wiki/Conventions)
    """

    # In order to number this item correctly, we need to decide how many tiers of the same type
    # there are. This is done by systematically adding filters to the list.
    filters = []

    # First, do we align with another item? (Either segmentation, alignment, or head/dep)
    if alignment is not None:
        filters.append(lambda x: aln_match(alignment)(x) or seg_match(alignment)(x) or ref_match(x, alignment, DS_HEAD_ATTRIBUTE))

    # Next, does the type match ours?
    if tier_type is not None:
        filters.append(type_match(tier_type))

    # Get the number of tiers that match this.
    num_tiers = len(inst.findall(others=filters))


    id_str = id_base
    # Now, if we have specified the alignment, we also want to prepend
    # that to the generated id string.
    if alignment is not None:
        if no_hyphenate:
            return '{}{}'.format(alignment, id_str)
        else:
            id_str = '{}-{}'.format(alignment, id_str)

    # Finally, if we have multiple tiers of the same type that annotate the
    # same item, we should append a letter for the different analyses.
    if num_tiers > 0:
        letters = string.ascii_lowercase
        assert num_tiers < 26, "More than 26 alternative analyses not currently supported"
        id_str += '_{}'.format(letters[num_tiers])

    return id_str


def get_id_base(id_str):
    """
    Return the "base" of the id string. This should either be everything leading up to the final numbering, or a hyphen-separated letter.

    :param id_str:
    :type id_str:
    """
    s = re.search('^(\S+?)(?:[0-9]+|-[a-z])?$', id_str).group(1)
    return s

# ===============================================================================


class RGCorpus(XigtCorpus, RecursiveFindMixin):

    def askIgtId(self):
        return gen_item_id('i', len(self.igts))

    def __len__(self):
        return len(self._list)

    def copy(self, limit=None):
        new_c = RGCorpus(id=self.id, attributes=copy.deepcopy(self.attributes), metadata=copy.copy(self.metadata), igts=None)

        for i, igt in enumerate(self.igts):
            new_c.append(igt.copy(parent=new_c))

            if limit and i >= limit:
                break

        return new_c


    @classmethod
    def from_txt(cls, text, require_trans = True, require_gloss = True, require_lang = True, require_1_to_1 = True, limit = None):
        """
        Read in a odin-style textfile to create the xigt corpus.

        """
        # Initialize the corpus
        xc = cls()

        # Replace invalid characters...
        _illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')
        data = re.sub(_illegal_xml_chars_RE, ' ', text)

        # Read all the text lines
        inst_txts = re.findall('doc_id=[\s\S]+?\n\n', data)

        #=======================================================================
        # Begin parsing...
        #=======================================================================

        parsed = 0
        PARSELOG.info('Beginning parse')
        for inst_num, inst_txt in enumerate(inst_txts):

            if parsed % 250 == 0:
                PARSELOG.info('Parsing instance %d...' % parsed)
                pass

            # Handle the requirement for 1_to_1 alignment.
            try:
                i = RGIgt.fromString(inst_txt, corpus=xc, require_1_to_1=require_1_to_1, idnum=inst_num)
            except GlossLangAlignException as glae:
                PARSELOG.warn('Gloss and language could not be automatically aligned for instance "%s". Skipping' % gen_item_id('i', inst_num))
                continue

            # Try to get the translation line. ---------------------------------
            try:
                hastrans = i.trans
            except NoTransLineException as ntle:
                PARSELOG.info(ntle)
                hastrans = False

            # Try to get the gloss line. --------------------------------------
            try:
                hasgloss = i.gloss
            except NoGlossLineException as ngle:
                PARSELOG.info(ngle)
                hasgloss = False

            # Try to get the language line. ------------------------------------
            try:
                haslang = i.lang
            except NoLangLineException as nlle:
                PARSELOG.info(nlle)
                haslang = False


            parsed +=1


            trans_constraint = (hastrans and require_trans) or (not require_trans)
            gloss_constraint = (hasgloss and require_gloss) or (not require_gloss)
            lang_constraint  = (haslang  and require_lang)  or (not require_lang)

            if trans_constraint and gloss_constraint and lang_constraint:
                xc.append(i)
            else:
                PARSELOG.info('Requirements for instance "%s" were not satisfied. Skipping' % i.id)

            # If we have reached the limit of instances that have been requested,
            # stop processing.
            if limit is not None and limit == parsed: break



        # Return the corpus
        return xc


    @classmethod
    def loads(cls, s):
        xc = xigtxml.loads(s)
        xc.__class__ = RGCorpus
        xc._finish_load()
        return xc

    @classmethod
    def load(cls, path, basic_processing = False):
        xc = xigtxml.load(path)
        xc.__class__ = RGCorpus
        xc._finish_load()

        # If asked, we will also do some
        # basic-level enrichment...
        if basic_processing:
            for inst in xc:
                try:
                    inst.basic_processing()
                except XigtFormatException as xfe:
                    PARSELOG.warn("Basic processing failed for instance {}".format(inst.id))
                except GlossLangAlignException as glae:
                    PARSELOG.warn(glae)
                    continue

                else:
                    try:
                        inst.add_gloss_lang_alignments()
                    except GlossLangAlignException as glae:
                        PARSELOG.warn(glae)
                    except (NoGlossLineException, NoLangLineException) as ngle:
                        PARSELOG.warn(ngle)
                        continue

        return xc

    def _finish_load(self):
        # Now, convert all the IGT instances to RGIgt instances.
        for igt in self.igts:
            igt.__class__ = RGIgt

            for tier in igt.tiers:
                tier.__class__ = RGTier

                for i, item in enumerate(tier):
                    item.__class__ = RGItem
                    item.index = i+1


    def filter(self, attr):
        new_igts = []
        for i in self:
            try:
                tier = getattr(i, attr)
            except XigtFormatException as tpe:
                PARSELOG.info(tpe)
            else:
                new_igts.append(i)

        self.igts = new_igts

    def require_trans_lines(self):
        self.filter('trans')<commit_msg>Adding exception handling for the XigtError when attempting to add glosses... Should be fixed.<commit_after>"""
Subclassing of the xigt package to add a few convenience methods.
"""

#===============================================================================
# Logging
#===============================================================================

import logging
import re
import copy
import string

from intent.utils.logconsts import XIGT_PARSE_LOG_STR
from intent.utils.logconsts import GIZA_LOG_STR
from xigt.errors import XigtError

from xigt.model import XigtCorpus, Igt, Item, Tier
from xigt.metadata import Metadata, Meta
from xigt.consts import ALIGNMENT, SEGMENTATION, CONTENT
from intent.igt.metadata import add_meta, find_meta_attr, del_meta_attr, set_intent_method, get_intent_method, \
    set_intent_proj_data
from xigt import ref





# Set up logging ---------------------------------------------------------------
from xigt.ref import ids

PARSELOG = logging.getLogger(XIGT_PARSE_LOG_STR)
GIZA_LOG = logging.getLogger(GIZA_LOG_STR)

# XIGT imports -----------------------------------------------------------------
from xigt.codecs import xigtxml

# INTERNAL imports -------------------------------------------------------------
from .igtutils import merge_lines, clean_lang_string, clean_gloss_string,\
    clean_trans_string, remove_hyphens, surrounding_quotes_and_parens, punc_re, rgencode, resolve_objects, rgp

from .consts import *

import intent.utils.token
from intent.utils.env import c
from intent.alignment.Alignment import Alignment, heur_alignments
from intent.utils.token import Token, POSToken
from intent.interfaces.giza import GizaAligner
from intent.utils.dicts import DefaultOrderedDict

# Other imports ----------------------------------------------------------------
from collections import defaultdict





#===============================================================================
# Exceptions
#===============================================================================

class RGXigtException(Exception): pass

# • Format Exceptions ------------------------------------------------------------

class XigtFormatException(RGXigtException): pass
class NoNormLineException(XigtFormatException): pass
class MultipleNormLineException(XigtFormatException): pass

class NoTransLineException(XigtFormatException): pass
class NoLangLineException(XigtFormatException):	pass
class NoGlossLineException(XigtFormatException): pass

class NoODINRawException(XigtFormatException):	pass

# • Alignment and Projection Exceptions ------------------------------------------

class GlossLangAlignException(RGXigtException):	pass
class ProjectionException(RGXigtException): pass
class ProjectionTransGlossException(ProjectionException): pass
class PhraseStructureProjectionException(RGXigtException): pass

class AlignerError(Exception): pass
class GizaAlignerError(Exception): pass

def project_creator_except(msg_start, msg_end, created_by):

    if created_by:
        msg_start += ' by the creator "%s".' % created_by
    else:
        msg_start += '.'
    raise ProjectionException(msg_start + ' ' + msg_end)

#===============================================================================
# Mixins
#===============================================================================



def ref_match(o, target_ref, ref_type):
    if hasattr(o, ref_type):
        my_ref = getattr(o, ref_type)
        if my_ref and target_ref in ref.ids(my_ref):
            return True
    return False

def seg_match(seg): return lambda o: ref_match(o, seg, SEGMENTATION)
def cnt_match(cnt): return lambda o: ref_match(o, cnt, CONTENT)
def aln_match(aln): return lambda o: ref_match(o, aln, ALIGNMENT)

def type_match(type): return lambda o: o.type == type
def id_match(id): return lambda o: o.id == id
def id_base_match(id_base): return lambda o: get_id_base(o.id) == id_base
def attr_match(attr): return lambda o: set(attr.items()).issubset(set(o.attributes.items()))

class FindMixin():
    """
    Extension of the recursive search for non-iterable elements.
    """

    # FindMixin objects should have an index.
    index = None

    def find_self(self, filters=list):
        """
        Check to see if this object matches all of the filter functions in filters.

        :param filters: List of functions to apply to this object. All filters have a logical and
                        applied to them.
        :type filters: list
        """

        assert len(filters) > 0, "Must have selected some attribute to filter."

        # Iterate through the filters...
        for filter in filters:
            if not filter(self): # If one evaluates to false...
                return None      # ..we're done. Exit with "None"

        # If we make it through all the iteration, we're a match. Return.
        return self

    def _build_filterlist(self, **kwargs):
        filters = []
        for kw, val in kwargs.items():
            if kw == 'id':
                filters += [id_match(val)]
            elif kw == 'content':
                filters += [cnt_match(val)]
            elif kw == 'segmentation':
                filters += [seg_match(val)]
            elif kw == 'id_base':
                filters += [id_base_match(val)]
            elif kw == 'attributes':
                filters += [attr_match(val)]
            elif kw == 'type':
                filters += [type_match(val)]

            elif kw == 'others': # Append any other filters...
                filters += val
            else:
                raise ValueError('Invalid keyword argument "%s"' % kw)
        return filters

    def find(self, **kwargs):
        return self.find_self(self._build_filterlist(**kwargs))

    def findall(self, **kwargs):
        found = self.find_self(self._build_filterlist(**kwargs))
        if found:
            return [found]
        else:
            return []

class RecursiveFindMixin(FindMixin):

    # Define the FindMixin as iterable,
    # but don't implement the class.
    def __iter__(self): pass

    def find(self, **kwargs):
        """
        Generic find function for non-iterable elements. NOTE: This version stops on the first match.

        :param id: id of an element to find, or None to search by attribute.
        :type id: str
        :param attributes: key:value pairs that are an inclusive subset of those found in the desired item.
        :type attributes: dict
        """
        if super().find(**kwargs) is not None:
            return self
        else:
            found = None
            for child in self:
                found = child.find(**kwargs)
                if found is not None:
                    break
            return found

    def findall(self, **kwargs):
        """
        Find function that does not terminate on the first match.
        """
        found = []
        if super().find(**kwargs) is not None:
            found = [self]

        for child in self:
            found += child.findall(**kwargs)

        return found


#===============================================================================
# • Parse Tree Functions ---
#===============================================================================
def read_pt(tier):

    # Assume that we are reading from a phrase structure tier.
    assert tier.type == PS_TIER_TYPE

    # Provide a way to look up the nodes by their ID so we can
    # pair them directly later...
    node_dict = {}

    # Also, keep track of the child-parent relationships that need
    # to be constructed.
    children_dict = defaultdict(list)

    for node in tier:

        # 1) If the node has an alignment, that means it's a terminal ----------
        aln = node.attributes.get(ALIGNMENT)
        if aln:
            w = tier.igt.find(id=aln)
            n = IdTree(node.value(), [Terminal(w.value(), w.index)])

            # If this is a preterminal, it shouldn't have children.
            assert not node.attributes.get(PS_CHILD_ATTRIBUTE)


        else:
            n = IdTree(node.value(), [])

            # 2) If there is a "children" attribute, split it on whitespace and store ---
            #    those IDs to revisit, with the current node as the parent.
            childids = node.attributes.get(PS_CHILD_ATTRIBUTE, '').split()

            for childid in childids:
                children_dict[node.id].append(childid)


        node_dict[node.id] = n


    # 3) Revisit the children and make the linkages.
    child_n = None
    for parent_id in children_dict.keys():
        parent_n = node_dict[parent_id]
        for child_id in children_dict[parent_id]:
            child_n = node_dict[child_id]
            parent_n.append(child_n)


    # Finally, pick an arbitrary node, and try to find the root.
    assert child_n, "There should have been at least one child found..."

    return child_n.root()


def gen_item_id(id_base, num):
    return '{}{}'.format(id_base, num+1)

def gen_tier_id(inst, id_base, tier_type=None, alignment=None, no_hyphenate=False):
    """
    Unified method to generate a tier ID string. (See: https://github.com/goodmami/xigt/wiki/Conventions)
    """

    # In order to number this item correctly, we need to decide how many tiers of the same type
    # there are. This is done by systematically adding filters to the list.
    filters = []

    # First, do we align with another item? (Either segmentation, alignment, or head/dep)
    if alignment is not None:
        filters.append(lambda x: aln_match(alignment)(x) or seg_match(alignment)(x) or ref_match(x, alignment, DS_HEAD_ATTRIBUTE))

    # Next, does the type match ours?
    if tier_type is not None:
        filters.append(type_match(tier_type))

    # Get the number of tiers that match this.
    num_tiers = len(inst.findall(others=filters))


    id_str = id_base
    # Now, if we have specified the alignment, we also want to prepend
    # that to the generated id string.
    if alignment is not None:
        if no_hyphenate:
            return '{}{}'.format(alignment, id_str)
        else:
            id_str = '{}-{}'.format(alignment, id_str)

    # Finally, if we have multiple tiers of the same type that annotate the
    # same item, we should append a letter for the different analyses.
    if num_tiers > 0:
        letters = string.ascii_lowercase
        assert num_tiers < 26, "More than 26 alternative analyses not currently supported"
        id_str += '_{}'.format(letters[num_tiers])

    return id_str


def get_id_base(id_str):
    """
    Return the "base" of the id string. This should either be everything leading up to the final numbering, or a hyphen-separated letter.

    :param id_str:
    :type id_str:
    """
    s = re.search('^(\S+?)(?:[0-9]+|-[a-z])?$', id_str).group(1)
    return s

# ===============================================================================


class RGCorpus(XigtCorpus, RecursiveFindMixin):

    def askIgtId(self):
        return gen_item_id('i', len(self.igts))

    def __len__(self):
        return len(self._list)

    def copy(self, limit=None):
        new_c = RGCorpus(id=self.id, attributes=copy.deepcopy(self.attributes), metadata=copy.copy(self.metadata), igts=None)

        for i, igt in enumerate(self.igts):
            new_c.append(igt.copy(parent=new_c))

            if limit and i >= limit:
                break

        return new_c


    @classmethod
    def from_txt(cls, text, require_trans = True, require_gloss = True, require_lang = True, require_1_to_1 = True, limit = None):
        """
        Read in a odin-style textfile to create the xigt corpus.

        """
        # Initialize the corpus
        xc = cls()

        # Replace invalid characters...
        _illegal_xml_chars_RE = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')
        data = re.sub(_illegal_xml_chars_RE, ' ', text)

        # Read all the text lines
        inst_txts = re.findall('doc_id=[\s\S]+?\n\n', data)

        #=======================================================================
        # Begin parsing...
        #=======================================================================

        parsed = 0
        PARSELOG.info('Beginning parse')
        for inst_num, inst_txt in enumerate(inst_txts):

            if parsed % 250 == 0:
                PARSELOG.info('Parsing instance %d...' % parsed)
                pass

            # Handle the requirement for 1_to_1 alignment.
            try:
                i = RGIgt.fromString(inst_txt, corpus=xc, require_1_to_1=require_1_to_1, idnum=inst_num)
            except GlossLangAlignException as glae:
                PARSELOG.warn('Gloss and language could not be automatically aligned for instance "%s". Skipping' % gen_item_id('i', inst_num))
                continue

            # Try to get the translation line. ---------------------------------
            try:
                hastrans = i.trans
            except NoTransLineException as ntle:
                PARSELOG.info(ntle)
                hastrans = False

            # Try to get the gloss line. --------------------------------------
            try:
                hasgloss = i.gloss
            except NoGlossLineException as ngle:
                PARSELOG.info(ngle)
                hasgloss = False

            # Try to get the language line. ------------------------------------
            try:
                haslang = i.lang
            except NoLangLineException as nlle:
                PARSELOG.info(nlle)
                haslang = False


            parsed +=1


            trans_constraint = (hastrans and require_trans) or (not require_trans)
            gloss_constraint = (hasgloss and require_gloss) or (not require_gloss)
            lang_constraint  = (haslang  and require_lang)  or (not require_lang)

            if trans_constraint and gloss_constraint and lang_constraint:
                xc.append(i)
            else:
                PARSELOG.info('Requirements for instance "%s" were not satisfied. Skipping' % i.id)

            # If we have reached the limit of instances that have been requested,
            # stop processing.
            if limit is not None and limit == parsed: break



        # Return the corpus
        return xc


    @classmethod
    def loads(cls, s):
        xc = xigtxml.loads(s)
        xc.__class__ = RGCorpus
        xc._finish_load()
        return xc

    @classmethod
    def load(cls, path, basic_processing = False):
        xc = xigtxml.load(path)
        xc.__class__ = RGCorpus
        xc._finish_load()

        # If asked, we will also do some
        # basic-level enrichment...
        if basic_processing:
            for inst in xc:
                try:
                    inst.basic_processing()
                except XigtFormatException as xfe:
                    PARSELOG.warn("Basic processing failed for instance {}".format(inst.id))
                except GlossLangAlignException as glae:
                    PARSELOG.warn(glae)
                    continue

                else:
                    try:
                        inst.add_gloss_lang_alignments()
                    except GlossLangAlignException as glae:
                        PARSELOG.warn(glae)
                    except (NoGlossLineException, NoLangLineException) as ngle:
                        PARSELOG.warn(ngle)
                        continue

                    # TODO: Figure out why we are getting "g" already in collection.
                    # But for now, just skip it.
                    except XigtError as xe:
                        PARSELOG.warn(xe)

        return xc

    def _finish_load(self):
        # Now, convert all the IGT instances to RGIgt instances.
        for igt in self.igts:
            igt.__class__ = RGIgt

            for tier in igt.tiers:
                tier.__class__ = RGTier

                for i, item in enumerate(tier):
                    item.__class__ = RGItem
                    item.index = i+1


    def filter(self, attr):
        new_igts = []
        for i in self:
            try:
                tier = getattr(i, attr)
            except XigtFormatException as tpe:
                PARSELOG.info(tpe)
            else:
                new_igts.append(i)

        self.igts = new_igts

    def require_trans_lines(self):
        self.filter('trans')
{'fraction_non_alphanumeric': 0.16052161045687158, 'fraction_numerical': 0.0024045872125285158, 'mean_word_length': 2.158924919661116, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2183981', 'n_tokens_mistral': 9068, 'n_tokens_neox': 8509, 'n_words': 2684}