# Path: citation_extractor/io/iob.py
# def count_tokens(instances):
#     """Short summary.
# 
#     :param type instances: Description of parameter `instances`.
#     :return: Description of returned object.
#     :rtype: type
# 
#     """
#     return sum([1 for instance in instances for token in instance])
# 
# def file_to_instances(inp_file):
#     """Reads a IOB file a converts it into a list of instances.
# 
#     :param type inp_file: Path to the input IOB file.
#     :return: A list of tuples, where tuple[0] is the token and tuple[1]
#         contains its assigned label.
#     :rtype: list
# 
#     """
#     f = codecs.open(inp_file,"r","utf-8")
#     inp_text = f.read()
#     f.close()
#     out=[]
#     comment=re.compile(r'#.*?')
#     for i in inp_text.split("\n\n"):
#         inst=[]
#         for j in i.split("\n"):
#             if(not comment.match(j)):
#                 temp = tuple(j.split("\t"))
#                 if(len(temp)>1):
#                     inst.append(temp)
#         if(inst and len(inst)>1):
#             out.append(inst)
#     return out
# 
# def instance_to_string(instance):
#     """Converts a feature dictionary into a string representation.
# 
#     :param type instance: Description of parameter `instance`.
#     :return: Description of returned object.
#     :rtype: type
# 
#     """
#     def token_to_string(tok_dict):
#         return [tok_dict[k] for k in sorted(tok_dict.keys())]
# 
#     return [
#         "\t".join(token_to_string(feature_set))
#         for feature_set in instance
#     ]


import json
import logging
import os
import pickle
import re
import string
import types
import citation_extractor
        import re
    import glob
    import codecs
            import codecs
        import codecs
        import codecs
    import doctest
from sklearn_crfsuite import CRF
from stop_words import safe_get_stop_words
from citation_extractor.crfpp_wrap import *
from citation_extractor.io.iob import (count_tokens, file_to_instances,
                                       instance_to_string)
        from nltk.classify.scikitlearn import SklearnClassifier
        from sklearn.ensemble import AdaBoostClassifier
        from sklearn.naive_bayes import GaussianNB
        from sklearn.ensemble import RandomForestClassifier
        from citation_extractor.Utils.FastDict import LookupDictionary
        Args:
            instance:
                the instance to be classified, represented as a list of tokens.
            labels:
                ...
            outp_label:
                ...

        Example:
            >>> fe = FeatureExtractor()
            >>> test = ['cf.', 'Hom', 'Il.', '1.1', ';']
            >>> postags = [('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A')]
            >>> tmp = fe.get_features(test,outp_label=False,legacy_features=postags)
        """
        features = [self.extract_features(tok) for tok in instance]
        tok1 = features[0]
        keys = [f[0] for f in tok1]
        res = [dict(r) for r in features]
        logger = logging.getLogger('CREX.FeatureExtractor')
        for n,x in enumerate(res):
            # transform the numeric values into strings
            for key in keys:
                if(type(x[key]) is type(12)):
                    x[key] = self.feat_labels[x[key]] # get the string label corresponding to a given int value
                    #x[key] = str(x[key]) # leave the numeric feature value
            if(outp_label is True):
                x['z_gt_label']=labels[n]
        if(legacy_features is not None):
            for n,token in enumerate(res):
                token[legacy_features[n][0]] = legacy_features[n][1]

# Path: citation_extractor/io/iob.py
# def count_tokens(instances):
#     """Short summary.
# 
#     :param type instances: Description of parameter `instances`.
#     :return: Description of returned object.
#     :rtype: type
# 
#     """
#     return sum([1 for instance in instances for token in instance])
# 
# def file_to_instances(inp_file):
#     """Reads a IOB file a converts it into a list of instances.
# 
#     :param type inp_file: Path to the input IOB file.
#     :return: A list of tuples, where tuple[0] is the token and tuple[1]
#         contains its assigned label.
#     :rtype: list
# 
#     """
#     f = codecs.open(inp_file,"r","utf-8")
#     inp_text = f.read()
#     f.close()
#     out=[]
#     comment=re.compile(r'#.*?')
#     for i in inp_text.split("\n\n"):
#         inst=[]
#         for j in i.split("\n"):
#             if(not comment.match(j)):
#                 temp = tuple(j.split("\t"))
#                 if(len(temp)>1):
#                     inst.append(temp)
#         if(inst and len(inst)>1):
#             out.append(inst)
#     return out
# 
# def instance_to_string(instance):
#     """Converts a feature dictionary into a string representation.
# 
#     :param type instance: Description of parameter `instance`.
#     :return: Description of returned object.
#     :rtype: type
# 
#     """
#     def token_to_string(tok_dict):
#         return [tok_dict[k] for k in sorted(tok_dict.keys())]
# 
#     return [
#         "\t".join(token_to_string(feature_set))
#         for feature_set in instance
#     ]


# Path: citation_extractor/core.py
import json
import logging
import os
import pickle
import re
import string
import types
import citation_extractor
        import re
    import glob
    import codecs
            import codecs
        import codecs
        import codecs
    import doctest
from sklearn_crfsuite import CRF
from stop_words import safe_get_stop_words
from citation_extractor.crfpp_wrap import *
from citation_extractor.io.iob import (count_tokens, file_to_instances,
                                       instance_to_string)
        from nltk.classify.scikitlearn import SklearnClassifier
        from sklearn.ensemble import AdaBoostClassifier
        from sklearn.naive_bayes import GaussianNB
        from sklearn.ensemble import RandomForestClassifier
        from citation_extractor.Utils.FastDict import LookupDictionary

        Args:
            instance:
                the instance to be classified, represented as a list of tokens.
            labels:
                ...
            outp_label:
                ...

        Example:
            >>> fe = FeatureExtractor()
            >>> test = ['cf.', 'Hom', 'Il.', '1.1', ';']
            >>> postags = [('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A'),('z_POS','N/A')]
            >>> tmp = fe.get_features(test,outp_label=False,legacy_features=postags)
        """
        features = [self.extract_features(tok) for tok in instance]
        tok1 = features[0]
        keys = [f[0] for f in tok1]
        res = [dict(r) for r in features]
        logger = logging.getLogger('CREX.FeatureExtractor')
        for n,x in enumerate(res):
            # transform the numeric values into strings
            for key in keys:
                if(type(x[key]) is type(12)):
                    x[key] = self.feat_labels[x[key]] # get the string label corresponding to a given int value
                    #x[key] = str(x[key]) # leave the numeric feature value
            if(outp_label is True):
                x['z_gt_label']=labels[n]
        if(legacy_features is not None):
            for n,token in enumerate(res):
                token[legacy_features[n][0]] = legacy_features[n][1]

        logger.debug("\n"+"\n".join(instance_to_string(res)))