# Path: citation_extractor/settings/crf.py
# DEBUG = False
# POS = True
# LOG_FILE = ""
# DATA_DIRS = (
# pkg_resources.resource_filename('citation_extractor','data/aph_corpus/goldset/iob/'),
# )
# CLASSIFIER = None
# TEST_DIR = ()
# TRAIN_COLLECTIONS = ()
# TEST_COLLECTIONS = ()
# DATA_FILE = ""
# TEMP_DIR = ""
# OUTPUT_DIR = ""
# CROSS_VAL_FOLDS = 10
# CRFPP_TEMPLATE_DIR = pkg_resources.resource_filename('citation_extractor','crfpp_templates/')
# CRFPP_TEMPLATE = "template_5.tpl"
# CRFPP_PARAM_C = ''
# CRFPP_PARAM_A = ''
# CRFPP_PARAM_F = ''
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():
# 
# Path: citation_extractor/io/iob.py
# def read_iob_files(inp_dir, extension=".iob"):
#     instances = []
#     for infile in glob.glob( os.path.join(inp_dir, '*%s'%extension) ):
#         temp = file_to_instances(infile)
#         instances +=temp
#         logger.debug("read %i instances from file %s"%(len(temp),infile))
#     return instances
# 
# def filter_IOB(instances, tag_name):
#     """docstring for filter_IOB"""
#     out=[]
#     res=[]
#     temp = []
#     count = 0
#     for instance in instances:
#         temp = []
#         open = False
#         for i in instance:
#                 if(i[2]=='B-%s'%tag_name):
#                     temp.append(i[0])
#                     open = True
#                 elif(i[2]=='I-%s'%tag_name):
#                     if(open):
#                         temp.append(i[0])
#                 elif(i[2]=='O'):
#                     if(open):
#                         out.append(temp)
#                         temp = []
#                         open = False
#                     else:
#                         pass
#         if(len(temp) > 0 and open):
#             out.append(temp)
#     for r in out:
#         res.append(' '.join(r))
#     return res


import pickle
import logging
from citation_extractor.settings import crf
from citation_extractor import pipeline
from citation_extractor.io.iob import read_iob_files, filter_IOB
    postags = [
        [
            ("z_POS", token[1])
            for token in sentence
        ] for sentence in iob_data if len(sentence) > 0
    ]

    # put the tokens into a separate nested list
    tokens = [
        [
            token[0]
            for token in sentence
        ] for sentence in iob_data if len(sentence) > 0
    ]

    # invoke the citation extractor
    tagged_sents = crfsuite_citation_extractor.extract(tokens, postags)

    # convert the (verbose) output into an IOB structure
    output = [
        [
            (
                res[n]["token"].decode('utf-8'),
                postags[i][n][1], res[n]["label"]
            )
            for n, d_res in enumerate(res)
        ]
        for i, res in enumerate(tagged_sents)
    ]
    authors = filter_IOB(output, "AAUTHOR")

# Path: citation_extractor/settings/crf.py
# DEBUG = False
# POS = True
# LOG_FILE = ""
# DATA_DIRS = (
# pkg_resources.resource_filename('citation_extractor','data/aph_corpus/goldset/iob/'),
# )
# CLASSIFIER = None
# TEST_DIR = ()
# TRAIN_COLLECTIONS = ()
# TEST_COLLECTIONS = ()
# DATA_FILE = ""
# TEMP_DIR = ""
# OUTPUT_DIR = ""
# CROSS_VAL_FOLDS = 10
# CRFPP_TEMPLATE_DIR = pkg_resources.resource_filename('citation_extractor','crfpp_templates/')
# CRFPP_TEMPLATE = "template_5.tpl"
# CRFPP_PARAM_C = ''
# CRFPP_PARAM_A = ''
# CRFPP_PARAM_F = ''
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():
# 
# Path: citation_extractor/io/iob.py
# def read_iob_files(inp_dir, extension=".iob"):
#     instances = []
#     for infile in glob.glob( os.path.join(inp_dir, '*%s'%extension) ):
#         temp = file_to_instances(infile)
#         instances +=temp
#         logger.debug("read %i instances from file %s"%(len(temp),infile))
#     return instances
# 
# def filter_IOB(instances, tag_name):
#     """docstring for filter_IOB"""
#     out=[]
#     res=[]
#     temp = []
#     count = 0
#     for instance in instances:
#         temp = []
#         open = False
#         for i in instance:
#                 if(i[2]=='B-%s'%tag_name):
#                     temp.append(i[0])
#                     open = True
#                 elif(i[2]=='I-%s'%tag_name):
#                     if(open):
#                         temp.append(i[0])
#                 elif(i[2]=='O'):
#                     if(open):
#                         out.append(temp)
#                         temp = []
#                         open = False
#                     else:
#                         pass
#         if(len(temp) > 0 and open):
#             out.append(temp)
#     for r in out:
#         res.append(' '.join(r))
#     return res


# Path: tests/test_citation_extractor.py
import pickle
import logging
from citation_extractor.settings import crf
from citation_extractor import pipeline
from citation_extractor.io.iob import read_iob_files, filter_IOB

    postags = [
        [
            ("z_POS", token[1])
            for token in sentence
        ] for sentence in iob_data if len(sentence) > 0
    ]

    # put the tokens into a separate nested list
    tokens = [
        [
            token[0]
            for token in sentence
        ] for sentence in iob_data if len(sentence) > 0
    ]

    # invoke the citation extractor
    tagged_sents = crfsuite_citation_extractor.extract(tokens, postags)

    # convert the (verbose) output into an IOB structure
    output = [
        [
            (
                res[n]["token"].decode('utf-8'),
                postags[i][n][1], res[n]["label"]
            )
            for n, d_res in enumerate(res)
        ]
        for i, res in enumerate(tagged_sents)
    ]
    authors = filter_IOB(output, "AAUTHOR")

    works = filter_IOB(output, "AWORK")