# Path: citation_extractor/Utils/IO.py
# NIL_ENTITY = "urn:cts:GreekLatinLit:NIL"
# def init_logger(log_file=None, loglevel=logging.DEBUG):
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():


import sys
import re
import os
import codecs
	import argparse
from citation_extractor.Utils import IO
from citation_extractor.pipeline import read_ann_file
            token, tag = m.groups()

            # parse tag
            m = re.match(r'^([BIO])((?:-[A-Za-z_]+)?)$', tag)
            assert m, "ERROR: failed to parse tag '%s' in %s" % (tag, fn)
            ttag, ttype = m.groups()
            if len(ttype) > 0 and ttype[0] == "-":
                ttype = ttype[1:]

            current.append((token, ttag, ttype))

        # process leftovers, if any
        if len(current) > 0:
            sentences.append(current)
    return sentences

def get_start_end(sentences):
	# function borrowed from https://github.com/nlplab/brat/blob/master/tools/conll02tostandoff.py
	# this function returns the start and end index for each token of an IOB file.
	# in order to do so, it uses the same logic of brat's algorithm (to convert from
	# IOB to StandOff format. The tokenisation of reference is the one contained in the
	# IOB file

	# output (list of lists): 
	#		[[(u'Hesiod', 0, 6), (u'Th.', 7, 10),...]

    offset, idnum = 0, 1

    doctext = ""
    sents = []

# Path: citation_extractor/Utils/IO.py
# NIL_ENTITY = "urn:cts:GreekLatinLit:NIL"
# def init_logger(log_file=None, loglevel=logging.DEBUG):
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():


# Path: citation_extractor/data/aph_corpus/align_IOB_to_ST_annotations.py
import sys
import re
import os
import codecs
	import argparse
from citation_extractor.Utils import IO
from citation_extractor.pipeline import read_ann_file

            token, tag = m.groups()

            # parse tag
            m = re.match(r'^([BIO])((?:-[A-Za-z_]+)?)$', tag)
            assert m, "ERROR: failed to parse tag '%s' in %s" % (tag, fn)
            ttag, ttype = m.groups()
            if len(ttype) > 0 and ttype[0] == "-":
                ttype = ttype[1:]

            current.append((token, ttag, ttype))

        # process leftovers, if any
        if len(current) > 0:
            sentences.append(current)
    return sentences

def get_start_end(sentences):
	# function borrowed from https://github.com/nlplab/brat/blob/master/tools/conll02tostandoff.py
	# this function returns the start and end index for each token of an IOB file.
	# in order to do so, it uses the same logic of brat's algorithm (to convert from
	# IOB to StandOff format. The tokenisation of reference is the one contained in the
	# IOB file

	# output (list of lists): 
	#		[[(u'Hesiod', 0, 6), (u'Th.', 7, 10),...]

    offset, idnum = 0, 1

    doctext = ""
    sents = []

    for si, sentence in enumerate(sentences):