# Path: citation_extractor/Utils/IO.py
# NIL_ENTITY = "urn:cts:GreekLatinLit:NIL"
# def init_logger(log_file=None, loglevel=logging.DEBUG):
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():


import sys
import re
import os
import codecs
	import argparse
from citation_extractor.Utils import IO
from citation_extractor.pipeline import read_ann_file

        # store (token, BIO-tag, type) triples for sentence
        current = []

        lines = f.readlines()

        for ln, l in enumerate(lines):
            l = l.strip()

            if re.match(r'^\s*$', l):
                # blank lines separate sentences
                if len(current) > 0:
                    sentences.append(current)
                current = []
                continue
            elif (re.match(r'^===*\s+O\s*$', l) or
                  re.match(r'^-DOCSTART-', l)):
                # special character sequence separating documents
                if len(sentences) > 0:
                    output(fn, docnum, sentences)
                sentences = []
                docnum += 1
                continue

            if (ln + 2 < len(lines) and 
                re.match(r'^\s*$', lines[ln+1]) and
                re.match(r'^-+\s+O\s*$', lines[ln+2])):
                # heuristic match for likely doc before current line
                if len(sentences) > 0:
                    output(fn, docnum, sentences)

# Path: citation_extractor/Utils/IO.py
# NIL_ENTITY = "urn:cts:GreekLatinLit:NIL"
# def init_logger(log_file=None, loglevel=logging.DEBUG):
# 
# Path: citation_extractor/pipeline.py
# def extract_entity_mentions(text, citation_extractor, postaggers, norm=False):
# def recover_segmentation_errors(text, abbreviation_list, verbose=False):
#     def is_abbreviation(token,abbreviations):
# def get_taggers(treetagger_dir='/Applications/treetagger/cmd/', abbrev_file=None):
# def get_extractor(settings):
# def detect_language(text, return_probability=False):
# def compact_abbreviations(abbreviation_dir):
# def split_sentences(filename, outfilename=None):
# def preproc_document(doc_id, inp_dir, interm_dir, out_dir, abbreviations, taggers, split_sentences=True):
# def validate_configuration(configuration_parameters, task="all"):
#     def is_valid_configuration_ner(configuration_parameters):
#     def is_valid_configuration_relex(configuration_parameters):
#     def is_valid_configuration_ned(configuration_parameters):
# def initialize(configuration):
# def init_working_dir(path, overwrite=False):
# def find_input_documents(task, processing_dirs, doc_id):
# def do_ner(doc_id, inp_dir, interm_dir, out_dir, extractor):
# def do_relex(doc_id, input_dir, output_dir, prefix=None):
# def do_ned(doc_id, input_dir, output_dir, matcher):
# def main():


# Path: citation_extractor/data/aph_corpus/align_IOB_to_ST_annotations.py
import sys
import re
import os
import codecs
	import argparse
from citation_extractor.Utils import IO
from citation_extractor.pipeline import read_ann_file


        # store (token, BIO-tag, type) triples for sentence
        current = []

        lines = f.readlines()

        for ln, l in enumerate(lines):
            l = l.strip()

            if re.match(r'^\s*$', l):
                # blank lines separate sentences
                if len(current) > 0:
                    sentences.append(current)
                current = []
                continue
            elif (re.match(r'^===*\s+O\s*$', l) or
                  re.match(r'^-DOCSTART-', l)):
                # special character sequence separating documents
                if len(sentences) > 0:
                    output(fn, docnum, sentences)
                sentences = []
                docnum += 1
                continue

            if (ln + 2 < len(lines) and 
                re.match(r'^\s*$', lines[ln+1]) and
                re.match(r'^-+\s+O\s*$', lines[ln+2])):
                # heuristic match for likely doc before current line
                if len(sentences) > 0:
                    output(fn, docnum, sentences)

                sentences = []