# Path: citation_extractor/io/brat.py
# def read_ann_file(fileid, ann_dir, suffix="-doc-1.ann"):
#     """Reads annotations from a brat standoff file.
# 
#     :param fileid:
#     :type fileid:
#     :param ann_dir:
#     :type ann_dir:
#     :returns:
#     :rtype: tuple
#     """
#     ann_file = "%s%s%s"%(ann_dir,fileid,suffix)
#     f = codecs.open(ann_file,'r','utf-8')
#     data = f.read()
#     f.close()
#     rows = data.split('\n')
#     entities = {}
#     ent_count = 0
#     relations = {}
#     annotations = []
#     for row in rows:
#         cols = row.split("\t")
#         ann_id = cols[0]
# 
#         if(u"#" in cols[0]):
#             # it's a text annotation
#             tmp = {
#                 "ann_id":"%s%s"%(cols[1].split()[0],cols[0])
#                 ,"anchor":cols[1].split()[1:][0]
#                 ,"text":cols[2]
#             }
#             annotations.append(tmp)
# 
#         elif(len(cols)==3 and u"T" in cols[0]):
#             # it's an entity
#             ent_count += 1
#             ent_type = cols[1].split()[0]
#             ranges = cols[1].replace("%s"%ent_type,"")
#             entities[cols[0]] = {"ann_id":ann_id
#                                 ,"entity_type": ent_type
#                                 ,"offset_start":ranges.split()[0]
#                                 ,"offset_end":ranges.split()[1]
#                                 ,"surface":cols[2]}
# 
#         elif(len(cols)>=2 and u"R" in cols[0]):
#             # it's a relation
#             rel_type, arg1, arg2 = cols[1].split()
#             relations[cols[0]] = {
#                 "ann_id": ann_id,
#                 "arguments": (arg1.split(":")[1], arg2.split(":")[1]),
#                 "relation_type":rel_type
#             }
# 
#     return entities, relations, annotations
# 
# Path: citation_extractor/io/iob.py
# def file_to_instances(inp_file):
#     """Reads a IOB file a converts it into a list of instances.
# 
#     :param type inp_file: Path to the input IOB file.
#     :return: A list of tuples, where tuple[0] is the token and tuple[1]
#         contains its assigned label.
#     :rtype: list
# 
#     """
#     f = codecs.open(inp_file,"r","utf-8")
#     inp_text = f.read()
#     f.close()
#     out=[]
#     comment=re.compile(r'#.*?')
#     for i in inp_text.split("\n\n"):
#         inst=[]
#         for j in i.split("\n"):
#             if(not comment.match(j)):
#                 temp = tuple(j.split("\t"))
#                 if(len(temp)>1):
#                     inst.append(temp)
#         if(inst and len(inst)>1):
#             out.append(inst)
#     return out
# 
# def write_iob_file(instances, dest_file):
#     """Write a set of instances to an IOB file.
# 
#     :param list instances: Description of parameter `instances`.
#     :param str dest_file: Description of parameter `dest_file`.
#     :return: Description of returned object.
#     :rtype: boolean
# 
#     """
#     to_write = "\n\n".join(
#         [
#             "\n".join(["\t".join(token) for token in instance])
#             for instance in instances
#         ]
#     )
#     try:
#         with codecs.open(dest_file, 'w', 'utf-8') as f:
#             f.write(to_write)
#         return True
#     except Exception, e:
#         raise e
# 
# Path: citation_extractor/io/brat.py
# def _find_newlines(text, newline=u'\n'):
#     """
#     TODO
#     """
#     positions = []
#     last_position = 0
#     if(text.find(newline) == -1):
#         return positions
#     else:
#         while(text.find(newline, last_position + 1) > -1):
#             last_position = text.find(newline, last_position + 1)
#             positions.append((last_position, last_position + len(newline)))
#         return positions


from docopt import docopt
from knowledge_base import KnowledgeBase
from citation_extractor.io.brat import read_ann_file
from citation_extractor.io.iob import file_to_instances, write_iob_file
from citation_extractor.io.brat import _find_newlines
from pyCTS import CTS_URN
import logging
import codecs
import json
import os
import re
            for sentence_n, sentence in enumerate(self._iob_standoff)
            for token_n, token in enumerate(sentence)
        ]

        if self._standoff_dir is not None:
            so_entities, so_relations, so_annotations = read_ann_file(
                self.document_id,
                self._standoff_dir
            )

            self.entities = {
                ent_id: {
                    'ann_id': so_entities[ent_id]["ann_id"],
                    'entity_type': so_entities[ent_id]["entity_type"],
                    'surface': so_entities[ent_id]["surface"],
                    'start_offset': int(so_entities[ent_id]["offset_start"]),
                    'end_offset': int(so_entities[ent_id]["offset_end"]),
                }
                for ent_id in so_entities
            }

            self.relations = so_relations
            self.disambiguations = {
                ann['anchor']: ann
                for ann in so_annotations
            }

            text = read_text(self.document_id, self._standoff_dir)
            self.text = text


# Path: citation_extractor/io/brat.py
# def read_ann_file(fileid, ann_dir, suffix="-doc-1.ann"):
#     """Reads annotations from a brat standoff file.
# 
#     :param fileid:
#     :type fileid:
#     :param ann_dir:
#     :type ann_dir:
#     :returns:
#     :rtype: tuple
#     """
#     ann_file = "%s%s%s"%(ann_dir,fileid,suffix)
#     f = codecs.open(ann_file,'r','utf-8')
#     data = f.read()
#     f.close()
#     rows = data.split('\n')
#     entities = {}
#     ent_count = 0
#     relations = {}
#     annotations = []
#     for row in rows:
#         cols = row.split("\t")
#         ann_id = cols[0]
# 
#         if(u"#" in cols[0]):
#             # it's a text annotation
#             tmp = {
#                 "ann_id":"%s%s"%(cols[1].split()[0],cols[0])
#                 ,"anchor":cols[1].split()[1:][0]
#                 ,"text":cols[2]
#             }
#             annotations.append(tmp)
# 
#         elif(len(cols)==3 and u"T" in cols[0]):
#             # it's an entity
#             ent_count += 1
#             ent_type = cols[1].split()[0]
#             ranges = cols[1].replace("%s"%ent_type,"")
#             entities[cols[0]] = {"ann_id":ann_id
#                                 ,"entity_type": ent_type
#                                 ,"offset_start":ranges.split()[0]
#                                 ,"offset_end":ranges.split()[1]
#                                 ,"surface":cols[2]}
# 
#         elif(len(cols)>=2 and u"R" in cols[0]):
#             # it's a relation
#             rel_type, arg1, arg2 = cols[1].split()
#             relations[cols[0]] = {
#                 "ann_id": ann_id,
#                 "arguments": (arg1.split(":")[1], arg2.split(":")[1]),
#                 "relation_type":rel_type
#             }
# 
#     return entities, relations, annotations
# 
# Path: citation_extractor/io/iob.py
# def file_to_instances(inp_file):
#     """Reads a IOB file a converts it into a list of instances.
# 
#     :param type inp_file: Path to the input IOB file.
#     :return: A list of tuples, where tuple[0] is the token and tuple[1]
#         contains its assigned label.
#     :rtype: list
# 
#     """
#     f = codecs.open(inp_file,"r","utf-8")
#     inp_text = f.read()
#     f.close()
#     out=[]
#     comment=re.compile(r'#.*?')
#     for i in inp_text.split("\n\n"):
#         inst=[]
#         for j in i.split("\n"):
#             if(not comment.match(j)):
#                 temp = tuple(j.split("\t"))
#                 if(len(temp)>1):
#                     inst.append(temp)
#         if(inst and len(inst)>1):
#             out.append(inst)
#     return out
# 
# def write_iob_file(instances, dest_file):
#     """Write a set of instances to an IOB file.
# 
#     :param list instances: Description of parameter `instances`.
#     :param str dest_file: Description of parameter `dest_file`.
#     :return: Description of returned object.
#     :rtype: boolean
# 
#     """
#     to_write = "\n\n".join(
#         [
#             "\n".join(["\t".join(token) for token in instance])
#             for instance in instances
#         ]
#     )
#     try:
#         with codecs.open(dest_file, 'w', 'utf-8') as f:
#             f.write(to_write)
#         return True
#     except Exception, e:
#         raise e
# 
# Path: citation_extractor/io/brat.py
# def _find_newlines(text, newline=u'\n'):
#     """
#     TODO
#     """
#     positions = []
#     last_position = 0
#     if(text.find(newline) == -1):
#         return positions
#     else:
#         while(text.find(newline, last_position + 1) > -1):
#             last_position = text.find(newline, last_position + 1)
#             positions.append((last_position, last_position + len(newline)))
#         return positions


# Path: citation_extractor/io/converters.py
from docopt import docopt
from knowledge_base import KnowledgeBase
from citation_extractor.io.brat import read_ann_file
from citation_extractor.io.iob import file_to_instances, write_iob_file
from citation_extractor.io.brat import _find_newlines
from pyCTS import CTS_URN
import logging
import codecs
import json
import os
import re

            for sentence_n, sentence in enumerate(self._iob_standoff)
            for token_n, token in enumerate(sentence)
        ]

        if self._standoff_dir is not None:
            so_entities, so_relations, so_annotations = read_ann_file(
                self.document_id,
                self._standoff_dir
            )

            self.entities = {
                ent_id: {
                    'ann_id': so_entities[ent_id]["ann_id"],
                    'entity_type': so_entities[ent_id]["entity_type"],
                    'surface': so_entities[ent_id]["surface"],
                    'start_offset': int(so_entities[ent_id]["offset_start"]),
                    'end_offset': int(so_entities[ent_id]["offset_end"]),
                }
                for ent_id in so_entities
            }

            self.relations = so_relations
            self.disambiguations = {
                ann['anchor']: ann
                for ann in so_annotations
            }

            text = read_text(self.document_id, self._standoff_dir)
            self.text = text


            self._line_breaks = _find_newlines(text)