<commit_before>from __future__ import with_statement


######################################################################
#  CliNER - note_xml.py                                              #
#                                                                    #
#  <NAME>                                      <EMAIL> #
#                                                                    #
#  Purpose: Derived note object for reading xml formatted data.      #
######################################################################


__author__ = '<NAME>'
__date__   = 'Nov. 6, 2014'



import re
import string
from copy import copy


from abstract_note       import AbstractNote
from utilities_for_notes import classification_cmp, lineno_and_tokspan


class Note_xml(AbstractNote):

    def __init__(self):
        # Internal representation natural for i2b2 format
        self.data            = []  # list of list of tokens
        self.classifications = []  # list of concept tuples
        self.line_inds       = []  # list of (start,end) line character offsets


    def getExtension(self):
        return 'xml'


    def getText(self):
        return self.text


    def getTokenizedSentences(self):
        return self.data


    def getClassificationTuples(self):
        # return value
        retVal = []

        # Indices of each line
        line_inds = []
        start = 0
        end = 0
        for sent in self.data:
            for word in sent:
                end += len(word) + 1
            line_inds.append( (start,end-1) )
            start = end

        # Build list of standardized classification tuples
        for classification in self.classifications:
            concept,lineno,tok_start,tok_end = classification

            # character offset of beginning of line
            begin = line_inds[lineno-1][0]

            # Sweep through line to get character offsets from line start
            start = 0
            for word in self.data[lineno-1][:tok_start]:
                start += len(word) + 1

            # Length of concept span
            end = start
            for word in self.data[lineno-1][tok_start:tok_end+1]:
                end += len(word) + 1
            end -= 1

            #print begin
            #print begin+start, begin+end
            #print '~~' + self.text[begin+start:begin+end] + '~~'

            retVal.append( (concept,[(begin+start,begin+end)]) )

        return retVal


    def getLineIndices(self):
        return self.line_inds


    def read(self, txt, con=None):

        """
        Note_xml::read()

        @param txt. A file path for the tokenized medical record
        @param con. A file path for the xml annotated concepts for txt
        """

        start = 0
        end = 0

        # Read in the medical text
        with open(txt) as f:
            for line in f:
<commit_msg>tore whole file in Note object<commit_after>from __future__ import with_statement


######################################################################
#  CliNER - note_xml.py                                              #
#                                                                    #
#  <NAME>                                      <EMAIL> #
#                                                                    #
#  Purpose: Derived note object for reading xml formatted data.      #
######################################################################


__author__ = '<NAME>'
__date__   = 'Nov. 6, 2014'



import re
import string
from copy import copy


from abstract_note       import AbstractNote
from utilities_for_notes import classification_cmp, lineno_and_tokspan


class Note_xml(AbstractNote):

    def __init__(self):
        # Internal representation natural for i2b2 format
        self.data            = []  # list of list of tokens
        self.classifications = []  # list of concept tuples
        self.line_inds       = []  # list of (start,end) line character offsets


    def getExtension(self):
        return 'xml'


    def getText(self):
        return self.text


    def getTokenizedSentences(self):
        return self.data


    def getClassificationTuples(self):
        # return value
        retVal = []

        # Indices of each line
        line_inds = []
        start = 0
        end = 0
        for sent in self.data:
            for word in sent:
                end += len(word) + 1
            line_inds.append( (start,end-1) )
            start = end

        # Build list of standardized classification tuples
        for classification in self.classifications:
            concept,lineno,tok_start,tok_end = classification

            # character offset of beginning of line
            begin = line_inds[lineno-1][0]

            # Sweep through line to get character offsets from line start
            start = 0
            for word in self.data[lineno-1][:tok_start]:
                start += len(word) + 1

            # Length of concept span
            end = start
            for word in self.data[lineno-1][tok_start:tok_end+1]:
                end += len(word) + 1
            end -= 1

            #print begin
            #print begin+start, begin+end
            #print '~~' + self.text[begin+start:begin+end] + '~~'

            retVal.append( (concept,[(begin+start,begin+end)]) )

        return retVal


    def getLineIndices(self):
        return self.line_inds


    def read(self, txt, con=None):

        """
        Note_xml::read()

        @param txt. A file path for the tokenized medical record
        @param con. A file path for the xml annotated concepts for txt
        """

        start = 0
        end = 0

        # Read in the medical text
        with open(txt) as f:

            # Get entire file
            text = f.read()
            self.text = text

            for line in text.split('\n'):

{'fraction_non_alphanumeric': 0.12212570260602963, 'fraction_numerical': 0.007494464316130131, 'mean_word_length': 1.5959328028293545, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3360670', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1432, 'n_words': 456}