<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Feb, 27 2013

@author: heinz-peterlang
         <NAME>

Handles the new (http://www.weblyzard.com/wl/2013#) weblyzard
XML format.

Functions added:
 - support for sentence tokens and pos iterators

Remove functions:
 - compatibility fixes for namespaces, encodings etc.
 - support for the old POS tags mapping.
'''

import logging
from lxml import etree

DOCUMENT_NAMESPACE  = {'wl': 'http://www.weblyzard.com/wl/2013#',
                       'dc': 'http://purl.org/dc/elements/1.1/',
                       'xml': 'http://www.w3.org/XML/1998/namespace',
                       }

SENTENCE_ATTRIBUTES = {
    'pos_tag_string': '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'pos'), 
    'token_indices' : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'token'),
    'dependencies'  : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'dependencies'),
    'significance'  : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'significance'),
    'sem_orient'    : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'sem_orient'),
    'md5sum'        : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'id'),
}.items()

logger = logging.getLogger('wl_core.xml_content')

class Sentence(object):
    '''
    The sentence class used for accessing single sentences.

    Note: the class provides convenient properties for accessing pos tags 
          and tokens:
            .sentence: sentence text
            .tokens  : provides a list of tokens (e.g. ['A', 'new', 'day'])
            .pos_tags: provides a list of pos tags (e.g. ['DET', 'CC', 'NN'])
    '''
    # 'value' is used for the @XMLValue field
    ATTRIBUTE_TO_DICT_MAPPING = { 'sentence'      : 'value', 
                                  'is_title'      : 'is_title',
                                  'md5sum'        : 'id',
                                  'token_indices' : 'token',
                                  'dependencies'  : 'dependencies',
                                  'pos_tag_string': 'pos',
                                  'sem_orient'    : 'sem_orient',
                                  'significance'  : 'significance' }.items()

    XML_MAPPING = {'is_title': '{%s}is_title'  % DOCUMENT_NAMESPACE['wl'], 
                   'md5sum': '{%s}id' % DOCUMENT_NAMESPACE['wl'], 
                   'token_indices': '{%s}token' % DOCUMENT_NAMESPACE['wl'],
                   'pos_tag_string': '{%s}pos' % DOCUMENT_NAMESPACE['wl'], 
                   'dependencies': '{%s}dependencies' % DOCUMENT_NAMESPACE['wl'],
                   'sem_orient': '{%s}sem_orient' % DOCUMENT_NAMESPACE['wl'], 
                   'significance': '{%s}significance' % DOCUMENT_NAMESPACE['wl']}

    def __init__(self, md5sum, pos_tag_string=None, token_indices=None, 
                 sem_orient=None, sentence=None, significance=None, is_title=False, dependencies=None):
        '''
        @param pos_tag_string: a string containing the sentences pos
                               tags (e.g. 'NN VB NN')
        @param token_indices: a string containing the token indices.
                              (e.g. '0:1 3:12')
        @param dependencies: the dependency structure of the sentence as
                             head_index:dependency_type. An index of -1 is
                             used to indicate that no head is present.
                             (e.g. 1:NMOD 3:NMOD 1:SUFFIX 4:SBJ)
        '''
        self.md5sum = md5sum
        self._sentence = None # access via property 
        self.pos_tag_string = pos_tag_string
        self.sem_orient = sem_orient
        self.sentence = sentence
        self.significance = significance
        self.token_indices = token_indices
        self.dependencies = dependencies
        self.is_title = is_title
    
    def as_dict(self):
        '''
        @return: a dictionary representation of the given sentence object
              that can be used for REST services.
        '''
        return { dictattr: getattr(self, key) for key, dictattr in 
                     self.ATTRIBUTE_TO_DICT_MAPPING if getattr(self, key) }

    def get_xml_attributes(self, skip_none_values=False):
        xml_dict = {}
        
        for obj_attr, xml_attr in self.XML_MAPPING.iteritems():
            value = getattr(self, obj_attr)
            if skip_none_values and not value:
                continue    
            xml_dict[xml_attr] = str(value)
            
        return xml_dict

    def get_pos_tags(self):
        '''
        @return: a list of the sentence's POS tags
        '''
        return [] if not self.pos_tag_string else self.pos_tag_string.split(' ')

    def get_token(self):
        '''
        @return: an iterator providing the sentence's tokens 
        '''
        if not self.token_indices:
            raise StopIteration

        tokens = self.token_indices.split(" ")
        for token_pos in tokens:
            start, end = map(int, token_pos.split(","))
            yield self.sentence[start:end]

    def get_sentence(self):
        return self._sentence
    
    def set_sentence(self, sentence):
        if isinstance(sentence, str):
            sentence = sentence.decode('utf-8')
        self._sentence = sentence

    # convenient functions for directly accessing 
    # pos tags and tokens
    pos_tags = property(get_pos_tags)
    tokens   = property(get_token)
    sentence = property(get_sentence, set_sentence)
    
class XMLContent(object):
    SENTENCE_XPATH = './/{%s}sentence' % DOCUMENT_NAMESPACE['wl']

    def __init__(self, xml_content):
        ''' '''
        if xml_content and xml_content.find(DOCUMENT_NAMESPACE['wl']) == -1:
            raise ValueError("Unsupported XML format.")
        self.root, self.attributes = self._set_root(xml_content)
        self.sentence_objects = []
        self.sentence_objects = self.get_sentences()
    
    def __repr__(self):
        return self.get_xml_document()
    
    def __len__(self):
        return len(self.sentences)
    
    @classmethod
    def get_xml_from_dict(cls, attributes, sentences):
        ''' '''
        root = etree.Element('{%s}page' % DOCUMENT_NAMESPACE['wl'], attrib=attributes, 
                             nsmap=DOCUMENT_NAMESPACE)
        
        for sentence in sentences: 
            sent_obj = etree.SubElement(root, 
                                        '{%s}sentence' % DOCUMENT_NAMESPACE['wl'], 
                                        attrib=sentence.get_xml_attributes(), 
                                        nsmap=DOCUMENT_NAMESPACE)
            sent_obj.text = etree.CDATA(sentence.sentence)
        
        xml_content = etree.tostring(root, encoding='UTF-8', pretty_print=True)
        
        return XMLContent(xml_content)

    def as_dict(self):
        '''
        @return: a dictionary representation of the given<commit_msg>fix: added is_title to the sentence attributes<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Feb, 27 2013

@author: heinz-peterlang
         <NAME>

Handles the new (http://www.weblyzard.com/wl/2013#) weblyzard
XML format.

Functions added:
 - support for sentence tokens and pos iterators

Remove functions:
 - compatibility fixes for namespaces, encodings etc.
 - support for the old POS tags mapping.
'''

import logging
from lxml import etree

DOCUMENT_NAMESPACE  = {'wl': 'http://www.weblyzard.com/wl/2013#',
                       'dc': 'http://purl.org/dc/elements/1.1/',
                       'xml': 'http://www.w3.org/XML/1998/namespace',
                       }

SENTENCE_ATTRIBUTES = {
    'pos_tag_string': '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'pos'), 
    'token_indices' : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'token'),
    'dependencies'  : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'dependencies'),
    'significance'  : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'significance'),
    'sem_orient'    : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'sem_orient'),
    'md5sum'        : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'id'),
    'is_title'        : '{%s}%s' % (DOCUMENT_NAMESPACE['wl'], 'is_title'),
}.items()

logger = logging.getLogger('wl_core.xml_content')

class Sentence(object):
    '''
    The sentence class used for accessing single sentences.

    Note: the class provides convenient properties for accessing pos tags 
          and tokens:
            .sentence: sentence text
            .tokens  : provides a list of tokens (e.g. ['A', 'new', 'day'])
            .pos_tags: provides a list of pos tags (e.g. ['DET', 'CC', 'NN'])
    '''
    # 'value' is used for the @XMLValue field
    ATTRIBUTE_TO_DICT_MAPPING = { 'sentence'      : 'value', 
                                  'is_title'      : 'is_title',
                                  'md5sum'        : 'id',
                                  'token_indices' : 'token',
                                  'dependencies'  : 'dependencies',
                                  'pos_tag_string': 'pos',
                                  'sem_orient'    : 'sem_orient',
                                  'significance'  : 'significance' }.items()

    XML_MAPPING = {'is_title': '{%s}is_title'  % DOCUMENT_NAMESPACE['wl'], 
                   'md5sum': '{%s}id' % DOCUMENT_NAMESPACE['wl'], 
                   'token_indices': '{%s}token' % DOCUMENT_NAMESPACE['wl'],
                   'pos_tag_string': '{%s}pos' % DOCUMENT_NAMESPACE['wl'], 
                   'dependencies': '{%s}dependencies' % DOCUMENT_NAMESPACE['wl'],
                   'sem_orient': '{%s}sem_orient' % DOCUMENT_NAMESPACE['wl'], 
                   'significance': '{%s}significance' % DOCUMENT_NAMESPACE['wl']}

    def __init__(self, md5sum, pos_tag_string=None, token_indices=None, 
                 sem_orient=None, sentence=None, significance=None, is_title=False, dependencies=None):
        '''
        @param pos_tag_string: a string containing the sentences pos
                               tags (e.g. 'NN VB NN')
        @param token_indices: a string containing the token indices.
                              (e.g. '0:1 3:12')
        @param dependencies: the dependency structure of the sentence as
                             head_index:dependency_type. An index of -1 is
                             used to indicate that no head is present.
                             (e.g. 1:NMOD 3:NMOD 1:SUFFIX 4:SBJ)
        '''
        self.md5sum = md5sum
        self._sentence = None # access via property 
        self.pos_tag_string = pos_tag_string
        self.sem_orient = sem_orient
        self.sentence = sentence
        self.significance = significance
        self.token_indices = token_indices
        self.dependencies = dependencies
        self.is_title = is_title
    
    def as_dict(self):
        '''
        @return: a dictionary representation of the given sentence object
              that can be used for REST services.
        '''
        return { dictattr: getattr(self, key) for key, dictattr in 
                     self.ATTRIBUTE_TO_DICT_MAPPING if getattr(self, key) }

    def get_xml_attributes(self, skip_none_values=False):
        xml_dict = {}
        
        for obj_attr, xml_attr in self.XML_MAPPING.iteritems():
            value = getattr(self, obj_attr)
            if skip_none_values and not value:
                continue    
            xml_dict[xml_attr] = str(value)
            
        return xml_dict

    def get_pos_tags(self):
        '''
        @return: a list of the sentence's POS tags
        '''
        return [] if not self.pos_tag_string else self.pos_tag_string.split(' ')

    def get_token(self):
        '''
        @return: an iterator providing the sentence's tokens 
        '''
        if not self.token_indices:
            raise StopIteration

        tokens = self.token_indices.split(" ")
        for token_pos in tokens:
            start, end = map(int, token_pos.split(","))
            yield self.sentence[start:end]

    def get_sentence(self):
        return self._sentence
    
    def set_sentence(self, sentence):
        if isinstance(sentence, str):
            sentence = sentence.decode('utf-8')
        self._sentence = sentence

    # convenient functions for directly accessing 
    # pos tags and tokens
    pos_tags = property(get_pos_tags)
    tokens   = property(get_token)
    sentence = property(get_sentence, set_sentence)
    
class XMLContent(object):
    SENTENCE_XPATH = './/{%s}sentence' % DOCUMENT_NAMESPACE['wl']

    def __init__(self, xml_content):
        ''' '''
        if xml_content and xml_content.find(DOCUMENT_NAMESPACE['wl']) == -1:
            raise ValueError("Unsupported XML format.")
        self.root, self.attributes = self._set_root(xml_content)
        self.sentence_objects = []
        self.sentence_objects = self.get_sentences()
    
    def __repr__(self):
        return self.get_xml_document()
    
    def __len__(self):
        return len(self.sentences)
    
    @classmethod
    def get_xml_from_dict(cls, attributes, sentences):
        ''' '''
        root = etree.Element('{%s}page' % DOCUMENT_NAMESPACE['wl'], 
                             attrib=attributes, 
                             nsmap=DOCUMENT_NAMESPACE)
        
        for sentence in sentences: 
            sent_obj = etree.SubElement(root, 
                                        '{%s}sentence' % DOCUMENT_NAMESPACE['wl'], 
                                        attrib=sentence.get_xml_attributes(), 
                                        nsmap=DOCUMENT_NAMESPACE)
            sent_obj.text = etree.CDATA(sentence.sentence)
        
        xml_content = etree.tostring(root, encoding='UTF-8', pretty_print=True)
        
        return XMLContent(xml_content)

    def as_dict(self):
        '''
        @return: a dictionary representation of the given
{'fraction_non_alphanumeric': 0.1170104590354445, 'fraction_numerical': 0.005955839628123184, 'mean_word_length': 1.964900947459087, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 36}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2127711', 'n_tokens_mistral': 4168, 'n_tokens_neox': 3923, 'n_words': 911}