<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on 07.04.2014

@author: heinz-peterlang
'''
import unittest
from weblyzard_api.xml_content.parsers import XMLParser

class XML2005(XMLParser):
    
    SUPPORTED_NAMESPACE = 'http://www.weblyzard.com/wl/2005'
    DOCUMENT_NAMESPACES = {'wl': SUPPORTED_NAMESPACE}
    SENTENCE_MAPPING = {'id': 'md5sum'} 
    VERSION = 2005

    @classmethod
    def pre_xml_dump(cls, titles, attributes, sentences):
        
        if not 'title' in attributes: 
            attributes['title'] = ' '.join([t.value for t in titles])
                
        return attributes, sentences

class TestXML2005(unittest.TestCase):
    
    def test(self):
        xml = ''' <wl:page xmlns:wl="http://www.weblyzard.com/wl/2005" 
             lang="de" 
             title="Freihandelsgespr??che - Erleichterungen f??r kleine Firmen geplant" 
             content_type="text/html" 
             content_id="578351358" <commit_msg>fix: mapping pos_tags -> pos<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on 07.04.2014

@author: heinz-peterlang
'''
import unittest
from weblyzard_api.xml_content.parsers import XMLParser

class XML2005(XMLParser):
    
    SUPPORTED_NAMESPACE = 'http://www.weblyzard.com/wl/2005'
    DOCUMENT_NAMESPACES = {'wl': SUPPORTED_NAMESPACE}
    SENTENCE_MAPPING = {'id': 'md5sum',
                        'pos_tags': 'pos'} 
    VERSION = 2005

    @classmethod
    def pre_xml_dump(cls, titles, attributes, sentences):
        
        if not 'title' in attributes: 
            attributes['title'] = ' '.join([t.value for t in titles])
                
        return attributes, sentences

class TestXML2005(unittest.TestCase):
    
    def test(self):
        xml = ''' <wl:page xmlns:wl="http://www.weblyzard.com/wl/2005" 
             lang="de" 
             title="Freihandelsgespr??che - Erleichterungen f??r kleine Firmen geplant" 
             content_type="text/html" 
             content_id="578351358" 
{'fraction_non_alphanumeric': 0.12786069651741294, 'fraction_numerical': 0.03880597014925373, 'mean_word_length': 2.540492957746479, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6954089', 'n_tokens_mistral': 728, 'n_tokens_neox': 644, 'n_words': 129}