<commit_before>from __future__ import absolute_import, division, print_function, unicode_literals

from django.core.files.storage import default_storage
from lxml import objectify, etree

from . import LANGUAGES, INVALID_XML_ERROR


def construct_MultiLingualText_from_xml(xml, instance):
    u"""
    Arguments:
    `xml` : A block of XML formatted like this:
        <languages>
            <language code="en">
                Hello
            </language>
            <language code="es">
                Hola
            </language>
        </languages>

    * `instance`: A MultiLingualText instance

    If the above block of XML was passed to this function (as `xml`) `instance` will now have two attributes:

    * `en` with a value of 'Hello'
    * `es` with a value of 'Hola'
    """
    try:
        xml_as_python_object = objectify.fromstring(xml)
    except etree.XMLSyntaxError:
        raise Exception(INVALID_XML_ERROR + ' MultiLingualText')
    else:
        # Creating a dictionary of all the languages passed in the value XML
        # with the language code (i.e. 'en', 'de', 'fr') as the key
        text_dict = {}
        try:
            text_dict = {unicode(l.get(u'code')): unicode(l.text or u'') for l in xml_as_python_object.language}
        except AttributeError:
            # Empty fields throw-off lxml and cause an AttributeError
            pass
        for code, verbose in LANGUAGES:
            setattr(instance, code, text_dict.get(code, u''))


def construct_MultiLingualFile_from_xml(xml, instance, storage=default_storage):<commit_msg>Another python 2.6 compatible dict constructor conversion<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

from django.core.files.storage import default_storage
from lxml import objectify, etree

from . import LANGUAGES, INVALID_XML_ERROR


def construct_MultiLingualText_from_xml(xml, instance):
    u"""
    Arguments:
    `xml` : A block of XML formatted like this:
        <languages>
            <language code="en">
                Hello
            </language>
            <language code="es">
                Hola
            </language>
        </languages>

    * `instance`: A MultiLingualText instance

    If the above block of XML was passed to this function (as `xml`) `instance` will now have two attributes:

    * `en` with a value of 'Hello'
    * `es` with a value of 'Hola'
    """
    try:
        xml_as_python_object = objectify.fromstring(xml)
    except etree.XMLSyntaxError:
        raise Exception(INVALID_XML_ERROR + ' MultiLingualText')
    else:
        # Creating a dictionary of all the languages passed in the value XML
        # with the language code (i.e. 'en', 'de', 'fr') as the key
        text_dict = {}
        try:
            text_dict = dict(
                (unicode(l.get(u'code')), unicode(l.text or u''))
                for l in xml_as_python_object.language
            )
        except AttributeError:
            # Empty fields throw-off lxml and cause an AttributeError
            pass
        for code, verbose in LANGUAGES:
            setattr(instance, code, text_dict.get(code, u''))


def construct_MultiLingualFile_from_xml(xml, instance, storage=default_storage):
{'fraction_non_alphanumeric': 0.08584615384615385, 'fraction_numerical': 0.0006153846153846154, 'mean_word_length': 2.5107991360691146, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4907963', 'n_tokens_mistral': 953, 'n_tokens_neox': 912, 'n_words': 296}