<commit_before># coding=utf-8
"""
Created on 03/02/2014

@author: Dani
"""
from __future__ import unicode_literals
from lpentities.computation import Computation
from lpentities.country import Country

try:
    from xml.etree.cElementTree import Element, ElementTree
except:
    from xml.etree.ElementTree import Element, ElementTree
from lpentities.interval import Interval
from lpentities.instant import Instant
from lpentities.month_interval import MonthInterval
from lpentities.year_interval import YearInterval
from lpentities.time import Time
from lpentities.region import Region

import codecs

import urllib
import urllib2


class ModelToXMLTransformer(object):
    """
    classdocs
    """

    def __init__(self, dataset, import_process, user, indicator_relations=None):
        """
        Constructor:
         - dataset: lpentities.dataset object containing most of the info
         - user: user that strat the importation process. According to the model, it is not possible to
        obtain it form the dataset
         - import_proces: string chain that summarizes the way in which data were taken from the source: excell, api,...
         - indicator_relations: list with lpentities.IndicatorRelationship objects

        """
        self._datasource = dataset.source
        self._dataset = dataset
        self._user = user
        self._import_process = import_process
        self._root = None
        self._indicator_dic = {}  # It will store an indicator object with it id as key.
        self._group_dic = {}
        self._indicator_relations = indicator_relations
        # One per indicator referred by the observations

        self._root = self._build_root()

    #
    # Const containing names of tag or attributtes
    # to use in the XML file
    #
    ROOT = "dataset"
    ROOT_ID = "id"
    INDICATORS = "indicators"

    SLICES = "slices"
    SLICE = "slice"
    SLICE_METADATA = "sli_metadata"
    SLICE_ATT_ID = "id"
    SLICE_REFERRED = "referred"

    OBSERVATIONS = "observations"

    OBSERVATION = "observation"
    OBSERVATION_ATT_ID = "id"
    OBSERVATION_ISSUED = "issued"
    OBSERVATION_OBS_STATUS = "obs-status"
    OBSERVATION_COMPUTATION = "computation"
    OBSERVATION_COMPUTATION_TYPE_ATT = "type"
    OBSERVATION_REGION = "region"
    OBSERVATION_COUNTRY = "country"
    OBSERVATION_VALUE = "value"
    OBSERVATION_INDICATOR = "indicator"
    OBSERVATION_TIME = "time"
    OBSERVATION_ATT_GROUP = "group"

    OBSERVATION_ATT_COUNTRY_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    OBSERVATION_ATT_INDICATOR_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    OBSERVATION_ATT_TIME_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    IMPORT_PROCESS = "import_process"
    IMPORT_PROCESS_ORGANIZATION_NAME = "organization_name"
    IMPORT_PROCESS_ORGANIZATION_URL = "organization_url"
    IMPORT_PROCESS_DATASOURCE = "datasource"
    IMPORT_PROCESS_DATASOURCE_ID_ATT = "id"
    IMPORT_PROCESS_TYPE = "type"
    IMPORT_PROCESS_TIMESTAMP = "timestamp"
    IMPORT_PROCESS_USER = "user"
    IMPORT_PROCESS_SDMX_FREQUENCY = "sdmx_frequency"
    IMPORT_PROCESS_DATASOURCE_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    IMPORT_PROCESS_TYPE_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    IMPORT_PROCESS_USER_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    INDICATOR = "indicator"
    INDICATOR_ATT_ID = "id"
    INDICATOR_NAME_EN = "ind_name_en"
    INDICATOR_NAME_ES = "ind_name_es"
    INDICATOR_NAME_FR = "ind_name_fr"
    INDICATOR_DESCRIPTION_EN = "ind_description_en"
    INDICATOR_DESCRIPTION_ES = "ind_description_es"
    INDICATOR_DESCRIPTION_FR = "ind_description_fr"
    INDICATOR_MEASURE_UNIT = "measure_unit"
    INDICATOR_MEASURE_UNIT_CONVERT_TO_ATT = "convertible_to"
    INDICATOR_MEASURE_UNIT_FACTOR_ATT = "factor"
    INDICATOR_TOPIC = "topic-ref"
    INDICATOR_SPLITS_IN = "splitsIn"
    INDICATOR_PREFERABLE_TENDENCY = "preferable_tendency"

    INDICATOR_REF = "indicator-ref"

    INDICATOR_ATT_ID_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    INDICATOR_ATT_MEASURE_UNIT_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    LICENSE = "license"
    LICENSE_NAME = "lic_name"
    LICENSE_DESCRIPTION = "lic_description"
    LICENSE_REPUBLISH = "republish"
    LICENSE_URL = "lic_url"

    TIME = "time"
    TIME_INTERVAL = "interval"
    TIME_INTERVAL_BEG = "beginning"
    TIME_INTERVAL_END = "end"
    TIME_ATT_UNIT = "unit"

    INDICATOR_GROUPS = "indicator_groups"
    GROUP = "indicator_group"
    GROUP_ATT_ID = "id"
    GROUP_ATT_INDICATOR = "indicator"

    OBSERVATION_REF = "observation-ref"
    OBSERVATION_REF_ID = "id"


    def run(self):
        #Building xml
        #The order calling these methods should not be changed
        try:
            self.build_import_process_node()  # Done
            self.build_license_node()  # Done
            self.build_observations_node()  # Done
            self.build_indicators_node()  # Done
            self.build_indicator_groups_node()  # Done
            self.build_slices_node()  # Done
            self.include_indicator_relations()  # Done
        except BaseException as e:
            raise RuntimeError("Error while building xml file. " + e.message)
        paths = None

        #Persisting xml
        try:
            paths = self._persist_tree()  #
        except BaseException as e:
            raise RuntimeError("Error while persisting the builded xml. " + e.message)

        #Sending xml to the receiver
        try:
            # self._send_to_receiver(paths)
            pass
        except BaseException as e:
            raise RuntimeError("Error while sending xml to the receiver module. " + e.message)


    def _build_root(self):
        result = Element(self.ROOT)
        result.attrib[self.ROOT_ID] = self._dataset.dataset_id
        return result

    def _send_to_receiver(self, paths):
        url = "http://192.168.3.11/receiver"
        exceptions = []
        for file_path in paths:
            try:
                with codecs.open(file_path, encoding="utf-8") as xml:
                    file_content = xml.read()
                    data = urllib.urlencode({'xml': unicode(file_content).encode('utf-8')})
                    req = urllib2.Request(url, data)
                    resp = urllib2.urlopen(req)
            except BaseException as e:
                e.message = 'File "{0}": {1}'.format(file_path, e.message)
                exceptions.append(e)
        self._process_sending_exceptions(exceptions)
<commit_msg>Send zipped original path<commit_after># coding=utf-8
"""
Created on 03/02/2014

@author: Dani
"""
from __future__ import unicode_literals
from lpentities.computation import Computation
from lpentities.country import Country
from zipfile import ZipFile

try:
    from xml.etree.cElementTree import Element, ElementTree
except:
    from xml.etree.ElementTree import Element, ElementTree
from lpentities.interval import Interval
from lpentities.instant import Instant
from lpentities.month_interval import MonthInterval
from lpentities.year_interval import YearInterval
from lpentities.time import Time
from lpentities.region import Region

import codecs

import urllib
import urllib2


class ModelToXMLTransformer(object):
    """
    classdocs
    """

    XML = "xml"
    XLS = "excell"
    CSV = "csv"
    API = "api"
    JSON = "json"
    SCRAP = "scrap"

    def __init__(self, dataset, import_process, user, path_to_original_file, indicator_relations=None):
        """
        Constructor:
         - dataset: lpentities.dataset object containing most of the info
         - user: user that strat the importation process. According to the model, it is not possible to
        obtain it form the dataset
         - import_proces: string chain that summarizes the way in which data were taken from the source: excell, api,...
         - indicator_relations: list with lpentities.IndicatorRelationship objects

        """
        self._datasource = dataset.source
        self._dataset = dataset
        self._user = user
        self._import_process = import_process
        self._root = None
        self._indicator_dic = {}  # It will store an indicator object with it id as key.
        self._group_dic = {}
        self._indicator_relations = indicator_relations
        self._path_to_original_file = path_to_original_file
        # One per indicator referred by the observations

        self._root = self._build_root()

    #
    # Const containing names of tag or attributtes
    # to use in the XML file
    #
    ROOT = "dataset"
    ROOT_ID = "id"
    INDICATORS = "indicators"

    SLICES = "slices"
    SLICE = "slice"
    SLICE_METADATA = "sli_metadata"
    SLICE_ATT_ID = "id"
    SLICE_REFERRED = "referred"

    OBSERVATIONS = "observations"

    OBSERVATION = "observation"
    OBSERVATION_ATT_ID = "id"
    OBSERVATION_ISSUED = "issued"
    OBSERVATION_OBS_STATUS = "obs-status"
    OBSERVATION_COMPUTATION = "computation"
    OBSERVATION_COMPUTATION_TYPE_ATT = "type"
    OBSERVATION_REGION = "region"
    OBSERVATION_COUNTRY = "country"
    OBSERVATION_VALUE = "value"
    OBSERVATION_INDICATOR = "indicator"
    OBSERVATION_TIME = "time"
    OBSERVATION_ATT_GROUP = "group"

    OBSERVATION_ATT_COUNTRY_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    OBSERVATION_ATT_INDICATOR_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    OBSERVATION_ATT_TIME_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    IMPORT_PROCESS = "import_process"
    IMPORT_PROCESS_ORGANIZATION_NAME = "organization_name"
    IMPORT_PROCESS_ORGANIZATION_URL = "organization_url"
    IMPORT_PROCESS_DATASOURCE = "datasource"
    IMPORT_PROCESS_DATASOURCE_ID_ATT = "id"
    IMPORT_PROCESS_TYPE = "type"
    IMPORT_PROCESS_TIMESTAMP = "timestamp"
    IMPORT_PROCESS_USER = "user"
    IMPORT_PROCESS_SDMX_FREQUENCY = "sdmx_frequency"
    IMPORT_PROCESS_DATASOURCE_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    IMPORT_PROCESS_TYPE_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    IMPORT_PROCESS_USER_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    INDICATOR = "indicator"
    INDICATOR_ATT_ID = "id"
    INDICATOR_NAME_EN = "ind_name_en"
    INDICATOR_NAME_ES = "ind_name_es"
    INDICATOR_NAME_FR = "ind_name_fr"
    INDICATOR_DESCRIPTION_EN = "ind_description_en"
    INDICATOR_DESCRIPTION_ES = "ind_description_es"
    INDICATOR_DESCRIPTION_FR = "ind_description_fr"
    INDICATOR_MEASURE_UNIT = "measure_unit"
    INDICATOR_MEASURE_UNIT_CONVERT_TO_ATT = "convertible_to"
    INDICATOR_MEASURE_UNIT_FACTOR_ATT = "factor"
    INDICATOR_TOPIC = "topic-ref"
    INDICATOR_SPLITS_IN = "splitsIn"
    INDICATOR_PREFERABLE_TENDENCY = "preferable_tendency"

    INDICATOR_REF = "indicator-ref"

    INDICATOR_ATT_ID_PREFIX = ""  # Empty value, but it remains here because it could still be changed
    INDICATOR_ATT_MEASURE_UNIT_PREFIX = ""  # Empty value, but it remains here because it could still be changed

    LICENSE = "license"
    LICENSE_NAME = "lic_name"
    LICENSE_DESCRIPTION = "lic_description"
    LICENSE_REPUBLISH = "republish"
    LICENSE_URL = "lic_url"

    TIME = "time"
    TIME_INTERVAL = "interval"
    TIME_INTERVAL_BEG = "beginning"
    TIME_INTERVAL_END = "end"
    TIME_ATT_UNIT = "unit"

    INDICATOR_GROUPS = "indicator_groups"
    GROUP = "indicator_group"
    GROUP_ATT_ID = "id"
    GROUP_ATT_INDICATOR = "indicator"

    OBSERVATION_REF = "observation-ref"
    OBSERVATION_REF_ID = "id"


    def run(self):
        #Building xml
        #The order calling these methods should not be changed
        try:
            self.build_import_process_node()  # Done
            self.build_license_node()  # Done
            self.build_observations_node()  # Done
            self.build_indicators_node()  # Done
            self.build_indicator_groups_node()  # Done
            self.build_slices_node()  # Done
            self.include_indicator_relations()  # Done
        except BaseException as e:
            raise RuntimeError("Error while building xml file. " + e.message)
        paths = None

        #Persisting xml
        try:
            paths = self._persist_tree()  #
        except BaseException as e:
            raise RuntimeError("Error while persisting the builded xml. " + e.message)

        #Sending xml to the receiver
        try:
            self._send_to_receiver(paths)
            pass
        except BaseException as e:
            raise RuntimeError("Error while sending xml to the receiver module. " + e.message)


    def _build_root(self):
        result = Element(self.ROOT)
        result.attrib[self.ROOT_ID] = self._dataset.dataset_id
        return result

    def _send_to_receiver(self, paths):
        url = "http://156.35.82.103/receiver"
        exceptions = []
        for file_path in paths:
            try:
                with codecs.open(file_path, encoding="utf-8") as xml:
                    file_content = xml.read()
                    data = urllib.urlencode({'xml': unicode(file_content).encode('utf-8')},
                                            {'file': self._obtain_content_of_original_path()})
                    req = urllib2.Request(url, data)
                    resp = urllib2.urlopen(req)
            except BaseException as e:
                e.message = 'File "{0}": {1}'.format(file_path, e.message)
                exceptions.append(e)
        self._process_sending_exceptions(exceptions)


    def _obtain_content_of_original_path(self):
        file_zip = ZipFile(self._path_to_original_file + ".zip", "w")
        file_zip.write(self._path_to_original_file)
        file_zip.close()
        with open(self._path_to_original_file + ".zip") as file_content:
            return file_content
{'fraction_non_alphanumeric': 0.07471102340005639, 'fraction_numerical': 0.003594586974908373, 'mean_word_length': 2.9304709141274237, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 23}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3930862', 'n_tokens_mistral': 4588, 'n_tokens_neox': 4355, 'n_words': 1079}