<commit_before>from jcudc24ingesterapi.ingester_exceptions import InvalidObjectError
__author__ = '<NAME>'
import xmlrpclib
import inspect
import datetime
import httplib
import urlparse
import logging
import base64

from jcudc24ingesterapi import parse_timestamp, format_timestamp, typed,\
    ValidationError, ingester_exceptions
import jcudc24ingesterapi.models.system
import jcudc24ingesterapi.models.dataset
import jcudc24ingesterapi.models.locations
import jcudc24ingesterapi.models.sampling
import jcudc24ingesterapi.models.data_sources
import jcudc24ingesterapi.models.metadata
import jcudc24ingesterapi.schemas.metadata_schemas
import jcudc24ingesterapi.schemas.data_entry_schemas
import jcudc24ingesterapi.schemas.data_types
from jcudc24ingesterapi.models.data_entry import FileObject, DataEntry

logger = logging.getLogger(__name__)

def get_properties(obj):
    """Returns a list of valid property names for this object"""
    return [k for k,v in inspect.getmembers(type(obj)) if isinstance(v, property)]

class Marshaller(object):
    """A Marshaller object is responsible for converting between real objects
    and dicts. This is used as a helper for the XMLRPC service.
    """
    def __init__(self):
        self._classes = {}
        self._class_factories = {}

        self.scanPackage(jcudc24ingesterapi.models.locations)
        self.scanPackage(jcudc24ingesterapi.models.dataset)
        self.scanPackage(jcudc24ingesterapi.models.sampling)
        self.scanPackage(jcudc24ingesterapi.models.data_sources)
        self.scanPackage(jcudc24ingesterapi.models.data_entry)
        self.scanPackage(jcudc24ingesterapi.models.metadata)
        self.scanPackage(jcudc24ingesterapi.models.system)
        self.scanPackage(jcudc24ingesterapi.schemas.metadata_schemas)
        self.scanPackage(jcudc24ingesterapi.schemas.data_entry_schemas)
        self.scanPackage(jcudc24ingesterapi.schemas.data_types)
        self.scanPackage(jcudc24ingesterapi.ingester_platform_api)
        
    def scanPackage(self, pkg):
        """Scan through the given package and find classes that are eligable for 
        marshalling."""
        for cls in dir(pkg):
            cls = getattr(pkg, cls)
            if isinstance(cls, type) and hasattr(cls, "__xmlrpc_class__"):
                self._classes[cls] = cls.__xmlrpc_class__
                self._class_factories[cls.__xmlrpc_class__] = cls

    def class_for(self, klass):
        return self._class_factories[klass]

    def obj_to_dict(self, obj, special_attrs=[]):
        """Maps an object of base class BaseManagementObject to a dict.
        """
        if type(obj) in (str, int, float, unicode, bool, type(None), tuple):
            return obj
        elif type(obj) == list:
            return [self.obj_to_dict(o, special_attrs=special_attrs) for o in obj] 
        elif not self._classes.has_key(type(obj)):
            raise ValueError("This object class is not supported: " + str(obj.__class__))
        ret = {}
        if isinstance(obj, jcudc24ingesterapi.schemas.Schema):
            ret["attributes"] = []
            for k in obj.attrs:
                attr = obj.attrs[k]
                ret["attributes"].append({"class":attr.__xmlrpc_class__, "name":attr.name, 
                                          "description":attr.description, "units":attr.units})
            if hasattr(obj, "id"):
                ret["id"] = obj.id
        else:
            data_keys = get_properties(obj)

            for k in data_keys:
                v = getattr(obj, k)
                if type(v) == datetime.datetime:
                    ret[k] = format_timestamp(v)
                elif isinstance(v, dict):
                    ret[k] = {}
                    for k1 in v:
                        ret[k][k1] = self.obj_to_dict(v[k1])
                else:
                    ret[k] = self.obj_to_dict(v)
        for k in special_attrs:
            ret[k] = getattr(obj, k)
            
        ret["class"] = self._classes[type(obj)]
        return ret

    def dict_to_obj(self, x, obj=None):
        """Maps a dict back to an object, created based on the 'class' element.
        
        :param x: 
        :param obj: an optional object to use as the destination
        """
        if isinstance(x, list):
            return [self.dict_to_obj(obj) for obj in x]
        elif type(x) in (str, int, float, unicode, bool, type(None)):
            return x
        elif not x.has_key("class"):
            raise ValueError("There is no class element")
        
        if obj == None:
            try:
                obj = self._class_factories[x["class"]]()
            except TypeError, e:
                raise TypeError(e.message + " for " + x["class"], *e.args[1:])

        # Create a dict of the properties, and the valid types allowed
        data_keys = dict([ (k,v.fset.valid_types if v.fset != None and hasattr(v.fset, "valid_types") else []) for k,v in inspect.getmembers(type(obj)) if isinstance(v, property)])
<commit_msg>fix the marshalling of schemas to not loose the name<commit_after>from jcudc24ingesterapi.ingester_exceptions import InvalidObjectError
__author__ = '<NAME>'
import xmlrpclib
import inspect
import datetime
import httplib
import urlparse
import logging
import base64

from jcudc24ingesterapi import parse_timestamp, format_timestamp, typed,\
    ValidationError, ingester_exceptions
import jcudc24ingesterapi.models.system
import jcudc24ingesterapi.models.dataset
import jcudc24ingesterapi.models.locations
import jcudc24ingesterapi.models.sampling
import jcudc24ingesterapi.models.data_sources
import jcudc24ingesterapi.models.metadata
import jcudc24ingesterapi.schemas.metadata_schemas
import jcudc24ingesterapi.schemas.data_entry_schemas
import jcudc24ingesterapi.schemas.data_types
from jcudc24ingesterapi.models.data_entry import FileObject, DataEntry

logger = logging.getLogger(__name__)

def get_properties(obj):
    """Returns a list of valid property names for this object"""
    return [k for k,v in inspect.getmembers(type(obj)) if isinstance(v, property)]

class Marshaller(object):
    """A Marshaller object is responsible for converting between real objects
    and dicts. This is used as a helper for the XMLRPC service.
    """
    def __init__(self):
        self._classes = {}
        self._class_factories = {}

        self.scanPackage(jcudc24ingesterapi.models.locations)
        self.scanPackage(jcudc24ingesterapi.models.dataset)
        self.scanPackage(jcudc24ingesterapi.models.sampling)
        self.scanPackage(jcudc24ingesterapi.models.data_sources)
        self.scanPackage(jcudc24ingesterapi.models.data_entry)
        self.scanPackage(jcudc24ingesterapi.models.metadata)
        self.scanPackage(jcudc24ingesterapi.models.system)
        self.scanPackage(jcudc24ingesterapi.schemas.metadata_schemas)
        self.scanPackage(jcudc24ingesterapi.schemas.data_entry_schemas)
        self.scanPackage(jcudc24ingesterapi.schemas.data_types)
        self.scanPackage(jcudc24ingesterapi.ingester_platform_api)
        
    def scanPackage(self, pkg):
        """Scan through the given package and find classes that are eligable for 
        marshalling."""
        for cls in dir(pkg):
            cls = getattr(pkg, cls)
            if isinstance(cls, type) and hasattr(cls, "__xmlrpc_class__"):
                self._classes[cls] = cls.__xmlrpc_class__
                self._class_factories[cls.__xmlrpc_class__] = cls

    def class_for(self, klass):
        return self._class_factories[klass]

    def obj_to_dict(self, obj, special_attrs=[]):
        """Maps an object of base class BaseManagementObject to a dict.
        """
        if type(obj) in (str, int, float, unicode, bool, type(None), tuple):
            return obj
        elif type(obj) == list:
            return [self.obj_to_dict(o, special_attrs=special_attrs) for o in obj] 
        elif not self._classes.has_key(type(obj)):
            raise ValueError("This object class is not supported: " + str(obj.__class__))
        ret = {}

        data_keys = get_properties(obj)

        for k in data_keys:
            v = getattr(obj, k)
            if k in ("attrs", "extends") and isinstance(obj, jcudc24ingesterapi.schemas.Schema): continue
            if type(v) == datetime.datetime:
                ret[k] = format_timestamp(v)
            elif isinstance(v, dict):
                ret[k] = {}
                for k1 in v:
                    ret[k][k1] = self.obj_to_dict(v[k1])
            else:
                ret[k] = self.obj_to_dict(v)

        for k in special_attrs:
            ret[k] = getattr(obj, k)

        if isinstance(obj, jcudc24ingesterapi.schemas.Schema):
            ret["attributes"] = []
            for k in obj.attrs:
                attr = obj.attrs[k]
                ret["attributes"].append({"class":attr.__xmlrpc_class__, "name":attr.name, 
                                          "description":attr.description, "units":attr.units})
            ret["extends"] = [] + obj.extends
            
        ret["class"] = self._classes[type(obj)]
        return ret

    def dict_to_obj(self, x, obj=None):
        """Maps a dict back to an object, created based on the 'class' element.
        
        :param x: 
        :param obj: an optional object to use as the destination
        """
        if isinstance(x, list):
            return [self.dict_to_obj(obj) for obj in x]
        elif type(x) in (str, int, float, unicode, bool, type(None)):
            return x
        elif not x.has_key("class"):
            raise ValueError("There is no class element")
        
        if obj == None:
            try:
                obj = self._class_factories[x["class"]]()
            except TypeError, e:
                raise TypeError(e.message + " for " + x["class"], *e.args[1:])

        # Create a dict of the properties, and the valid types allowed
        data_keys = dict([ (k,v.fset.valid_types if v.fset != None and hasattr(v.fset, "valid_types") else []) for k,v in inspect.getmembers(type(obj)) if isinstance(v, property)])

{'fraction_non_alphanumeric': 0.09561159650516282, 'fraction_numerical': 0.010921366163621922, 'mean_word_length': 2.6155778894472363, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1778160', 'n_tokens_mistral': 3107, 'n_tokens_neox': 2962, 'n_words': 728}