<commit_before>import datetime,json,logging,sys,decimal
from types import *
from afs.exceptions.AfsError import AfsError
import afs

# log-level is set in AfsConfig
Logger=logging.getLogger("afs.model")

class BaseModel(object):
    """
    The mother of all models.
    """

    def setByDict(self,objByDict):      
        """
        fill in object by dict.
        It is an error to try to create new attributes with this method
        It is also an error to pass json representations here.
        """
        Logger.debug("setByDict: got dict=%s" % objByDict)
        # inject ignAttrList is not present, happens in hand-craftet dicts..
        if not objByDict.has_key("ignAttrList") : objByDict["ignAttrList"]=self.getDict()["ignAttrList"]
        for attr, value in objByDict.iteritems():
            if not hasattr(self,attr) and not attr in objByDict["ignAttrList"] + ["ignAttrList"]  :
                raise AfsError("Cannot create new attribute '%s' to object '%s'" %(attr, self.__class__.__name__))
            elif attr[-3:] == "_js" :
                raise AfsError("Cannot set json representational attributes here.")
            else :# do not alter DB internal id and creation date
                if attr != "id" and attr != "cdate":
                    Logger.debug("setByDict: setting %s to %s" % (attr,value) )
                    setattr(self,attr,value)
        return True

    def updateAppRepr(self) :
        """
        update Complex attributes from their json encoded counterparts
        """
        cmplxAttrs={}
        for attr, value in self.__dict__.iteritems() :
            Logger.debug("updateAppRepr: attr=%s, value=%s" % (attr,value))
            if attr[-3:]=="_js" :
                if len(value) == 0 :
                    cmplxAttrs[attr[:-3]]= '""'
                else :
                    cmplxAttrs[attr[:-3]]=json.loads(value)
                Logger.debug("updateAppRepr: complxAttr=%s" % (cmplxAttrs[attr[:-3]]))
        for attr,value in cmplxAttrs.iteritems() :
            Logger.debug("updateAppRepr: setting %s=%s, type=%s" % (attr,value,type(value)))
            setattr(self,attr,value)
        return

    def updateDBRepr(self) :
        """
        update the all attributes holding json represenations of complex attributes
        """
        jsonReps={}
        Logger.debug("in updateDBRepr" )
        for attr, value in self.__dict__.iteritems() :
            ignore=False
            if attr[0] == "_" : continue
            if isinstance(value, datetime.datetime) :
                ignore = True
            elif isinstance(value,decimal.Decimal) :
                ignore = True
            elif type(value) in [StringType,IntType,LongType,FloatType,BooleanType,NoneType] :
                ignore=True

            if not ignore :
                Logger.debug("attr=%s type(value)=%s" % (attr,type(value)))
                jsonReps["%s_js" % attr] = json.dumps(value)
            else :
                Logger.debug("Ignoring attr=%s type(value)=%s" % (attr,type(value)))
        for attr,value in jsonReps.iteritems() :
            Logger.debug("setting json rep %s to '%s'" % (attr,value) )
            setattr(self,attr,value)
        return
 
    def copyObj(self, newObj):
        """
        copies one object onto another.
        """
        Logger.debug("Copying newObj=%s to onto self." % newObj)
        ObjDict=newObj.getDict()
        self.setByDict(ObjDict)
        return
    
    def getJson(self):
        """
        Get a json representation of the model object
        """
        res={}
        for attr,value in self.getDict() :
            if isinstance(value, datetime.datetime):
                res[attr] = json.dumps(value.isoformat('-'))
            elif type(value) in [StringType,IntType,UnicodeType,ListType,DictType,NoneType] : 
                res[attr] = json.dumps(value)
        return res
    
    def getDict(self):
        """
        Get a dictionary representation of the model object.
        json encoded attributes ending in "_js" are not set.
        """
        res = {}
        for attr, value in self.__dict__.iteritems() :
            if attr[0] == "_" : continue
            if attr[-3:]=="_js" : continue
            if isinstance(value, datetime.datetime):
                res[attr] = value.isoformat('-')
            elif isinstance(value,decimal.Decimal) :
                res[attr] = long(value)
            elif type(value) in [StringType,IntType,LongType,FloatType,BooleanType,UnicodeType,DictType,ListType,NoneType] :
                Logger.debug("getDict: attr=%s, value=%s, type(value)=%s" % (attr,value,type(value)))
                res[attr] = value
            else : # ignore anything else
                Logger.warn("getDict: ignoring attr %s with type(value)=%s" % (attr,type(value)) )
                pass
        Logger.debug("getDict: returning : %s" % res)
        return res
    
    def __repr__(self):
        """
        Get a string representation of the model object.
        no python built-ins nor BaseModel stuff
        """
        BaseModelAttributes=dir(BaseModel())
        repr="<%s(" % self.__class__.__name__
        ModelAttributes=dir(self)
        for a in ModelAttributes :
            if a in BaseModelAttributes : continue
            repr += "%s=%s, " % (a, eval("self.%s" % a))
        repr += ")>"
        return repr<commit_msg>BaseModel: lower debug level for ignore-output<commit_after>import datetime,json,logging,sys,decimal
from types import *
from afs.exceptions.AfsError import AfsError
import afs

# log-level is set in AfsConfig
Logger=logging.getLogger("afs.model")

class BaseModel(object):
    """
    The mother of all models.
    """

    def setByDict(self,objByDict):      
        """
        fill in object by dict.
        It is an error to try to create new attributes with this method
        It is also an error to pass json representations here.
        """
        Logger.debug("setByDict: got dict=%s" % objByDict)
        # inject ignAttrList is not present, happens in hand-craftet dicts..
        if not objByDict.has_key("ignAttrList") : objByDict["ignAttrList"]=self.getDict()["ignAttrList"]
        for attr, value in objByDict.iteritems():
            if not hasattr(self,attr) and not attr in objByDict["ignAttrList"] + ["ignAttrList"]  :
                raise AfsError("Cannot create new attribute '%s' to object '%s'" %(attr, self.__class__.__name__))
            elif attr[-3:] == "_js" :
                raise AfsError("Cannot set json representational attributes here.")
            else :# do not alter DB internal id and creation date
                if attr != "id" and attr != "cdate":
                    Logger.debug("setByDict: setting %s to %s" % (attr,value) )
                    setattr(self,attr,value)
        return True

    def updateAppRepr(self) :
        """
        update Complex attributes from their json encoded counterparts
        """
        cmplxAttrs={}
        for attr, value in self.__dict__.iteritems() :
            Logger.debug("updateAppRepr: attr=%s, value=%s" % (attr,value))
            if attr[-3:]=="_js" :
                if len(value) == 0 :
                    cmplxAttrs[attr[:-3]]= '""'
                else :
                    cmplxAttrs[attr[:-3]]=json.loads(value)
                Logger.debug("updateAppRepr: complxAttr=%s" % (cmplxAttrs[attr[:-3]]))
        for attr,value in cmplxAttrs.iteritems() :
            Logger.debug("updateAppRepr: setting %s=%s, type=%s" % (attr,value,type(value)))
            setattr(self,attr,value)
        return

    def updateDBRepr(self) :
        """
        update the all attributes holding json represenations of complex attributes
        """
        jsonReps={}
        Logger.debug("in updateDBRepr" )
        for attr, value in self.__dict__.iteritems() :
            ignore=False
            if attr[0] == "_" : continue
            if isinstance(value, datetime.datetime) :
                ignore = True
            elif isinstance(value,decimal.Decimal) :
                ignore = True
            elif type(value) in [StringType,IntType,LongType,FloatType,BooleanType,NoneType] :
                ignore=True

            if not ignore :
                Logger.debug("attr=%s type(value)=%s" % (attr,type(value)))
                jsonReps["%s_js" % attr] = json.dumps(value)
            else :
                Logger.debug("Ignoring attr=%s type(value)=%s" % (attr,type(value)))
        for attr,value in jsonReps.iteritems() :
            Logger.debug("setting json rep %s to '%s'" % (attr,value) )
            setattr(self,attr,value)
        return
 
    def copyObj(self, newObj):
        """
        copies one object onto another.
        """
        Logger.debug("Copying newObj=%s to onto self." % newObj)
        ObjDict=newObj.getDict()
        self.setByDict(ObjDict)
        return
    
    def getJson(self):
        """
        Get a json representation of the model object
        """
        res={}
        for attr,value in self.getDict() :
            if isinstance(value, datetime.datetime):
                res[attr] = json.dumps(value.isoformat('-'))
            elif type(value) in [StringType,IntType,UnicodeType,ListType,DictType,NoneType] : 
                res[attr] = json.dumps(value)
        return res
    
    def getDict(self):
        """
        Get a dictionary representation of the model object.
        json encoded attributes ending in "_js" are not set.
        """
        res = {}
        for attr, value in self.__dict__.iteritems() :
            if attr[0] == "_" : continue
            if attr[-3:]=="_js" : continue
            if isinstance(value, datetime.datetime):
                res[attr] = value.isoformat('-')
            elif isinstance(value,decimal.Decimal) :
                res[attr] = long(value)
            elif type(value) in [StringType,IntType,LongType,FloatType,BooleanType,UnicodeType,DictType,ListType,NoneType] :
                Logger.debug("getDict: attr=%s, value=%s, type(value)=%s" % (attr,value,type(value)))
                res[attr] = value
            else : # ignore anything else
                Logger.debug("getDict: ignoring attr %s with type(value)=%s" % (attr,type(value)) )
                pass
        Logger.debug("getDict: returning : %s" % res)
        return res
    
    def __repr__(self):
        """
        Get a string representation of the model object.
        no python built-ins nor BaseModel stuff
        """
        BaseModelAttributes=dir(BaseModel())
        repr="<%s(" % self.__class__.__name__
        ModelAttributes=dir(self)
        for a in ModelAttributes :
            if a in BaseModelAttributes : continue
            repr += "%s=%s, " % (a, eval("self.%s" % a))
        repr += ")>"
        return repr
{'fraction_non_alphanumeric': 0.1193423240347312, 'fraction_numerical': 0.001662663957140218, 'mean_word_length': 2.031923830859703, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '882674', 'n_tokens_mistral': 3102, 'n_tokens_neox': 3025, 'n_words': 782}