<commit_before>
import pickle
from StringIO import StringIO
import shelve
from mapping import Collection,Mapping,Graph


class OneTimeDescriptor(object):
    'provides shadow attribute based on schema'
    def __init__(self,attrName,**kwargs):
        self.attr=attrName
    def __get__(self,obj,objtype):
        try:
            id=obj._persistent_id # GET ITS RESOURCE ID
        except AttributeError:
            raise AttributeError('attempt to access pygr.Data attr on non-pygr.Data object')
        target=getResource.schemaAttr(id,self.attr) # ATTEMPT TO GET FROM pygr.Data
        obj.__dict__[self.attr]=target # PROVIDE DIRECTLY TO THE __dict__
        return target

class ItemDescriptor(object):
    'provides shadow attribute for items in a db, based on schema'
    def __init__(self,attrName,invert=False,getEdges=False,mapAttr=None,
                 targetAttr=None,**kwargs):
        self.attr=attrName
        self.invert=invert
        self.getEdges=getEdges
        self.mapAttr=mapAttr
        self.targetAttr=targetAttr
    def __get__(self,obj,objtype):
        try:
            id=obj.db._persistent_id # GET RESOURCE ID OF DATABASE
        except AttributeError:
            raise AttributeError('attempt to access pygr.Data attr on non-pygr.Data object')
        targetDict=getResource.schemaAttr(id,self.attr) # ATTEMPT TO GET FROM pygr.Data
        if self.invert:
            targetDict= ~targetDict
        if self.getEdges:
            targetDict=targetDict.edges
        if self.mapAttr is not None: # USE mapAttr TO GET ID FOR MAPPING obj
            result=targetDict[getattr(obj,self.mapAttr)]
        else:
            result=targetDict[obj] # NOW PERFORM MAPPING IN THAT RESOURCE...
        if self.targetAttr is not None:
            result=getattr(result,self.targetAttr) # GET ATTRIBUTE OF THE result
        obj.__dict__[self.attr]=result # PROVIDE DIRECTLY TO THE __dict__
        return result

class ForwardingDescriptor(object):
    'forward an attribute request to item from another container'
    def __init__(self,targetDB,attr):
        self.targetDB=targetDB # CONTAINER TO GET ITEMS FROM
        self.attr=attr # ATTRIBUTE TO MAP TO
    def __get__(self,obj,objtype):
        target=self.targetDB[obj.id] # GET target FROM CONTAINER
        return getattr(target,self.attr) # GET DESIRED ATTRIBUTE

class SpecialMethodDescriptor(object):
    'enables shadowing of special methods like __invert__'
    def __init__(self,attrName):
        self.attr=attrName
    def __get__(self,obj,objtype):
        try:
            return obj.__dict__[self.attr]
        except KeyError:
            raise AttributeError('%s has no method %s'%(obj,self.attr))

def addSpecialMethod(obj,attr,f):
    '''bind function f as special method attr on obj.
       obj cannot be an builtin or extension class
       (if so, just subclass it)'''
    import new
    m=new.instancemethod(f,obj,obj.__class__)
    try:
        if getattr(obj,attr) == m: # ALREADY BOUND TO f
            return # ALREADY BOUND, NOTHING FURTHER TO DO
    except AttributeError:
        pass
    else:
        raise AttributeError('%s already bound to a different function' %attr)
    setattr(obj,attr,m) # SAVE BOUND METHOD TO __dict__
    setattr(obj.__class__,attr,SpecialMethodDescriptor(attr)) # DOES FORWARDING

def getInverseDB(self):
    'default shadow __invert__ method'
    return self.inverseDB # TRIGGER CONSTRUCTION OF THE TARGET RESOURCE


class PygrDataNotPortableError(ValueError):
    'indicates that object has a local data dependency and cannnot be transferred to a remote client'
    pass
class PygrDataNotFoundError(KeyError):
    'unable to find a loadable resource for the requested pygr.Data identifier from PYGRDATAPATH'
    pass
class PygrDataMismatchError(ValueError):
    '_persistent_id attr on object no longer matches its assigned pygr.Data ID?!?'
    pass
class PygrDataEmptyError(ValueError):
    "user hasn't queued anything, so trying to save or rollback is an error"
    pass
class PygrDataReadOnlyError(ValueError):
    'attempt to write data to a read-only resource database'
    pass


class PygrPickler(pickle.Pickler):
    def persistent_id(self,obj):
        'convert objects with _persistent_id to PYGR_ID strings during pickling'
        import types
        try:
            if not isinstance(obj,types.TypeType) and obj is not self.root:
                try:
                    return 'PYGR_ID:%s' % self.sourceIDs[id(obj)]
                except KeyError:
                    if obj._persistent_id is not None:
                        return 'PYGR_ID:%s' % obj._persistent_id
        except AttributeError:
            pass
        for klass in self.badClasses: # CHECK FOR LOCAL DEPENDENCIES
            if isinstance(obj,klass):
                raise PygrDataNotPortableError('this object has a local data dependency and cannnot be transferred to a remote client')
        return None
    def setRoot(self,obj,sourceIDs={},badClasses=()):
        'set obj as root of pickling tree: genuinely pickle it (not just its id)'
        self.root=obj
        self.sourceIDs=sourceIDs
        self.badClasses = badClasses


class ResourceDBServer(object):
    'simple XMLRPC resource database server'
    xmlrpc_methods={'getResource':0,'registerServer':0,'delResource':0,
                    'getName':0,'dir':0,'get_version':0}
    _pygr_data_version=(0,1,0)
    def __init__(self,name,readOnly=False):
        self.name=name
        self.d={}
        self.docs={}
        if readOnly: # LOCK THE INDEX.  DON'T ACCEPT FOREIGN DATA!!
            self.xmlrpc_methods={'getResource':0,'getName':0,'dir':0,
                                 'get_version':0} # ONLY ALLOW THESE METHODS!
    def getName(self):
        return self.name
    def getResource(self,id):<commit_msg>made readOnly the default for ResourceDBServer<commit_after>
import pickle
from StringIO import StringIO
import shelve
from mapping import Collection,Mapping,Graph


class OneTimeDescriptor(object):
    'provides shadow attribute based on schema'
    def __init__(self,attrName,**kwargs):
        self.attr=attrName
    def __get__(self,obj,objtype):
        try:
            id=obj._persistent_id # GET ITS RESOURCE ID
        except AttributeError:
            raise AttributeError('attempt to access pygr.Data attr on non-pygr.Data object')
        target=getResource.schemaAttr(id,self.attr) # ATTEMPT TO GET FROM pygr.Data
        obj.__dict__[self.attr]=target # PROVIDE DIRECTLY TO THE __dict__
        return target

class ItemDescriptor(object):
    'provides shadow attribute for items in a db, based on schema'
    def __init__(self,attrName,invert=False,getEdges=False,mapAttr=None,
                 targetAttr=None,**kwargs):
        self.attr=attrName
        self.invert=invert
        self.getEdges=getEdges
        self.mapAttr=mapAttr
        self.targetAttr=targetAttr
    def __get__(self,obj,objtype):
        try:
            id=obj.db._persistent_id # GET RESOURCE ID OF DATABASE
        except AttributeError:
            raise AttributeError('attempt to access pygr.Data attr on non-pygr.Data object')
        targetDict=getResource.schemaAttr(id,self.attr) # ATTEMPT TO GET FROM pygr.Data
        if self.invert:
            targetDict= ~targetDict
        if self.getEdges:
            targetDict=targetDict.edges
        if self.mapAttr is not None: # USE mapAttr TO GET ID FOR MAPPING obj
            result=targetDict[getattr(obj,self.mapAttr)]
        else:
            result=targetDict[obj] # NOW PERFORM MAPPING IN THAT RESOURCE...
        if self.targetAttr is not None:
            result=getattr(result,self.targetAttr) # GET ATTRIBUTE OF THE result
        obj.__dict__[self.attr]=result # PROVIDE DIRECTLY TO THE __dict__
        return result

class ForwardingDescriptor(object):
    'forward an attribute request to item from another container'
    def __init__(self,targetDB,attr):
        self.targetDB=targetDB # CONTAINER TO GET ITEMS FROM
        self.attr=attr # ATTRIBUTE TO MAP TO
    def __get__(self,obj,objtype):
        target=self.targetDB[obj.id] # GET target FROM CONTAINER
        return getattr(target,self.attr) # GET DESIRED ATTRIBUTE

class SpecialMethodDescriptor(object):
    'enables shadowing of special methods like __invert__'
    def __init__(self,attrName):
        self.attr=attrName
    def __get__(self,obj,objtype):
        try:
            return obj.__dict__[self.attr]
        except KeyError:
            raise AttributeError('%s has no method %s'%(obj,self.attr))

def addSpecialMethod(obj,attr,f):
    '''bind function f as special method attr on obj.
       obj cannot be an builtin or extension class
       (if so, just subclass it)'''
    import new
    m=new.instancemethod(f,obj,obj.__class__)
    try:
        if getattr(obj,attr) == m: # ALREADY BOUND TO f
            return # ALREADY BOUND, NOTHING FURTHER TO DO
    except AttributeError:
        pass
    else:
        raise AttributeError('%s already bound to a different function' %attr)
    setattr(obj,attr,m) # SAVE BOUND METHOD TO __dict__
    setattr(obj.__class__,attr,SpecialMethodDescriptor(attr)) # DOES FORWARDING

def getInverseDB(self):
    'default shadow __invert__ method'
    return self.inverseDB # TRIGGER CONSTRUCTION OF THE TARGET RESOURCE


class PygrDataNotPortableError(ValueError):
    'indicates that object has a local data dependency and cannnot be transferred to a remote client'
    pass
class PygrDataNotFoundError(KeyError):
    'unable to find a loadable resource for the requested pygr.Data identifier from PYGRDATAPATH'
    pass
class PygrDataMismatchError(ValueError):
    '_persistent_id attr on object no longer matches its assigned pygr.Data ID?!?'
    pass
class PygrDataEmptyError(ValueError):
    "user hasn't queued anything, so trying to save or rollback is an error"
    pass
class PygrDataReadOnlyError(ValueError):
    'attempt to write data to a read-only resource database'
    pass


class PygrPickler(pickle.Pickler):
    def persistent_id(self,obj):
        'convert objects with _persistent_id to PYGR_ID strings during pickling'
        import types
        try:
            if not isinstance(obj,types.TypeType) and obj is not self.root:
                try:
                    return 'PYGR_ID:%s' % self.sourceIDs[id(obj)]
                except KeyError:
                    if obj._persistent_id is not None:
                        return 'PYGR_ID:%s' % obj._persistent_id
        except AttributeError:
            pass
        for klass in self.badClasses: # CHECK FOR LOCAL DEPENDENCIES
            if isinstance(obj,klass):
                raise PygrDataNotPortableError('this object has a local data dependency and cannnot be transferred to a remote client')
        return None
    def setRoot(self,obj,sourceIDs={},badClasses=()):
        'set obj as root of pickling tree: genuinely pickle it (not just its id)'
        self.root=obj
        self.sourceIDs=sourceIDs
        self.badClasses = badClasses


class ResourceDBServer(object):
    'simple XMLRPC resource database server'
    xmlrpc_methods={'getResource':0,'registerServer':0,'delResource':0,
                    'getName':0,'dir':0,'get_version':0}
    _pygr_data_version=(0,1,0)
    def __init__(self,name,readOnly=True):
        self.name=name
        self.d={}
        self.docs={}
        if readOnly: # LOCK THE INDEX.  DON'T ACCEPT FOREIGN DATA!!
            self.xmlrpc_methods={'getResource':0,'getName':0,'dir':0,
                                 'get_version':0} # ONLY ALLOW THESE METHODS!
    def getName(self):
        return self.name
    def getResource(self,id):
{'fraction_non_alphanumeric': 0.07952013710368466, 'fraction_numerical': 0.0022279348757497856, 'mean_word_length': 2.8442028985507246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1010243', 'n_tokens_mistral': 3407, 'n_tokens_neox': 3325, 'n_words': 992}