<commit_before>import networkx as nx
import pprint
import collections
import time
import autonetkit.log as log
import ank_json
import functools
import string

try:
    import cPickle as pickle
except ImportError:
    import pickle

class interface_data_dict(collections.MutableMapping):
    """A dictionary which allows access as dict.key as well as dict['key']
    Based on http://stackoverflow.com/questions/3387691
    only allows read only acess
    """

    def __repr__(self):
        return ", ".join(self.store.keys())

    def __init__(self, data):
#Note this won't allow updates in place
        self.store = data
        #self.data = parent[index]
        #self.update(dict(*args, **kwargs)) # use the free update to set keys
#TODO: remove duplicate of self.store and parent

    def __getitem__(self, key):
        return self.store[self.__keytransform__(key)]

    def __setitem__(self, key, value):
        self.store[key] = value # store locally

    def __delitem__(self, key):
        del self.store[self.__keytransform__(key)]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def __keytransform__(self, key):
        return key

    def __getattr__(self, key):
        return self.store.get(key)

    def __setattr__(self, key, value):
        if key == "store":
            object.__setattr__(self, 'store', value)
        else:
            self.store[key] = value # store locally

    def dump(self):
        return self.store

class overlay_data_dict(collections.MutableMapping):
    """A dictionary which allows access as dict.key as well as dict['key']
    Based on http://stackoverflow.com/questions/3387691
    """

    def __repr__(self):
        return ", ".join(self.store.keys())

    def __init__(self, parent, index, data):
#Note this won't allow updates in place
        self.parent = parent
        self.index = index
        self.store = data
        #self.data = parent[index]
        #self.update(dict(*args, **kwargs)) # use the free update to set keys
#TODO: remove duplicate of self.store and parent

    def __getitem__(self, key):
        return self.store[self.__keytransform__(key)]

    def __setitem__(self, key, value):
        self.store[key] = value # store locally
        self.parent._update_dict(self.index, key, value)

    def __delitem__(self, key):
        del self.store[self.__keytransform__(key)]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def __keytransform__(self, key):
        return key

    def __getattr__(self, key):
        return self.store.get(key)

#TODO: fix __setattr__ so doesn't cause recursion - and so doesn't silently not set values in compiler
#same as for interfaces

    def dump(self):
        return self.store

class overlay_data_list_of_dicts(object):
    #NOTE: this is just a presentation of the data - so can't modify the original
# but can do in place sort, append, etc
    #TODO: make this inherit from list - so can remove .append() etc
    def __init__(self, parent, key):
        self.parent = parent
        self.key = key
        self.data = parent[key]

    def __getstate__(self):
        return (self.data)

    def __getnewargs__(self):
        return ()

    def __setstate__(self, state):
        self.data = state

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        return str(self.data)

    def __nonzero__(self):
        """Allows for checking if data exists """
        if len(self.data):
            return True
        else:
            return False

    def dump(self):
        return self.data

    def append(self, *args, **kwargs):
        if args:
            item = args[0]
            self.data.append(item)
        if kwargs:
            self.data.append(kwargs)

    def sort(self, key, reverse = False):
        #TODO: allow multiple keys
        self.data.sort(key = lambda x: x.get(key), reverse = reverse)

    def __iter__(self):
        #TODO: want to introduce some sorting here.... how?
        #return iter(overlay_data_dict(self, item) for item in self.data)
        for index, item in enumerate(self.data):
            yield overlay_data_dict(self, index, item)

    def __getitem__(self, index):
        return overlay_data_dict(self, index, self.data[index])

    def _update_dict(self, index, key, value):
        """Used to feed changes from overlay_data_dict back"""
        self.parent[self.key][index][key] = value


class overlay_interface(object):
    def __init__(self, nidb, node_id, interface_id):
        object.__setattr__(self, 'nidb', nidb)
        object.__setattr__(self, 'node_id', node_id)
        object.__setattr__(self, 'interface_id', interface_id)

    def __key(self):
        # based on http://stackoverflow.com/q/2909106
        return (self.interface_id, self.node_id)

    def __hash__(self):
        """"""
        return hash(self.__key())

    def __eq__(self, other):
        return self.__key() == other.__key()

    def __repr__(self):
        description = self.description or self.interface_id
        return "(%s, %s)" % (self.node_id, description)

    def __nonzero__(self):
        """Allows for checking if node exists
        """
        return len(self._interface) > 0  # if interface data set

    def __str__(self):
        return self.__repr__()

    @property
    def _graph(self):
        """Return graph the node belongs to"""
        return self.nidb._graph

    @property
    def _node(self):
        """Return graph the node belongs to"""<commit_msg>add is_bound property for nidb interfaces for parity with anm<commit_after>import networkx as nx
import pprint
import collections
import time
import autonetkit.log as log
import ank_json
import functools
import string

try:
    import cPickle as pickle
except ImportError:
    import pickle

class interface_data_dict(collections.MutableMapping):
    """A dictionary which allows access as dict.key as well as dict['key']
    Based on http://stackoverflow.com/questions/3387691
    only allows read only acess
    """

    def __repr__(self):
        return ", ".join(self.store.keys())

    def __init__(self, data):
#Note this won't allow updates in place
        self.store = data
        #self.data = parent[index]
        #self.update(dict(*args, **kwargs)) # use the free update to set keys
#TODO: remove duplicate of self.store and parent

    def __getitem__(self, key):
        return self.store[self.__keytransform__(key)]

    def __setitem__(self, key, value):
        self.store[key] = value # store locally

    def __delitem__(self, key):
        del self.store[self.__keytransform__(key)]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def __keytransform__(self, key):
        return key

    def __getattr__(self, key):
        return self.store.get(key)

    def __setattr__(self, key, value):
        if key == "store":
            object.__setattr__(self, 'store', value)
        else:
            self.store[key] = value # store locally

    def dump(self):
        return self.store

class overlay_data_dict(collections.MutableMapping):
    """A dictionary which allows access as dict.key as well as dict['key']
    Based on http://stackoverflow.com/questions/3387691
    """

    def __repr__(self):
        return ", ".join(self.store.keys())

    def __init__(self, parent, index, data):
#Note this won't allow updates in place
        self.parent = parent
        self.index = index
        self.store = data
        #self.data = parent[index]
        #self.update(dict(*args, **kwargs)) # use the free update to set keys
#TODO: remove duplicate of self.store and parent

    def __getitem__(self, key):
        return self.store[self.__keytransform__(key)]

    def __setitem__(self, key, value):
        self.store[key] = value # store locally
        self.parent._update_dict(self.index, key, value)

    def __delitem__(self, key):
        del self.store[self.__keytransform__(key)]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def __keytransform__(self, key):
        return key

    def __getattr__(self, key):
        return self.store.get(key)

#TODO: fix __setattr__ so doesn't cause recursion - and so doesn't silently not set values in compiler
#same as for interfaces

    def dump(self):
        return self.store

class overlay_data_list_of_dicts(object):
    #NOTE: this is just a presentation of the data - so can't modify the original
# but can do in place sort, append, etc
    #TODO: make this inherit from list - so can remove .append() etc
    def __init__(self, parent, key):
        self.parent = parent
        self.key = key
        self.data = parent[key]

    def __getstate__(self):
        return (self.data)

    def __getnewargs__(self):
        return ()

    def __setstate__(self, state):
        self.data = state

    def __len__(self):
        return len(self.data)

    def __repr__(self):
        return str(self.data)

    def __nonzero__(self):
        """Allows for checking if data exists """
        if len(self.data):
            return True
        else:
            return False

    def dump(self):
        return self.data

    def append(self, *args, **kwargs):
        if args:
            item = args[0]
            self.data.append(item)
        if kwargs:
            self.data.append(kwargs)

    def sort(self, key, reverse = False):
        #TODO: allow multiple keys
        self.data.sort(key = lambda x: x.get(key), reverse = reverse)

    def __iter__(self):
        #TODO: want to introduce some sorting here.... how?
        #return iter(overlay_data_dict(self, item) for item in self.data)
        for index, item in enumerate(self.data):
            yield overlay_data_dict(self, index, item)

    def __getitem__(self, index):
        return overlay_data_dict(self, index, self.data[index])

    def _update_dict(self, index, key, value):
        """Used to feed changes from overlay_data_dict back"""
        self.parent[self.key][index][key] = value


class overlay_interface(object):
    def __init__(self, nidb, node_id, interface_id):
        object.__setattr__(self, 'nidb', nidb)
        object.__setattr__(self, 'node_id', node_id)
        object.__setattr__(self, 'interface_id', interface_id)

    def __key(self):
        # based on http://stackoverflow.com/q/2909106
        return (self.interface_id, self.node_id)

    def __hash__(self):
        """"""
        return hash(self.__key())

    def __eq__(self, other):
        return self.__key() == other.__key()

    @property
    def is_bound(self):
        """Returns if this interface is bound to an edge on this layer"""
        return len(self.edges()) > 0

    def __repr__(self):
        description = self.description or self.interface_id
        return "(%s, %s)" % (self.node_id, description)

    def __nonzero__(self):
        """Allows for checking if node exists
        """
        return len(self._interface) > 0  # if interface data set

    def __str__(self):
        return self.__repr__()

    @property
    def _graph(self):
        """Return graph the node belongs to"""
        return self.nidb._graph

    @property
    def _node(self):
        """Return graph the node belongs to"""
{'fraction_non_alphanumeric': 0.10495882249868582, 'fraction_numerical': 0.004117750131417557, 'mean_word_length': 2.597541758588087, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1731130', 'n_tokens_mistral': 3457, 'n_tokens_neox': 3300, 'n_words': 1026}