<commit_before>import uuid
from collections import namedtuple

import numpy as np
from . import config, log, Parser

from .hnumpy import StorageNumpy
from .IStorage import IStorage, AlreadyPersistentError

from .tools import discrete_token_ranges, extract_ks_tab, count_name_collision, get_istorage_attrs, build_remotely, \
    storage_id_from_name, basic_types, valid_types


class StorageObj(IStorage):
    args_names = ["name", "tokens", "storage_id", "istorage_props", "class_name", "built_remotely"]
    args = namedtuple('StorageObjArgs', args_names)
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba' +
                                                  '.istorage (storage_id, class_name, name, tokens,istorage_props) '
                                                  ' VALUES (?,?,?,?,?)')

    """
    This class is where information will be stored in Hecuba.
    The information can be in memory, stored in a python dictionary or local variables, or saved in a
    DB(Cassandra), depending on if it's persistent or not.
    """

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:

            config.session.execute(StorageObj._prepared_store_meta,
                                   [storage_args.storage_id,
                                    storage_args.class_name,
                                    storage_args.name,
                                    storage_args.tokens,
                                    storage_args.istorage_props])
        except Exception as ex:
            log.warn("Error creating the StorageDict metadata: %s, %s", str(storage_args), ex)
            raise ex

    @classmethod
    def _parse_comments(cls, comments):
        parser = Parser("ClassField")
        return parser._parse_comments(comments)

    def __init__(self, name="", tokens=None, storage_id=None, istorage_props=None, built_remotely=False, **kwargs):
        """
            Creates a new storageobj.
            Args:
                name (string): the name of the Cassandra Keyspace + table where information can be found
                tokens (list of tuples): token ranges assigned to the new StorageObj
                storage_id (string):  an unique storageobj identifier
                istorage_props dict(string,string): a map with the storage id of each contained istorage object.
                kwargs: more optional parameters
        """
        log.debug("CREATED StorageObj(%s)", name)

        super().__init__()

        # Assign private attributes
        self._built_remotely = built_remotely
        self._persistent_props = StorageObj._parse_comments(self.__doc__)
        self._persistent_attrs = self._persistent_props.keys()
        self._class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)

        if name:
            self._ksp, self._table = extract_ks_tab(name)
            name = self._ksp + '.' + self._table
            if storage_id is None:
                # Rebuild storage id
                storage_id = storage_id_from_name(name)

        self.storage_id = storage_id
        if self.storage_id:
            # Retrieve from hecuba istorage the data
            metas = get_istorage_attrs(self.storage_id)

            # If found data, replace the constructor data
            if len(metas) != 0:
                tokens = metas[0].tokens
                istorage_props = metas[0].istorage_props
                name = metas[0].name
                self._ksp, self._table = extract_ks_tab(name)

        if tokens is None:
            # log.info('using all tokens')
            tokens = [token.value for token in config.cluster.metadata.token_map.ring]
            tokens = discrete_token_ranges(tokens)

        self._tokens = tokens
        self._istorage_props = istorage_props

        # Arguments used to build objects remotely
        self._build_args = self.args(name,
                                     self._tokens,
                                     self.storage_id,
                                     self._istorage_props,
                                     self._class_name,
                                     built_remotely)

        if self.storage_id:
            # If never existed, must create the tables and register
            if not self._built_remotely:
                self._create_tables()
            super().make_persistent(name)
            StorageObj._store_meta(self._build_args)
            self._is_persistent = True

        self._load_attributes()

    def _load_attributes(self):
        """
            Loads the IStorage objects into memory by creating them or retrieving from the backend.
        """
        for attribute, value_info in self._persistent_props.items():
            if value_info['type'] not in basic_types:
                # The attribute is an IStorage object
                getattr(self, attribute)

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.getID() == other.getID()

    def _create_tables(self):
        """
            Setups the python structures used to communicate with the backend.
            Creates the necessary tables on the backend to store the object data.
        """

        log.info("CREATING KEYSPACE AND TABLE %s %s", self._ksp, self._table)

        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (self._ksp, config.replication)
        config.session.execute(query_keyspace)

        query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
                       '( storage_id uuid PRIMARY KEY, '
        for key, entry in self._persistent_props.items():
            query_simple += str(key) + ' '
            if entry['type'] != 'dict' and entry['type'] in valid_types:
                if entry['type'] == 'list' or entry['type'] == 'tuple':
                    query_simple += entry['type'] + '<' + entry['columns'] + '>, '<commit_msg>Fix loading attrs on StorageObj<commit_after>import uuid
from collections import namedtuple

import numpy as np
from . import config, log, Parser

from .hnumpy import StorageNumpy
from .IStorage import IStorage, AlreadyPersistentError

from .tools import discrete_token_ranges, extract_ks_tab, count_name_collision, get_istorage_attrs, build_remotely, \
    storage_id_from_name, basic_types, valid_types


class StorageObj(IStorage):
    args_names = ["name", "tokens", "storage_id", "istorage_props", "class_name", "built_remotely"]
    args = namedtuple('StorageObjArgs', args_names)
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba' +
                                                  '.istorage (storage_id, class_name, name, tokens,istorage_props) '
                                                  ' VALUES (?,?,?,?,?)')

    """
    This class is where information will be stored in Hecuba.
    The information can be in memory, stored in a python dictionary or local variables, or saved in a
    DB(Cassandra), depending on if it's persistent or not.
    """

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:

            config.session.execute(StorageObj._prepared_store_meta,
                                   [storage_args.storage_id,
                                    storage_args.class_name,
                                    storage_args.name,
                                    storage_args.tokens,
                                    storage_args.istorage_props])
        except Exception as ex:
            log.warn("Error creating the StorageDict metadata: %s, %s", str(storage_args), ex)
            raise ex

    @classmethod
    def _parse_comments(cls, comments):
        parser = Parser("ClassField")
        return parser._parse_comments(comments)

    def __init__(self, name="", tokens=None, storage_id=None, istorage_props=None, built_remotely=False, **kwargs):
        """
            Creates a new storageobj.
            Args:
                name (string): the name of the Cassandra Keyspace + table where information can be found
                tokens (list of tuples): token ranges assigned to the new StorageObj
                storage_id (string):  an unique storageobj identifier
                istorage_props dict(string,string): a map with the storage id of each contained istorage object.
                kwargs: more optional parameters
        """
        log.debug("CREATED StorageObj(%s)", name)

        super().__init__()

        # Assign private attributes
        self._built_remotely = built_remotely
        self._persistent_props = StorageObj._parse_comments(self.__doc__)
        self._persistent_attrs = self._persistent_props.keys()
        self._class_name = '%s.%s' % (self.__class__.__module__, self.__class__.__name__)

        if name:
            self._ksp, self._table = extract_ks_tab(name)
            name = self._ksp + '.' + self._table
            if storage_id is None:
                # Rebuild storage id
                storage_id = storage_id_from_name(name)

        self.storage_id = storage_id
        if self.storage_id:
            # Retrieve from hecuba istorage the data
            metas = get_istorage_attrs(self.storage_id)

            # If found data, replace the constructor data
            if len(metas) != 0:
                tokens = metas[0].tokens
                istorage_props = metas[0].istorage_props
                name = metas[0].name
                self._ksp, self._table = extract_ks_tab(name)

        if tokens is None:
            # log.info('using all tokens')
            tokens = [token.value for token in config.cluster.metadata.token_map.ring]
            tokens = discrete_token_ranges(tokens)

        self._tokens = tokens
        self._istorage_props = istorage_props

        # Arguments used to build objects remotely
        self._build_args = self.args(name,
                                     self._tokens,
                                     self.storage_id,
                                     self._istorage_props,
                                     self._class_name,
                                     built_remotely)

        if self.storage_id:
            # If never existed, must create the tables and register
            if not self._built_remotely:
                self._create_tables()
            super().make_persistent(name)
            StorageObj._store_meta(self._build_args)
            self._is_persistent = True

        self._load_attributes()

    def _load_attributes(self):
        """
            Loads the IStorage objects into memory by creating them or retrieving from the backend.
        """
        for attribute, value_info in self._persistent_props.items():
            if value_info['type'] not in basic_types:
                # The attribute is an IStorage object
                try:
                    getattr(self, attribute)
                except AttributeError:
                    pass

    def __eq__(self, other):
        return self.__class__ == other.__class__ and self.getID() == other.getID()

    def _create_tables(self):
        """
            Setups the python structures used to communicate with the backend.
            Creates the necessary tables on the backend to store the object data.
        """

        log.info("CREATING KEYSPACE AND TABLE %s %s", self._ksp, self._table)

        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (self._ksp, config.replication)
        config.session.execute(query_keyspace)

        query_simple = 'CREATE TABLE IF NOT EXISTS ' + self._ksp + '.' + self._table + \
                       '( storage_id uuid PRIMARY KEY, '
        for key, entry in self._persistent_props.items():
            query_simple += str(key) + ' '
            if entry['type'] != 'dict' and entry['type'] in valid_types:
                if entry['type'] == 'list' or entry['type'] == 'tuple':
                    query_simple += entry['type'] + '<' + entry['columns'] + '>, '
{'fraction_non_alphanumeric': 0.08271575424336601, 'fraction_numerical': 0.0006375009960953064, 'mean_word_length': 1.911157504059383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1857052', 'n_tokens_mistral': 3291, 'n_tokens_neox': 3162, 'n_words': 920}