<commit_before>from __future__ import print_function, absolute_import, division

import os

try:
    import urlparse
except ImportError:
    # python > 3.1 has not module urlparse
    import urllib.parse as urlparse

import gnodeclient.util.hdf5io as hdf5io
import gnodeclient.util.helper as helper
import gnodeclient.store.convert as convert
from gnodeclient.store.basic_store import BasicStore
from gnodeclient.util.cache import Cache


class CacheStore(BasicStore):
    """
    A simple cache store.
    """

    def __init__(self, location=None):
        super(CacheStore, self).__init__(location)
        self.__cache = Cache(self.location)

    def connect(self):
        if self.__cache is None:
            self.__cache = Cache(self.location)

    def is_connected(self):
        return self.__cache is not None

    def disconnect(self):
        if self.__cache is not None:
            del self.__cache
        self.__cache = None

    def get(self, location):
        obj = self.__cache.get(location)
        if obj is not None:
            entity = convert.collections_to_model(obj)
            return entity
        else:
            return None

    def get_file(self, location):
        """
        Get raw file data (bytestring) from the cache.

        :param location: The locations of all entities as path or URL.
        :type location: str

        :returns: The raw file data.
        :rtype: str
        """
        data = self.__cache.get_file(location)
        return data

    def get_array(self, location):
        """
        Read array data from an hdf5 file in the cache.

        :param location: The locations of all entities as path or URL.
        :type location: str

        :returns: The raw file data.
        :rtype: numpy.ndarray|list
        """
        ident = helper.id_from_location(location)
        path = self.__cache.file_cache_path(ident)

        if os.path.isfile(path):
            data = hdf5io.read_array_data(path)
            return data
        else:
            return None

    def set(self, entity):
        if entity is not None:
            obj = convert.model_to_collections(entity)
            self.__cache.set(entity.location, obj)
        return entity

    def set_file(self, location, data):
        """
        Save raw file data in the cache.

        :param location: The location of the file.
        :type location: str
        :param data: The raw data of the file.
        :type data: str
        """
        self.__cache.set_file(location, data)

    def set_array(self, location, array_data):
        """
        Save array data in a cached HDF5 file.

        :param location: The location of the file.
        :type location: str
        :param array_data: The raw data to store.
        :type array_data: numpy.ndarray|list
        """
        ident = helper.id_from_location(location)
        path = self.__cache.file_cache_path(ident)

        data = hdf5io.store_array_data(path, array_data)
        return data

    def delete(self, entity):
        if entity is not None:
            self.__cache.delete(entity.location)

    def clear_cache(self):
        self.__cache.clear()<commit_msg>CacheStore: implement file and array methods<commit_after>from __future__ import print_function, absolute_import, division

import os

try:
    import urlparse
except ImportError:
    # python > 3.1 has not module urlparse
    import urllib.parse as urlparse

import gnodeclient.util.hdf5io as hdf5io
import gnodeclient.util.helper as helper
import gnodeclient.store.convert as convert
from gnodeclient.store.basic_store import BasicStore
from gnodeclient.util.cache import Cache


class CacheStore(BasicStore):
    """
    A simple cache store.
    """

    def __init__(self, location=None):
        super(CacheStore, self).__init__(location)
        self.__cache = Cache(self.location)

    def connect(self):
        if self.__cache is None:
            self.__cache = Cache(self.location)

    def is_connected(self):
        return self.__cache is not None

    def disconnect(self):
        if self.__cache is not None:
            del self.__cache
        self.__cache = None

    def get(self, location):
        obj = self.__cache.get(location)
        if obj is not None:
            entity = convert.collections_to_model(obj)
            return entity
        else:
            return None

    def get_file(self, location):
        """
        Get raw file data (bytestring) from the cache.

        :param location: The locations of all entities as path or URL.
        :type location: str

        :returns: The raw file data.
        :rtype: str
        """
        data = self.__cache.get_file(location)
        return data

    def get_array(self, location):
        """
        Read array data from an hdf5 file in the cache.

        :param location: The locations of all entities as path or URL.
        :type location: str

        :returns: The raw file data.
        :rtype: numpy.ndarray|list
        """
        ident = helper.id_from_location(location)
        path = self.__cache.file_cache_path(ident)

        if os.path.isfile(path):
            data = hdf5io.read_array_data(path)
            return data
        else:
            return None

    def set(self, entity):
        if entity is not None:
            obj = convert.model_to_collections(entity)
            self.__cache.set(entity.location, obj)
        return entity

    def set_file(self, data, location=None):
        """
        Save raw file data in the cache.

        :param data: The raw data of the file.
        :type data: str
        :param location: The location of the file.
        :type location: str

        :returns: The url to the uploaded file.
        :rtype: str
        """
        if location is None:
            location = "datafiles/datafile/" + helper.random_str()

        self.__cache.set_file(location, data)

    def set_array(self, array_data, location=None):
        """
        Save array data in a cached HDF5 file.

        :param array_data: The raw data to store.
        :type array_data: numpy.ndarray|list
        :param location: The location of the file.
        :type location: str

        :returns: The url to the uploaded file.
        :rtype: str
        """
        if location is None:
            ident = helper.random_str()
            location = "datafiles/datafile/" + ident
        else:
            ident = helper.id_from_location(location)

        path = self.__cache.file_cache_path(ident)
        hdf5io.store_array_data(path, array_data)

        return location

    def delete(self, entity):
        if entity is not None:
            self.__cache.delete(entity.location)

    def delete_file(self, location):
        self.__cache.delete_file(location)

    def clear_cache(self):
        self.__cache.clear()
{'fraction_non_alphanumeric': 0.0794972965073798, 'fraction_numerical': 0.002338155779628818, 'mean_word_length': 2.1094956837801, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3081504', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1875, 'n_words': 608}