<commit_before>"""
funsize.database.cache
~~~~~~~~~~~~~~~~~~~~~~~~~~~

This is currently a stub file that contains function prototypes for the
caching layer core

"""

import os
from boto.s3.connection import S3Connection

import funsize.utils.oddity as oddity


class Cache(object):
    """ Class that provides access to cache
        Assumes all keys are hex-encoded SHA512s
        Internally converts  hex to base64 encoding
    """

    def __init__(self, _bucket=os.environ.get('FUNSIZE_S3_UPLOAD_BUCKET')):
        """ _bucket : bucket name to use for S3 resources """
        if not _bucket:
            raise oddity.CacheError("Amazon S3 bucket not set")
        # open a connection and get the bucket
        self.conn = S3Connection()
        self.bucket = self.conn.get_bucket(_bucket)

    def _get_cache_internals(self, identifier, category):
        """ Method to return cache bucket key based on identifier """
        if not identifier:
            raise oddity.CacheError('Save object failed without identifier')
        if category not in ('diff', 'partial', 'patch'):
            raise oddity.CacheError("Category failed for S3 uploading")
        bucket_key = "files/%s/%s" % (category, identifier)
        return bucket_key

    def _create_new_bucket_key(self, identifier, category):
        """ Based on identifier and category create a new key in the bucket"""
        _key = self._get_cache_internals(identifier, category)
        return self.bucket.new_key(_key)

    def _get_bucket_key(self, identifier, category):
        """ Based on identifier and category retrieve key from bucket """
        _key = self._get_cache_internals(identifier, category)
        return self.bucket.get_key(_key)

    def save(self, string, identifier, category, isfile=False):
        """ Saves given file to cache, treats string as a local filepath if
            isfile is true. returns hash of file.
        """
        # FIXME: What should the behaviour be when we try to save to a
        # pre-existing key?
        key = self._create_new_bucket_key(identifier, category)
        if isfile:
            key.set_contents_from_filename(string)
        else:
            key.set_contents_from_string(string)
        key.set_acl('public-read')

    def save_blank_file(self, identifier, category):
        """ Method to save a blank file to show a partial has been triggered and
            it is being in progress
        """
        key = self._create_new_bucket_key(identifier, category)
        key.set_contents_from_string('')
        key.set_acl('public-read')

    def is_blank_file(self, identifier, category):
        """ Function to check if the file is empty or not. To be used to ensure
            no second triggering is done for the same partial
            Returns True is file exists and is blank, False otherwise
        """
        key = self._get_bucket_key(identifier, category)
        if not key:
            return False
        return key.size == 0

    def find(self, identifier, category):
        """ Checks if file with specified key is in cache
            returns True or False depending on whether the file exists
        """
        key = self._get_bucket_key(identifier, category)
        return bool(key)

    def retrieve(self, identifier, category, output_file=None):
        """ Retrieve file with the given key
            writes the file to the path specified by output_file if present
            otherwise returns the file as a binary string/file object
        """
        key = self._get_bucket_key(identifier, category)
        if output_file:
            key.get_contents_to_filename(output_file)<commit_msg>Default to private policy in cache S3 objects.<commit_after>"""
funsize.database.cache
~~~~~~~~~~~~~~~~~~~~~~~~~~~

This is currently a stub file that contains function prototypes for the
caching layer core

"""

import os
from boto.s3.connection import S3Connection

import funsize.utils.oddity as oddity


class Cache(object):
    """ Class that provides access to cache
        Assumes all keys are hex-encoded SHA512s
        Internally converts  hex to base64 encoding
    """

    def __init__(self, _bucket=os.environ.get('FUNSIZE_S3_UPLOAD_BUCKET')):
        """ _bucket : bucket name to use for S3 resources """
        if not _bucket:
            raise oddity.CacheError("Amazon S3 bucket not set")
        # open a connection and get the bucket
        self.conn = S3Connection()
        self.bucket = self.conn.get_bucket(_bucket)

    def _get_cache_internals(self, identifier, category):
        """ Method to return cache bucket key based on identifier """
        if not identifier:
            raise oddity.CacheError('Save object failed without identifier')
        if category not in ('diff', 'partial', 'patch'):
            raise oddity.CacheError("Category failed for S3 uploading")
        bucket_key = "files/%s/%s" % (category, identifier)
        return bucket_key

    def _create_new_bucket_key(self, identifier, category):
        """ Based on identifier and category create a new key in the bucket"""
        _key = self._get_cache_internals(identifier, category)
        return self.bucket.new_key(_key)

    def _get_bucket_key(self, identifier, category):
        """ Based on identifier and category retrieve key from bucket """
        _key = self._get_cache_internals(identifier, category)
        return self.bucket.get_key(_key)

    def save(self, string, identifier, category, isfile=False):
        """ Saves given file to cache, treats string as a local filepath if
            isfile is true. returns hash of file.
        """
        # FIXME: What should the behaviour be when we try to save to a
        # pre-existing key?
        key = self._create_new_bucket_key(identifier, category)
        if isfile:
            key.set_contents_from_filename(string)
        else:
            key.set_contents_from_string(string)

    def save_blank_file(self, identifier, category):
        """ Method to save a blank file to show a partial has been triggered and
            it is being in progress
        """
        key = self._create_new_bucket_key(identifier, category)
        key.set_contents_from_string('')

    def is_blank_file(self, identifier, category):
        """ Function to check if the file is empty or not. To be used to ensure
            no second triggering is done for the same partial
            Returns True is file exists and is blank, False otherwise
        """
        key = self._get_bucket_key(identifier, category)
        if not key:
            return False
        return key.size == 0

    def find(self, identifier, category):
        """ Checks if file with specified key is in cache
            returns True or False depending on whether the file exists
        """
        key = self._get_bucket_key(identifier, category)
        return bool(key)

    def retrieve(self, identifier, category, output_file=None):
        """ Retrieve file with the given key
            writes the file to the path specified by output_file if present
            otherwise returns the file as a binary string/file object
        """
        key = self._get_bucket_key(identifier, category)
        if output_file:
            key.get_contents_to_filename(output_file)
{'fraction_non_alphanumeric': 0.08128855380397533, 'fraction_numerical': 0.0037011651816312545, 'mean_word_length': 2.64253619570644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1839687', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1865, 'n_words': 702}