<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com <EMAIL>

import os
from json import dumps, loads
from datetime import datetime
from os.path import splitext

from os.path import exists, dirname, join, getmtime

from thumbor.storages import BaseStorage

class Storage(BaseStorage):

    def ensure_dir(self, path):
        if not exists(path):
            os.makedirs(path)

    def put(self, path, bytes):
        file_abspath = self.__normalize_path(path)
        file_dir_abspath = dirname(file_abspath)

        self.ensure_dir(file_dir_abspath)

        with open(file_abspath, 'w') as _file:
            _file.write(bytes)

    def put_crypto(self, path):
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        file_abspath = self.__normalize_path(path)
        file_dir_abspath = dirname(file_abspath)

        self.ensure_dir(file_dir_abspath)

        if not self.context.server.security_key:
            raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        crypto_path = '%s.txt' % splitext(file_abspath)[0]
        with open(crypto_path, 'w') as _file:
            _file.write(self.context.server.security_key)

    def put_detector_data(self, path, data):
        file_abspath = self.__normalize_path(path)

        path = '%s.detectors.txt' % splitext(file_abspath)[0]
        with open(path, 'w') as _file:
            _file.write(dumps(data))

    def get_crypto(self, path):
        file_abspath = self.__normalize_path(path)
        crypto_file = "%s.txt" % (splitext(file_abspath)[0])

        if not exists(crypto_file):
            return None
        return file(crypto_file).read()

    def get(self, path):
        file_abspath = self.__normalize_path(path)

        if not exists(file_abspath) or self.__is_expired(file_abspath):
            return None
        return open(file_abspath, 'r').read()

    def get_detector_data(self, path):
        file_abspath = self.__normalize_path(path)
        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        if not exists(path) or self.__is_expired(path):
            return None

        return loads(open(path, 'r').read())

    def __normalize_path(self, path):
        return join(self.context.config.FILE_STORAGE_ROOT_PATH.rstrip('/'), path.lstrip('/'))<commit_msg>Exposing resolve path method to improve usage of the file storage. People can now inherit from it and change the way the files get stored.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# thumbor imaging service
# https://github.com/globocom/thumbor/wiki

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 globo.com <EMAIL>

import os
from json import dumps, loads
from datetime import datetime
from os.path import splitext

from os.path import exists, dirname, join, getmtime

from thumbor.storages import BaseStorage

class Storage(BaseStorage):

    def ensure_dir(self, path):
        if not exists(path):
            os.makedirs(path)

    def put(self, path, bytes):
        file_abspath = self.normalize_path(path)
        file_dir_abspath = dirname(file_abspath)

        self.ensure_dir(file_dir_abspath)

        with open(file_abspath, 'w') as _file:
            _file.write(bytes)

    def put_crypto(self, path):
        if not self.context.config.STORES_CRYPTO_KEY_FOR_EACH_IMAGE:
            return

        file_abspath = self.normalize_path(path)
        file_dir_abspath = dirname(file_abspath)

        self.ensure_dir(file_dir_abspath)

        if not self.context.server.security_key:
            raise RuntimeError("STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no SECURITY_KEY specified")

        crypto_path = '%s.txt' % splitext(file_abspath)[0]
        with open(crypto_path, 'w') as _file:
            _file.write(self.context.server.security_key)

    def put_detector_data(self, path, data):
        file_abspath = self.normalize_path(path)

        path = '%s.detectors.txt' % splitext(file_abspath)[0]
        with open(path, 'w') as _file:
            _file.write(dumps(data))

    def get_crypto(self, path):
        file_abspath = self.normalize_path(path)
        crypto_file = "%s.txt" % (splitext(file_abspath)[0])

        if not exists(crypto_file):
            return None
        return file(crypto_file).read()

    def get(self, path):
        file_abspath = self.normalize_path(path)

        if not exists(file_abspath) or self.__is_expired(file_abspath):
            return None
        return open(file_abspath, 'r').read()

    def get_detector_data(self, path):
        file_abspath = self.normalize_path(path)
        path = '%s.detectors.txt' % splitext(file_abspath)[0]

        if not exists(path) or self.__is_expired(path):
            return None

        return loads(open(path, 'r').read())

    def normalize_path(self, path):
        return join(self.context.config.FILE_STORAGE_ROOT_PATH.rstrip('/'), path.lstrip('/'))
{'fraction_non_alphanumeric': 0.10611614894848544, 'fraction_numerical': 0.003472892147404978, 'mean_word_length': 2.9785111281657715, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2342234', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1700, 'n_words': 363}