<commit_before>import logging.handlers
import os
import redis
from pystemon.storage import PastieStorage

logger = logging.getLogger('pystemon')

class RedisStorage(PastieStorage):
    def __getconn(self):
        # LATER: implement pipelining
        return redis.StrictRedis(host=self.server, port=self.port, db=self.database)

    def __init_storage__(self, **kwargs):
        self.save_dir = kwargs['save_dir']
        self.archive_dir = kwargs['archive_dir']
        self.server = kwargs['server']
        self.port = kwargs['port']
        self.database = kwargs['database']
        self.save_all = kwargs['save-all']
        r = self.__getconn()
        r.ping()

    def __save_pastie__(self, pastie):
        directories = []
        directories.append(self.archive_dir)
        if pastie.matched:
            directories.append(self.save_dir)
        for directory in directories:
            if directory is None:
                continue
            directory = directory + os.sep + pastie.site.name
            full_path = self.format_directory(directory) + os.sep + pastie.filename
            if pastie.matched or self.save_all:
                self.__getconn().lpush('pastes', full_path)
                logger.debug('Site[{site}]: Sent pastie[{id}][{disk}] to redis.'.format(site=pastie.site.name, id=pastie.id, disk=full_path))

<commit_msg>fix: [redis] fixes #119 - file path incorrect with redis<commit_after>import logging.handlers
import os
import redis
from pystemon.storage import PastieStorage

logger = logging.getLogger('pystemon')

class RedisStorage(PastieStorage):
    def __getconn(self):
        # LATER: implement pipelining
        return redis.StrictRedis(host=self.server, port=self.port, db=self.database)

    def __init_storage__(self, **kwargs):
        self.save_dir = kwargs['save_dir']
        self.archive_dir = kwargs['archive_dir']
        self.server = kwargs['server']
        self.port = kwargs['port']
        self.database = kwargs['database']
        self.save_all = kwargs['save-all']
        r = self.__getconn()
        r.ping()

    def __save_pastie__(self, pastie):
        directories = []
        directories.append(self.archive_dir)
        if pastie.matched:
            directories.append(self.save_dir)
        for directory in directories:
            if directory is None:
                continue
            directory = directory + os.sep + pastie.site.site
            full_path = self.format_directory(directory) + os.sep + pastie.filename
            if pastie.matched or self.save_all:
                self.__getconn().lpush('pastes', full_path)
                logger.debug('Site[{site}]: Sent pastie[{id}][{disk}] to redis.'.format(site=pastie.site.name, id=pastie.id, disk=full_path))


{'fraction_non_alphanumeric': 0.11408909815284317, 'fraction_numerical': 0.0010865628395508873, 'mean_word_length': 2.677762982689747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4490696', 'n_tokens_mistral': 841, 'n_tokens_neox': 818, 'n_words': 166}