<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""cassandra support for unwound spider using pycassa"""

import pycassa

from arachne import utils
from arachne.conf import settings, merge, require

defaults = {
    'port': 9160,
    'timeout': 5,
    'max_retries': 10,
    'pool_timeout': 60,
    'recycle': 1000,
    'prefill': False,
}

encode = utils.encode

def decode(data, colname=None):
    """Decode data coming out of storage."""
    if colname:
        return utils.decode(data[colname])
    return utils.decode(data)


class Cassandra(object):
    def __init__(self, **kwargs):
        config = merge(defaults, settings.like('cassandra'), kwargs)
        require(self, config, ('cf_content', 'keyspace', 'servers', 'port'))
        self.__dict__.update(config)
        self.pool_size = len(self.servers) * 2
        self.pool = pycassa.ConnectionPool(
            self.keyspace,
            self.servers,
            timeout=self.timeout,
            max_retries=self.max_retries,
            pool_timeout=self.pool_timeout,
            pool_size=self.pool_size,
            recycle=self.recycle,
            prefill=self.prefill,
        )
        self.client = pycassa.ColumnFamily(self.pool, self.cf_content)

    def set(self, user_id, data, uuid=None):
        self.client.insert(str(user_id), {uuid: encode(data)})

<commit_msg>adding cassandra.get<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""cassandra support for unwound spider using pycassa"""

import pycassa

from arachne import utils
from arachne.conf import settings, merge, require

defaults = {
    'port': 9160,
    'timeout': 5,
    'max_retries': 10,
    'pool_timeout': 60,
    'recycle': 1000,
    'prefill': False,
}

encode = utils.encode

def decode(data, colname=None):
    """Decode data coming out of storage."""
    if colname:
        return utils.decode(data[colname])
    return utils.decode(data)


class Cassandra(object):
    def __init__(self, **kwargs):
        config = merge(defaults, settings.like('cassandra'), kwargs)
        require(self, config, ('cf_content', 'keyspace', 'servers', 'port'))
        self.__dict__.update(config)
        self.pool_size = len(self.servers) * 2
        self.pool = pycassa.ConnectionPool(
            self.keyspace,
            self.servers,
            timeout=self.timeout,
            max_retries=self.max_retries,
            pool_timeout=self.pool_timeout,
            pool_size=self.pool_size,
            recycle=self.recycle,
            prefill=self.prefill,
        )
        self.client = pycassa.ColumnFamily(self.pool, self.cf_content)

    def set(self, user_id, data, uuid=None):
        return self.client.insert(str(user_id), {uuid: encode(data)})

    def get(self, user_id, uuid):
        return decode(self.client.get(str(user_id), columns=[uuid]).values()[0])

{'fraction_non_alphanumeric': 0.13153279551034724, 'fraction_numerical': 0.010873377762188706, 'mean_word_length': 2.8026666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1263321', 'n_tokens_mistral': 939, 'n_tokens_neox': 911, 'n_words': 195}