<commit_before>#! /usr/bin/env python

'''Our code to connect to the Riak backend'''

import struct
import pymongo
from pymongo import common
from . import BaseClient
import calendar
from datetime import datetime
from dateutil.relativedelta import relativedelta
import dateutil.parser

pymongo.common.VALIDATORS['days'] = pymongo.common.validate_positive_integer


def unsigned_to_signed(integer):
    '''Convert an unsigned integer into a signed integer with the same bits'''
    return struct.unpack('!q', struct.pack('!Q', integer))[0]


def signed_to_unsigned(integer):
    '''Convert an unsigned integer into a signed integer with the same bits'''
    return struct.unpack('!Q', struct.pack('!q', integer))[0]


class Client(BaseClient):
    '''Our Mongo backend client'''
    def __init__(self, name, num_blocks, num_bits, *args, **kwargs):
        BaseClient.__init__(self, name, num_blocks, num_bits)
        self.client = pymongo.Connection(*args, **kwargs)
        self.days = kwargs.pop('days', None)
        self.namePrefix = name + '-'
        if self.days is None:
            self.names = [name]
            self.docsList = [getattr(self.client, name).documents]
        else:
            today = datetime.now()
            self.names = [self.namePrefix
                          + (today -
                             relativedelta(days=i)).strftime(
                                 '%Y-%m-%d')
                          for i in range(self.days)]
            self.docsList = [getattr(self.client, n).documents
                             for n in self.names]

        # Create the indexes (if they exist it's ok)
        for j in range(len(self.names)):
            for i in range(self.num_tables):
                self.docsList[j].create_index(str(i), pymongo.ASCENDING)

    def delete(self):
        '''Delete this database of simhashes'''
        for name in self.names:
            self.client.drop_database(name)

    def delete_old(self):
        '''Delete data that's older than the retention period.'''
        if self.days is None:
            return<commit_msg>pymongo doesn't like the days argument.<commit_after>#! /usr/bin/env python

'''Our code to connect to the Riak backend'''

import struct
import pymongo
from pymongo import common
from . import BaseClient
import calendar
from datetime import datetime
from dateutil.relativedelta import relativedelta
import dateutil.parser

pymongo.common.VALIDATORS['days'] = pymongo.common.validate_positive_integer


def unsigned_to_signed(integer):
    '''Convert an unsigned integer into a signed integer with the same bits'''
    return struct.unpack('!q', struct.pack('!Q', integer))[0]


def signed_to_unsigned(integer):
    '''Convert an unsigned integer into a signed integer with the same bits'''
    return struct.unpack('!Q', struct.pack('!q', integer))[0]


class Client(BaseClient):
    '''Our Mongo backend client'''
    def __init__(self, name, num_blocks, num_bits, *args, **kwargs):
        BaseClient.__init__(self, name, num_blocks, num_bits)
        self.days = kwargs.pop('days', None)
        self.client = pymongo.Connection(*args, **kwargs)
        self.namePrefix = name + '-'
        if self.days is None:
            self.names = [name]
            self.docsList = [getattr(self.client, name).documents]
        else:
            today = datetime.now()
            self.names = [self.namePrefix
                          + (today -
                             relativedelta(days=i)).strftime(
                                 '%Y-%m-%d')
                          for i in range(self.days)]
            self.docsList = [getattr(self.client, n).documents
                             for n in self.names]

        # Create the indexes (if they exist it's ok)
        for j in range(len(self.names)):
            for i in range(self.num_tables):
                self.docsList[j].create_index(str(i), pymongo.ASCENDING)

    def delete(self):
        '''Delete this database of simhashes'''
        for name in self.names:
            self.client.drop_database(name)

    def delete_old(self):
        '''Delete data that's older than the retention period.'''
        if self.days is None:
            return
{'fraction_non_alphanumeric': 0.11068883610451306, 'fraction_numerical': 0.0009501187648456057, 'mean_word_length': 2.3394131641554323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008787', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1168, 'n_words': 344}