<commit_before>#!/usr/bin/env python

__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Rackspace Inc."
__email__ = "<EMAIL>"

import sys
import bson
from bson.json_util import loads
from logging import getLogger

class MongoReaderMock:
    """ Similiar interface to MongoReader. For test purposes. """
    def __init__(self, list_of_raw_bson_data, query=None):
        self.current_raw_bson_data_idx = None
        self.list_of_raw_bson_data = list_of_raw_bson_data
        self.query = query
        self.failed = False
        self.load_next_test_dataset()

    def next_dataset_idx(self):
        if self.current_raw_bson_data_idx is None:
            return 0
        elif self.current_raw_bson_data_idx+1 < len(self.list_of_raw_bson_data):
            return self.current_raw_bson_data_idx + 1
        else:
            return None

    def load_next_test_dataset(self):
        if self.next_dataset_idx() is not None:
            self.current_raw_bson_data_idx = self.next_dataset_idx()
            self.rec_i = 0
            getLogger(__name__).info("MockMongoReader load dataset idx=%d"
                                     % self.current_raw_bson_data_idx)
            data =  loads(self.list_of_raw_bson_data[self.current_raw_bson_data_idx])
            getLogger(__name__).info("MockMongoReader loaded dataset %d recs"  % len(data))
            self.array_data = data
            return data
        else:
            self.array_data = []
            getLogger(__name__).info('No more dataset available')
            return None

    def reset_dataset(self):
        self.current_raw_bson_data_idx = None
        self.load_next_test_dataset()

    def connauthreq(self):
        return None

    def make_new_request(self, query):
        self.query = query

    def next(self):
        rec = None
        getLogger(__name__).info("reader.next query=" + str(self.query))
        if self.query and len(self.query):
            for item_idx in xrange(len(self.array_data)):<commit_msg>renamed reader's mock class members<commit_after>#!/usr/bin/env python

__author__ = "<NAME>"
__copyright__ = "Copyright 2016, Rackspace Inc."
__email__ = "<EMAIL>"

import sys
import bson
from bson.json_util import loads
from logging import getLogger

class MongoReaderMock:
    """ Similiar interface to MongoReader. For test purposes. """
    def __init__(self, datasets_list, query=None):
        self.current_dataset_idx = None
        self.datasets_list = datasets_list
        self.query = query
        self.failed = False
        self.load_next_test_dataset()

    def next_dataset_idx(self):
        if self.current_dataset_idx is None:
            return 0
        elif self.current_dataset_idx+1 < len(self.datasets_list):
            return self.current_dataset_idx + 1
        else:
            return None

    def load_next_test_dataset(self):
        if self.next_dataset_idx() is not None:
            self.current_dataset_idx = self.next_dataset_idx()
            self.rec_i = 0
            getLogger(__name__).info("MockMongoReader load dataset idx=%d"
                                     % self.current_dataset_idx)
            data =  loads(self.datasets_list[self.current_dataset_idx])
            getLogger(__name__).info("MockMongoReader loaded dataset %d recs"  % len(data))
            self.array_data = data
            return data
        else:
            self.array_data = []
            getLogger(__name__).info('No more dataset available')
            return None

    def reset_dataset(self):
        self.current_dataset_idx = None
        self.load_next_test_dataset()

    def connauthreq(self):
        return None

    def make_new_request(self, query):
        self.query = query

    def next(self):
        rec = None
        getLogger(__name__).info("reader.next query=" + str(self.query))
        if self.query and len(self.query):
            for item_idx in xrange(len(self.array_data)):
{'fraction_non_alphanumeric': 0.085155058464667, 'fraction_numerical': 0.004067107269954245, 'mean_word_length': 2.4366812227074237, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5688432', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1160, 'n_words': 257}