<commit_before>import unittest
import pymongo
from app import MongoInputSplit as MI
from app import MongoSplitter as MS
import datetime
import bson

config = {
        "db_name": "test",
        "collection_name": "in",
        "splitSize": 1, #MB
        "inputURI": "mongodb://localhost/test.in",
        "createInputSplits": True,
        "splitKey": {'_id' : 1},
        }

class TestSplits(unittest.TestCase):
    def runTest(self):
        #put 20000 objects in a database, call for a split by hand, then a split by the class
        conn = pymongo.Connection()
        db = conn[config.get('db_name')]
        coll = db[config.get('collection_name')]
        #print db.command("collstats", coll.full_name)
        '''
        for i in range(20000):
            post = {"name" : i, "date": datetime.datetime.utcnow()}
            coll.insert(post)
        '''

        #print coll.count()

        command = bson.son.SON()
        command['splitVector'] = coll.full_name
        command['maxChunkSize'] = config.get('splitSize')
        command['force'] = False
        command['keyPattern'] = {'_id' : 1}
        results = db.command(command)

        man_splits = results.get("splitKeys")
        assert results.get('ok') == 1.0, 'split command did not return with 1.0 ok'
        #print results
        #print len(man_splits)
        assert man_splits, 'no splitKeys returned'

        #now do it through MongoSplit
        splits = MS.calculate_splits(config)

        assert splits, "MongoSplitter did not return the right splits"
        #print len(splits)
        assert len(man_splits) + 1 == len(splits) , "MongoSplitter returned a different number of splits than manual splits"

if __name__ == '__main__':
    unittest.main()<commit_msg>tested splits inserting into MongoDB<commit_after>import unittest
import pymongo
from app import MongoInputSplit as MI
from app import MongoSplitter as MS
import datetime
import bson
import logging

config = {
        "db_name": "test",
        "collection_name": "in",
        "splitSize": 1, #MB
        "inputURI": "mongodb://localhost/test.in",
        "createInputSplits": True,
        "splitKey": {'_id' : 1},
        }

config2 = {
        "db_name": "test",
        "collection_name": "tempSplit",
        "splitSize": 1, #MB
        "inputURI": "mongodb://localhost/test.in",
        "createInputSplits": True,
        "splitKey": {'_id' : 1},
        }

class TestSplits(unittest.TestCase):
    def runTest(self):
        #put 20000 objects in a database, call for a split by hand, then a split by the class
        conn = pymongo.Connection()
        db = conn[config.get('db_name')]
        coll = db[config.get('collection_name')]
        print coll.full_name
        
        '''for i in range(20000):
            post = {"name" : i, "date": datetime.datetime.utcnow()}
            coll.insert(post)
        '''

        #print coll.count()

        command = bson.son.SON()
        command['splitVector'] = coll.full_name
        command['maxChunkSize'] = config.get('splitSize')
        command['force'] = False
        command['keyPattern'] = {'_id' : 1}
        results = db.command(command)

        man_splits = results.get("splitKeys")
        assert results.get('ok') == 1.0, 'split command did not return with 1.0 ok'
        #print results
        print 'man_splits = ', len(man_splits)
        assert man_splits, 'no splitKeys returned'

        #now do it through MongoSplit
        splits = MS.calculate_splits(config)

        assert splits, "MongoSplitter did not return the right splits"
        print len(splits)
        assert len(man_splits) + 1 == len(splits) , "MongoSplitter returned a different number of splits than manual splits"

        base_name = config2.get('collection_name')
        for j, i in enumerate(splits):
            coll_name = base_name + str(j)
	    coll = db[coll_name]
            coll.insert(i.cursor)
            
             

if __name__ == '__main__':
    #logging.getLogger().setLevel(logging.DEBUG)
    unittest.main()
{'fraction_non_alphanumeric': 0.12029702970297029, 'fraction_numerical': 0.009900990099009901, 'mean_word_length': 2.433305012744265, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3773375', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1201, 'n_words': 332}