<commit_before>#! /usr/bin/python

import sys
import functools
import time

from functools import partial
from benchmark import Benchmark
from bulk_insert import BulkInserter
from hyperdex.mongo import HyperDatabase, HyperSpace 

### CONFIG ###
# Modify this to change the behaviour of the benchmark

# How many atomic add calls?
numCalls = 1000
# How many documents in the space ? 
numDocs = 100000
# Gaps between calling 
addGap = 1
##############

# Verify Config
assert numCalls > 0
assert numDocs > 0
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))

def create_doc(num_elems, prefix, pos):
        id = prefix + str(pos)
        doc = {'_id' : id}

        for i in range(num_elems):
                doc['elem' + str(i)] = i

        return doc

# Run
for n in range(1000, 10100, 1000):
        space = db.bench
    
        generator = partial(create_doc, n, 'myval')
        bulk_inserter = BulkInserter(numDocs, space, generator)
        bulk_inserter.run()

        pendingOps = []

        bench = Benchmark()
        bench.start()

        for i in range(numCalls):
                p = space.async_atomic_add('myval' + str(i*addGap), {'elem50' : 10})
                pendingOps.append(p)

        while len(pendingOps) > 0:
                pendingOps[0].wait()
                pendingOps.pop(0)

        elapsed = bench.stop()

        # Print simple CSV
        print str(n) + ',' + str(elapsed)

        space.clear()
        time.sleep(5)
<commit_msg>Don't vary document size in benchmark<commit_after>#! /usr/bin/python

import sys
import functools
import time

from functools import partial
from benchmark import Benchmark
from bulk_insert import BulkInserter
from hyperdex.mongo import HyperDatabase, HyperSpace 

### CONFIG ###
# Modify this to change the behaviour of the benchmark

# How big are the documents (in elements?
documentSize = 1000
# How many atomic add calls?
numCalls = 1000
# How many documents in the space ? 
numDocs = 100000
# Gaps between calling 
addGap = 1
##############

# Verify Config
assert numCalls > 0
assert numDocs > 0
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))
assert addGap > 0
assert (numCalls * addGap) < numDocs

# Setup
db = HyperDatabase(sys.argv[1], int(sys.argv[2]))

def create_doc(num_elems, prefix, pos):
        id = prefix + str(pos)
        doc = {'_id' : id}

        for i in range(num_elems):
                doc['elem' + str(i)] = i

        return doc

# Run
for n in range(0, 10, 1):
        space = db.bench
    
        generator = partial(create_doc, documentSize, 'myval')
        bulk_inserter = BulkInserter(numDocs, space, generator)
        bulk_inserter.run()

        pendingOps = []

        bench = Benchmark()
        bench.start()

        for i in range(numCalls):
                p = space.async_atomic_add('myval' + str(i*addGap), {'elem50' : 10})
                pendingOps.append(p)

        while len(pendingOps) > 0:
                pendingOps[0].wait()
                pendingOps.pop(0)

        elapsed = bench.stop()

        # Print simple CSV
        print str(elapsed)

        space.clear()
        time.sleep(5)

{'fraction_non_alphanumeric': 0.11239964317573595, 'fraction_numerical': 0.02230151650312221, 'mean_word_length': 2.567338282078473, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2256576', 'n_tokens_mistral': 1163, 'n_tokens_neox': 1074, 'n_words': 315}