<commit_before>import sys

def calc(base, reps):
	if reps == 1:
                return base
        if base == 1:
                return 1
	total=0
	for i in range(0, base):
		total += calc(base-i, reps-1)
	return total

def size(base, reps):
	if reps == 1:
		return base
	if base == 1:
		return 1
	return (size(base,reps-1))*((base+(reps-1))/float(reps))

def walk(base, reps, target, accum, stack):
        if reps == 1:
                accum += base
		return accum
        if base == 1:
                accum += 1
		return accum
        for i in range(0, base):
		stack.append(i)
                accum = walk(base-i, reps-1, target, accum, stack)
		if accum > target:
			stack.append(base-i-1)
			print "*** " + str(stack) + ": " + str(accum)
			accum = 0
			stack.pop()
		stack.pop()
        return accum

cores=sys.argv[1]
jobsize=float(calc(int(sys.argv[2]),int(sys.argv[3])))
workunit=jobsize/int(cores)
print "Cores=" + str(cores)
print "Total job size=" + str(jobsize)
print "Target work unit=" + str(workunit)

walk(int(sys.argv[2]),int(sys.argv[3]), workunit, 0, [])



#Slower
#print str(size(int(sys.argv[2]),int(sys.argv[3])))

#Faster
#print str(calc(int(sys.argv[2]),int(sys.argv[3])))
<commit_msg>Further development on parallelization algorithm<commit_after>import sys

def calc(base, reps):
	if reps == 1:
                return base
        if base == 1:
                return 1
	total=0
	for i in range(0, base):
		total += calc(base-i, reps-1)
	return total

def size(base, reps):
	if reps == 1:
		return base
	if base == 1:
		return 1
	return (size(base,reps-1))*((base+(reps-1))/float(reps))

def walk(base, reps, target, accum, stack, basebase, repsreps):
        if reps == 1:
                accum += base
		return accum
        if base == 1:
                accum += 1
		return accum
        for i in range(0, base):
		stack.append(i)
                accum = walk(base-i, reps-1, target, accum, stack, basebase, repsreps)
		if accum > target:
			stack.append(base-i-1)
			#stacklength = len(stack)
			#while len(stack) < repsreps:
			#	stack.append(0)
			offset = repsreps - 1
			index = 0
			for j in stack:
				index += j * (basebase**offset)
				offset -= 1
			print str(index)
			accum = 0
			#while len(stack) > stacklength:
			#	stack.pop()
			stack.pop()
		stack.pop()
        return accum

cores=sys.argv[1]
if len(sys.argv) > 4:
	cores=sys.argv[4]
jobsize=float(calc(int(sys.argv[2]),int(sys.argv[3])))
workunit=jobsize/int(cores)
print "Cores=" + str(cores)
print "Total job size=" + str(jobsize)
print "Target work unit=" + str(workunit)
print "0"
walk(int(sys.argv[2]),int(sys.argv[3]), workunit, 0, [],int(sys.argv[2]),int(sys.argv[3]))



#Slower
#print str(size(int(sys.argv[2]),int(sys.argv[3])))

#Faster
#print str(calc(int(sys.argv[2]),int(sys.argv[3])))

{'fraction_non_alphanumeric': 0.1666072065643953, 'fraction_numerical': 0.023189439885836605, 'mean_word_length': 2.8675862068965516, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174176', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1030, 'n_words': 260}