<commit_before>#!/usr/bin/env python

"""
Read in a simple file of the format
    epochtime	event	IP_address
where epochtime is an int, event is "join" or "fail" and IP_address is
something like "172.16.58.3" (such as generated by plupdown.py).
Generate a bunch of events corresponding to vnodes that might have
something to do with that plus a corresponding number of inserts.

We will still simulate by the "minute".
"""

import sha
import sys
from utils import random_id, random_interval, make_chordID

mu_i	= 5
sd_i	= 5
maxblocks = 1000

def str2bigint(s):
    a = map(ord, s)
    v = 0L
    for d in a:
	v = v << 8
	v = v | d
    return v

inserted = {}
def random_blockid ():
    s = sha.sha("%d" % random_id ())
    id = str2bigint (s.digest ())
    while id in inserted:
	# Damn birthdays.
	s = sha.sha("%d" % random_id ())
	id = str2bigint (s.digest ())
    inserted[id] = 1
    return id


firsttime = None
lasttime  = 0
fh = open (sys.argv[1])
for line in fh.readlines ():
    (t, e, ip) = line.strip ().split ()

    # normalize time
    t = int (t)
    if not firsttime:
	firsttime = t
	nnode = make_chordID (ip, 11977, 0)
    t = t - firsttime
    t /= 60

    if t != lasttime:
	if maxblocks > 0 and t > 200:
	    for nt in xrange(lasttime+1,t-1):
		ni = random_interval (mu_i, sd_i)
		while ni > 0 and maxblocks > 0:
		    print nt, "insert", nnode, random_blockid ()
		    ni -= 1
		    maxblocks -= 1
	lasttime = t
    print t, e, make_chordID (ip, 11977, 0)<commit_msg>Hack to work with comon trace --- don't wait for a certain amount of time before adding the insert block events, wait for a certain number of churn events instead.<commit_after>#!/usr/bin/env python

"""
Read in a simple file of the format
    epochtime	event	IP_address
where epochtime is an int, event is "join" or "fail" and IP_address is
something like "172.16.58.3" (such as generated by plupdown.py).
Generate a bunch of events corresponding to vnodes that might have
something to do with that plus a corresponding number of inserts.

We will still simulate by the "minute".
"""

import sha
import sys
from utils import random_id, random_interval, make_chordID

mu_i	= 5
sd_i	= 5
maxblocks = 1000

def str2bigint(s):
    a = map(ord, s)
    v = 0L
    for d in a:
	v = v << 8
	v = v | d
    return v

inserted = {}
def random_blockid ():
    s = sha.sha("%d" % random_id ())
    id = str2bigint (s.digest ())
    while id in inserted:
	# Damn birthdays.
	s = sha.sha("%d" % random_id ())
	id = str2bigint (s.digest ())
    inserted[id] = 1
    return id


firsttime = None
lasttime  = 0
fh = open (sys.argv[1])
count = 0
onode = 0
for line in fh.readlines ():
    (t, e, ip) = line.strip ().split ()

    # normalize time
    t = int (t)
    if not firsttime:
	firsttime = t
	nnode = make_chordID (ip, 11977, 0)
    t = t - firsttime
    t /= 60

    if e == 'join':
	count += 1
	onode = make_chordID (ip, 11977, 0)

    if t != lasttime:
	if maxblocks > 0 and count > 1000:
	    for nt in xrange(lasttime+1,t-1):
		ni = random_interval (mu_i, sd_i)
		while ni > 0 and maxblocks > 0:
		    print nt, "insert", nnode, random_blockid ()
		    ni -= 1
		    maxblocks -= 1
	lasttime = t
    print t, e, make_chordID (ip, 11977, 0)
    nnode = onode

{'fraction_non_alphanumeric': 0.11141975308641976, 'fraction_numerical': 0.03148148148148148, 'mean_word_length': 2.8129411764705883, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5793730', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1174, 'n_words': 443}