<commit_before># Site Model

import networkx as nx
from gen_model import *
from random import *
import datetime
import pickle 
import datetime
import os

BLOG_TEST = 1
WIKI_TEST = 0

if BLOG_TEST:    
    VISIT_RATE = 4
    VISIT_UNIT = "days"
    NUM_USERS  = 20
    PERCENT_NEW = 0.3
    FROM_DATE = datetime.datetime(2010, 05, 01)
    TO_DATE = datetime.datetime(2010, 05, 07)
    SITE = OnePageBlog('')
    TEMPLATE_ENNDPOINT = "/static/pages/blog.html"
    SYNCKIT_ENNDPOINT = "/blog/entries"
    TOKYO_ENNDPOINT = "/blog/flying_entries"
    PRERENDERED_ENNDPOINT = "/blog/traditional"

if WIKI_TEST:    
#    VISIT_RATE = 4
    VISIT_UNIT = "days"
#    NUM_USERS  = 20
#    PERCENT_NEW = 0.3
    FROM_DATE = datetime.datetime(2010, 05, 01)
    TO_DATE = datetime.datetime(2010, 05, 15)
    SITE = Wiki("", 0.5)
#    TEMPLATE_ENNDPOINT = "/static/pages/blog.html"
#    DATA_ENNDPOINT = "/blog/entries"
#    PRERENDERED_ENNDPOINT = "/blog/traditional"

#tick_hash = {VISIT_UNIT : 1}
tick_hash = {"minutes" : 1}

def create_users(number, percent_new, visit_rate, visit_unit):
    # Generate the users
    users = []
    print "%d, %d, %s" % (number, visit_rate, visit_unit)
    for i in range(number):
        last_time = None
        if (random() >= percent_new):        
            time_delta = expovariate(visit_rate)
            delta_hash = {visit_unit : time_delta}
            last_time = FROM_DATE - datetime.timedelta(**delta_hash)
            user = User(visit_rate, visit_unit, last_time)
            user.plan_next_visit()
            users.append(user)
        else:
            last_time = None
            user = User(visit_rate, visit_unit, last_time)
            delta = random() * (FROM_DATE - TO_DATE).microseconds
            user.next_visit_time = FROM_DATE + datetime.timedelta(microseconds=delta)
            users.append(user)
    return users

def saveToFile(obj, filename):
    output = open(filename, 'wb')
    pickle.dump(obj, output)
    output.close()

def loadFromFile(filename):
    input = open(filename, 'rb')
    data = pickle.load(input)
    input.close()
    return data

def run_test(site, users):
    all_visits = []
    # Now step forward in time
    now = FROM_DATE
    while now < TO_DATE:
        # For all users who have a planned visit 
        for user in users:
            if user.next_visit_time <= now:
                all_visits.append(user.perform_next_visit(SITE))
        # Advance the clock    
        now += datetime.timedelta(**tick_hash)
    return all_visits

def query_for_visit(visit, strategy):
    if strategy == 'tokyo':
        return 'queries={"Posts":{"now":"%s"}}' % (str(visit.this_time))
    elif strategy == 'traditional':
#        return 'queries={"Posts":{"now":"%s"}}' % (str(visit.this_time))
        return 'now=%s' % (str(visit.this_time))
    else:
        if visit.last_time == None:
            return 'queries={"Posts":{"now":"%s", "__vshash":"3c9977be25383449119b351ce5388e81"}}' % (str(visit.this_time))
        else:
            return 'queries={"Posts":{"now":"%s", "max":"%s", "__vshash":"3c9977be25383449119b351ce5388e81"}}' % (str(visit.this_time), str(visit.last_time))        

def url_strings_for_visit(visit, strategy):
    page = visit.click_trail.path[0]
    strings = []
	
    if strategy == 'traditional':
            strings.append("%s%s method=POST contents='%s'" % (page.url, PRERENDERED_ENNDPOINT, query_for_visit(visit, strategy)))
    else: 
        DATA_ENNDPOINT = TOKYO_ENNDPOINT
        if strategy == 'synckit':
            DATA_ENNDPOINT = SYNCKIT_ENNDPOINT       
        if visit.last_time == None:
            strings.append(page.url + TEMPLATE_ENNDPOINT)
            strings.append("      %s%s method=POST contents='%s'" % (page.url, DATA_ENNDPOINT, query_for_visit(visit, strategy)))
            strings.append("      /static/manifest")
        else:
            strings.append("%s%s method=POST contents='%s'" % (page.url, DATA_ENNDPOINT, query_for_visit(visit, strategy)))
    return strings

def write_httperf_file(urls, filename, header=""):
    output = open(filename, 'wb')
    if len(header) > 0:
        output.write(header)
    for url in urls:
        output.write(url + '\n')
    output.close()

def write_json_file(users, filename, baseurl, header=""):
    output = open(filename, 'wb')
    if len(header) > 0:
        output.write("// %s" % (header) + '\n')
    output.write("runTestWithUsers([\n")
    visits = [user.visits_to_json(baseurl) for user in users]
    output.write(",".join(visits))
    output.write("]);\n")
    output.close()
            
# --------------------------------------------------------------------------------                      
# | The Test
# --------------------------------------------------------------------------------                      
def ensure_directory(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def write_test_files(directory_name, test_name, num_users, percent_new, num_visits, in_period):
    ensure_directory(directory_name)
    
    users = create_users(num_users, percent_new, num_visits, in_period)    
    visits = run_test(SITE,users)
    
    for strategy in ('synckit', 'tokyo', 'traditional'):
        # Write The Server-Oriented Tests---write less than 1000 sessions
        # since httperf can't handle more
        urls = []
        for visit in visits[:998]:
            urls.extend(url_strings_for_visit(visit, strategy))
            urls.append("")
        comments = "# Test Name: %s\n# Strategy: %s\n# Number Users: %s\n# Percent New: %s\n# Number Visits: %s / %s\n" % (test_name, strategy, str(num_users), str(percent_new), str(num_visits), str(in_period))
        write_httperf_file(urls, "%s/%s_%s.txt" % (directory_name, test_name, strategy), header=comments)

        # Write The Client-Oriented Tests  
        if strategy == 'synckit':
            url = '/static/pages/blog.html'
        elif strategy == 'tokyo':
            url = '/static/pages/blog-flying.html'
        elif strategy == 'traditional':
            url = '/blog/traditional'     
                     
        comments = "# Test Name: %s  # Strategy: %s # Number Users: %s  # Percent New: %s   # Number Visits: %s / %s\n" % (test_name, strategy, str(num_users), str(percent_new), str(num_visits), str(in_period))
        write_json_file(users, "%s/%s_%s.js" % (directory_name, test_name, strategy), url, header=comments)

now = datetime.datetime.now()
dirname = now.strftime("%Y-%m-%d.%H:%M:%S")

if BLOG_TEST:
    # print "NOTE! need to make num users 100 and new usrs rate .5"
    write_test_files(dirname, "test_freq_4_per_update", 100, 0.5, 48, "days")
    write_test_files(dirname, "test_freq_3_per_update", 100, 0.5, 36, "days")
    write_test_files(dirname, "test_freq_2_per_update", 100, 0.5, 24, "days")<commit_msg>first stab at a client-side wiki workload<commit_after># Site Model

import networkx as nx
from gen_model import *
from random import *
import datetime
import pickle 
import datetime
import os

BLOG_TEST = 0
WIKI_TEST = 1

if BLOG_TEST:    
    VISIT_RATE = 4
    VISIT_UNIT = "days"
    NUM_USERS  = 20
    PERCENT_NEW = 0.3
    FROM_DATE = datetime.datetime(2010, 05, 01)
    TO_DATE = datetime.datetime(2010, 05, 07)
    SITE = OnePageBlog('')
    TEMPLATE_ENNDPOINT = "/static/pages/blog.html"
    SYNCKIT_ENNDPOINT = "/blog/entries"
    TOKYO_ENNDPOINT = "/blog/flying_entries"
    PRERENDERED_ENNDPOINT = "/blog/traditional"

if WIKI_TEST:    
#    VISIT_RATE = 4
    VISIT_UNIT = "days"
#    NUM_USERS  = 20
#    PERCENT_NEW = 0.3
    FROM_DATE = datetime.datetime(2010, 05, 01)
    TO_DATE = datetime.datetime(2010, 05, 15)
    SITE = Wiki("", 0.5)
#    TEMPLATE_ENNDPOINT = "/static/pages/blog.html"
#    DATA_ENNDPOINT = "/blog/entries"
#    PRERENDERED_ENNDPOINT = "/blog/traditional"

#tick_hash = {VISIT_UNIT : 1}
tick_hash = {"minutes" : 1}

def create_users(number, percent_new, visit_rate, visit_unit):
    # Generate the users
    users = []
    print "%d, %d, %s" % (number, visit_rate, visit_unit)
    for i in range(number):
        last_time = None
        if (random() >= percent_new):        
            time_delta = expovariate(visit_rate)
            delta_hash = {visit_unit : time_delta}
            last_time = FROM_DATE - datetime.timedelta(**delta_hash)
            user = User(visit_rate, visit_unit, last_time)
            user.plan_next_visit()
            users.append(user)
        else:
            last_time = None
            user = User(visit_rate, visit_unit, last_time)
            delta = random() * (FROM_DATE - TO_DATE).microseconds
            user.next_visit_time = FROM_DATE + datetime.timedelta(microseconds=delta)
            users.append(user)
    return users

def saveToFile(obj, filename):
    output = open(filename, 'wb')
    pickle.dump(obj, output)
    output.close()

def loadFromFile(filename):
    input = open(filename, 'rb')
    data = pickle.load(input)
    input.close()
    return data

def run_test(site, users):
    all_visits = []
    # Now step forward in time
    now = FROM_DATE
    while now < TO_DATE:
        # For all users who have a planned visit 
        for user in users:
            if user.next_visit_time <= now:
                all_visits.append(user.perform_next_visit(SITE))
        # Advance the clock    
        now += datetime.timedelta(**tick_hash)
    return all_visits

def query_for_visit(visit, strategy):
    if strategy == 'tokyo':
        return 'queries={"Posts":{"now":"%s"}}' % (str(visit.this_time))
    elif strategy == 'traditional':
#        return 'queries={"Posts":{"now":"%s"}}' % (str(visit.this_time))
        return 'now=%s' % (str(visit.this_time))
    else:
        if visit.last_time == None:
            return 'queries={"Posts":{"now":"%s", "__vshash":"3c9977be25383449119b351ce5388e81"}}' % (str(visit.this_time))
        else:
            return 'queries={"Posts":{"now":"%s", "max":"%s", "__vshash":"3c9977be25383449119b351ce5388e81"}}' % (str(visit.this_time), str(visit.last_time))        

def url_strings_for_visit(visit, strategy):
    page = visit.click_trail.path[0]
    strings = []
	
    if strategy == 'traditional':
            strings.append("%s%s method=POST contents='%s'" % (page.url, PRERENDERED_ENNDPOINT, query_for_visit(visit, strategy)))
    else: 
        DATA_ENNDPOINT = TOKYO_ENNDPOINT
        if strategy == 'synckit':
            DATA_ENNDPOINT = SYNCKIT_ENNDPOINT       
        if visit.last_time == None:
            strings.append(page.url + TEMPLATE_ENNDPOINT)
            strings.append("      %s%s method=POST contents='%s'" % (page.url, DATA_ENNDPOINT, query_for_visit(visit, strategy)))
            strings.append("      /static/manifest")
        else:
            strings.append("%s%s method=POST contents='%s'" % (page.url, DATA_ENNDPOINT, query_for_visit(visit, strategy)))
    return strings

def write_httperf_file(urls, filename, header=""):
    output = open(filename, 'wb')
    if len(header) > 0:
        output.write(header)
    for url in urls:
        output.write(url + '\n')
    output.close()

def write_json_file(users, filename, baseurl, header=""):
    output = open(filename, 'wb')
    if len(header) > 0:
        output.write("// %s" % (header) + '\n')
    output.write("runTestWithUsers([\n")
    visits = [user.visits_to_json(baseurl) for user in users]
    output.write(",".join(visits))
    output.write("]);\n")
    output.close()
            
# --------------------------------------------------------------------------------                      
# | The Test
# --------------------------------------------------------------------------------                      
def ensure_directory(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def write_test_files(directory_name, test_name, num_users, percent_new, num_visits, in_period):
    ensure_directory(directory_name)
    
    users = create_users(num_users, percent_new, num_visits, in_period)    
    visits = run_test(SITE,users)
    
    for strategy in ('synckit', 'tokyo', 'traditional'):
        # Write The Server-Oriented Tests---write less than 1000 sessions
        # since httperf can't handle more
        urls = []
        for visit in visits[:998]:
            urls.extend(url_strings_for_visit(visit, strategy))
            urls.append("")
        comments = "# Test Name: %s\n# Strategy: %s\n# Number Users: %s\n# Percent New: %s\n# Number Visits: %s / %s\n" % (test_name, strategy, str(num_users), str(percent_new), str(num_visits), str(in_period))
        write_httperf_file(urls, "%s/%s_%s.txt" % (directory_name, test_name, strategy), header=comments)

        # Write The Client-Oriented Tests  
        if strategy == 'synckit':
            url = '/static/pages/wiki.html'
        elif strategy == 'tokyo':
            url = '/static/pages/flying-wiki.html'
        elif strategy == 'traditional':
            url = '/wiki/traditional'     
                     
        comments = "# Test Name: %s  # Strategy: %s # Number Users: %s  # Percent New: %s   # Number Visits: %s / %s\n" % (test_name, strategy, str(num_users), str(percent_new), str(num_visits), str(in_period))
        write_json_file(users, "%s/%s_%s.js" % (directory_name, test_name, strategy), url, header=comments)

now = datetime.datetime.now()
dirname = now.strftime("%Y-%m-%d.%H:%M:%S")

if BLOG_TEST:
    # print "NOTE! need to make num users 100 and new usrs rate .5"
    write_test_files(dirname, "test_freq_4_per_update", 100, 0.5, 48, "days")
    write_test_files(dirname, "test_freq_3_per_update", 100, 0.5, 36, "days")
    write_test_files(dirname, "test_freq_2_per_update", 100, 0.5, 24, "days")
{'fraction_non_alphanumeric': 0.1546852122986823, 'fraction_numerical': 0.019912152269399706, 'mean_word_length': 2.5874474789915967, 'pattern_counts': {'":': 22, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779378', 'n_tokens_mistral': 4773, 'n_tokens_neox': 4462, 'n_words': 977}