<jupyter_start><jupyter_text>Scaling behavior<jupyter_code>%matplotlib inline

import subprocess
import json
import os
import pandas
import numpy<jupyter_output><empty_output><jupyter_text>ToolsFor generating scaling scenarios and running it through guv<jupyter_code>def normalize_scenario(flat):
    # Pop out messages
    messages = flat['messages']
    # Everything else assumed to be config
    roleconfig = flat.copy()
    del roleconfig['messages']
    config = { 'test': flat }
    s = {
        'role': 'test',
        'config': config,
        'messages': messages,
    }
    return s

def calculate_scaling(scenarios, timeout=10):
    normalized = [normalize_scenario(s) for s in scenarios]
    serialized = json.dumps(normalized)
        
    args = ['../node_modules/.bin/coffee', './scalescenarios.coffee']
    try:
        stdout = subprocess.check_output(args, input=serialized,
                                         encoding='utf-8', timeout=timeout, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        raise Exception(e.stdout)

    try:
        results = json.loads(stdout)
    except Exception as e:
        print('Could not parse stdout', std)
    
    return results

def merge(a, b):
    d = a.copy()
    d.update(b)
    return d<jupyter_output><empty_output><jupyter_text>Workers versus queue lengthFor different processing times (fixed deadline).Should generally be linear, as guv has a proportional scaling algorithm<jupyter_code>base = dict(minimum=0, maximum=20, processing=2, deadline=10, messages=-1)
messages = range(0, 50)

def versus_messages(processing):
    scenarios = [ merge(base, { 'messages': m, 'processing': processing }) for m in messages ] 
    res = calculate_scaling(scenarios)
    return res

df = pandas.DataFrame({
    'messages': messages,
    '1s processing time': versus_messages(processing=1),
    '2s processing time': versus_messages(processing=2),
    '3s processing time': versus_messages(processing=3),
    '4s processing time': versus_messages(processing=4),
})
ax = df.plot(x='messages', figsize=(10, 5))
ax.set_ylabel("workers")
ax<jupyter_output><empty_output><jupyter_text>Workers versus closeness to deadlineBy moving processing time towards deadline. Could alternatively have moved deadline down towards a fixed processing time.<jupyter_code>processing_times = numpy.arange(0.1, 10.0, 0.2)

def versus_processing(messages):
    base = dict(minimum=0, maximum=20, processing=2, deadline=10, concurrency=1, messages=-1)
    scenarios = [ merge(base, { 'messages': messages, 'processing': p }) for p in processing_times ]
    res = calculate_scaling(scenarios)
    return res

df = pandas.DataFrame({
    'processing': processing_times,
    '0 message': versus_processing(messages=0),
    '1 message': versus_processing(messages=1),
    '4 message': versus_processing(messages=4),
    '8 message': versus_processing(messages=8),
    '32 message': versus_processing(messages=32),
})
ax = df.plot(x='processing', figsize=(10, 5))
ax.set_ylabel("workers")
ax<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11598440545808966, 'fraction_numerical': 0.017543859649122806, 'mean_word_length': 4.299483648881239, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '549805', 'license': 'MIT', 'n_tokens_mistral': 949, 'n_tokens_neox': 923, 'n_words': 237, 'path': 'doc/Scaling.ipynb', 'repo_name': 'jonnor/gov', 'size': '3078'}