<jupyter_start><jupyter_text>MPI calibration of a Grid5000 cluster  Data exploration<jupyter_code>import os
import zipfile
import yaml
import pandas
import io

archive_name = 'grenoble_2018-08-29_1808878.zip'

def extract_zip(zip_name):
    '''Taken from https://stackoverflow.com/a/10909016/4110059'''
    input_zip = zipfile.ZipFile(zip_name)
    result = {}
    deployment = yaml.load(input_zip.read('info.yaml'))['deployment']
    experiment = zip_name
    if '/' in experiment:
        experiment = experiment[experiment.index('/')+1:]
    experiment = experiment[:experiment.index('_')]
    for name in input_zip.namelist():
        if name.endswith('.csv'):
            dataframe = pandas.read_csv(io.BytesIO(input_zip.read(name)), names = ['op', 'msg_size', 'start', 'duration'])
            dataframe['experiment'] = experiment
            dataframe['type'] = name
            dataframe['deployment'] = deployment
            dataframe['index'] = range(len(dataframe))
            result[name] = dataframe
        elif name.endswith('.yaml'):
            result[name] = yaml.load(input_zip.read(name))
        else:
            result[name] = input_zip.read(name)
    return result

def aggregate_dataframe(dataframe):
    df = dataframe.groupby('msg_size').mean().reset_index()
    for col in dataframe.columns:
        if col not in df.columns:
            value = list(dataframe[col].unique())
            if len(value) == 1:
                df[col] = value[0]
    return df

threshold = 8133

# I really hate that we have to define these two values:
expected_bandwidth = 1e11/8
expected_latency   = 1e-7

result = extract_zip(archive_name)
!unzip -p $archive_name info.yaml
# Special processing for the Pingpong dataset
df_pingpong = result['exp/exp_PingPong.csv']
pingpong_send = df_pingpong[df_pingpong.op=='MPI_Send'].reset_index(drop=True)
pingpong_recv = df_pingpong[df_pingpong.op=='MPI_Recv'].reset_index(drop=True)
df_pingpong = pandas.DataFrame(dict(op = 'PingPong',
                   msg_size = pingpong_send.msg_size,
                   start    = pingpong_send.start,
                   duration = pingpong_recv.duration + pingpong_send.duration
              ))

df_send = pingpong_send
df_isend = result['exp/exp_Isend.csv']
df_recv = result['exp/exp_Recv.csv']
df_wtime = result['exp/exp_Wtime.csv']
df_test = result['exp/exp_Test.csv']
df_iprobe = result['exp/exp_Iprobe.csv']<jupyter_output><empty_output><jupyter_text>STAN<jupyter_code>from plotnine import *
import plotnine
plotnine.options.figure_size = (12, 8)
import warnings
import pystan
import numpy
warnings.simplefilter(action='ignore', category=FutureWarning)
%%time

model_code = """
data {
    int<lower=1> N; // number of points
    real<lower=0> duration[N];
    real<lower=0> msg_size[N];
    int<lower=1> nb_modes;
    int<lower = 0, upper = 1> run_estimation; // a switch to evaluate the likelihood, see http://modernstatisticalworkflow.blogspot.com/2017/04/an-easy-way-to-simulate-fake-data-from.html
    real<lower=0> sd_intercept;
    real<lower=0> sd_coefficient;
    real<lower=0> prior_sigma;
    real prior_intercept[nb_modes];
    real prior_coefficient;
}

transformed data {
    real min_size = min(msg_size);
    real max_size = max(msg_size);
}

parameters {
    ordered[nb_modes] intercept;
    real coefficient;
    real<lower=0> sigma[nb_modes];
    simplex[nb_modes] theta;
}

model {
    // Priors
    for (i in 1:nb_modes) {
        intercept[i] ~ normal(prior_intercept[i], sd_intercept);
    }
    coefficient ~ normal(prior_coefficient, sd_coefficient);
    sigma ~ normal(prior_sigma, prior_sigma);
    theta ~ dirichlet(rep_vector(1, nb_modes));
    
    // Likelihood
    if(run_estimation==1) {
        for (i in 1:N) {
            vector[nb_modes] contributions;
            for(k in 1:nb_modes) {
                contributions[k] = log(theta[k]) + normal_lpdf(duration[i] | intercept[k] + coefficient*msg_size[i], sigma[k]);
            }
            target += log_sum_exp(contributions);
        }
    }
}

generated quantities {
    real duration_sim;
    int mode;
    real size_sim;
    mode = categorical_rng(theta);
    size_sim = msg_size[categorical_rng(rep_vector(1,N) / N)];
    duration_sim = normal_rng(intercept[mode] + coefficient*size_sim, sigma[mode]);
}
"""

sm = pystan.StanModel(model_code=model_code)
%%time

nb_modes = 3
prior_intercept = [1e-6, 2.2e-6, 6e-6]
prior_coefficient = 8.456e-11
prior_sigma = [1e-7, 1e-7, 1e-6]

def mean(l):
    return sum(l)/len(l)

def initfunction():
    return {
        'intercept': prior_intercept,
        'coefficient': prior_coefficient,
        'sigma': prior_sigma,
        'theta': [0.49, 0.49, 0.02]
    }


df = df_recv[df_recv.msg_size < 8133].sample(n=10000) # taking only a (random) subset of the data
data=dict(N=len(df), duration=df.duration, msg_size=df.msg_size, nb_modes=nb_modes, run_estimation=0,
          sd_intercept=mean(prior_intercept)/10, sd_coefficient=prior_coefficient/10, prior_sigma=mean(prior_sigma),
          prior_intercept = prior_intercept, prior_coefficient=prior_coefficient)

fit = sm.sampling(data=data, iter=500, chains=8, init=initfunction)
print(fit)

fit.plot()
def plot_posterior(df, fit):
    generated_df = fit.to_dataframe()
    generated_df['mode'] = generated_df['mode'].astype(int)
    plot = ggplot() + geom_point(generated_df, aes(x='size_sim', y='duration_sim', color='factor(mode)'))
    plot = plot + geom_point(df, aes(x='msg_size', y='duration'), alpha=0.1)
    plot = plot + theme_bw() + scale_x_log10() + scale_y_log10()
    return plot
plot_posterior(df, fit)
%%time

data['run_estimation'] = 1
fit = sm.sampling(data=data, iter=500, chains=8, init=initfunction)
print(fit)
fit.plot()
plot_posterior(df, fit)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13146514443867843, 'fraction_numerical': 0.022487458917142364, 'mean_word_length': 3.7161500815660684, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '622566', 'license': 'MIT', 'n_tokens_mistral': 2146, 'n_tokens_neox': 2002, 'n_words': 373, 'path': 'dahu/mpi/remote_calibration_stan.ipynb', 'repo_name': 'Ezibenroc/calibration_analysis', 'size': '5781'}