<jupyter_start><jupyter_text>Concurrency Value Analysis By $\lambda/N$<jupyter_code># let's analyze the relationship between conc_window_average and lambda_over_n
plt.plot(df_logger_merged_lon.index, df_logger_merged_lon['conc_window_average'], 'x', label='Windowed CC')
plt.plot(df_logger_merged_lon.index, df_logger_merged_lon['current_cc'], 'x', label='Current CC')
plt.xlabel('Arrival Rate per Container (reqs/s)')
plt.ylabel('Measured Concurrency')
plt.legend()
plt.grid(True)
# imports
from sklearn.pipeline import Pipeline
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
class CCTransformer():
    def __init__(self, columns=None, ):
        self.columns = columns
        # default columns if not provided
        if columns is None:
            self.columns = ['lon', 'cc']
        
    # here you define the operation it should perform
    def transform(self, X, y=None, **fit_params):
        cpy_df = X[self.columns].copy()
        cpy_df['lon2'] = cpy_df['lon'] ** 2
        return cpy_df

    # just return self
    def fit(self, X, y=None, **fit_params):
        return self

def cc_model_get_x_y(df_cc_orig, cc_pipe, output_col_name='cc'):
    df_cc = cc_pipe.fit_transform(df_cc_orig)
    X = df_cc[['lon', 'lon2']].values
    y = df_cc[output_col_name].values
    return X, y

def cc_model_train(df_cc, cc_pipe, output_col_name='cc', regression_options=None):
    if regression_options is None:
        regression_options = {
            'fit_intercept': False,
        }

    X,y = cc_model_get_x_y(df_cc, cc_pipe, output_col_name)
    # no intercept might be better because when lambda over n is zero, conc is zero
    regr = linear_model.LinearRegression(**regression_options)
    # Train the model using the training sets
    regr.fit(X, y)
    # get estimate for goodness of fit
    yhat = regr.predict(X)
    print('MSE:', mean_squared_error(y, yhat))
    print('R2:', r2_score(y, yhat))
    return regr

def get_linear_model_params(regr):
    # get model params
    model_params = [regr.intercept_] + list(regr.coef_)
    return model_params

def cc_get_model_predictions(plt_x, cc_pipe, cc_regr, output_col_name='cc'):
    plt_df = pd.DataFrame(data={ 'lon': plt_x, output_col_name: None })
    plt_df = cc_pipe.fit_transform(plt_df)
    test_X, test_Y = cc_model_get_x_y(plt_df, cc_pipe, output_col_name)
    plt_y = cc_regr.predict(test_X)
    return plt_y
cc_pipe = Pipeline([
    ('cctransformer', CCTransformer())
])

df_cc = df_logger_merged_lon.reset_index().rename(columns={
    'lambda_over_n': 'lon',
    'current_cc': 'cc',
})
cc_regr = cc_model_train(df_cc, cc_pipe)
print("Model Params:", get_linear_model_params(cc_regr))

plt_x = np.linspace(df_cc['lon'].min(), df_cc['lon'].max(), 20)
plt_y = cc_get_model_predictions(plt_x, cc_pipe, cc_regr)

plt.figure(figsize=(4,2.5))
plt.plot(df_cc['lon'], df_cc['cc'], 'x', label="Measured")
plt.plot(plt_x, plt_y, label="Predicted")
plt.xlabel('Arrival Rate per Container (reqs/s)')
plt.ylabel('Measured Concurrency')
plt.legend()
plt.grid(True)
plt.gcf().subplots_adjust(left=0.13, bottom=0.20)
parse_util.tmp_fig_save('05_cc_vs_lambda_over_n_regression', exp_config_name)<jupyter_output>MSE: 1.0106832987958918
R2: 0.649842088280487
Model Params: [0.0, -0.45515753415253646, 0.5650811995925363]<jupyter_text>Response Time Analysis By $\lambda/N$<jupyter_code>resp_pipe = Pipeline([
    ('cctransformer', CCTransformer(columns=['lon', 'resp']))
])

df_resp = df_reqs_merged_lon.reset_index().rename(columns={
    'lambda_over_n': 'lon',
    'resp_time_avg': 'resp',
})

# df_resp = df_resp[df_resp['resp'] < 0.8]

resp_regr = cc_model_train(df_resp, resp_pipe, 'resp', {})
print("Model Params:", get_linear_model_params(resp_regr))

plt_x = np.linspace(df_resp['lon'].min(), df_resp['lon'].max(), 20)
plt_y = cc_get_model_predictions(plt_x, resp_pipe, resp_regr, 'resp')

plt.figure(figsize=(4,2.5))
plt.plot(df_resp['lon'], df_resp['resp'], 'x', label="Measured")
plt.plot(plt_x, plt_y, label="Predicted")
plt.xlabel('Arrival Rate per Container (reqs/s)')
plt.ylabel('Response Time')
plt.legend()
plt.grid(True)
plt.gcf().subplots_adjust(left=0.13, bottom=0.20)
parse_util.tmp_fig_save('06_resp_vs_lambda_over_n_regression', exp_config_name)<jupyter_output>MSE: 0.00024177735414149194
R2: 0.8569035328548754
Model Params: [0.21371083355202003, -0.04080678570544321, 0.035958460346475485]
{'fraction_non_alphanumeric': 0.13836762378476147, 'fraction_numerical': 0.0449920868189012, 'mean_word_length': 5.329041487839771, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '82072', 'license': 'MIT', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1682, 'n_words': 315, 'path': 'experiments/parse_multiple_merged.ipynb', 'repo_name': 'pacslab/conc-value-perf-modelling', 'size': '4423'}