<jupyter_start><jupyter_text>GoolyBib MMM[www.vexpower.com](www.vexpower.com)<jupyter_code># Set the right folder

import sys
import os

if not os.path.isdir("mmm"):
    module_path = os.path.abspath(os.path.join('..'))
    if module_path not in sys.path:
        sys.path.append(module_path)

import mmm
import pandas as pd
pd.set_option('display.float_format', lambda x: '%.3f' % x) # suppress scientific notation

# Load dataset

file_name = "GoolyBib-ABT - Sheet1.csv"
data = pd.read_csv('../data/'+file_name)
data.head()
from IPython.display import display

from mmm.clean import make_column_index
from mmm.engineer import add_constant
from mmm.select import get_all_X_labels, guess_date_column, guess_y_column
from mmm.build import run_regression, create_results_df
from mmm.validate import calculate_r2
from mmm.display import display_accuracy_chart

# Run regression analysis

df = data.copy()

date_label = guess_date_column(df)
make_column_index(df, date_label)
add_constant(df)

y_label = guess_y_column(df)
X_labels = get_all_X_labels(df, y_label)

y_actual, y_pred, coefficients, p_values = run_regression(df, y_label, X_labels)

r2 = calculate_r2(y_actual, y_pred)

results_df = create_results_df(X_labels, coefficients, p_values)
display(results_df)

display_accuracy_chart(y_actual, y_pred, y_label, accuracy=("R-squared", r2))
from mmm.display import save_model

# Save model to spreadsheet
save_model(y_label, r2, X_labels, coefficients)
from mmm.build import run_regression_with_test_split

# Run regression with test split

y_test, y_test_pred, coefficients, p_values, y_actual, y_pred = run_regression_with_test_split(df, y_label, X_labels)

r2 = calculate_r2(y_test, y_test_pred)

results_df = create_results_df(X_labels, coefficients, p_values)
display(results_df)

display_accuracy_chart(y_actual, y_pred, y_label, accuracy=("R-squared", r2))
save_model(y_label, r2, X_labels, coefficients)
from mmm.engineer import add_day_of_week_dummies

# Add day of week dummies

day_of_week_columns, df = add_day_of_week_dummies(df)
df.head()
from mmm.build import run_regression_with_test_split
from mmm.select import backwards_feature_elimination
from mmm.select import add_X_labels

# Run regression with day_of_week and backwards feature elimination

X_labels = add_X_labels(X_labels, day_of_week_columns)

bfe_keep, bfe_df = backwards_feature_elimination(df, y_label, X_labels)

display(bfe_df)

y_test, y_test_pred, coefficients, p_values, y_actual, y_pred = run_regression_with_test_split(df, y_label, bfe_keep)

r2 = calculate_r2(y_test, y_test_pred)

results_df = create_results_df(bfe_keep, coefficients, p_values)
display(results_df)

display_accuracy_chart(y_actual, y_pred, y_label, accuracy=("R-squared", r2))
save_model(y_label, r2, X_labels, coefficients)
from mmm.build import create_pred_df
from mmm.display import display_contrib_chart, display_decomp_chart

pred_df = create_pred_df(df, results_df)

display_contrib_chart(pred_df)
display_decomp_chart(pred_df)
from mmm.display import load_models

# Compare models
    
models = load_models()
models.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08656620711766592, 'fraction_numerical': 0.0048092337287592175, 'mean_word_length': 7.188976377952756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '367377', 'license': 'MIT', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1138, 'n_words': 247, 'path': 'notebooks/GoolyBib MMM.ipynb', 'repo_name': 'jamesaphoenix/VEX-MMM', 'size': '3119'}