<jupyter_start><jupyter_text>Find the markdown blocks that say interaction required! The notebook should take care of the rest!  Import libs<jupyter_code>import sys
import os
sys.path.append('..')
from eflow.foundation import DataPipeline,DataFrameTypes
from eflow.model_analysis import RegressionAnalysis
from eflow.utils.modeling_utils import optimize_model_grid
from eflow.utils.eflow_utils import get_type_holder_from_pipeline, remove_unconnected_pipeline_segments
from eflow.utils.pandas_utils import data_types_table

import pandas as pd
import numpy as np
import scikitplot as skplt
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
import copy
import pickle
from IPython.display import clear_output
# # Additional add ons
# !pip install pandasgui
# !pip install pivottablejs
# clear_output()
%matplotlib notebook
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Declare Project Variables  Interaction required<jupyter_code>dataset_path = "Datasets/titanic_train.csv"

# -----
dataset_name = "Titanic Data"
pipeline_name = "Titanic Pipeline"

# -----


# -----
notebook_mode = True<jupyter_output><empty_output><jupyter_text>Clean out segment space<jupyter_code>remove_unconnected_pipeline_segments()<jupyter_output><empty_output><jupyter_text>Import dataset<jupyter_code>df = pd.read_csv(dataset_path)
shape_df = pd.DataFrame.from_dict({'Rows': [df.shape[0]],
                                   'Columns': [df.shape[1]]})
display(shape_df)
display(df.head(30))
data_types_table(df)<jupyter_output><empty_output><jupyter_text>Loading and init df_features<jupyter_code># Option: 1
# df_features = get_type_holder_from_pipeline(pipeline_name)
# Option: 2
df_features = DataFrameTypes()
df_features.init_on_json_file(os.getcwd() + f"/eflow Data/{dataset_name}/df_features.json")
df_features.display_features(display_dataframes=True,
                             notebook_mode=notebook_mode)<jupyter_output><empty_output><jupyter_text>Any extra processing before eflow DataPipeline  Setup pipeline structure  Interaction Required<jupyter_code>main_pipe = DataPipeline(pipeline_name,
                         df,
                         df_features)
main_pipe.perform_pipeline(df,
                           df_features)
df<jupyter_output><empty_output><jupyter_text>Seperate out data into train and test sets<jupyter_code>X = df.drop(columns="Age").values
y = df["Age"].values
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.3, random_state=517,
)
feature_order = list(df.columns)
del df<jupyter_output><empty_output><jupyter_text>Fit Models and view results<jupyter_code># Find best parameters for model
param_grid = {
    "max_depth": list(range(1, 10)),
    'criterion': ["mse", "friedman_mse", "mae"]
}

model, best_params = optimize_model_grid(
    model=DecisionTreeRegressor(),
    X_train=X_train,
    y_train=y_train,
    param_grid=param_grid,
    scoring="r2"
)
model.fit(X,y)
model_name = repr(model).split("(")[0]
model_analysis = RegressionAnalysis(dataset_name=dataset_name,
                                    model=model,
                                    model_name=model_name,
                                    feature_order=feature_order,
                                    target_feature="Age",
                                    pred_funcs_dict={"Predict function":model.predict},
                                    notebook_mode=notebook_mode,
                                    df_features=df_features)
model_analysis.perform_analysis(X=X_train,
                                y=y_train,
                                dataset_name="Train Data",
                                regression_error_analysis=True,
                                regression_correct_analysis=True,
                                display_visuals=False,
                                mse_score=.2)
for pred_name in model_analysis.get_predictions_names():
    for mse_score in [.2]:
        print(f"Prediction name: {pred_name} with mse_score greater than {mse_score}")
        infile = open(model_analysis.folder_path + "Train Data" + f"/{pred_name}/MSE score greater than {mse_score}/Train Data/_Extras/Statistics/Stat methods of features dataframes.pkl",'rb')
        stat_methods_dict = pickle.load(infile)
        infile.close()

        for stats_method in stat_methods_dict.keys():
            print(stats_method)
            display(stat_methods_dict[stats_method].round(6))
            all_feature_relationship = set()
            for feature_relationship in stat_methods_dict[stats_method][:10].index.to_list():
                for feature in feature_relationship.split(" compared to "):
                    all_feature_relationship.add(feature)
            print(all_feature_relationship)
            print("-----" * 12 + "\n\n")

        del stat_methods_dict
model_analysis.perform_analysis(X=X_test,
                                y=y_test,
                                dataset_name="Test Data",
                                regression_error_analysis=True,
                                regression_correct_analysis=True,
                                display_visuals=False,
                                mse_score=.2)
for pred_name in model_analysis.get_predictions_names():
    for mse_score in [.2]:
        print(f"Prediction name: {pred_name} with mse_score less than {mse_score}")
        infile = open(model_analysis.folder_path + "Test Data" + f"/{pred_name}/MSE score less than {mse_score}/Test Data/_Extras/Statistics/Stat methods of features dataframes.pkl",'rb')
        stat_methods_dict = pickle.load(infile)
        infile.close()

        for stats_method in stat_methods_dict.keys():
            print(stats_method)
            display(stat_methods_dict[stats_method].round(6))
            all_feature_relationship = set()
            for feature_relationship in stat_methods_dict[stats_method][:10].index.to_list():
                for feature in feature_relationship.split(" compared to "):
                    all_feature_relationship.add(feature)
            print(all_feature_relationship)
            print("-----" * 12 + "\n\n")

        del stat_methods_dict<jupyter_output>Prediction name: Predict function with mse_score less than 0.2
Kolmogorov-Smirnov statistic
{'fraction_non_alphanumeric': 0.08892036512433113, 'fraction_numerical': 0.005036197670758577, 'mean_word_length': 2.8398791540785497, 'pattern_counts': {'":': 2, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '101522', 'license': 'MIT', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1855, 'n_words': 350, 'path': 'testing/3.) Regression of data.ipynb', 'repo_name': 'EricCacciavillani/eFlow', 'size': '6354'}