<jupyter_start><jupyter_text>Find the markdown blocks that say interaction required! The notebook should take care of the rest!  Import libs<jupyter_code>import sys
import os
sys.path.append('..')
from eflow.foundation import DataPipeline,DataFrameTypes
from eflow.data_analysis import FeatureAnalysis, NullAnalysis
from eflow.model_analysis import ClassificationAnalysis
from eflow.data_pipeline_segments import FeatureTransformer, TypeFixer, DataEncoder, FeatureDataCleaner
from eflow.utils.modeling_utils import optimize_model_grid
from eflow.utils.eflow_utils import get_type_holder_from_pipeline, remove_unconnected_pipeline_segments
from eflow.utils.math_utils import get_unbalanced_threshold
from eflow.utils.sys_utils import create_dir_structure
from eflow.utils.eflow_utils import create_color_dict_for_features
from eflow.utils.pandas_utils import missing_values_table,data_types_table, value_counts_table, suggest_removal_features 
from eflow.widgets import ColorLabelingWidget

import pandas as pd
import numpy as np
import scikitplot as skplt
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
import copy
from IPython.display import clear_output
from IPython.core.getipython import get_ipython
import ipython_blocking
# # Additional add ons
# !pip install pandasgui
# !pip install pivottablejs
# clear_output()
%matplotlib notebook
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Declare Project Variables  Interaction required<jupyter_code>dataset_path = "Datasets/titanic_train.csv"

# -----
dataset_name = "Titanic Data"
pipeline_name = "Titanic Pipeline"

# -----


# -----
notebook_mode = True<jupyter_output><empty_output><jupyter_text>Clean out segment space<jupyter_code>remove_unconnected_pipeline_segments()<jupyter_output><empty_output><jupyter_text>Import dataset<jupyter_code>df = pd.read_csv(dataset_path)
shape_df = pd.DataFrame.from_dict({'Rows': [df.shape[0]],
                                   'Columns': [df.shape[1]]})
display(shape_df)
display(df.head(30))
data_types_table(df)<jupyter_output><empty_output><jupyter_text>Loading and init df_features<jupyter_code># Option: 1
# df_features = get_type_holder_from_pipeline(pipeline_name)
# Option: 2
df_features = DataFrameTypes()
df_features.init_on_json_file(os.getcwd() + f"/eflow Data/{dataset_name}/df_features.json")
df_features.display_features(display_dataframes=True,
                             notebook_mode=notebook_mode)<jupyter_output><empty_output><jupyter_text>Any extra processing before eflow DataPipeline  Setup pipeline structure  Interaction Required<jupyter_code>main_pipe = DataPipeline(pipeline_name,
                         df,
                         df_features)
main_pipe.perform_pipeline(df,
                           df_features)
df<jupyter_output><empty_output><jupyter_text>Seperate out data into train and test sets<jupyter_code>X = df.drop(columns=df_features.target_feature()).values
y = df[df_features.target_feature()].values
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.80, random_state=517, stratify=y,
)<jupyter_output><empty_output><jupyter_text>Fit Models and view results<jupyter_code># Find best parameters for model
param_grid = {
    "max_depth": list(range(1, 4)),
#     "min_samples_leaf": list(range(80, 130, 5)),
#     "criterion": ["gini", "entropy"],
#     "n_splits": [20, 30]
}

model, best_params = optimize_model_grid(
    model=DecisionTreeClassifier(),
    X_train=X_train,
    y_train=y_train,
    param_grid=param_grid,
    scoring="f1_micro"
)
model_name = repr(model).split("(")[0]
dt_model_analysis = ClassificationAnalysis(dataset_name=dataset_name,
                                           model=model,
                                           model_name=model_name,
                                           target_feature=df_features.target_feature(),
                                           pred_funcs_dict={"Probabilities function":model.predict_proba,
                                                            "Predict function":model.predict},
                                           sample_data=X_train[0],
                                           project_sub_dir='Classification Analysis',
                                           notebook_mode=notebook_mode,
                                           df_features=df_features)
dt_model_analysis.perform_analysis(X=X_train,
                                   y=y_train,
                                   dataset_name="Train Data",
                                   thresholds_matrix=[[.0,.0],],
                                   display_visuals=True)
dt_model_analysis.perform_analysis(X=X_test,
                                   y=y_test,
                                   dataset_name="Test Data",
                                   display_visuals=True)<jupyter_output>------------------------------Test Data------------------------------
Now running classification on Probabilities function on no thresholds.
---------------------------------------------------------------------

Now running classification on Probabilities function on no thresholds.
---------------------------------------------------------------------





Now running classification on Predict function
---------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.12395929694727105, 'fraction_numerical': 0.005365402405180388, 'mean_word_length': 2.5753968253968256, 'pattern_counts': {'":': 6, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '26579', 'license': 'MIT', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1459, 'n_words': 285, 'path': 'testing/.ipynb_checkpoints/3.) Regression of data-checkpoint 2.ipynb', 'repo_name': 'EricCacciavillani/eFlow', 'size': '5405'}