<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Init2&nbsp;&nbsp;Notes3&nbsp;&nbsp;Systematic go through stuff3.1&nbsp;&nbsp;Meta analysis3.2&nbsp;&nbsp;Split data3.3&nbsp;&nbsp;Transformer  Init<jupyter_code># Model libs (you can use any of the models below)
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.neural_network import MLPRegressor

# Data
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import re
import os.path

# Text processing
# import nltk
# nltk.download('stopwords')

# Validation/scoring
from sklearn.metrics import mean_squared_error

# Plotting
from matplotlib import pyplot as plt

# Other
from IPython.display import HTML, display

# Config
pd.set_option('display.max_columns', 500)
pd.set_option('max_colwidth', 100)
y_col = "points"<jupyter_output><empty_output><jupyter_text>https://www.kaggle.com/zynicide/wine-reviews<jupyter_code>if os.path.isfile("../data/wine.csv"):
    df_src = pd.read_csv("../data/wine.csv", encoding="utf-8", sep=";")
elif os.path.isfile("../data/winemag-data_first150k.json"): 
    df_src = pd.read_json("../data/winemag-data_first150k.json", encoding="utf-8")

#(""../data/winemag-data_first150k.json")

#https://www.kaggle.com/zynicide/wine-reviews#winemag-data-130k-v2.csv
df_src.head()<jupyter_output><empty_output><jupyter_text>DS explore- "meta analysis"- What pipelines to build, for which coloumns<jupyter_code>def ds_explore(df_src,target,cols_rm,**kwargs):
    unique_lim = kwargs.get("unique_lim",50)
    
    target=[target]
    
    cols_num = [i for i in df_src._get_numeric_data().columns if i not in cols_rm + target]
    cols_str = [i for i in df_src.columns if i not in cols_num + cols_rm + target]

    unique_count = df_src[cols_str].nunique()
    

    cols_str_cat = list(unique_count[unique_count<unique_lim].index)
    cols_str_nlp = [i for i in cols_str if i not in cols_str_cat]

    meta = {
        "cols_str_cat" : cols_str_cat
        ,"cols_str_nlp" : cols_str_nlp
        ,"cols_num" : cols_num
        ,"target" : target
    }
    # we now have split the dataset in 3 categories
    # num_cols = numeric columns
    # str_cat = string columns that can be treated as categorical
    # str_nlp = string columns that can be treated as nlp

    return meta
    
# Auto categorize columns
target = ["points"]
cols_rm = ["id"]

meta = ds_explore(df_src=df_src
           ,target =target
           ,cols_rm = cols_rm)
meta
df_src.isnull().sum()/len(df_src.index)<jupyter_output><empty_output><jupyter_text>Split data<jupyter_code>import scipy
test_size = 0.3
random_state = 1

X_train, X_test, y_train, y_test = train_test_split(
        df_src[meta["cols_num"] + meta["cols_str_cat"] + meta["cols_str_nlp"]],
        df_src[target].values.ravel(),
        test_size = test_size, 
        random_state = random_state
    )

X_train_clean = X_train.dropna()<jupyter_output><empty_output><jupyter_text>Transformer<jupyter_code># https://stackoverflow.com/questions/54160370/how-to-use-sklearn-column-transformer
# https://medium.com/vickdata/a-simple-guide-to-scikit-learn-pipelines-4ac0d974bdcf

from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder

meta["trans_num"] = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())])

meta["trans_str_cat"] = Pipeline(steps=[
    #('imputer', SimpleImputer(missing_values=[None,float('nan')],strategy='constant', fill_value='missing')),
    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
    ('onehot', OneHotEncoder(handle_unknown='ignore'))])

preprocessor = ColumnTransformer(
    transformers=[
        ('num', meta["trans_num"], meta["cols_num"]),
        ('cat', meta["trans_str_cat"], meta["cols_str_cat"])
    ])


X_train_prep = preprocessor.fit_transform(X_train.copy())
meta
#GradientBoostingRegressor
model = RandomForestRegressor(n_estimators=10)


model.fit(X_train_prep,y_train.ravel())
X_test_prep = preprocessor.transform(X_test)

y_test_model = model.predict(X_test_prep)

mse = mean_squared_error(y_test, y_test_model)
mse

df = pd.DataFrame({"target":[1,0,1,0], "a":[1,2,3,4],"b":["a","a","b","b"]})


df[[]].nunique()
if not []:
    print(True)<jupyter_output>True
{'fraction_non_alphanumeric': 0.1331241595696997, 'fraction_numerical': 0.013222770058269833, 'mean_word_length': 5.172890733056708, 'pattern_counts': {'":': 3, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '25845', 'license': 'MIT', 'n_tokens_mistral': 1679, 'n_tokens_neox': 1606, 'n_words': 289, 'path': '03_test_automl/notebooks/wine_review_automl.ipynb', 'repo_name': 'EmilMachine/ml_club', 'size': '4462'}