<jupyter_start><jupyter_text>Environment Setup<jupyter_code># Essential modules for data manipulation
import pandas as pd
import numpy as np

# Custom modules to assist the commom data exploration and preparation tasks
import src.data.explore as dataexp

# Modules to preprocess data
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler

# Modules to persist classes
import joblib

# Modules to create pipelines and transformers
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer<jupyter_output><empty_output><jupyter_text>4. Preprocessing Pipeline  4.1 Collect FInal Unprocessed Data<jupyter_code>df, dfNone = dataexp.read_csv_data('beer_reviews.csv', None, '../data/interim/')
df.head(5)<jupyter_output><empty_output><jupyter_text>4.2 Create Preprocessing Data pipeline  4.2.1 Setup Pipeline to encode brewery_name<jupyter_code># Setup abv columns and pipeline to impute most trequent value
brewery_name_encoder = joblib.load('../models/brewery_name_encoder.joblib')
bn_cols = ['brewery_name']
bn_pipe = Pipeline (
    steps=[
        ('bnencode', brewery_name_encoder)
    ]
)<jupyter_output><empty_output><jupyter_text>4.2.2 Setup Pipeline to encode and scale review rating columns<jupyter_code># Define cat columns and pipeline to do ordinal encoding
rev_cols = ['review_aroma', 'review_appearance', 'review_palate', 'review_taste']
rev_pipe = Pipeline (
    steps = [
        #('ordencoder', OrdinalEncoder()),
        #('revscaler', MinMaxScaler())
        ('revpass','passthrough')
    ]
)<jupyter_output><empty_output><jupyter_text>4.2.3 Setup Pipeline to impute most frequent value and scale abv_col<jupyter_code># Setup abv columns and pipeline to impute most trequent value
abv_cols = ['beer_abv']
abv_pipe = Pipeline (
    steps=[
        ('imputer', SimpleImputer(strategy='most_frequent'))
        #('abvscaler', MinMaxScaler())
    ]
)<jupyter_output><empty_output><jupyter_text>4.2.4 Setup Pipeline to encode review month<jupyter_code># Setup One Hot Encoded columns and pipeline to perform one hot encodind
#ohe_cols = ['review_month']
#ohe_pipe = Pipeline(
#    steps=[
#        ('one_hot_encoder', OneHotEncoder(sparse=False, drop='first'))
#    ]
#)<jupyter_output><empty_output><jupyter_text>4.2.5 Define Transformer for specific pipelines and columns<jupyter_code># Define a column transformer to apply defined pipelines to specific columns
preprocessor = ColumnTransformer (
    transformers = [
        ('bn_trans', bn_pipe, bn_cols),
        ('rev_trans', rev_pipe, rev_cols),
        ('abv_trans', abv_pipe, abv_cols)
    ]
)<jupyter_output><empty_output><jupyter_text>4.2.6 Fit the preprocessor to the data<jupyter_code>npdata = preprocessor.fit_transform(df)<jupyter_output><empty_output><jupyter_text>4.2.7 Review Transformed data<jupyter_code># Convery NP Array to dataframe and display data
df_npdata = pd.DataFrame(npdata, columns=['brewery_name', 'review_aroma', 'review_appearance', 'review_palate', 'review_taste', 'beer_abv'])
df_npdata.head(10)
df_npdata.describe()<jupyter_output><empty_output><jupyter_text>4.2.8 Save the preprocessor for use in the API<jupyter_code>joblib.dump(preprocessor, '../models/preproc_beer_type_prediction.joblib', compress=9)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10011641443538999, 'fraction_numerical': 0.00960419091967404, 'mean_word_length': 6.130705394190872, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '544777', 'license': 'FTL', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1124, 'n_words': 274, 'path': 'notebooks/pipeline.ipynb', 'repo_name': 'seanbwilliams/beer_type_prediction', 'size': '3436'}