<jupyter_start><jupyter_text>Praca domowa nr 3 bomus <NAME><jupyter_code>#Pakiety
import pandas as pd
import numpy as np
import sklearn
import category_encoders
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import Ridge, LinearRegression, Lasso, BayesianRidge
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor
import matplotlib.pyplot as plt
np.random.seed(123)<jupyter_output><empty_output><jupyter_text>Część z PD2 + niezbędne operacje  Wczytanie zbioru danych<jupyter_code>df = pd.read_csv("allegro-api-transactions.csv")<jupyter_output><empty_output><jupyter_text>Krótkie obejrzenie zbioru<jupyter_code>df.head(5)<jupyter_output><empty_output><jupyter_text>+ rozbicie kolumny 'date'<jupyter_code>df['day'] = int(df.date[0][8:10])
df['month'] = int(df.date[0][5:7])
df['year'] = int(df.date[0][0:4])
df['hour'] = int(df.date[0][10:13])
df = df.drop('date', axis=1)<jupyter_output><empty_output><jupyter_text>+ usunięcie kolumn 'lp' i 'id'<jupyter_code>df = df.drop(['lp', 'item_id'], axis=1)
df.info()
#Transformacja string
df['it_location'] = df['it_location'].str.lower()
df['seller'] = df['seller'].str.lower()<jupyter_output><empty_output><jupyter_text>+ podział na zbiór treningowy i testowy<jupyter_code>X = df.drop('price', axis=1)
y = df['price']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 123)<jupyter_output><empty_output><jupyter_text>Encoding zmiennych kategorycznychMuszę zmienić nieco metody na takie, które będą funkcjonować dla zmiennych o dużym zbiorze unikalnych kategorii.<jupyter_code>#Kolumny do encodingu
cols = list(X.select_dtypes('object').columns)
cols<jupyter_output><empty_output><jupyter_text>Target encoding<jupyter_code>te = category_encoders.TargetEncoder(cols=cols)
te_train = te.fit_transform(X_train, y_train)
te_test = te.transform(X_test)<jupyter_output><empty_output><jupyter_text>James-Stein encoder<jupyter_code>js_en = category_encoders.JamesSteinEncoder(cols = cols)
js_train = js_en.fit_transform(X_train, y_train)
js_test = js_en.transform(X_test)<jupyter_output><empty_output><jupyter_text>Leave-one-out<jupyter_code>leo = category_encoders.leave_one_out.LeaveOneOutEncoder(cols=cols)
leo_train = leo.fit_transform(X_train, y_train)
leo_test = leo.transform(X_test)<jupyter_output><empty_output><jupyter_text>Modelowanie W celu zbadania wpływu encodingu porównam miary osiągnięte przez model lasso regression.<jupyter_code>def model(X_train, X_test, y_train, y_test):
    model = Lasso()
    
    model.fit(X_train, y_train)
    predict = model.predict(X_test)
    
    rmse = np.sqrt(mean_squared_error(y_test, predict))
    r2 = r2_score(y_test, predict)
    
    return(rmse, r2)
te_score = model(te_train, te_test, y_train, y_test)
js_score = model(js_train, js_test, y_train, y_test)
leo_score = model(leo_train, leo_test, y_train, y_test)
result = pd.DataFrame({'target-encoding': te_score,
                      'James-Stein': js_score,
                      'Leave-one-out': leo_score},
                      index=['RMSE', 'R2'])
result<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1229380641145347, 'fraction_numerical': 0.010893246187363835, 'mean_word_length': 7.1367088607594935, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '293524', 'license': 'Apache-2.0', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1275, 'n_words': 203, 'path': 'Prace_domowe/Praca_domowa3/Grupa1/Piotr_Fic/pd3_bonus.ipynb', 'repo_name': 'niladrem/2020L-WUM', 'size': '3216'}