<jupyter_start><jupyter_text>Data load & exploration<jupyter_code>file = '../data/classification.xlsx'
D = pd.read_excel(file)
D.describe()
D.info()
D['category'].value_counts().iloc[:5] / D.shape[0]
D['macrocategory_name'].value_counts() / D.shape[0]
D['dependent variable'].value_counts() / D.shape[0]<jupyter_output><empty_output><jupyter_text>Handle null values<jupyter_code>D.dropna(inplace=True)<jupyter_output><empty_output><jupyter_text>Encoding classes<jupyter_code>from sklearn import preprocessing as pre
target = 'dependent variable'
X, target_values = D[[x for x in D.columns if x != target]], D[target].values
target_values.reshape(-1, 1)
encoder = pre.OrdinalEncoder()
y = encoder.fit_transform(target_values.reshape(-1, 1))
y
print(encoder.categories_)
print(encoder.inverse_transform(np.array(
    list(range(len(encoder.categories_[0])))).reshape(-1, 1)))<jupyter_output>[array(['Non venduto', 'Venduto su Subito', 'Venduto su altri canali',
       'active', 'cancellato senza alcuna ragione di cancellazione'],
      dtype=object)]
[['Non venduto']
 ['Venduto su Subito']
 ['Venduto su altri canali']
 ['active']
 ['cancellato senza alcuna ragione di cancellazione']]<jupyter_text>Train and test split<jupyter_code>from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Unstratified split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, 
                                                    y, 
                                                    test_size=0.2, 
                                                    random_state=42)
fig, ax = plt.subplots(figsize=(14, 4))
df = pd.DataFrame({
    'original': X.macrocategory_name.value_counts() / X.shape[0],
    'shuffle': X_train.macrocategory_name.value_counts() / X_train.shape[0]})
df.plot.barh(ax=ax, rot=0)
plt.show()
S_train, S_test, s_train, s_test = train_test_split(
    X, y, stratify=X[['category', 'macrocategory_name']], 
    test_size=0.2, random_state=42)
fig, ax = plt.subplots(figsize=(14, 4))
df = pd.DataFrame({
    'original': X.macrocategory_name.value_counts() / X.shape[0],
    'shuffle': X_train.macrocategory_name.value_counts() / X_train.shape[0],
    'stratified': S_train.macrocategory_name.value_counts() / S_train.shape[0],
})
df.plot.barh(ax=ax, rot=0)
plt.show()
df
df.corr()
S_train.head()<jupyter_output><empty_output><jupyter_text>Numeric vs categorical features<jupyter_code>n_cols, c_cols = [], []
for i, x in S_train.dtypes.items():
    if i != 'ad_id' and x in [np.int64, np.float64]:
        n_cols.append(i)
    elif i != 'ad_id':
        c_cols.append(i)
    else:
        pass
N_train, C_train = S_train[n_cols], S_train[c_cols]
N_test, C_test = S_test[n_cols], S_test[c_cols]<jupyter_output><empty_output><jupyter_text>Deal with numeric values<jupyter_code>N_train.corr()<jupyter_output><empty_output><jupyter_text>Linear transformations<jupyter_code>N_std = pre.StandardScaler().fit_transform(N_train)
N_mms = pre.MinMaxScaler().fit_transform(N_train)
N_mab = pre.MaxAbsScaler().fit_transform(N_train)<jupyter_output><empty_output><jupyter_text>Non linear transformations<jupyter_code>N_qtl = pre.QuantileTransformer().fit_transform(N_train)
N_pow = pre.PowerTransformer().fit_transform(N_train)
N_norm = pre.Normalizer().fit_transform(N_train)
fig, ax = plt.subplots(ncols=3, nrows=2, figsize=(14, 6))
ax[0,0].boxplot(N_std)
ax[0,1].boxplot(N_mms)
ax[0,2].boxplot(N_mab)
ax[1,0].boxplot(N_qtl)
ax[1,1].boxplot(N_pow)
ax[1,2].boxplot(N_norm)
plt.tight_layout()
plt.show()
print(N_std.mean(axis=0), N_std.std(axis=0))
print(N_mms.mean(axis=0), N_mms.std(axis=0))
print(N_mab.mean(axis=0), N_mab.std(axis=0))
print(N_qtl.mean(axis=0), N_qtl.std(axis=0))
print(N_pow.mean(axis=0), N_pow.std(axis=0))
print(N_norm.mean(axis=0), N_norm.std(axis=0))<jupyter_output>[ 4.44635999e-17 -6.23725498e-17 -4.11699999e-18] [1. 1. 1.]
[0.05718903 0.12612008 0.00233659] [0.08955431 0.11938215 0.01373122]
[0.05718903 0.12612008 0.00233659] [0.08955431 0.11938215 0.01373122]
[0.49977571 0.49971549 0.50003892] [0.28907347 0.28567414 0.28859997]
[-1.66738500e-16  4.58016249e-16 -4.03465999e-17] [1. 1. 1.]
[0.49781998 0.02318968 0.65093613] [0.41809777 0.04843616 0.38828837]<jupyter_text>Discretize numbers<jupyter_code>N_bin = pre.KBinsDiscretizer(n_bins=[2, 4, 3], 
                             encode='ordinal').fit_transform(N_train)
N_thr = pre.Binarizer(threshold=0.02).fit_transform(N_mms)
np.array(N_train)[:2,:]
N_bin[:2,:]
N_train.iloc[:5]
N_bin[:5,:]
N_thr[:5,:]<jupyter_output><empty_output><jupyter_text>Deal with categorical values<jupyter_code>hot = pre.OneHotEncoder()
C_hot = hot.fit_transform(C_train)
print(C_hot[[0,1000]])
print(type(C_hot))
print(hot.categories_)
hot2 = pre.OneHotEncoder(sparse=False)
C_hot2 = hot2.fit_transform(C_train)
C_hot2[[0,1000]]
C_train.iloc[0]
for x in np.where(C_hot2[0] > 0)[0]:
    print(hot2.get_feature_names()[x])<jupyter_output>x0_False
x1_Biciclette
x2_SPORTS E HOBBY
x3_Lazio<jupyter_text>Correlation with target<jupyter_code>encoder.categories_[0]
fig, ax = plt.subplots(ncols=len(encoder.categories_[0]), nrows=len(N_train.columns), figsize=(12, 8))
for j, cat in enumerate(encoder.categories_[0]):
    cat_indexes = np.where(y_train.ravel() == j)[0]
    for i, c in enumerate(N_train.columns):
        V = N_train.iloc[cat_indexes][[c]]
        ax[i,j].boxplot(pre.PowerTransformer().fit_transform(V.values))
        ax[i,j].set_title("{}, {}".format(i, c))
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>For categorical features, we aim at estimating $P(f \mid C)$<jupyter_code>C_train.head()
P = {}
column = 'macrocategory_name'
for j, cat in enumerate(encoder.categories_[0]):
    P[cat] = {}
    cat_indexes = np.where(y_train.ravel() == j)[0]
    subset = C_train.iloc[cat_indexes]
    count = subset.groupby(column).size()
    for i, v in count.items():
        P[cat][i] = v
P = pd.DataFrame(P)
P / P.sum(axis=0)
(P.T / P.sum(axis=1)).sum(axis=0)<jupyter_output><empty_output><jupyter_text>Dimensionality reduction<jupyter_code>from sklearn.decomposition import PCA
pca = PCA(n_components=2)
N2_train = pca.fit_transform(C_hot2)
#    pre.PowerTransformer().fit_transform(N_train))
fig, ax = plt.subplots(figsize=(6, 6))
ax.scatter(N2_train[:,0], N2_train[:,1], alpha=0.2,
          c=s_train.ravel())
plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16162718204488777, 'fraction_numerical': 0.06405860349127182, 'mean_word_length': 5.378727634194831, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '324153', 'license': 'MIT', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2604, 'n_words': 376, 'path': 'classification/exploration.ipynb', 'repo_name': 'afflint/subito', 'size': '6416'}