<jupyter_start><jupyter_text>離群值處理  Detect<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

data_path = './data/house_train.csv'
df_train = pd.read_csv(data_path)
ori_series = df_train['土地移轉總面積(平方公尺)']
display(ori_series)
print(len(ori_series))
qt1 = ori_series.quantile(q=0.25)
qt3 = ori_series.quantile(q=0.75)
iqr = qt3-qt1


display(ori_series[ori_series<(qt1 - 1.5*iqr)])
display(ori_series[ori_series>(qt3 + 1.5*iqr)])
plt.boxplot(ori_series)
plt.show()<jupyter_output><empty_output><jupyter_text>drop outliers<jupyter_code>mask = (ori_series>(qt1 - 1.5*iqr)).tolist() and (ori_series<(qt3 + 1.5*iqr)).tolist()
series_drop_outliers = ori_series[mask]
print(len(series_drop_outliers))
plt.boxplot(series_drop_outliers)
plt.show()<jupyter_output><empty_output><jupyter_text>fill outliers<jupyter_code>series_fill = ori_series.copy()
series_fill[series_fill>(qt3 + 1.5*iqr)] = qt3 + 1.5*iqr
print(len(series_fill))
plt.boxplot(series_fill)
plt.show()<jupyter_output><empty_output><jupyter_text>Normalize  MinMaxScaler<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler
import numpy as np
X = np.array([[ 1., -1.,  2.],
              [ 2.,  0.,  0.],
              [ 0.,  1., -1.]])

print(f"min of X: {X.min(axis=0)}")
print(f"max of X: {X.max(axis=0)}\n")
min_max_scaler = MinMaxScaler().fit(X)
x_minmax_sk = min_max_scaler.transform(X)
print(x_minmax_sk)<jupyter_output><empty_output><jupyter_text>StandardScaler<jupyter_code>X = np.array([[ 1., -1.,  2.],
              [ 2.,  0.,  0.],
              [ 0.,  1., -1.]])
print(f"mean of X: {X.mean(axis=0)}")
print(f"std of X: {X.std(axis=0)}\n")

scaler = StandardScaler().fit(X)
# apply mean and std to standardize data
x_sc_sk = scaler.transform(X)
print(f"mean of scaler: {scaler.mean_}")
print(f"std of scaler: {scaler.scale_}\n")
print(x_sc_sk)<jupyter_output><empty_output><jupyter_text>資料合併  pd.concat()<jupyter_code>df1 = pd.DataFrame([['A1','B1'], ['A2','B2']],
             columns=['A', 'B'],
             index=[1,2])

df2 = pd.DataFrame([['A3','B3'], ['A4','B4']],
             columns=['A', 'B'],
             index=[1,2])

df3 = pd.DataFrame([['B5','C5'], ['B6','C6']],
             columns=['B', 'C'],
             index=[5,6])

df4 = pd.concat([df1, df2])
df5 = pd.concat([df1, df2],axis = 1)
df6 = pd.concat([df1, df3])

display(df1)
display(df2)
display(df3)
display(df4)
display(df5)
display(df6)<jupyter_output><empty_output><jupyter_text>pd.merge()<jupyter_code>df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],
                    'B': ['B0', 'B1', 'B2', 'B3'],
                    'C': ['C0', 'C1', 'C2', 'C3'],
                    'D': ['D0', 'D1', 'D2', 'D3']},
                   index=[0, 1, 2, 3])
df2 = pd.DataFrame({'B': ['B2', 'B3', 'B6', 'B7'],
                    'D': ['D2', 'D10000', 'D6', 'D7'],
                    'F': ['F2', 'F3', 'F6', 'F7']},
                   index=[2, 3, 6, 7])
display(df1)
display(df2)
result_inner = pd.merge(df1, df2, how='inner')
result_inner_B = pd.merge(df1, df2, how='inner', on = 'B')
result_outer = pd.merge(df1, df2, how='outer')
display(result_inner)
display(result_inner_B)
display(result_outer)<jupyter_output><empty_output><jupyter_text>groupby()<jupyter_code>df_train.groupby('鄉鎮市區').mean()<jupyter_output><empty_output><jupyter_text>補充:<jupyter_code>df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],
                    'B': ['B0', 'B1', 'B2', 'B3'],
                    'C': ['C0', 'C1', 'C2', 'C3'],
                    'D': ['D0', 'D1', 'D2', 'D3']},
                   index=[0, 1, 2, 3])
df_melt = df1.melt()
display(df_melt)
df_pivot=df_melt.set_index(pd.Index([0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]))
df_pivot=df_pivot.pivot(columns='variable')
display(df_pivot)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21296537360062484, 'fraction_numerical': 0.04556105180942463, 'mean_word_length': 4.095490716180372, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '489652', 'license': 'MIT', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1680, 'n_words': 254, 'path': '.ipynb_checkpoints/mod04-checkpoint.ipynb', 'repo_name': 'theabc50111/machine_learning_tibame', 'size': '3841'}