<jupyter_start><jupyter_text>Import Library and Dataset<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
data = pd.read_csv('https://raw.githubusercontent.com/algonacci/DataWarehouse/main/activity_km_07_01.csv')
data<jupyter_output><empty_output><jupyter_text>Normalization<jupyter_code>data.rename(columns={
    "intensitas air" : "intensitas_air",
    "tinggi air" : "tinggi_air"
}, inplace=True)
data.head()
data.dtypes
data["cahaya"].value_counts()
data["intensitas_air"].value_counts()
data['aksi'].value_counts()
kategorikal = {"cahaya":
               {"Tidak ada": 0, "Ada": 1},
               "intensitas_air":
               {"Rendah Sekali": 0, "Rendah": 1, "Cukup": 2, "Tinggi": 3,},
               "aksi":
               {"Tidak melakukan apa-apa" : 0, "Hidupkan Lampu" : 1, "Hidupkan Lampu dan Pompa nutrisi TDS" : 2, "Hidupkan Pompa nutrisi TDS" : 3}
               ,}
data = data.replace(kategorikal)
data.head()<jupyter_output><empty_output><jupyter_text>Handle Missing Value<jupyter_code>data.isnull().sum()
data.isnull().sum().sum()<jupyter_output><empty_output><jupyter_text>Detect Outliers<jupyter_code>data.boxplot()
plt.show()
data['PH']=data['PH'].fillna(data['PH'].mean())
data.isnull().sum()
data['cahaya'].fillna(method='bfill', inplace=True)
data.isnull().sum()
data['cahaya'].head(187)
data['suhu'].describe()
data.drop([109,118], axis=0, inplace=True)
data['suhu'].describe()
data['suhu']=data['suhu'].fillna(data['suhu'].mean())
data.isnull().sum()
data['PPM']=data['PPM'].fillna(data['PPM'].median())
data.isnull().sum()
data['tinggi_air'].describe()
data['tinggi_air'].iloc[[3,77]]
data.drop([3,77], axis=0, inplace=True)
data['tinggi_air'].describe()
data.boxplot()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19760820045558086, 'fraction_numerical': 0.01765375854214123, 'mean_word_length': 5.890196078431373, 'pattern_counts': {'":': 9, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '595329', 'license': 'MIT', 'n_tokens_mistral': 720, 'n_tokens_neox': 692, 'n_words': 94, 'path': '04 - Data Cleansing and Normalization/Data_Cleansing_Normalization.ipynb', 'repo_name': 'algonacci/Projects', 'size': '1756'}