<jupyter_start><jupyter_text>Session 5 Missing values<jupyter_code>import numpy as np
missing_values=['--','na','n/a']
df6 = pd.read_csv('https://raw.githubusercontent.com/dataoptimal/posts/master/data%20cleaning%20with%20python%20and%20pandas/property%20data.csv', na_values=missing_values)
df6
df6['OWN_OCCUPIED'] = df6['OWN_OCCUPIED'].replace(r'[0-9]+',np.nan,regex=True)
df6
df6['NUM_BATH'] = df6['NUM_BATH'].replace(r'[A-Z]+',np.nan,regex=True)
df6
df6.isnull().sum()
df6.isnull().sum().sum()
#fillna() - you could give val, backward fill, forward fill
df6.fillna(0)
df6
#using backward fill, if last value is missing it wont get filled
df6.fillna(method='bfill')
df6.fillna(method='ffill')
df6.replace(to_replace=np.nan, value= -99) #same as fillna(-99)
df6.dropna() #no parameters - if there are nan values, the row will get deleted
df6.dropna(how='all') #if the row fully consists of nan values, the row gets deleted
df6.dropna(axis=1) #drop columnwise, if nan value present
new_data = df6.dropna(axis=0, how='any')
data_set = pd.read_csv("https://gist.githubusercontent.com/ktisha/c21e73a1bd1700294ef790c56c8aec1f/raw/819b69b5736821ccee93d05b51de0510bea00294/pima-indians-diabetes.csv", skiprows=9, header=None)
data_set
data_set[[1,2,3,4,5]]=data_set[[1,2,3,4,5]].replace(0,np.nan)
print(data_set.isnull().sum())
data_set.head()
data_set.fillna(data_set.mean(),inplace=True)
data_set<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16584680252986647, 'fraction_numerical': 0.06957132817990162, 'mean_word_length': 7.790123456790123, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '30928', 'license': 'MIT', 'n_tokens_mistral': 622, 'n_tokens_neox': 583, 'n_words': 97, 'path': 'Introduction_to_handling_missing_values.ipynb', 'repo_name': 'ColdCoffee21/Foundations-of-Data-Science', 'size': '1423'}