<jupyter_start><jupyter_text>Handling Missing Values<jupyter_code># import pandas 
import pandas as pd 
# read ufo data 
ufo = pd.read_csv("http://bit.ly/uforeports")
# last 5 rows 
ufo.tail() 
# check missing values 
ufo.isnull()<jupyter_output><empty_output><jupyter_text>__Note__1. True: Missing 2. False: Not Missing<jupyter_code># using notnull() 
ufo.notnull()<jupyter_output><empty_output><jupyter_text>__Note__1. axis = 0: Rows 2. axis = 1: Columns<jupyter_code># sum of missing values: by default axis = 0
ufo.isnull().sum() 
# Let's create a series 
pd.Series([True, False, True]).sum() 
# filtering using isnull() 
ufo[ufo.City.isnull()]
# Check specific column
ufo.City.isnull().sum()<jupyter_output><empty_output><jupyter_text>Drop Missing Values<jupyter_code># shape 
ufo.shape 
# drop missing: drop row contains missing values 
# it is inplace = False 
ufo.dropna(how='any').shape
# how=all 
ufo.dropna(how='all').shape 
# subset: any 
ufo.dropna(subset=['City', 'Shape Reported'], how='any').shape
# subset: all 
ufo.dropna(subset=['City', 'Shape Reported'], how='all').shape<jupyter_output><empty_output><jupyter_text>Filling Missing Values<jupyter_code># value counts: by default drop = True 
ufo["Shape Reported"].value_counts() 
# value counts: false 
ufo["Shape Reported"].value_counts(dropna=False) 
# fillna() 
ufo["Shape Reported"].fillna(value="VARIOUS", inplace=True)
# now take a look 
ufo["Shape Reported"].value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14739688979039892, 'fraction_numerical': 0.005409060175794456, 'mean_word_length': 6.551020408163265, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '276325', 'license': 'CC0-1.0', 'n_tokens_mistral': 575, 'n_tokens_neox': 557, 'n_words': 117, 'path': 'book/pandas/09-Handling Missing Values.ipynb', 'repo_name': 'hossainlab/dsnotes', 'size': '1479'}