<jupyter_start><jupyter_text>Missing Values<jupyter_code>df.count()
df.isna().sum()
df.count()/df.shape[0] # % of rows with non-null values
df.shape[0] - df.count() # number of rows with non-null values
## other methods for finding null values
df.isna()
pd.isna(df).head()
pd.isna(df).sum()
pd.notnull(df).head()
## Handling missing value
# Method 1 by dropping
df.dropna(subset=['Embarked'],inplace=True)
df.isna().sum()
df.shape
df.drop('Cabin',axis=1,inplace=True)
df.head()
# Method 2 by replacing wtih stastical measures- Mean, median or mode
df['Age']=df['Age'].fillna(df['Age'].mean())
df['Age'].isnull().sum()
df['age']='NaN'
df.head()
## replacing missing values from another column
df['age'] = df['age'].fillna( df['Age'] )<jupyter_output><empty_output><jupyter_text>Impute infinite values<jupyter_code>dict={'a':np.arange(1,4,0.5),
     'b':np.arange(5,8,0.5),
     'c':np.arange(3,6,0.5)}
dff=pd.DataFrame(dict)
dff
## introducing infite values
dff=dff.replace([3,5,7],-np.inf)
dff
##converting infinite values to NaN values
#method 1
dff[np.isfinite(dff)]
# method 2
dff=dff.replace([np.inf,-np.inf],np.nan)
dff<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code>dict_={'Names':['John', 'Paul', "Mark", "Sarah", "Morgan", "Mike", "Ram",
            "Jennifer", "Amy", "Chris", "Mitchel", "Alex"],
    'Age':[25, 45, 12, 95, 6, 21, 34, 24, 41, 31, 19, 16]}
dff=pd.DataFrame(dict_)
dff
from scipy import stats
stats.zscore(dff['Age'])
dff=dff[abs(stats.zscore(dff['Age'])) < 1.96]
dff=dff.reset_index(drop=True)
dff<jupyter_output><empty_output><jupyter_text>Erroneous Values<jupyter_code>dict_ = { "Names": ['John', 'Paul', "Mark", "Sarah", "Morgan", "Mike", "Ram",
                "Jennifer", "Amy", "Chris", "Mitchel", "Alex"],
        "Age": [25, 45, 12, 167, 6, 21, 34, 24, 410, 31, 19, 16]}
dff=pd.DataFrame(dict_)
dff
dff[dff['Age']<100] 
dff.loc[dff['Age']>100,'Age']=np.NaN
dff
dff['Age']=dff['Age'].fillna(dff['Age'].mode())
dff<jupyter_output><empty_output><jupyter_text>Values in different format<jupyter_code>dict_ = { "Names": ['John', 'Paul', "Mark", "Sarah", "Morgan", "Mike", "Ram",
                    "Jennifer", "Amy", "Chris", "Mitchel", "Alex"],
           "Age": [25, 45, 12, 0.1, 6, 21, "03/20/1980", 24, 410, 31, 19, 16]}
dff=pd.DataFrame(dict_)
dff
dff.dtypes
import traceback
# Method: 1
try:
    dff['Age'].astype(int)
except:
    traceback.print_exc()
# method 2
all(isinstance(x,int) for x in dff['Age'])
## Filter out the rows that do not match the expected format
index=[not isinstance(x,int)for x in dff['Age']]
dff[index]
# method 2
index=[type(x)!=int for x in dff['Age']]
dff[index]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2132435465768799, 'fraction_numerical': 0.041900486344930786, 'mean_word_length': 5.635235732009925, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '620863', 'license': 'MIT', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1095, 'n_words': 228, 'path': 'Pandas_ Data Cleansing.ipynb', 'repo_name': 'pallabi-sahoo/python', 'size': '2673'}