<jupyter_start><jupyter_text>Missing data<jupyter_code>from numpy import nan

df.iloc[2:4,2]=nan
df.iloc[2,3]=nan

df
df.dropna()
df.fillna(0)
df.fillna({'age':0,'gender':'Unknown'})
df.fillna(method='ffill', limit=1)
df.fillna(df.mean())<jupyter_output><empty_output><jupyter_text>Duplicated values<jupyter_code>newitem = {'id':6,'name':'Rose', 'age':18.0, 'gender':'Female','grade':100}
df_dup = df.append(newitem, ignore_index=True)
df_dup
df_dup.duplicated()
df_dup.drop_duplicates()
# specify a subset of column to drop the duplicated
df_dup.drop_duplicates(['age', 'gender'])<jupyter_output><empty_output><jupyter_text>Data Transform<jupyter_code>df.gender.apply(lambda x:1 if x == 'Female' else 0)
df
df['gender'] = df.gender.apply(lambda x:1 if x == 'Female' else 0)
df
df = pd.read_csv('data/demo.table', sep='\s+')

#map
grade_mapper = {
    100:'A+',
    90:'A',
    80:'B',
    70:'C'
}
df_graded = df.grade.map(grade_mapper)
df_graded
df['grade']=df_graded
df
df.id.map(lambda x:x*10)
df.grade.replace('A+', 'A')
df.age.replace([0,20], 20)
df.columns = df.columns.map(lambda x:x.upper())
# to modify index, use df.index.map
df
df
df.rename(index={5:55555},columns={'GRADE':'GRA'})<jupyter_output><empty_output><jupyter_text>Discretization and Binning<jupyter_code>import json
df = pd.read_json('data/acc-demo1.json')
df
bins = [-3,-1,-0.5, 0, 0.5, 1,3]
interval = pd.cut(df.z, bins)
interval
pd.value_counts(interval)
# pass an integer nunber of bins to cut instead of explicit bin edges
# it will compute equal-length bins based on the minimum and maximum values in the data
interval = pd.cut(df.z, 10, precision=2)
interval
pd.value_counts(interval)
# cut with quanties
interval = pd.qcut(df.z, 4)
pd.value_counts(interval)<jupyter_output><empty_output><jupyter_text>Sampling<jupyter_code>df.sample(n=10)<jupyter_output><empty_output><jupyter_text>Permutation<jupyter_code>import numpy as np

neworder = np.random.permutation(len(df))
neworder
df.take(neworder)<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code>import json
df = pd.read_json('data/acc-demo1.json')
df
df.describe()
import numpy as np

df.z[np.abs(df.z)>1]
df[(np.abs(df)>0.7).any(axis=1)]
np.abs(df)>0.7
(np.abs(df)>0.7).any(0)
(np.abs(df)>0.7).any(1)
df<df.quantile(0.75)
df.std()
df.mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17817083692838653, 'fraction_numerical': 0.031492666091458156, 'mean_word_length': 6.996551724137931, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '230284', 'license': 'MIT', 'n_tokens_mistral': 1013, 'n_tokens_neox': 965, 'n_words': 164, 'path': 'dsp-session4-pandas2.ipynb', 'repo_name': 'dichangyan/dscourse', 'size': '2318'}