<jupyter_start><jupyter_text>Manipulating DataframeThe features of Pandas that make it indispensable:To be able to harness the true power of something as versatile as the Pandas library, one should know the following features.1. Great data handling2. Handling of missing data3. Indexing and Alignment4. Tools for input and output5. Data clean-up6. Support for multiple file formats7. Multiple features for Time Series8. Joining and Merging Datasets9. Support for Python10. Optimal performance11. Grouping of data12. Visualization of data13. Data is Unique14. Masking data15. Mathematical Operations<jupyter_code>import pandas as pd<jupyter_output><empty_output><jupyter_text>Creating dataframe<jupyter_code># Create a DataFrame manually from a dictionary of Pandas Series

# create a dictionary of Pandas Series 
items = {'Bob' : pd.Series(data = [245, 25, 55], index = ['bike', 'pants', 'watch']),
         'Alice' : pd.Series(data = [40, 110, 500, 45], index = ['book', 'glasses', 'bike', 'pants'])}

# print the type of items to see that it is a dictionary
print(type(items)) # class 'dict'

# create a Pandas DataFrame by passing it a dictionary of Series
shopping_carts = pd.DataFrame(items)
shopping_carts.head()
# create a DataFrame that only has a subset of the data/columns
bob_shopping_cart = pd.DataFrame(items, columns=['Bob'])

# create a DataFrame that only has selected keys
sel_shopping_cart = pd.DataFrame(items, index = ['pants', 'book'])

# combine both of the above - selected keys for selected columns
alice_sel_shopping_cart = pd.DataFrame(items, index = ['glasses', 'bike'], columns = ['Alice'])

print('shopping_carts has shape:', shopping_carts.shape)
print('shopping_carts has dimension:', shopping_carts.ndim)
print('shopping_carts has a total of:', shopping_carts.size, 'elements')
print()
print('The data in shopping_carts is:\n', shopping_carts.values)
print()
print('The row index in shopping_carts is:', shopping_carts.index)
print()
print('The column index in shopping_carts is:', shopping_carts.columns)
# create DataFrames from a dictionary of lists (arrays)
# In this case, however, all the lists (arrays) in the dictionary must be of the same length

# create a dictionary of lists (arrays)
data = {'Integers' : [1,2,3],
        'Floats' : [4.5, 8.2, 9.6]}

# create a DataFrame 
df = pd.DataFrame(data)

# create a DataFrame and provide the row index
df = pd.DataFrame(data, index = ['label 1', 'label 2', 'label 3'])

df.head()
# create DataFrames from a list of Python dictionaries
# create a list of Python dictionaries
items2 = [{'bikes': 20, 'pants': 30, 'watches': 35}, 
          {'watches': 10, 'glasses': 50, 'bikes': 15, 'pants':5}]

# create a DataFrame 
store_items = pd.DataFrame(items2)

# create a DataFrame and provide the row index
store_items = pd.DataFrame(items2, index = ['store 1', 'store 2'])
store_items.head()<jupyter_output><empty_output><jupyter_text>Loading Data into DF<jupyter_code># Loading Data into DF
filename = '../data/data.csv'

df = pd.read_csv(filename)

# limit which rows are read when reading in a file
pd.read_csv(filename, nrows=10)        
# only read first 10 rows

pd.read_csv(filename, skiprows=[1, 2]) 
# skip the first two rows of data

# randomly sample a DataFrame
train = df.sample(frac=0.75) 
# will contain 75% of the rows

test = df[~df.index.isin(train.index)] 
# will contain the other 25%

# change the maximum number of rows and columns printed ('None' means unlimited)
pd.set_option('max_rows', None) 
# default is 60 rows

pd.set_option('max_columns', None) 
# default is 20 columns
#print (df)

# reset options to defaults
pd.reset_option('max_rows')
pd.reset_option('max_columns')

# change the options temporarily (settings are restored when you exit the ‘with’ block)
#with pd.option_context('max_rows', None, 'max_columns', None):
df.head()<jupyter_output><empty_output><jupyter_text>Dealing with NaN values (missing data)<jupyter_code># Dealing with NaN values (missing data)

# create a list of Python dictionaries
items2 = [{'bikes': 20, 'pants': 30, 'watches': 35, 'shirts': 15, 'shoes':8, 'suits':45},
{'watches': 10, 'glasses': 50, 'bikes': 15, 'pants':5, 'shirts': 2, 'shoes':5, 'suits':7},
{'bikes': 20, 'pants': 30, 'watches': 35, 'glasses': 4, 'shoes':10}]

# We create a DataFrame and provide the row index
store_items = pd.DataFrame(items2, index = ['store 1', 'store 2', 'store 3'])

# check if we have any NaN values in our dataset
# .any() performs an or operation. If any of the values along the
# specified axis is True, this will return True.
df.isnull().any()
'''
Date   False
Open   True
High   False
Low    False
Close  False
Volume False
dtype: bool
'''

# count the number of NaN values in DataFrame
x =  store_items.isnull().sum().sum()
print("Number of NaN values is:", x)
# count the number of non-NaN values in DataFrame
x = store_items.count()
print("Number of non-NaN values is:", x)
# remove rows or columns from our DataFrame that contain any NaN values

# drop any rows with NaN values
new_store = store_items.dropna(axis = 0)

print(store_items)
print("\n")
print(new_store)
# drop any columns with NaN values
new_store = store_items.dropna(axis = 1)

print(store_items)
print("\n")
print(new_store)
original_store = store_items.copy()
# the original DataFrame is not modified by default
# to remove missing values from original df, use inplace = True
store_items.dropna(axis = 0, inplace = True)

print(store_items)
print(original_store)
# replace all NaN values with 0
original_store.fillna(0)
# forward filling: replace NaN values with previous values in the df,
# this is known as . When replacing NaN values with forward filling,
# we can use previous values taken from columns or rows.
# replace NaN values with the previous value in the column
original_store.fillna(method = 'ffill', axis = 0)
# backward filling: replace the NaN values with the values that
# go after them in the DataFrame
# replace NaN values with the next value in the row
original_store.fillna(method = 'backfill', axis = 1)
# replace NaN values by using linear interpolation using column values
original_store.interpolate(method = 'linear', axis = 0)
# the original DataFrame is not modified. replace the NaN values
# in place by setting inplace = True inside function
original_store.fillna(method = 'ffill', axis = 0, inplace = True)
original_store.interpolate(method = 'linear', axis = 0, inplace = True)
original_store<jupyter_output><empty_output><jupyter_text>head, tail, describe, max, memory_usage<jupyter_code>data = pd.read_csv(filename)
data.head()
data.head()
data.tail()
data.describe()
# prints max value in each column
data.max()

# display the memory usage of a DataFrame

# total usage
data.info()

# usage by column
data.memory_usage()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 569 entries, 0 to 568
Data columns (total 32 columns):
 #   Column                   Non-Null Count  Dtype  
---  ------                   --------------  -----  
 0   id                       569 non-null    int64  
 1   diagnosis                569 non-null    object 
 2   radius_mean              569 non-null    float64
 3   texture_mean             569 non-null    float64
 4   perimeter_mean           569 non-null    float64
 5   area_mean                569 non-null    float64
 6   smoothness_mean          569 non-null    float64
 7   compactness_mean         569 non-null    float64
 8   concavity_mean           569 non-null    float64
 9   concave points_mean      569 non-null    float64
 10  symmetry_mean            569 non-null    float64
 11  fractal_dimension_mean   569 non-null    float64
 12  radius_se                569 non-null    float64
 13  texture_se               569 non-null    float64
 14  perimeter_se             5[...]<jupyter_text>corr<jupyter_code># get the correlation between different columns
data.corr().head()<jupyter_output><empty_output><jupyter_text>Groupby<jupyter_code># Groupby
data.groupby(['diagnosis'])
data.groupby(['diagnosis'])['id']

# display the average radius_mean per diagnosis type
print(data.groupby(['diagnosis'])['radius_mean'].mean())

# display the total radius mean by diagnosis type
print(data.groupby(['diagnosis'])['radius_mean'].sum())<jupyter_output>diagnosis
B    12.146524
M    17.462830
Name: radius_mean, dtype: float64
diagnosis
B    4336.309
M    3702.120
Name: radius_mean, dtype: float64<jupyter_text>Replace Values<jupyter_code># Replace Values
import numpy as np

s = pd.Series(['cat', 'dog', np.nan, 'rabbit'])
print(s)
s.map({'cat': 'kitten', 'dog': 'puppy'})<jupyter_output>0       cat
1       dog
2       NaN
3    rabbit
dtype: object<jupyter_text>Selecting data using Labels (Column Headings)<jupyter_code>print("Index is :", data[(data.radius_mean == 20.29)].index.values)

datadict = data.to_dict()

data_1 = datadict["texture_mean"]
data_2 = data[["texture_mean", "concavity_mean", "diagnosis"]]
data_2.head()<jupyter_output><empty_output><jupyter_text>Slicing dataframe<jupyter_code>data_3 = data[0:3]
data_3
data_4 = data[-1:]
data_4<jupyter_output><empty_output><jupyter_text>Copying Objects vs Referencing Objects in Python<jupyter_code># Using the 'copy() method'
true_copy_data = data.copy()

# Using the '=' operator
ref_data = data
true_copy_data[0:3] = 0

true_copy_data.head()
data.head()
# Assign the value `0` to the first three rows of data in the DataFrame
ref_data[0:3] = 0

ref_data.head()
data.head()<jupyter_output><empty_output><jupyter_text>Slicing Subsets of Rows and Columns in Python- We can select specific ranges of our data in both the row and column directions using either label or integer-based indexing.    - loc is primarily label based indexing. Integers may be used but they are interpreted as a label.    - iloc is primarily integer based indexing<jupyter_code># iloc[row slicing, column slicing]
#data.iloc[row, col]
data.iloc[0:3, 1:4]
# Select all columns for rows of index values 0 and 10
data.loc[[0, 10], ["id","diagnosis"]]
# Select all columns for rows of index values 0 and 10
data.loc[[0, 10], "id"]
# Select all columns for rows of index values 0 and 10
data.loc[[0,6], ["id", "compactness_mean", "area_mean"]]
data[data.diagnosis == "B"]
data[data.area_mean < 200]
data[(data.diagnosis == "M") & (data.area_mean > 1500)]
import numpy as np

data.loc[0, ["diagnosis"]] = np.NaN
data.loc[1, ["diagnosis"]] = np.NaN

data.head()
# To select just the rows with NaN values, we can use the 'any()' method
data[pd.isnull(data).any(axis=1)]
# What does this do?
empty_diagnosis = data[pd.isnull(data).any(axis=1)]['diagnosis']
empty_diagnosis
import seaborn as sns

data_extracted = data["perimeter_mean"]
perimeter_mean = sns.distplot(data_extracted, hist=True, kde=True, 
                bins=20, color = 'darkblue', 
                hist_kws={'edgecolor':'black'},
                kde_kws={'linewidth': 4})
import seaborn as sns

data_extracted = data[(data.diagnosis == "M")]["perimeter_mean"]
perimeter_mean = sns.distplot(data_extracted, hist=True, kde=True, 
                bins=20, color = 'darkblue', 
                hist_kws={'edgecolor':'black'},
                kde_kws={'linewidth': 4})
import seaborn as sns

data_extracted = data[(data.diagnosis == "M") & (data.area_mean > 1500)]["perimeter_mean"]
perimeter_mean = sns.distplot(data_extracted, hist=True, kde=True, 
                bins=20, color = 'darkblue', 
                hist_kws={'edgecolor':'black'},
                kde_kws={'linewidth': 4})
data_extracted = data[["smoothness_worst", "concavity_mean", "symmetry_worst"]]
row, col = data_extracted.shape

Density = data_extracted.plot(kind='density', subplots=True, layout=(col,1), sharex=False, figsize=(7,7))
Box = data_extracted.plot(kind='box', subplots=True, layout=(3,2), sharex=False, sharey=False, figsize=(7,7) )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11736890990458498, 'fraction_numerical': 0.028624503926370008, 'mean_word_length': 4.34235453315291, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '617996', 'license': 'MIT', 'n_tokens_mistral': 4117, 'n_tokens_neox': 3818, 'n_words': 1250, 'path': 'Lessons/4- pandas dataframe.ipynb', 'repo_name': 'SakaiMarouane/python-bootcamp', 'size': '11843'}