<jupyter_start><jupyter_text>status1 =<jupyter_code>#concatenating two series or column and creating a new colummn to the dataframe.

uforeport['Location']  =  uforeport['City'] + ',' + uforeport['State']
uforeport.head()
#Why do some pandas commands end with parentheses () for methods no parantheses for atttributes
import pandas as pd
imdb = pd.read_csv('http://bit.ly/imdbratings')

imdb.head()
imdb.describe()
imdb.shape
imdb.dtypes
#additioonal parms with describe
#getting top available values , count & its cardinality value

imdb.describe(include=['object'])
import pandas as pd
ufo = pd.read_csv('http://bit.ly/uforeports')

ufo.head()
#getting column name using the attribute
ufo.columns
#renaming the columns
ufo.rename(columns = {'Colors Reported':'Colors_Reported' ,'Shape Reported':'Shape_Reported'} , inplace=True)
ufo.columns
#removing all column and adding new column name
ufo_cols = ['City', 'Colors Reported', 'Shape Reported', 'State', 'Time']
import pandas as pd
ufo = pd.read_csv('http://bit.ly/uforeports', names=ufo_cols , header=0)
ufo.head()
ufo.columns
ufo.columns = ufo.columns.str.replace(' ','_')
ufo.columns
#how to remove columns using pandas
import pandas as pd
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.head()
#To remove single column
ufo.drop('Colors Reported', axis=1, inplace=True)

ufo.head()
#To remove multiple column - we need to pass the names as list
ufo.drop(['City','State'], axis=1, inplace=True)
ufo.head()
#to remove the rows
ufo.drop([0,1], axis=0, inplace=True)
ufo.head()
ufo.shape
#How to sort in pandaas
import pandas as pd
imdb = pd.read_csv('http://bit.ly/imdbratings')
imdb.head()
imdbsort  = imdb['title'].sort_values()   
imdbsort.head()
#sort and get the series or column.
imdbsort  = imdb['title'].sort_values(ascending=False)   
imdbsort.head()
imdb.head()
#sort and get the complete dataframe.
imdb.sort_values('title')
#sorting by multiple columns
imdb.sort_values(['title', 'duration'])
#how to filter row using pandas with  column value
import pandas as pd
imdb = pd.read_csv('http://bit.ly/imdbratings')
imdb.head()
imdb.shape
booleans = []
for length in imdb.duration:
    if length >= 200:
        booleans.append(True)
    else:
        booleans.append(False)
booleans[:5]
len(booleans)
is_long = pd.Series(booleans)
is_long.head()
imdb[is_long]
#second way
Is_long = imdb.duration >=200
Is_long.head()
#method 3 to extract row based on column value
#imdb[imdb['duration'] >= 200].genre
imdb[imdb['duration'] >= 200]
#another method to extract rows based on column values using loc method
imdb.loc[imdb['duration'] >= 200, 'genre']
#multiple filter criteria to a dataframe
import pandas as pd
multifilter = pd.read_csv('http://bit.ly/imdbratings')
multifilter.head()
multifilter[multifilter.duration >= 200]
multifilter[(multifilter.duration >= 200) &  (multifilter.genre == 'Drama')]
multifilter[(multifilter.duration >= 200) |  (multifilter.genre == 'Drama')]
(multifilter.duration >= 200) &  (multifilter.genre == 'Drama')
#filtering row based onn multiple values of a column
multifilter[multifilter.genre.isin(['Adventure','Crime','Drama'])]
#extracting only two columns while loading the dataframe

import pandas as pd
imdb = pd.read_csv('http://bit.ly/uforeports', usecols=[0,4])
#imdb = pd.read_csv('http://bit.ly/uforeports', usecols=['City','Time'])
imdb.head()
#selecting first 4 rows
import pandas as pd
imdb = pd.read_csv('http://bit.ly/uforeports', nrows=4)
#imdb = pd.read_csv('http://bit.ly/uforeports', usecols=['City','Time'])
imdb.head()
#getting unique values of a column
import pandas as pd
imdb = pd.read_csv('http://bit.ly/uforeports')
#imdb = pd.read_csv('http://bit.ly/uforeports', usecols=['City','Time'])

for index, row in imdb.iterrows():
    print(index, row.City, row.State)
#dropping non numeric column
import pandas as pd
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.dtypes
import numpy as np
drinks.select_dtypes(include=[np.number]).dtypes
drinks.describe(include='all')
drinks.describe(include=['object','float64'])
import pandas as pd
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()
drinks.drop('continent', axis=1).head()
drinks.drop(2, axis=0).head()
#mean of the numbers
drinks.mean()
drinks.mean(axis=0)  #top to bottom
drinks.mean(axis='index').shape #both will give same output

drinks.mean(axis=1)  #left to right
drinks.mean(axis='columns')
#using str methods in pandas
orders = pd.read_table('http://bit.ly/chiporders')
orders.head()
'hello'.upper()
# string methods for pandas Series are accessed via 'str'
orders.item_name.str.upper().head()
# string methods for pandas Series are accessed via 'str'
orders[orders.item_name.str.contains('Chicken')]
# string methods can be chained together
orders.choice_description.str.replace('[', '').str.replace(']', '').head()

# many pandas string methods support regular expressions (regex)
orders.choice_description.str.replace('[\[\]]', '').head()

#How do I change the data type of a pandas Series
import pandas as pd

# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()
# examine the data type of each Series
drinks.dtypes
# change the data type of an existing Series
drinks['beer_servings'] = drinks.beer_servings.astype(float)
drinks.dtypes
# alternatively, change the data type of a Series while reading in a file
drinks = pd.read_csv('http://bit.ly/drinksbycountry', dtype={'beer_servings':float})
drinks.dtypes
# read a dataset of Chipotle orders into a DataFrame
orders = pd.read_table('http://bit.ly/chiporders')
orders.head()
# examine the data type of each Series
orders.dtypes
# convert a string to a number in order to do math
variable = orders.item_price.str.replace('$', '').astype(float).mean()
# convert a boolean Series to an integer (False = 0, True = 1)
orders.item_name.str.contains('Chicken').astype(int).head()
#group by usage
import pandas as pd
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()
#mean of beer servings
drinks.beer_servings.mean()
#group by continent and find the avg beer servings
drinks.groupby('continent').beer_servings.mean()
drinks[drinks.continent =='Africa'].beer_servings.mean()
drinks.groupby('continent').beer_servings.max()
drinks.groupby('continent').beer_servings.min()
# multiple aggregation functions can be applied simultaneously
drinks.groupby('continent').beer_servings.agg(['count', 'mean', 'min', 'max'])

# specifying a column to which the aggregation function should be applied is not required
drinks.groupby('continent').mean()
%matplotlib inline
# side-by-side bar plot of the DataFrame directly above
drinks.groupby('continent').mean().plot(kind='bar')




# read a dataset of top-rated IMDb movies into a DataFrame
movies = pd.read_csv('http://bit.ly/imdbratings')
movies.head()

# examine the data type of each Series
movies.dtypes
# count the non-null values, unique values, and frequency of the most common value
movies.genre.describe()

movies.shape
# count how many times each value in the Series occurs
movies.genre.value_counts()
# display percentages instead of raw counts
movies.genre.value_counts(normalize=True)
# 'value_counts' (like many pandas methods) outputs a Series
type(movies.genre.value_counts())
# thus, you can add another Series method on the end
movies.genre.value_counts().head()

# display the unique values in the Series
movies.genre.unique()
# count the number of unique values in the Series
movies.genre.nunique()

pd.head()




# compute a cross-tabulation of two Series
pd.crosstab(movies.genre, movies.content_rating)

# calculate various summary statistics
movies.duration.describe()
# many statistics are implemented as Series methods
movies.duration.mean()
# 'value_counts' is primarily useful for categorical data, not numerical data
movies.duration.value_counts().head()
# allow plots to appear in the notebook
%matplotlib inline
# histogram of the 'duration' Series (shows the distribution of a numerical variable)
movies.duration.plot(kind='hist')
# bar plot of the 'value_counts' for the 'genre' Series
movies.genre.value_counts().plot(kind='bar')
# read a dataset of UFO reports into a DataFrame
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.tail()

# 'isnull' returns a DataFrame of booleans (True if missing, False if not missing)
ufo.isnull().tail()
# 'nonnull' returns the opposite of 'isnull' (True if not missing, False if missing)
ufo.notnull().tail()
# count the number of missing values in each Series
ufo.isnull().sum()
# use the 'isnull' Series method to filter the DataFrame rows
ufo[ufo.City.isnull()].head()
# examine the number of rows and columns
ufo.shape




# if 'any' values are missing in a row, then drop that row
ufo.dropna(how='any').shape

# 'inplace' parameter for 'dropna' is False by default, thus rows were only dropped temporarily
ufo.shape
# if 'all' values are missing in a row, then drop that row (none are dropped in this case)
ufo.dropna(how='all').shape
# if 'any' values are missing in a row (considering only 'City' and 'Shape Reported'), then drop that row
ufo.dropna(subset=['City', 'Shape Reported'], how='any').shape
# if 'all' values are missing in a row (considering only 'City' and 'Shape Reported'), then drop that row
ufo.dropna(subset=['City', 'Shape Reported'], how='all').shape
# 'value_counts' does not include missing values by default
ufo['Shape Reported'].value_counts().head()
# explicitly include missing values
ufo['Shape Reported'].value_counts(dropna=False).head()
# fill in missing values with a specified value
ufo['Shape Reported'].fillna(value='VARIOUS', inplace=True)
# confirm that the missing values were filled in
ufo['Shape Reported'].value_counts().head()
ufo.isnull().sum()
# fill in missing values with a specified value
ufo['Colors Reported'].fillna(value='Orange', inplace=True)
ufo.isnull().sum()
# read a dataset of alcohol consumption into a DataFrame
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
drinks.head()
# every DataFrame has an index (sometimes called the "row labels")
drinks.index
# column names are also stored in a special "index" object
drinks.columns
# neither the index nor the columns are included in the shape
drinks.shape
# index and columns both default to integers if you don't define them
pd.read_table('http://bit.ly/movieusers', header=None, sep='|').head()
# What is the index used for?
# 1.identification
# 2.selection
# 3.alignment (covered in the next video)
# identification: index remains with each row when filtering the DataFrame
drinks[drinks.continent=='South America']

# selection: select a portion of the DataFrame using the index
drinks.loc[23, 'beer_servings']
# set an existing column as the index
drinks.set_index('country', inplace=True)
drinks.head()
# 'country' is now the index
drinks.index
# 'country' is no longer a column
drinks.columns
# 'country' data is no longer part of the DataFrame contents
drinks.shape
# country name can now be used for selection
drinks.loc['Brazil', 'beer_servings']
# index name is optional
drinks.index.name = None
drinks.head()
# restore the index name, and move the index back to a column
drinks.index.name = 'country'
drinks.reset_index(inplace=True)
drinks.head()
# many DataFrame methods output a DataFrame
drinks.describe()
# you can interact with any DataFrame using its index and columns
drinks.describe().loc['25%', 'beer_servings']
import pandas as pd
drinks = pd.read_csv('http://bit.ly/drinksbycountry')
# every DataFrame has an index
drinks.index
# every Series also has an index (which carries over from the DataFrame)
drinks.continent.head()
# set 'country' as the index
drinks.set_index('country', inplace=True)
# Series index is on the left, values are on the right
drinks.continent.head()
# another example of a Series (output from the 'value_counts' method)
drinks.continent.value_counts()
# access the Series index
drinks.continent.value_counts().index
# access the Series values
drinks.continent.value_counts().values
# elements in a Series can be selected by index (using bracket notation)
drinks.continent.value_counts()['Africa']
# any Series can be sorted by its values
drinks.continent.value_counts().sort_values()
# any Series can also be sorted by its index
drinks.continent.value_counts().sort_index()

# 'beer_servings' Series contains the average annual beer servings per person
# alignment
drinks.beer_servings.head()
# create a Series containing the population of two countries
people = pd.Series([3000000, 85000], index=['Albania', 'Andorra'], name='population')
people
# calculate the total annual beer servings for each country
(drinks.beer_servings * people).head()
# •The two Series were aligned by their indexes.
# •If a value is missing in either Series, the result is marked as NaN.
# •Alignment enables us to easily work with incomplete data.
# concatenate the 'drinks' DataFrame with the 'population' Series (aligns by the index)
pd.concat([drinks, people], axis=1).head()
# read a dataset of UFO reports into a DataFrame
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.head(3)

# The loc method is used to select rows and columns by label. You can pass it:
# •A single label
# •A list of labels
# •A slice of labels
# •A boolean Series
# •A colon (which indicates "all labels")
# row 0, all columns
ufo.loc[0, :]

# rows 0 and 1 and 2, all columns
ufo.loc[[0, 1, 2], :]
# rows 0 through 2 (inclusive), all columns
ufo.loc[0:2, :]
# this implies "all columns", but explicitly stating "all columns" is better
ufo.loc[0:2]
# rows 0 through 2 (inclusive), column 'City'
ufo.loc[0:2, 'City']

# rows 0 through 2 (inclusive), columns 'City' and 'State'
ufo.loc[0:2, ['City', 'State']]
# accomplish the same thing using double brackets - but using 'loc' is preferred since it's more explicit
ufo[['City', 'State']].head(3)
# rows 0 through 2 (inclusive), columns 'City' through 'State' (inclusive)
ufo.loc[0:2, 'City':'State']
# accomplish the same thing using 'head' and 'drop'
ufo.head(3).drop('Time', axis=1)
# rows in which the 'City' is 'Oakland', column 'State'
ufo.loc[ufo.City=='Oakland', 'State']
# accomplish the same thing using "chained indexing" - but using 'loc' is preferred since chained indexing can cause problems
ufo[ufo.City=='Oakland'].State
# The iloc method is used to select rows and columns by integer position. You can pass it:
# •A single integer position
# •A list of integer positions
# •A slice of integer positions
# •A colon (which indicates "all integer positions")
# rows in positions 0 and 1, columns in positions 0 and 3
ufo.iloc[[0, 1], [0, 3]]
# rows in positions 0 through 2 (exclusive), columns in positions 0 through 4 (exclusive)
ufo.iloc[0:2, 0:4]

# rows in positions 0 through 2 (exclusive), all columns
ufo.iloc[0:2, :]
# accomplish the same thing - but using 'iloc' is preferred since it's more explicit
ufo[0:2]
#The ix method is used to select rows and columns by label or integer position, 
#and should only be used when you need to mix label-based and integer-based selection in the same call.
# read a dataset of alcohol consumption into a DataFrame and set 'country' as the index
drinks = pd.read_csv('http://bit.ly/drinksbycountry', index_col='country')
drinks.head()
# row with label 'Albania', column in position 0
drinks.ix['Albania', 0]
# row in position 1, column with label 'beer_servings'
drinks.ix[1, 'beer_servings']
#Rules for using numbers with ix:
#•If the index is strings, numbers are treated as integer positions, and thus slices are exclusive on the right.
#•If the index is integers, numbers are treated as labels, and thus slices are inclusive.
# rows 'Albania' through 'Andorra' (inclusive), columns in positions 0 through 2 (exclusive)
drinks.ix['Albania':'Andorra', 0:2]

# rows 0 through 2 (inclusive), columns in positions 0 through 2 (exclusive)
ufo.ix[0:2, 0:2]
#When should I use the "inplace" parameter in pandas?
# read a dataset of UFO reports into a DataFrame
ufo = pd.read_csv('http://bit.ly/uforeports')
ufo.head()
ufo.shape
# remove the 'City' column (doesn't affect the DataFrame since inplace=False)
ufo.drop('City', axis=1).head()
# confirm that the 'City' column was not actually removed
ufo.head()
# remove the 'City' column (does affect the DataFrame since inplace=True)
ufo.drop('City', axis=1, inplace=True)
# drop a row if any value is missing from that row (doesn't affect the DataFrame since inplace=False)
ufo.dropna(how='any').shape
# confirm that no rows were actually removed
ufo.shape
# use an assignment statement instead of the 'inplace' parameter
ufo = ufo.set_index('Time')
ufo.tail()
# fill missing values using "backward fill" strategy (doesn't affect the DataFrame since inplace=False)
ufo.fillna(method='bfill').tail()
# compare with "forward fill" strategy (doesn't affect the DataFrame since inplace=False)
ufo.fillna(method='ffill').tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1221432800473653, 'fraction_numerical': 0.00822972172883363, 'mean_word_length': 5.402956785443518, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '531429', 'license': 'MIT', 'n_tokens_mistral': 5543, 'n_tokens_neox': 5362, 'n_words': 1893, 'path': 'Data_school_pandas.ipynb', 'repo_name': 'aravindangit/Machine-learning', 'size': '16890'}