<jupyter_start><jupyter_text>🐼 Pandas Example - Stack Overflow Survey<jupyter_code>import pandas as pd
pd
# loading in the stack overflow survey data
survey = pd.read_csv('developer_survey_2020/survey_results_public.csv', index_col='Respondent')
survey.head(20)
# loading a particular row using the index
survey.loc[1]
# loading survey results 
schema = pd.read_csv('developer_survey_2020/survey_results_schema.csv', index_col='Column')
schema.sort_index(inplace=True)
schema
# We can now search for columns directly
schema.loc['Age']
schema.loc['Hobbyist']
schema.loc['MiscTechDesireNextYear', 'QuestionText']
schema.loc['ConvertedComp']<jupyter_output><empty_output><jupyter_text>Filtering Data<jupyter_code>salary_mask = (survey['ConvertedComp'] > 70000)
salary_mask
survey.loc[salary_mask]
# grabbing only a few columns for this data
survey.loc[salary_mask, ['Country', 'LanguageWorkedWith', 'ConvertedComp']]
# filtering by countries
countries = ('United States', 'India', 'United Kingdom', 'Germany', 'Canada')
countries
country_filter = survey['Country'].isin(countries)
country_filter
# respondents who belong to these countries and also have high salaries
survey.loc[country_filter & salary_mask, ['Country', 'LanguageWorkedWith', 'ConvertedComp']]<jupyter_output><empty_output><jupyter_text>People That answered they knew Python<jupyter_code>survey['LanguageWorkedWith']
python_filter = survey['LanguageWorkedWith'].str.contains('Python', na=False)
python_filter
survey.loc[python_filter, ['LanguageWorkedWith']]
# Renaming the ConvertedComp Column
survey.rename(columns={'ConvertedComp': 'SalaryUSD'}, inplace=True)
survey.columns
survey['SalaryUSD']
survey['Hobbyist']
# mapping values to boolean
survey['Hobbyist'].map({
    'Yes': True,
    'No': False
})
survey['Hobbyist'] = survey['Hobbyist'].map({
    'Yes': True,
    'No': False
})
survey<jupyter_output><empty_output><jupyter_text>Sorting Data<jupyter_code># sorting by Counry name
survey.sort_values(by='Country', inplace=True)
survey
survey['Country'].head(50)
survey['Country'].tail(20)
survey[['Country', 'SalaryUSD']]
survey.sort_values(by=['Country', 'SalaryUSD'], ascending=[True, False], inplace=True)
survey[['Country', 'SalaryUSD']]
# 10 largest salaries
survey['SalaryUSD'].nlargest(10)
survey.nlargest(10, 'SalaryUSD')
survey.nsmallest(10, 'SalaryUSD')<jupyter_output><empty_output><jupyter_text>Grouping and Data Aggregation<jupyter_code>survey['SalaryUSD'].head(30)
# computing the median salary
survey['SalaryUSD'].median()
# median of all columns
survey.median()
survey.mode()
# Broad overview of data
survey.describe()
# number of rows with no NaN data
survey['SalaryUSD'].count()
survey.count()
survey.shape
survey['Hobbyist']
survey['Hobbyist'].value_counts()
survey['Gender'].value_counts()
survey['UndergradMajor'].value_counts()
survey['UndergradMajor'].value_counts(normalize=True)
survey['Country'].value_counts()
country_group = survey.groupby(['Country'])
country_group
country_group.get_group('United States')
country_group.get_group('Switzerland')
country_group.get_group('Switzerland').value_counts('Gender')
country_group.get_group('India').value_counts('Gender')
country_group.get_group('United States').value_counts('WebframeWorkedWith')
country_group.get_group('India').value_counts('WebframeWorkedWith')
country_group.get_group('Germany').value_counts('WebframeWorkedWith')
country_group.get_group('France').value_counts('WebframeWorkedWith')
# group functions
country_group['WebframeWorkedWith'].value_counts()
country_group['WebframeWorkedWith'].value_counts(normalize=True)['India'] * 100
# Having a look at salary
country_group['SalaryUSD'].median()
country_group['SalaryUSD'].median()['Germany']
# Having a look at salary
country_group['SalaryUSD'].median()['India']
# Having a look at salary
country_group['SalaryUSD'].median()['United States']
# Having a look at salary
country_group['SalaryUSD'].median()['Switzerland']<jupyter_output><empty_output><jupyter_text>Applying Multiple Aggregate Functions<jupyter_code>country_group['SalaryUSD'].agg(['median', 'mean'])
country_group['SalaryUSD'].agg(['median', 'mean']).loc['Canada']<jupyter_output><empty_output><jupyter_text>How many people in a country know Python?<jupyter_code>python_filter = survey['LanguageWorkedWith'].str.contains('Python', na=False)
python_filter
# number of people who claim to know python
python_filter.sum()
country_group.get_group('India')[python_filter]['LanguageWorkedWith'].count()
# the follwoing will give an error
# country_group['LanguageWorkedWith'].str.contains('Python')
country_group['LanguageWorkedWith'].apply(lambda x: x.str.contains('Python').sum())
# total number of python user
total_python_users = country_group['LanguageWorkedWith'].apply(lambda x: x.str.contains('Python').sum()).sum()
total_python_users
100 * country_group['LanguageWorkedWith'].apply(lambda x: x.str.contains('Python').sum()) / total_python_users
# Percentage of people in each country who know python
country_population = survey['Country'].value_counts()
country_population
python_users = country_group['LanguageWorkedWith'].apply(lambda x: x.str.contains('Python').sum())
python_users
python_users_per_country = 100 * python_users / country_population
python_users_per_country
python_users_per_country['India']
python_users_per_country['United States']
python_users_per_country['Germany']
python_users_per_country['Switzerland']
python_users_per_country.nlargest(10)
country_population['Gabon']<jupyter_output><empty_output><jupyter_text>Creating a New Table Using Concatenation<jupyter_code>python = pd.concat([country_population, python_users], axis='columns', sort=False)
python
python.columns = ['population', 'python_users']
python
python['percentage_python'] = 100 * python['python_users'] / python['population']
python
# sorting on basis of python knowledge
python.sort_values(by='percentage_python', ascending=False)
python.loc['Japan']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13988245172124264, 'fraction_numerical': 0.007388748950461797, 'mean_word_length': 10.431861804222649, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '113140', 'license': 'MIT', 'n_tokens_mistral': 2014, 'n_tokens_neox': 1888, 'n_words': 348, 'path': 'notebooks/pandas/pandas-stack-overflow.ipynb', 'repo_name': 'Ishaan-99-cyber/ml-workshop-wac-1', 'size': '5955'}