<jupyter_start><jupyter_text>Simple Data Exploration of Top 500 Indian Cities >* The data is from census 2011 of Indian Cities with Population more than 1 Lac and City wise number of Graduates from the Census 2011. Link to dataset [Top 500 Indian Cities](https://www.kaggle.com/zed9941/top-500-indian-cities)<jupyter_code>%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

pd.set_option('display.max_rows', 16)
pd.set_option('display.max_columns',100)
plt.style.use('ggplot')<jupyter_output><empty_output><jupyter_text>Load the data<jupyter_code>cities = pd.read_csv('cities_r2.csv')
cities<jupyter_output><empty_output><jupyter_text>No missing values<jupyter_code>cities.isnull().sum()<jupyter_output><empty_output><jupyter_text>States by total population<jupyter_code>state_by_pop = cities.groupby('state_name').agg({'population_total': np.sum, 
                                                 'population_male': np.sum, 'population_female': np.sum})
state_by_pop.sort_values([('population_total')], ascending=False).head()<jupyter_output><empty_output><jupyter_text>Total male vs female literacy<jupyter_code>male_lit = sum(cities.literates_male) / 1000000
female_lit = sum(cities.literates_female) / 1000000
lit_df = pd.DataFrame({'Gender':['male', 'female'], 'Population': [male_lit, female_lit]})
plt.subplots(figsize=(7, 5))
ax = sns.barplot(data=lit_df, x='Gender', y='Population')
ax.set_title('Literate Population vs Gender', size=20, color='blue', alpha=0.5)
ax.set_xlabel('Gender', color='red', size=20, alpha=0.5)
ax.set_ylabel('Literate Population (in million)', color='red', size=20, alpha=0.5)<jupyter_output><empty_output><jupyter_text>Top 10 states by literacy<jupyter_code>lit_by_states  = cities.groupby('state_name').agg({'literates_total': np.sum}).sort_values([('literates_total')], 
                                                                                           ascending=False)[:10]
lit_by_states = lit_by_states / 1000000
plt.subplots(figsize=(20, 8))
ax = sns.barplot(data=lit_by_states, x=lit_by_states.index, y='literates_total')

ax.set_title('Top 10 states with most literates', size=35, alpha=0.5, color='green')
ax.set_xlabel('States', size=30, alpha=0.5, color='red')
ax.set_ylabel('Number of literates (millions)', size=30, alpha=0.5, color='red')<jupyter_output><empty_output><jupyter_text>States according to literacy rate>* Literacy rate in eastern states is higher as compared to rest of the India>* 5 out of 7 states in eastern India are in top 10>* Most of the northern states have low literacy rate<jupyter_code>lit_by_states  = cities.groupby('state_name').agg({'literates_total': np.sum})
pop_by_states  = cities.groupby('state_name').agg({'population_total': np.sum})
literate_rate = lit_by_states.literates_total * 100 / pop_by_states.population_total
literate_rate = literate_rate.sort_values(ascending=False)

plt.subplots(figsize=(10, 8))
ax = sns.barplot(x=literate_rate, y=literate_rate.index)
ax.set_title('States according to literacy rate', size=30, alpha=0.5, color='green')
ax.set_xlabel('Literacy Rate(as % of population)', size=25, alpha=0.5, color='red')
ax.set_ylabel('States', size=30, alpha=0.5, color='red')<jupyter_output><empty_output><jupyter_text>States by female literacy rate<jupyter_code>def proportion(group, col1, col2):
    col = group[col1].sum()
    tot_pop = group[col2].sum()
    return (col * 100 / tot_pop)

prop_female_lit = cities.groupby('state_name').apply(proportion, 'literates_female', 'population_female')
prop_male_lit = cities.groupby('state_name').apply(proportion, 'literates_male', 'population_male')

summary = pd.DataFrame({'literates_female': prop_female_lit, 'literates_male':prop_male_lit})
fem_summary = summary.sort_values([('literates_female')], ascending=False)
plt.subplots(figsize=(10, 8))
ax = sns.barplot(x='literates_female', y=fem_summary.index, data=fem_summary)
ax.set_title('States by female literacy rate', size=35, alpha=0.5, color='green')
ax.set_xlabel('Female literacy Rate', size=30, alpha=0.5, color='red')
ax.set_ylabel('States', size=30, alpha=0.5, color='red')<jupyter_output><empty_output><jupyter_text>Is there any relation between sex ratio and female literacy?<jupyter_code>prop_female_lit = cities.groupby('state_name').apply(proportion, 'literates_female', 'population_female')
prop_female_lit = prop_female_lit * 10
sex_ratio1 = cities.groupby('state_name').agg({'sex_ratio':np.mean})
prop_female_lit = pd.DataFrame({'female_lit':prop_female_lit})
df = pd.concat([prop_female_lit, sex_ratio1], axis='columns')
plt.subplots(figsize=(8, 6))
ax = sns.regplot(x='sex_ratio', y='female_lit', data=df, order=3, ci=50, 
                 scatter_kws={'alpha':0.5, 'color':'red'}, 
                 line_kws={'alpha':0.5,'color':'green'})
ax.set_title('Female literacy vs sex ratio', size=30, alpha=0.5, color='green')
ax.set_xlabel('Sex Ratio', size=25, alpha=0.5, color='red')
ax.set_ylabel('Female literacy(per thousand)', size=25, alpha=0.5, color='red')<jupyter_output><empty_output><jupyter_text>States according to number of cities<jupyter_code>most_cities_in_states = cities.state_name.value_counts().sort_values(ascending=False)

plt.subplots(figsize=(10, 8))
ax = sns.barplot(x=most_cities_in_states, y=most_cities_in_states.index)
ax.set_title('States with most top 500 cities', size=20, alpha=0.5, color='green')
ax.set_xlabel('Number of cities in state', size=25, alpha=0.5, color='red')
ax.set_ylabel('States', size=25, alpha=0.5, color='red')<jupyter_output><empty_output><jupyter_text>Top 10 states with highest population<jupyter_code># Top 10 states with highest population
state_by_pop = cities.groupby('state_name').agg({'population_total': np.sum, 
                                                 'population_male': np.sum, 
                                                 'population_female': np.sum}).sort_values([('population_total')], 
                                                                                           ascending=False)[:10]
state_by_pop = state_by_pop / 1000000
plt.subplots(figsize=(20, 8))
ax = sns.barplot(data=state_by_pop, x=state_by_pop.index, y='population_total')
ax.set_title('Top 10 states with highest population', size=30, alpha=0.5, color='green')
ax.set_xlabel('States', size=30, alpha=0.5, color='red')
ax.set_ylabel('Population (millions)', size=30, alpha=0.5, color='red')
cities.pivot_table(index=['state_name'], values=['literates_female', 'female_graduates'], aggfunc=[np.sum])
states_by_sexratio = cities.pivot_table(index=['state_name'], values=['literates_female', 'sex_ratio', 'child_sex_ratio'], 
                   aggfunc={'literates_female':np.sum, 'sex_ratio':np.mean, 'child_sex_ratio':np.mean})
states_by_sexratio = states_by_sexratio.sort_values(['sex_ratio', 'literates_female'], ascending=False)
states_by_sexratio<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14574606384515384, 'fraction_numerical': 0.02903365592951033, 'mean_word_length': 5.728862973760933, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 49, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '571396', 'license': 'MIT', 'n_tokens_mistral': 2562, 'n_tokens_neox': 2327, 'n_words': 429, 'path': 'top500indiancities.ipynb', 'repo_name': 'hpanwar08/data-analysis-indian-cities', 'size': '6923'}