<jupyter_start><jupyter_text>Exploring the Data<jupyter_code>#View first 5 instances
df = pd.read_csv('/home/chris/projects/capstone/higher_ed_summary.csv')
df.head()
#Data Information
df.info()
#Drop Duplicates
df = df.drop_duplicates(keep='first')
df.info()
df.shape
df.dropna(axis=1, how='all')
df.dropna(axis=0, how='all')
#Class Balance (Closed Schools vs. Open Schools)
np.sum(df['closed_ind'] == 1)
np.sum(df['closed_ind'] != 1)<jupyter_output><empty_output><jupyter_text>Feature EDA<jupyter_code>#Create Missing Value's Table
missfeatures_count = df.loc[:, ~df.columns.isin(['closed_ind','instname','unitid','sector'])].isnull().sum()
percentage = 100 * df.loc[:, ~df.columns.isin(['closed_ind','instname','unitid','sector'])].isnull().sum() / len(df)
missfeatures = pd.concat([missfeatures_count, percentage], axis=1)
missfeatures = missfeatures.rename(columns = {0 : 'Missing Values', 
1 : 'Percentage'})
missfeatures
sns.kdeplot(missfeatures['Missing Values'], shade=True);
#Features Missing more than 3%
missfeatures = missfeatures[missfeatures.iloc[:,1] > 3].sort_values(
'Percentage', ascending=False).round(1)
missfeatures
missfeatures['Percentage'].plot.barh()
column_drop_list = missfeatures.index.tolist()
df.drop(column_drop_list, axis=1, inplace=True)
df.shape<jupyter_output><empty_output><jupyter_text>School EDA<jupyter_code>schools_count = df.isnull().sum(axis=1)
percentages = 100 * df.isnull().sum(axis=1) / len(df.columns)
schools = pd.concat([schools_count, percentages], axis=1)
schools = schools.rename(columns = {0 : 'Missing Values', 
1 : 'Percentages'})
sns.kdeplot(schools['Missing Values'], shade=True, color="r");
#Instaces Missing more than 50%
schools = schools[schools.iloc[:,1] > 50].sort_values(
'Percentages', ascending=False).round(1)
schools
#Percentages is 1 , Missing Values is 0
df1 = pd.concat([df,schools_count, percentages], axis=1)
df2 = df1.rename({0: 'missing_values', 1: 'percentages'}, axis=1)
df2
df2.shape
#Drop instnances with 50% or more missing values
indexMissingValues = df2[df2['missing_values']>64].index
df3 = df2.drop(indexMissingValues)

df3.shape
np.sum(df3['closed_ind'] == 1)
np.sum(df3['closed_ind'] != 1)
df3.isnull().sum()
schoolsz_count = df3.isnull().sum(axis=1)
percentagesz = 100 * df3.isnull().sum(axis=1) / len(df3.columns)
schoolsz = pd.concat([schoolsz_count, percentagesz], axis=1)
schoolsz = schoolsz.rename(columns = {0 : 'Missing Values', 
1 : 'Percentages'})
schoolsz = schoolsz[schoolsz.iloc[:,1] !=0].sort_values(
'Percentages', ascending=False).round(1)
schoolsz
sns.kdeplot(schoolsz['Missing Values'], shade=True, color="r");
#Fill in NAN's
df4 = df3.fillna(df3.agg(lambda x: pd.Series.mode(x)[0], axis=0))
missfeaturesz_count = df3.loc[:, ~df3.columns.isin(['closed_ind','instname','unitid','sector'])].isnull().sum()
percentage = 100 * df3.loc[:, ~df3.columns.isin(['closed_ind','instname','unitid','sector'])].isnull().sum() / len(df3)
missfeaturesz = pd.concat([missfeaturesz_count, percentage], axis=1)
missfeaturesz = missfeaturesz.rename(columns = {0 : 'Missing Values', 
1 : 'Percentage'})
missfeaturesz
sns.kdeplot(missfeaturesz['Missing Values'], shade=True);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18093446221386014, 'fraction_numerical': 0.0266541235497021, 'mean_word_length': 7.96067415730337, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '601749', 'license': 'MIT', 'n_tokens_mistral': 1207, 'n_tokens_neox': 1153, 'n_words': 205, 'path': 'Exploratory Data Analysis/edafinal.ipynb', 'repo_name': 'georgetown-analytics/Higher-Ed-Closings', 'size': '3189'}