<jupyter_start><jupyter_text>**NOTES*** f test for most important ones* EDA on these defined features* do we have to run any MCAR trye tests? for missing data    ** .01119 percent of my data was missing    ** 751299/759807 - so i dropped the missing rows    * i also dropped the columns.... explain way<jupyter_code>#****************USE THIS ONE****************************
pd.value_counts(df['usertype'],normalize=True).plot.bar() 
plt.ylabel('Percent of Total Data Set')
plt.xlabel('User Type')

#unballanced dataset...

# violin plot

dataset = [df_preprocessed[df_preprocessed['usertype']==0.0]['tripduration'].values,
            df_preprocessed[df_preprocessed['usertype']==1.0]['tripduration'].values]
                                        
                                        
plt.violinplot(dataset = dataset)
plt.xlabel('usertype')
plt.ylabel('tripduration')
plt.show()

plt.hist(dataset,alpha=0.5, label=c,bins=15,density=True, range=bin_range)<jupyter_output><empty_output><jupyter_text>mi_select10 = SelectKBest(mutual_info_regression,k=5)X_mi10 = mi_select10.fit_transform(df,df["['usertype']"])mi_select10.get_support()mi_top10 = df_preprocessed.columns[mi_select10.get_support()==True]mi_top10<jupyter_code>#continuous features

#****************USE start time and end time****************************

categories = df_preprocessed["['usertype']"].unique()

for feat in cont_ftrs:
    bin_range = (df_preprocessed[feat].min(),df_preprocessed[feat].max())
    for c in categories:
        plt.hist(df_preprocessed[df_preprocessed["['usertype']"]==c][feat],alpha=0.5,
                 label=c,bins=15,density=True, range=bin_range)
    plt.legend()
    plt.ylabel('pdf')
    plt.xlabel(feat)
    plt.tight_layout()
    plt.show()

#categorical features

#****************USE THIS ONE****************************

count_matrix = df.groupby(["gender",'usertype']).size().unstack()
count_matrix_norm = count_matrix.div(count_matrix.sum(axis=1),axis=0)
count_matrix_norm.plot(kind = 'bar', stacked = True)
plt.ylabel('Percent Usertype')
plt.legend(loc=4)
plt.show()


    
pd.value_counts(df["tripduration"])
pd.value_counts(df_preprocessed["tripduration"])
categories = df['usertype'].unique()
bin_range = (df['tripduration'].min(),df['tripduration'].max())
for c in categories:
    plt.hist(df[df["usertype"]==c]['tripduration'],alpha=0.5,
                 label=c,bins=100,density=True, range=bin_range)
plt.legend()
plt.ylabel('pdf')
plt.xlabel('tripduration')
plt.tight_layout()
plt.show()
#**********Use this***********************

cont = ['tripduration','Start Time','End Time','birth year']

pd.plotting.scatter_matrix(df_preprocessed[cont],c = df_preprocessed["['usertype']"], figsize = (15, 15),
                        marker='o', hist_kwds={'bins': 20}, s=60, alpha=.8)
plt.show()
from sklearn.decomposition import PCA

#PCA
y = df_preprocessed["['usertype']"]
df_pca = df_preprocessed.drop(columns = "['usertype']")

pca = PCA(n_components=df_pca.shape[1])
X_pca = pca.fit_transform(df_pca)

#first 5 components
pca_2 = PCA(n_components=2)
X_pca_2 = pca_2.fit_transform(df_pca)

for i in range(5):
    print(X_pca_2[i])
plt.scatter(X_pca[:, 0], X_pca[:, 1],
            c=y, edgecolor='none', alpha=0.5,
            cmap=plt.cm.get_cmap('Spectral', 10))
plt.xlabel('component 1')
plt.ylabel('component 2')
plt.colorbar();
print(np.cumsum(pca.explained_variance_ratio_))
#*******use this one?***************
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('Nr. components')
plt.ylabel('variance explained')
plt.show()
df[['tripduration','usertype']].boxplot(by='usertype')
plt.xlabel('usertype')
plt.ylabel('tripduration')
plt.show()
#*****USE THIS ONE ******************
df[['tripduration','usertype']].boxplot(by='usertype')
plt.title('Trip Durration by User Type')
plt.ylim((0,4_000))
plt.suptitle('')
plt.xlabel('User Type')
plt.ylabel('Trip Duration')
plt.show()
df_preprocessed[['tripduration',"['usertype']"]].boxplot(by="['usertype']")
plt.title('Trip Durration by User Type')
plt.suptitle('')
plt.xlabel('User Type')
plt.ylabel('Trip Duration')
plt.show()
df_preprocessed[['tripduration',"['usertype']"]].boxplot(by="['usertype']")
plt.title('Trip Durration by User Type')
plt.ylim((-0.25,0.5))
plt.suptitle('')
plt.xlabel('User Type')
plt.ylabel('Trip Duration')
plt.show()
categories = df_preprocessed["['usertype']"].unique()

bin_range = (df_preprocessed['tripduration'].min(),1.5)
for c in categories:
    plt.hist(df_preprocessed[df_preprocessed["['usertype']"]==c]['tripduration'],alpha=0.5,
            label=c,bins=100,density=True, range=bin_range)
plt.ylabel('PDF')
plt.xlabel('Trip Durration')
plt.tight_layout()
plt.show()
#*****USE THIS ONE ******************

categories = df['usertype'].unique()

bin_range = (df['tripduration'].min(),3500)
for c in categories:
    plt.hist(df[df['usertype']==c]['tripduration'],alpha=0.5,
            label=c,bins=100,density=True, range=bin_range)
plt.title('Trip Durration PDF by Customer Type')
plt.legend()
plt.ylabel('PDF')
plt.xlabel('Trip Durration (in seconds)')
plt.tight_layout()
plt.savefig(d+'/figures/trip_durr_by_custtype_hist',dpi = 300)

#*****USE THIS ONE ****************** USE not preprocessed data
categories = df['usertype'].unique()

bin_range = (df['Start Time'].min(),df['Start Time'].max())
for c in categories:
    plt.hist(df[df['usertype']==c]['Start Time'],alpha=0.5,
            label=c,bins=100,density=True, range=bin_range)
plt.title('Trip Start Time PDF by Customer Type')
plt.legend()
plt.ylabel('PDF')
plt.xlabel('Start Time')
plt.tight_layout()
plt.savefig(d+'/figures/start_time_by_custtype_hist',dpi = 300)
plt.show()
#*****USE THIS ONE ****************** USE not preprocessed data
categories = df['usertype'].unique()

bin_range = (df['End Time'].min(),df['End Time'].max())
for c in categories:
    plt.hist(df[df['usertype']==c]['End Time'],alpha=0.5,
            label=c,bins=100,density=True, range=bin_range)
plt.title('Trip End Time PDF by Customer Type')
plt.legend()
plt.ylabel('PDF')
plt.xlabel('End Time')
plt.tight_layout()
plt.savefig(d+'/figures/end_time_by_custtype_hist',dpi = 300)
plt.show()
#*****USE THIS ONE ******************
df[['tripduration','usertype']].boxplot(by='usertype',showfliers = False)
plt.title('Trip Durration by User Type')
plt.ylim((0,4_000))
plt.suptitle('')
plt.xlabel('User Type')
plt.ylabel('Trip Duration (in seconds)')
plt.tight_layout()
plt.savefig(d+'/figures/trip_durr_by_custtype_boxplot',dpi = 300)
plt.show()

#categorical features

#****************USE THIS ONE**************************** another bar before to show the numbers?

count_matrix = df.groupby(["gender",'usertype']).size().unstack()
count_matrix_norm = count_matrix.div(count_matrix.sum(axis=1),axis=0)
count_matrix_norm.plot(kind = 'bar', stacked = True)
plt.title('Percent of Usertype by Gender')
plt.ylabel('Percent Usertype')
plt.legend(loc=4)
plt.tight_layout()
plt.savefig(d+'/figures/gender_by_custtype_norm_stackbar',dpi = 300)
plt.show()


#****************USE THIS ONE****************************
pd.value_counts(df['usertype'],normalize=True).plot.bar() 
plt.title('Percent of User Types Found in Data Set')
plt.ylabel('Percent of Total')
plt.xlabel('User Type')
plt.tight_layout()
plt.savefig(d+'/figures/balanceofData_bar',dpi = 300)
#unballanced dataset...


#*******use this one?***************
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.title('Variance Explained')
plt.xlabel('Number of Components')
plt.ylabel('Variance Explained')
plt.tight_layout()
plt.savefig(d+'/figures/var_explained_line',dpi = 300)
plt.show()
#*****USE THIS ONE ****************** trip durration 
df[['tripduration','start station name']].boxplot(by='start station name',showfliers = False)
plt.title('Trip Durration by Start Station')
plt.ylim((0,2_000))
plt.xticks(np.arange(len(df['start station name'].unique()))[::1000],sorted(df['start station name'].unique())[::1000])
plt.suptitle('')
plt.xlabel('Start Station')
plt.ylabel('Trip Duration (in seconds)')
plt.tight_layout()
plt.savefig(d+'/figures/trip_durr_by_start_station_boxplot',dpi = 300)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22004170244081933, 'fraction_numerical': 0.019379369557218203, 'mean_word_length': 6.399274047186933, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '569320', 'license': 'MIT', 'n_tokens_mistral': 3166, 'n_tokens_neox': 2820, 'n_words': 489, 'path': 'src/.ipynb_checkpoints/EDA-checkpoint.ipynb', 'repo_name': 'Michael-Harder/NYC_Citi_Bike_-Analysis', 'size': '8153'}