<jupyter_start><jupyter_text>Univariate Analysis  Target Variable<jupyter_code>train['is_pass'].value_counts(normalize=True)
# Around 70% trainee have passed the test<jupyter_output><empty_output><jupyter_text>Categorical Independent Variables<jupyter_code>plt.figure(1)
plt.subplot(221)
train['program_id'].value_counts(normalize=True).plot.bar(figsize=(20,10), title= 'program_id')

plt.subplot(222)
train['program_type'].value_counts(normalize=True).plot.bar(title= 'program_type')

plt.subplot(223)
train['test_type'].value_counts(normalize=True).plot.bar(title= 'test_type')

plt.subplot(224)
train['difficulty_level'].value_counts(normalize=True).plot.bar(title= 'difficulty_level')

plt.show()
# Most of the trainee are enrolled for Y and T program_type.
# More number of trainee enrolment for offline test than online test.
# Most of the test are easy in terms of difficulty level.
plt.figure(1)
plt.subplot(221)
train['gender'].value_counts(normalize=True).plot.bar(figsize=(20,10), title= 'gender')

plt.subplot(222)
train['education'].value_counts(normalize=True).plot.bar(title= 'education')

plt.subplot(223)
train['city_tier'].value_counts(normalize=True).plot.bar(title= 'city_tier')

plt.subplot(224)
train['is_handicapped'].value_counts(normalize=True).plot.bar(title= 'is_handicapped')

plt.show()
# More male trainee as compared to female trainee
# Most of the trainee have diploma
# Most of the trainee belongs to tier 3 city
# 10% of the trainee are handicapped<jupyter_output><empty_output><jupyter_text>Numerical Independent Variables<jupyter_code>df=train.dropna()
sns.distplot(df['age']);
# Most of the trainee are in the age range of 20-30 and 40-50<jupyter_output><empty_output><jupyter_text>Bivariate Analysis<jupyter_code># Correlation between numerical variables
matrix = train.corr()
f, ax = plt.subplots(figsize=(9, 6))
sns.heatmap(matrix, vmax=.8, square=True, cmap="BuPu");
# Not much correlation between the variables
# program_id vs is_pass
plt.figure(figsize=(12,4))
sns.barplot(train['program_id'], train['is_pass'])
# program_type vs is_pass
sns.barplot(train['program_type'], train['is_pass'])
# Trainee in X and Y program type have higher chances to pass the test
# test_type vs is_pass
sns.barplot(train['test_type'], train['is_pass'])
# Trainee attending online mode of test have higher chances to pass the test
# difficulty_level vs is_pass
sns.barplot(train['difficulty_level'], train['is_pass'])
# If the difficulty level of the test is easy, chances to pass the test are higher
# Gender vs is_pass
sns.barplot(train['gender'], train['is_pass'])
# Gender does not affect the chances to pass the test
# education vs is_pass
plt.figure(figsize=(12,4))
sns.barplot(train['education'], train['is_pass'])
# Trainee with Masters education level have more chances to pass the test
# is_handicapped vs is_pass
sns.barplot(train['is_handicapped'], train['is_pass'])
# Handicapped trainee have less chances to pass the test
# city_tier vs is_pass
sns.barplot(train['city_tier'], train['is_pass'])
# Trainee from city tier 1 have higher chances to pass the test
# trainee_engagement_rating vs is_pass
sns.barplot(train['trainee_engagement_rating'], train['is_pass'])
# As the trainee engagement rating increases, chances to pass the test also increases<jupyter_output><empty_output><jupyter_text>Missing Values Treatment<jupyter_code># Check the number of missing values in each variable
train.isnull().sum()
# age and trainee_engagement_rating variables have missing values in it.
# Imputing missing value in age variable with median and in trainee_engagement_rating variable with mode of that variable.
train['age'].fillna(train['age'].median(), inplace=True)
train['trainee_engagement_rating'].fillna(train['trainee_engagement_rating'].mode()[0], inplace=True)
test['age'].fillna(train['age'].median(), inplace=True)
test['trainee_engagement_rating'].fillna(train['trainee_engagement_rating'].mode()[0], inplace=True)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code># Dropping id variables
train=train.drop(['id', 'trainee_id', 'test_id'], axis=1)
test=test.drop(['id', 'trainee_id', 'test_id'], axis=1)
# Save target variable in separate dataset
X = train.drop('is_pass',1)
y = train.is_pass
# Apply dummies to the dataset
X=pd.get_dummies(X)
train=pd.get_dummies(train)
test=pd.get_dummies(test)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
# Logistic regression using 5 fold stratified cross validation
i=1
kf = StratifiedKFold(n_splits=5,random_state=1,shuffle=True)
for train_index,test_index in kf.split(X,y):
     print('\n{} of kfold {}'.format(i,kf.n_splits))
     xtr,xvl = X.loc[train_index],X.loc[test_index]
     ytr,yvl = y[train_index],y[test_index]
    
     model = LogisticRegression(random_state=1)
     model.fit(xtr, ytr)
     pred=model.predict_proba(xvl)[:,1]
     score = roc_auc_score(yvl,pred)
     print('roc_auc_score',score)
     i+=1
# Making predictions for the test data
pred=model.predict_proba(test)[:,1]
# Read the submission file
submission=pd.read_csv("sample_submission_vaSxamm.csv")
submission.head()
# Fill the is_pass variable with the predictions
submission['is_pass']=pred
# Converting the submission file to csv format
submission.to_csv('logistic_submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12313088425327672, 'fraction_numerical': 0.012922281705741185, 'mean_word_length': 7.518867924528302, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '329340', 'license': 'MIT', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1806, 'n_words': 437, 'path': 'Data Identity.ipynb', 'repo_name': 'twisstosin/DataIdentity', 'size': '5417'}