<jupyter_start><jupyter_text>Preparing Target variable<jupyter_code>#droping all the withdrawn status as it wouldn't be useful

df = df.drop(df[df.case_status == 'Withdrawn'].index)
df['case_status'].value_counts()
#Converting the Certified-expired to certified

df.loc[df.case_status == 'Certified-Expired', 'case_status'] = 'Certified'
df['case_status'].value_counts()
#Total Number of data points of each class
df['case_status'].value_counts().plot(kind = 'bar')<jupyter_output><empty_output><jupyter_text>Checking percent of values in each column as compared to case_status<jupyter_code>i = 0;
for col in df.columns:
    i = i+1;
    print (i-1, col, 100*df[col].count()/len(df['case_status']))
#Dropping all empty columns
df = df.dropna(axis=1, how='all');

#Dropping all empty rows
df = df.dropna(axis=0, how='all');

df.shape<jupyter_output><empty_output><jupyter_text>Merging 2 attributes that may contain similar information: case_no and case_numberReason why we are doing this is because the dataset contains 2 different columns and have found instances where there is value present in one column but not in the other so we have found such rows where these instances have occured and have created a new column which has merged values from both the column.<jupyter_code>df['case_number'].head()
df['case_no'].head()<jupyter_output><empty_output><jupyter_text>Checking if any values overlap<jupyter_code>df[(df['case_number'].notnull()) & (df['case_no'].notnull())][['case_number','case_no']]<jupyter_output><empty_output><jupyter_text>Checking NA values in same indexes<jupyter_code>df[(df['case_number'].isnull()) & (df['case_no'].isnull())][['case_number','case_no']]
df['case_number'] = df['case_number'].fillna(df['case_no'])
df['case_number'].isnull().sum()
del df['case_no']
df['case_number'].head()<jupyter_output><empty_output><jupyter_text>Merging the Values of  country_of_citzenship to country_of_citizenship<jupyter_code>df.country_of_citizenship.head()
df.country_of_citzenship.head()
df.shape<jupyter_output><empty_output><jupyter_text>Checking if any values overlap<jupyter_code>df[(df['country_of_citzenship'].notnull()) & (df['country_of_citizenship'].notnull())][['country_of_citzenship','country_of_citizenship']]<jupyter_output><empty_output><jupyter_text>Checking NA values in same indexes<jupyter_code>df[(df['country_of_citzenship'].isnull()) & (df['country_of_citizenship'].isnull())][['country_of_citzenship','country_of_citizenship']].index
df = df[(df['country_of_citzenship'].notnull()) | (df['country_of_citizenship'].notnull())]
df['country_of_citizenship'] = df['country_of_citizenship'].fillna(df['country_of_citzenship'])
df['country_of_citizenship'].isnull().sum()
del df['country_of_citzenship']
df['country_of_citizenship'].head()<jupyter_output><empty_output><jupyter_text>Merging the values of naics_2007_us_code, naics_code, naics_us_code, naics_us_code_2007<jupyter_code>df.naics_2007_us_code.head()
df.naics_code.head()
df.naics_us_code.head()
df.naics_us_code_2007.head()<jupyter_output><empty_output><jupyter_text>Checking if any values overlap<jupyter_code>df[(df['naics_2007_us_code'].notnull()) & (df['naics_code'].notnull()) & (df['naics_us_code'].notnull()) & (df['naics_us_code_2007'].notnull())][['naics_2007_us_code','naics_code','naics_us_code','naics_us_code_2007']]<jupyter_output><empty_output><jupyter_text>Checking NA values in same indexes<jupyter_code>df[(df['naics_2007_us_code'].isnull()) & (df['naics_code'].isnull()) & (df['naics_us_code'].isnull()) & (df['naics_us_code_2007'].isnull())][['naics_2007_us_code','naics_code','naics_us_code','naics_us_code_2007']].shape
df = df[(df['naics_2007_us_code'].notnull()) | (df['naics_code'].notnull()) | (df['naics_us_code'].notnull()) | (df['naics_us_code_2007'].notnull())]
df['naics_code'] = df['naics_code'].fillna(df['naics_2007_us_code'])
del df['naics_2007_us_code']
df['naics_code'] = df['naics_code'].fillna(df['naics_us_code'])
del df['naics_us_code']
df['naics_code'] = df['naics_code'].fillna(df['naics_us_code_2007'])
del df['naics_us_code_2007']
df['naics_code'].isnull().sum()<jupyter_output><empty_output><jupyter_text>Merging the values of naics_2007_us_title, naics_title, naics_us_title, naics_us_title_2007<jupyter_code>df.naics_2007_us_title.head()
df.naics_title.head()
df.naics_us_title.head()
df.naics_us_title_2007.head()<jupyter_output><empty_output><jupyter_text>Checking if any values overlap<jupyter_code>df[(df['naics_2007_us_title'].notnull()) & (df['naics_title'].notnull()) & (df['naics_us_title'].notnull()) & (df['naics_us_title_2007'].notnull())][['naics_2007_us_title','naics_title','naics_us_title','naics_us_title_2007']]<jupyter_output><empty_output><jupyter_text>Checking NA values in same indexes<jupyter_code>df[(df['naics_2007_us_title'].isnull()) & (df['naics_title'].isnull()) & (df['naics_us_title'].isnull()) & (df['naics_us_title_2007'].isnull())][['naics_2007_us_title','naics_title','naics_us_title','naics_us_title_2007']].shape
df = df[(df['naics_2007_us_title'].notnull()) | (df['naics_title'].notnull()) | (df['naics_us_title'].notnull()) | (df['naics_us_title_2007'].notnull())]
df['naics_title'] = df['naics_title'].fillna(df['naics_2007_us_title'])
del df['naics_2007_us_title']
df['naics_title'] = df['naics_title'].fillna(df['naics_us_title'])
del df['naics_us_title']
df['naics_title'] = df['naics_title'].fillna(df['naics_us_title_2007'])
del df['naics_us_title_2007']
df['naics_title'].isnull().sum()
df['naics_title'].head()<jupyter_output><empty_output><jupyter_text>Merging the values of foreign_worker_info_alt_edu_experience and fw_info_alt_edu_experience df.foreign_worker_info_alt_edu_experience.tail() df.fw_info_alt_edu_experience.tail()  Checking if any values overlap df[(df['foreign_worker_info_alt_edu_experience'].notnull()) & (df['fw_info_alt_edu_experience'].notnull())][['foreign_worker_info_alt_edu_experience','fw_info_alt_edu_experience']]  Checking NA values in same indexes df[(df['foreign_worker_info_alt_edu_experience'].isnull()) & (df['fw_info_alt_edu_experience'].isnull())][['foreign_worker_info_alt_edu_experience','fw_info_alt_edu_experience']].shape df.fw_info_alt_edu_experience = df.fw_info_alt_edu_experience.fillna(df.foreign_worker_info_alt_edu_experience) del df['foreign_worker_info_alt_edu_experience'] df.fw_info_alt_edu_experience.isnull().sum()  Merging the values of foreign_worker_info_birth_country and fw_info_birth_country df.foreign_worker_info_birth_country.tail() df.fw_info_birth_country.tail()  Checking if any values overlap df[(df['foreign_worker_info_birth_country'].notnull()) & (df['fw_info_birth_country'].notnull())][['foreign_worker_info_birth_country','fw_info_birth_country']]  Checking NA values in same indexes df[(df['foreign_worker_info_birth_country'].isnull()) & (df['fw_info_birth_country'].isnull())][['foreign_worker_info_birth_country','fw_info_birth_country']].shape df.fw_info_birth_country = df.fw_info_birth_country.fillna(df.foreign_worker_info_birth_country) del df['foreign_worker_info_birth_country'] df.fw_info_birth_country.isnull().sum() df.fw_info_birth_country = df.fw_info_birth_country.fillna(df.fw_info_birth_country.mode()[0])  Merging the values of foreign_worker_info_rel_occup_exp and fw_info_rel_occup_exp df.foreign_worker_info_rel_occup_exp.tail() df['fw_info_rel_occup_exp'].tail()  Checking if any values overlap df[(df['foreign_worker_info_rel_occup_exp'].notnull()) & (df['fw_info_rel_occup_exp'].notnull())][['foreign_worker_info_rel_occup_exp','fw_info_rel_occup_exp']]  Checking NA values in same indexes df[(df['foreign_worker_info_rel_occup_exp'].isnull()) & (df['fw_info_rel_occup_exp'].isnull())][['foreign_worker_info_rel_occup_exp','fw_info_rel_occup_exp']].shape df['fw_info_rel_occup_exp'] = df['fw_info_rel_occup_exp'].fillna(df['foreign_worker_info_rel_occup_exp']) del df['foreign_worker_info_rel_occup_exp'] df['fw_info_rel_occup_exp'].isnull().sum()  Merging the values of foreign_worker_info_req_experience and fw_info_req_experience df.foreign_worker_info_req_experience.tail() df['fw_info_req_experience'].tail()  Checking if any values overlap df[(df['foreign_worker_info_req_experience'].notnull()) & (df['fw_info_req_experience'].notnull())][['foreign_worker_info_req_experience','fw_info_req_experience']]  Checking NA values in same indexes df[(df['foreign_worker_info_req_experience'].isnull()) & (df['fw_info_req_experience'].isnull())][['foreign_worker_info_req_experience','fw_info_req_experience']].shape df['fw_info_req_experience'] = df['fw_info_req_experience'].fillna(df['foreign_worker_info_req_experience']) del df['foreign_worker_info_req_experience'] df['fw_info_req_experience'].isnull().sum() df['fw_info_rel_occup_exp'] = df['fw_info_rel_occup_exp'].fillna(df['fw_info_rel_occup_exp'].mode()[0]) df['fw_info_req_experience'] = df['fw_info_req_experience'].fillna(df['fw_info_req_experience'].mode()[0])  Merging the values of fw_info_training_comp and foreign_worker_info_training_comp df['fw_info_training_comp'].tail() df['foreign_worker_info_training_comp'].tail()  Checking if any values overlap df[(df['fw_info_training_comp'].notnull()) & (df['foreign_worker_info_training_comp'].notnull())][['fw_info_training_comp','foreign_worker_info_training_comp']]  Checking NA values in same indexes df[(df['fw_info_training_comp'].isnull()) & (df['foreign_worker_info_training_comp'].isnull())][['fw_info_training_comp','foreign_worker_info_training_comp']].shape df['fw_info_training_comp'] = df['fw_info_training_comp'].fillna(df['foreign_worker_info_training_comp']) del df['foreign_worker_info_training_comp'] df['fw_info_training_comp'].isnull().sum() df['fw_info_training_comp'] = df['fw_info_training_comp'].fillna(df['fw_info_training_comp'].mode()[0])  Merging the values of fw_ownership_interest and foreign_worker_ownership_interest  fw_ownership_interest : Identifies if the foreign worker has ownership interest with the Employer. Approx only 0.5% have ownership interest and approx 69.5% values are nulldf['fw_ownership_interest'].tail() df['foreign_worker_ownership_interest'].tail()  Checking if any values overlap df[(df['fw_ownership_interest'].notnull()) & (df['foreign_worker_ownership_interest'].notnull())][['fw_ownership_interest','foreign_worker_ownership_interest']]  Checking NA values in same indexes df[(df['fw_ownership_interest'].isnull()) & (df['foreign_worker_ownership_interest'].isnull())][['fw_ownership_interest','foreign_worker_ownership_interest']].shape df['fw_ownership_interest'] = df['fw_ownership_interest'].fillna(df['foreign_worker_ownership_interest']) del df['foreign_worker_ownership_interest'] df['fw_ownership_interest'].isnull().sum() df['fw_ownership_interest'] = df['fw_ownership_interest'].fillna(df['fw_ownership_interest'].mode()[0])  Merging the values of ji_offered_to_sec_j_foreign_worker and ji_offered_to_sec_j_fw df.ji_offered_to_sec_j_foreign_worker.tail() df.ji_offered_to_sec_j_fw.tail()  Checking if any values overlap df[(df['ji_offered_to_sec_j_foreign_worker'].notnull()) & (df['ji_offered_to_sec_j_fw'].notnull())][['ji_offered_to_sec_j_foreign_worker','ji_offered_to_sec_j_fw']]  Checking NA values in same indexes df[(df['ji_offered_to_sec_j_foreign_worker'].isnull()) & (df['ji_offered_to_sec_j_fw'].isnull())][['ji_offered_to_sec_j_foreign_worker','ji_offered_to_sec_j_fw']].shape df['ji_offered_to_sec_j_fw'] = df['ji_offered_to_sec_j_fw'].fillna(df['ji_offered_to_sec_j_foreign_worker']) del df['ji_offered_to_sec_j_foreign_worker'] df['ji_offered_to_sec_j_fw'].isnull().sum()<jupyter_code>### Merging the values of pw_job_title_908 and pw_job_title_9089<jupyter_output><empty_output><jupyter_text>df.pw_job_title_908 = df.pw_job_title_908.str.upper()df.pw_job_title_9089 = df.pw_job_title_9089.str.upper() df.pw_job_title_908.head() df.pw_job_title_9089.head()<jupyter_code>#### Checking if any values overlap<jupyter_output><empty_output><jupyter_text>df[(df['pw_job_title_908'].notnull()) & (df['pw_job_title_9089'].notnull())][['pw_job_title_908','pw_job_title_9089']]<jupyter_code>#### Checking NA values in same indexes<jupyter_output><empty_output><jupyter_text>df[(df['pw_job_title_908'].isnull()) & (df['pw_job_title_9089'].isnull())][['pw_job_title_908','pw_job_title_9089']].shape df = df[(df['pw_job_title_908'].notnull()) | (df['pw_job_title_9089'].notnull())] df['pw_job_title_9089'] = df['pw_job_title_9089'].fillna(df['pw_job_title_908']) del df['pw_job_title_908'] df['pw_job_title_9089'].isnull().sum()<jupyter_code>### Merging the values of rec_info_barg_rep_notified and recr_info_barg_rep_notified
df.rec_info_barg_rep_notified.tail()
df.recr_info_barg_rep_notified.tail()
#### Checking if any values overlap
df[(df['recr_info_barg_rep_notified'].notnull()) & (df['rec_info_barg_rep_notified'].notnull())][['recr_info_barg_rep_notified','rec_info_barg_rep_notified']]
#### Checking NA values in same indexes
df[(df['recr_info_barg_rep_notified'].isnull()) & (df['rec_info_barg_rep_notified'].isnull())][['recr_info_barg_rep_notified','rec_info_barg_rep_notified']].shape
df['recr_info_barg_rep_notified'] = df['recr_info_barg_rep_notified'].fillna(df['rec_info_barg_rep_notified'])
del df['rec_info_barg_rep_notified']
df['recr_info_barg_rep_notified'].isnull().sum()<jupyter_output><empty_output><jupyter_text>Merging the values of wage_offer_unit_of_pay_9089 and wage_offered_unit_of_pay_9089<jupyter_code>df.wage_offer_unit_of_pay_9089.tail()
df.wage_offered_unit_of_pay_9089.tail()<jupyter_output><empty_output><jupyter_text>Checking if any values overlap<jupyter_code>df[(df['wage_offer_unit_of_pay_9089'].notnull()) & (df['wage_offered_unit_of_pay_9089'].notnull())][['wage_offer_unit_of_pay_9089','wage_offered_unit_of_pay_9089']]<jupyter_output><empty_output><jupyter_text>Checking NA values in same indexes<jupyter_code>df[(df['wage_offer_unit_of_pay_9089'].isnull()) & (df['wage_offered_unit_of_pay_9089'].isnull())][['wage_offer_unit_of_pay_9089','wage_offered_unit_of_pay_9089']].shape
df['wage_offered_unit_of_pay_9089'] = df['wage_offered_unit_of_pay_9089'].fillna(df['wage_offer_unit_of_pay_9089'])
del df['wage_offer_unit_of_pay_9089']
df['wage_offered_unit_of_pay_9089'].isnull().sum()<jupyter_output><empty_output><jupyter_text>pw_amount_9089<jupyter_code>df.pw_amount_9089.isnull().sum()
df[['pw_amount_9089','pw_unit_of_pay_9089']].head(10)
#Replacing commas with whitespace character
df['pw_amount_9089'] = df['pw_amount_9089'].str.replace(",","") 


#Calculating pw_amount_9089 yearly for every unit
for unit in df.pw_unit_of_pay_9089.unique():
    if unit == "hr" or unit == "Hour":
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_amount_9089'] = df['pw_amount_9089'].apply(lambda x: float(x) * 8 * 250)
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_unit_of_pay_9089'] = df['pw_unit_of_pay_9089'].replace(to_replace = unit, value = "Year") 
    elif unit == "wk" or unit == "Week":
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_amount_9089'] = df['pw_amount_9089'].apply(lambda x: float(x) * 50)
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_unit_of_pay_9089'] = df['pw_unit_of_pay_9089'].replace(to_replace = unit, value = "Year")
    elif unit == "mth" or unit == "Month":
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_amount_9089'] = df['pw_amount_9089'].apply(lambda x: float(x) * 12)
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_unit_of_pay_9089'] = df['pw_unit_of_pay_9089'].replace(to_replace = unit, value = "Year")
    elif unit == "bi" or unit == "Bi-Weekly":  
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_amount_9089'] = df['pw_amount_9089'].apply(lambda x: float(x) * 25)
        df.loc[df['pw_unit_of_pay_9089'] == unit, 'pw_unit_of_pay_9089'] = df['pw_unit_of_pay_9089'].replace(to_replace = unit, value = "Year")
    elif unit =="yr":
         df.loc[df['pw_unit_of_pay_9089'] == unit ,'pw_unit_of_pay_9089'] = df['pw_unit_of_pay_9089'].replace(to_replace = unit, value = "Year")
    else:
        continue
        
#Filling missing values with median 
df['pw_amount_9089'] = df['pw_amount_9089'].fillna((df['pw_amount_9089'].median()))

#Changing format from string to float
df['pw_amount_9089'] = df.pw_amount_9089.astype(float)

#Displaying 10 first values
df[['pw_amount_9089','pw_unit_of_pay_9089']].head(10)
with pd.option_context('float_format', '{:.2f}'.format): print(df.pw_amount_9089.describe())
#Dividing our continuous income values into some categories to facilitate their visualization
df['remuneration'] = pd.cut(df['pw_amount_9089'], [0, 30000, 60000,90000,120000,150000,180000,210000,240000,270000,495748000], right=False, labels=["0-30k", "30-60k","60-90k","90-120k","120-150k","150-180k","180-210k","210-240k","240-270k","270k+"])
salary = df['remuneration'].value_counts()
salary.iloc[np.argsort(salary.index)]<jupyter_output><empty_output><jupyter_text>job_info_job_title<jupyter_code>df['job_info_job_title'] = df['job_info_job_title'].str.upper()<jupyter_output><empty_output><jupyter_text>list(df['job_info_job_title'].value_counts().values)dic = dict(zip(list(df['job_info_job_title'].value_counts().index), list(df['job_info_job_title'].value_counts().values)))dic<jupyter_code>#Converting values to upper case
df['job_info_job_title'] = df['job_info_job_title'].str.upper()

#Splitting job titles by '-'
df['job_info_job_title'] = df['job_info_job_title'].astype(str).str.split('-').str[0]
#Splitting job titles by 'ii'
df['job_info_job_title'] = df['job_info_job_title'].astype(str).str.split('ii').str[0]
#Splitting job titles by '/'
df['job_info_job_title'] = df['job_info_job_title'].astype(str).str.split('/').str[0]
#Removing leading and ending spaces
df['job_info_job_title'] = df['job_info_job_title'].astype(str).str.strip()
#Replacing "sr." values with "senior"
df['job_info_job_title'] = df['job_info_job_title'].str.replace('sr.', 'senior')
#Replacing "NaN", "NaT" and "nan" values with np.nan
df['job_info_job_title'].replace(["NaN", 'NaT','nan'], np.nan, inplace = True)


df['job_info_job_title'].value_counts(dropna=True)[:10]
lst1 = list(df['job_info_job_title'].value_counts().index)
lst2 = list(df['job_info_job_title'].value_counts().index)
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(lst1)
print(tfidf_matrix.shape)
tfidf_matrix.data
#from sklearn.metrics.pairwise import cosine_similarity
#cosine_similarity(tfidf_matrix, tfidf_matrix)

# MemoryError
df['job_info_job_title'].value_counts().values[df['job_info_job_title'].value_counts().values > 300]
(303/228722)*100<jupyter_output><empty_output><jupyter_text>Displaying percentage of non-null values for each feature<jupyter_code>i = 0;
for col in df.columns:
    i = i+1;
    print (i-1,"Column: '{}'".format(col),"contains ", np.round(100*df[col].count()/len(df['case_status']),decimals=2),"% non-null values" )<jupyter_output>0 Column: 'add_these_pw_job_title_9089' contains  10.99 % non-null values
1 Column: 'agent_city' contains  56.92 % non-null values
2 Column: 'agent_firm_name' contains  55.74 % non-null values
3 Column: 'agent_state' contains  56.05 % non-null values
4 Column: 'application_type' contains  35.61 % non-null values
5 Column: 'case_number' contains  100.0 % non-null values
6 Column: 'case_received_date' contains  64.39 % non-null values
7 Column: 'case_status' contains  100.0 % non-null values
8 Column: 'class_of_admission' contains  94.08 % non-null values
9 Column: 'country_of_citizenship' contains  99.98 % non-null values
10 Column: 'decision_date' contains  100.0 % non-null values
11 Column: 'employer_address_1' contains  99.99 % non-null values
12 Column: 'employer_address_2' contains  60.4 % non-null values
13 Column: 'employer_city' contains  100.0 % non-null values
14 Column: 'employer_country' contains  64.37 % non-null values
15 Column: 'employer_decl_info_title' contains  64.37 [...]<jupyter_text>Feature selection and data cleaning<jupyter_code>#Leaving columns which have more than 330000 non-missing observations
df2 = df.loc[:,df.count() >= 330000]
df2.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 351526 entries, 0 to 374353
Data columns (total 21 columns):
case_number               351526 non-null object
case_status               351526 non-null object
class_of_admission        331278 non-null object
country_of_citizenship    351526 non-null object
decision_date             351526 non-null datetime64[ns]
employer_address_1        351498 non-null object
employer_city             351523 non-null object
employer_name             351525 non-null object
employer_postal_code      351509 non-null object
employer_state            351502 non-null object
job_info_work_city        351498 non-null object
job_info_work_state       351497 non-null object
naics_code                351526 non-null object
naics_title               351526 non-null object
pw_amount_9089            351526 non-null float64
pw_job_title_9089         351526 non-null object
pw_soc_code               351448 non-null object
pw_soc_title              351405 non-null objec[...]<jupyter_text>class_of_admission<jupyter_code>df2.class_of_admission.isnull().sum()
#Replacing missing values with mode
df2['class_of_admission'] = df2['class_of_admission'].fillna(df2['class_of_admission'].mode()[0])
df2.class_of_admission.isnull().sum()
df2.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 351526 entries, 0 to 374353
Data columns (total 21 columns):
case_number               351526 non-null object
case_status               351526 non-null object
class_of_admission        351526 non-null object
country_of_citizenship    351526 non-null object
decision_date             351526 non-null datetime64[ns]
employer_address_1        351498 non-null object
employer_city             351523 non-null object
employer_name             351525 non-null object
employer_postal_code      351509 non-null object
employer_state            351502 non-null object
job_info_work_city        351498 non-null object
job_info_work_state       351497 non-null object
naics_code                351526 non-null object
naics_title               351526 non-null object
pw_amount_9089            351526 non-null float64
pw_job_title_9089         351526 non-null object
pw_soc_code               351448 non-null object
pw_soc_title              351405 non-null objec[...]<jupyter_text>country_of_citizenship<jupyter_code>df2['country_of_citizenship'].isnull().sum()<jupyter_output><empty_output><jupyter_text>decision_date<jupyter_code>df2['decision_date'].head()
import datetime as dt

df2['decision_day'] = df2.decision_date.dt.day
df2['decision_month'] = df2.decision_date.dt.month
df2['decision_year'] = df2.decision_date.dt.year
df2['decision_dayofweek'] = df2.decision_date.dt.dayofweek
df2['decision_day'] = df2['decision_day'].astype(object)
df2['decision_month'] = df2['decision_month'].astype(object)
df2['decision_year'] = df2['decision_year'].astype(object)
df2['decision_dayofweek'] = df2['decision_dayofweek'].astype(object)
del df2['decision_date']
#np.sort(df2['decision_day'].unique())
#np.sort(df2['decision_month'].unique())
#np.sort(df2['decision_year'].unique())
np.sort(df2['decision_dayofweek'].unique())<jupyter_output><empty_output><jupyter_text>employer_address_1<jupyter_code>df2.employer_address_1.str.upper().value_counts()[:10]
df2.employer_address_1 = df2.employer_address_1.str.upper()
df2.employer_address_1.isnull().sum()
df2.employer_address_1 = df2.employer_address_1.fillna(df2.employer_address_1.mode()[0])
df2.employer_address_1.value_counts()<jupyter_output><empty_output><jupyter_text>lst1 = []for v in df2['employer_address_1'].value_counts():    if v > 500:        lst1.append(v)len(lst1) list(df2.employer_address_1.value_counts().index)[:50]  Taking values which have count more than 1000df2 = df2.loc[df2.employer_address_1.isin(['211 QUALITY CIRCLE','ONE MICROSOFT WAY','2200 MISSION COLLEGE BLVD.','1600 AMPHITHEATRE PARKWAY',                                         '440 TERRY AVE N','170 WEST TASMAN DRIVE','6100 TENNYSON PARKWAY','500 ORACLE PARKWAY',                                         '1 INFINITE LOOP','1700 MARKET STREET','5775 MOREHOUSE DRIVE','330 POTRERO AVENUE','1 HACKER WAY',                                         '701 FIRST AVENUE','200 PLAZA DRIVE','2 TOWER CENTER BLVD.','100 N. TRYON STREET',                                     '10 SOUTH DEARBORN STREET','3039 CORNWALLIS ROAD','3401 HILLVIEW AVENUE','5300 CALIFORNIA AVENUE',                                     '6400 SHAFER COURT','2065 HAMILTON AVE.','500 JACKSON STREET','731 LEXINGTON AVE.',                                     '9031 GARNERS FERRY RD','150 PRESIDENTIAL WAY','17890 BLANCO ROAD','1001 EAST PALM AVENUE',                                     '2211 NORTH FIRST STREET','6300 LEGACY DR., R1-C10','ONE DELL WAY','2701 SAN TOMAS EXPRESSWAY',                                     '176 SOUTH ST','300 MADISON AVENUE','3333 HWY 117 N.','LANDMARK BUILDING @ ONE MARKET STREET',                                     '2029 STIERLIN COURT','702 SOUTHWEST 8TH ST.','100 SOMERSET CORPORATE BOULEVARD, SUITE 5000, 5TH FLOOR',                                     '3500 WEST BLVD','1680 CAPITAL ONE DRIVE','103 MORGAN LANE','3000 HANOVER STREET',                                     '8816 HIGHWAY 301 NORTH','2035 LINCOLN HIGHWAY','1095 AVENUE OF THE AMERICAS','385 PILCH ROAD',                                     '3333 US HIGHWAY 117 SOUTH','460 PARK AVENUE SOUTH'])]<jupyter_code>### #employer_city<jupyter_output><empty_output><jupyter_text>df2.employer_city.isnull().sum() df2.employer_city = df2.employer_city.fillna(df2.employer_city.mode()[0]) df2.employer_city = df2.employer_city.str.upper()<jupyter_code>df2.employer_city.value_counts()
list(df2.employer_city.value_counts().index)
# Taking values which have count more than 500

df2 = df2.loc[df2.employer_city.isin(['COLLEGE STATION','REDMOND','SANTA CLARA','MOUNTAIN VIEW','SAN JOSE','SEATTLE','PLANO','SUNNYVALE',
                                'NEW YORK','REDWOOD SHORES','CUPERTINO','PHILADELPHIA','SAN DIEGO','PALO ALTO','MENLO PARK','ROSE HILL',
                                'SECAUCUS','EAST BRUNSWICK','CHICAGO','CHARLOTTE','RTP','IRVINE','ROSEMONT','COLUMBUS','HOPKINS','WOBURN',
                                'SAN ANTONIO','TAMPA','ROUND ROCK','HOPKINTON','SAN FRANCISCO','BENTONVILLE','BRIDGEWATER','MONTGOMERY',
                                'MCLEAN','PLAINSBORO','CLAXTON','EDISON','TROUTMAN'])]<jupyter_output><empty_output><jupyter_text>employer_name<jupyter_code>df2.employer_name.isnull().sum()
df2.employer_name = df2.employer_name.str.upper()
df2.employer_name = df2.employer_name.fillna(df2.employer_name.mode()[0])<jupyter_output><empty_output><jupyter_text>list(df2.employer_name.value_counts().index)  Taking values which have count more than 700df2 = df2.loc[df2.employer_name.isin(['COGNIZANT TECHNOLOGY SOLUTIONS US CORPORATION','MICROSOFT CORPORATION','INTEL CORPORATION','GOOGLE INC.',                                'AMAZON CORPORATE LLC','CISCO SYSTEMS, INC.','INFOSYS LTD.','ORACLE AMERICA, INC.','APPLE INC.',                                'QUALCOMM TECHNOLOGIES INC.','DELOITTE CONSULTING LLP','HCL AMERICA INC.','FACEBOOK, INC.','YAHOO! INC.',                                'ERNST & YOUNG U.S. LLP','WIPRO LIMITED','JP MORGAN CHASE & CO','IBM CORPORATION',                                'HOUSE OF RAEFORD FARMS, INC.','BROADCOM CORPORATION','CAPGEMINI FINANCIAL SERVICES USA INC.','EBAY INC.',                                'VMWARE, INC.','BLOOMBERG L.P.','DEFENDER SERVICES, INC.','RANDSTAD TECHNOLOGIES, LP','KFORCE INC.',                                'CUMMINS INC.','BANK OF AMERICA','ERICSSON INC.'])]<jupyter_code>### #employer_postal_code<jupyter_output><empty_output><jupyter_text>df2.employer_postal_code.isnull().sum() del df2['employer_postal_code']<jupyter_code>### #employer_state<jupyter_output><empty_output><jupyter_text>df2.employer_state.isnull().sum() Filling missing values in "employer_state" column with modedf2['employer_state'] = df2['employer_state'].fillna(df2['employer_state'].mode()[0]);Mapping from state name to abbreviationstate_abbrevs = {    'Alabama': 'AL',    'Alaska': 'AK',    'Arizona': 'AZ',    'Arkansas': 'AR',    'California': 'CA',    'Colorado': 'CO',    'Connecticut': 'CT',    'Delaware': 'DE',    'Florida': 'FL',    'Georgia': 'GA',    'Hawaii': 'HI',    'Idaho': 'ID',    'Illinois': 'IL',    'Indiana': 'IN',    'Iowa': 'IA',    'Kansas': 'KS',    'Kentucky': 'KY',    'Louisiana': 'LA',    'Maine': 'ME',    'Maryland': 'MD',    'Massachusetts': 'MA',    'Michigan': 'MI',    'Minnesota': 'MN',    'Mississippi': 'MS',    'Missouri': 'MO',    'Montana': 'MT',    'Nebraska': 'NE',    'Nevada': 'NV',    'New Hampshire': 'NH',    'New Jersey': 'NJ',    'New Mexico': 'NM',    'New York': 'NY',    'North Carolina': 'NC',    'North Dakota': 'ND',    'Ohio': 'OH',    'Oklahoma': 'OK',    'Oregon': 'OR',    'Pennsylvania': 'PA',    'Rhode Island': 'RI',    'South Carolina': 'SC',    'South Dakota': 'SD',    'Tennessee': 'TN',    'Texas': 'TX',    'Utah': 'UT',    'Vermont': 'VT',    'Virginia': 'VA',    'Washington': 'WA',    'West Virginia': 'WV',    'Wisconsin': 'WI',    'Wyoming': 'WY',    'Northern Mariana Islands':'MP',     'Palau': 'PW',     'Puerto Rico': 'PR',     'Virgin Islands': 'VI',     'District of Columbia': 'DC'}Capitalizing Keysus_state_abbrev = {k.upper(): v for k, v in state_abbrevs.items()}df2['employer_state'].replace(us_state_abbrev, inplace=True)df2.employer_state = df2.employer_state.astype(str) df2['employer_state'].head() df2.employer_state.value_counts()[:10]<jupyter_code>### #job_info_work_city<jupyter_output><empty_output><jupyter_text>df2.job_info_work_city.isnull().sum() df2.job_info_work_city.str.upper().value_counts()[:10] df2.job_info_work_city = df2.job_info_work_city.str.upper() df2.job_info_work_city = df2.job_info_work_city.fillna(df2.job_info_work_city.mode()[0])<jupyter_code>df2.job_info_work_city.value_counts()
# Taking values which have count more than 600

df2 = df2.loc[df2.job_info_work_city.isin(['COLLEGE STATION',
 'REDMOND',
 'MOUNTAIN VIEW',
 'SEATTLE',
 '<NAME>',
 'PLANO',
 'HILLSBORO',
 'NEW YORK',
 'CUPERTINO',
 '<NAME>',
 'SUNNYVALE',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 'FOLSOM',
 'EAST BRUNSWICK',
 'TEANECK',
 '<NAME>',
 'ROSEMONT',
 'TAMPA',
 'WOBURN',
 'COLUMBUS',
 'ROSE HILL',
 'CHANDLER',
 'JERSEY CITY',
 'SOMERS'])]<jupyter_output><empty_output><jupyter_text>job_info_work_state<jupyter_code>df2.job_info_work_state.isnull().sum()
df2.job_info_work_state = df2.job_info_work_state.fillna(df2.job_info_work_state.mode()[0])
df2['job_info_work_state'].replace(us_state_abbrev, inplace=True)
df2.job_info_work_state = df2.job_info_work_state.astype(str)
df2.job_info_work_state.isnull().sum()
df2.job_info_work_state.value_counts()[:10]<jupyter_output><empty_output><jupyter_text>Taking values which have count more than 500df2 = df2.loc[df2.job_info_work_state.isin(["CA", "TX", "WA", "NY", "NJ", "OR", "IL", "FL", "MA", "NC", "AZ", "IN"])]<jupyter_code>### #naics_code<jupyter_output><empty_output><jupyter_text>df2.naics_code.isnull().sum() df2.naics_code = df2.naics_code.fillna(df2.naics_code.mode()[0]) del df2['naics_code']<jupyter_code>### #naics_title<jupyter_output><empty_output><jupyter_text>df2.naics_title.isnull().sum() df2.naics_title = df2.naics_title.fillna(df2.naics_title.mode()[0]) df2.naics_title = df2.naics_title.str.upper() df2.naics_title.value_counts()[:10]<jupyter_code>## Taking values which have count more than 200

df2 = df2.loc[df2.naics_title.isin(['COMPUTER SYSTEMS DESIGN SERVICES',
 'SOFTWARE PUBLISHERS',
 'CUSTOM COMPUTER PROGRAMMING SERVICES',
 'SEMICONDUCTOR AND OTHER ELECTRONIC COMPONENT MANUFACTURING',
 'ELECTRONIC COMPUTER MANUFACTURING',
 'ELECTRONIC SHOPPING',
 'WEB SEARCH PORTALS',
 'RADIO AND TELEVISION BROADCASTING AND WIRELESS COMMUNICATIONS EQUIPMENT MANUFACTURING',
 'OTHER COMPUTER PERIPHERAL EQUIPMENT MANUFACTURING',
 'MANAGEMENT, SCIENTIFIC, AND TECHNICAL CONSULTING SERVICES',
 'ALL OTHER INFORMATION SERVICES',
 'ELECTRONIC AUCTIONS',
 'COMPUTER SYSTEMS DESIGN AND RELATED SERVICES',
 'POULTRY PROCESSING',
 'OTHER COMPUTER RELATED SERVICES',
 'OFFICES OF CERTIFIED PUBLIC ACCOUNTANTS',
 'ENGINE, TURBINE, AND POWER TRANSMISSION EQUIPMENT MANUFACTURING',
 'OFFICES OF OTHER HOLDING COMPANIES',
 'SEMICONDUCTOR AND RELATED DEVICE MANUFACTURING',
 'MANAGEMENT CONSULTING SERVICES'])]<jupyter_output><empty_output><jupyter_text>pw_job_title_9089<jupyter_code>df2.pw_job_title_9089.isnull().sum()
del df2['pw_job_title_9089']<jupyter_output><empty_output><jupyter_text>pw_level_9089<jupyter_code>#df2.pw_level_9089.isnull().sum()
df2.pw_level_9089.value_counts()
df2.pw_level_9089 = df2.pw_level_9089.fillna(df2.pw_level_9089.mode()[0])<jupyter_output><empty_output><jupyter_text>pw_soc_code<jupyter_code>df2.pw_soc_code.isnull().sum()
df2['pw_soc_code'] = df2['pw_soc_code'].str.replace('.','')
df2['pw_soc_code'] = df2['pw_soc_code'].str.replace('-','')
df2['pw_soc_code'] = df2['pw_soc_code'].astype(str).str[0:6]
del df2['pw_soc_code']<jupyter_output><empty_output><jupyter_text>Finding "nan" values in "pw_soc_code" column and filling them with modedf2.loc[df2['pw_soc_code'] == "nan",'pw_soc_code'] = df2['pw_soc_code'].mode()[0] Finding "None" values in "pw_soc_code" column and filling them with modedf2.loc[df2['pw_soc_code'] == "None",'pw_soc_code'] = df2['pw_soc_code'].mode()[0] Changing type from string to intdf2['pw_soc_code'] = df2['pw_soc_code'].astype(object) df2['pw_soc_code'].isnull().sum() df2['pw_soc_code'].value_counts() del df2['pw_soc_code']<jupyter_code>### #pw_soc_title<jupyter_output><empty_output><jupyter_text>df2.pw_soc_title.isnull().sum() df2.pw_soc_title.value_counts()[:10]<jupyter_code>df2.pw_soc_title = df2.pw_soc_title.str.upper()
df2.pw_soc_title = df2.pw_soc_title.fillna(df2.pw_soc_title.mode()[0])<jupyter_output><empty_output><jupyter_text>del df2['pw_soc_title']<jupyter_code>### #pw_source_name_9089<jupyter_output><empty_output><jupyter_text>df2.pw_source_name_9089.isnull().sum() df2.pw_source_name_9089 = df2.pw_source_name_9089.fillna(df2.pw_source_name_9089.mode()[0]) df2.pw_source_name_9089.value_counts()<jupyter_code>df2 = df2.loc[df2.pw_source_name_9089 != 'CBA']<jupyter_output><empty_output><jupyter_text>pw_unit_of_pay_9089<jupyter_code>df2.pw_unit_of_pay_9089.isnull().sum()
df2.pw_unit_of_pay_9089.value_counts()
del df2['pw_unit_of_pay_9089']<jupyter_output><empty_output><jupyter_text>wage_offered_unit_of_pay_9089<jupyter_code>#df2.wage_offered_unit_of_pay_9089.isnull().sum()
#df2.wage_offered_unit_of_pay_9089.value_counts()
for unit in df2.wage_offered_unit_of_pay_9089.unique():
    if unit == "hr":
        df2.loc[df2['wage_offered_unit_of_pay_9089'] == unit, 'wage_offered_unit_of_pay_9089'] = "Hour" 
    elif unit == "wk":
        df2.loc[df2['wage_offered_unit_of_pay_9089'] == unit, 'wage_offered_unit_of_pay_9089'] = "Week"
    elif unit == "mth":
        df2.loc[df2['wage_offered_unit_of_pay_9089'] == unit, 'wage_offered_unit_of_pay_9089'] = "Month"
    elif unit == "bi":  
        df2.loc[df2['wage_offered_unit_of_pay_9089'] == unit, 'wage_offered_unit_of_pay_9089'] = "Bi-Weekly"
    elif unit == "yr":
        df2.loc[df2['wage_offered_unit_of_pay_9089'] == unit, 'wage_offered_unit_of_pay_9089'] = "Year"
    else:
        continue
df2.wage_offered_unit_of_pay_9089 = df2.wage_offered_unit_of_pay_9089.fillna(df2.wage_offered_unit_of_pay_9089.mode()[0])
df2.wage_offered_unit_of_pay_9089.value_counts()<jupyter_output><empty_output><jupyter_text>remuneration<jupyter_code>df2.remuneration.isnull().sum()
df2.remuneration = df2.remuneration.fillna(df2.remuneration.mode()[0])
##############################################################################################################################################

df2.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 351526 entries, 0 to 374353
Data columns (total 18 columns):
case_number               351526 non-null object
case_status               351526 non-null object
class_of_admission        351526 non-null object
country_of_citizenship    351526 non-null object
employer_address_1        351526 non-null object
employer_city             351526 non-null object
employer_name             351526 non-null object
employer_state            351526 non-null object
job_info_work_city        351526 non-null object
job_info_work_state       351526 non-null object
naics_title               351526 non-null object
pw_amount_9089            351526 non-null float64
pw_source_name_9089       351526 non-null object
remuneration              351526 non-null category
decision_day              351526 non-null object
decision_month            351526 non-null object
decision_year             351526 non-null object
decision_dayofweek        351526 non-null object
dtyp[...]<jupyter_text>Data type conversion<jupyter_code>df3 = df2.copy()
from sklearn.preprocessing import LabelEncoder
categorical_variables = {}

#Creating categories denoted by integers from column values
for col in df3.columns:
    cat_var_name = "cat_"+ col
    cat_var_name = LabelEncoder()
    cat_var_name.fit(df3[col])
    df3[col] = cat_var_name.transform(df3[col])
    categorical_variables[col] = cat_var_name

df3.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 351526 entries, 0 to 374353
Data columns (total 18 columns):
case_number               351526 non-null int32
case_status               351526 non-null int32
class_of_admission        351526 non-null int32
country_of_citizenship    351526 non-null int32
employer_address_1        351526 non-null int32
employer_city             351526 non-null int32
employer_name             351526 non-null int32
employer_state            351526 non-null int32
job_info_work_city        351526 non-null int32
job_info_work_state       351526 non-null int32
naics_title               351526 non-null int32
pw_amount_9089            351526 non-null int64
pw_source_name_9089       351526 non-null int32
remuneration              351526 non-null int32
decision_day              351526 non-null int32
decision_month            351526 non-null int32
decision_year             351526 non-null int32
decision_dayofweek        351526 non-null int32
dtypes: int32(17), int64([...]<jupyter_text>Applying Tree-based models<jupyter_code>#Dividing our final dataset into features(explanatory variables) and labels(target variable)
X = df3.loc[:, df3.columns != 'case_status']
y = df3.case_status

print("The shape of X is: {}".format(X.shape))
print("The shape of y is: {}".format(y.shape))
X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.3)
print (X_train.shape, y_train.shape)
print (X_test.shape, y_test.shape)<jupyter_output>(246068, 17) (246068,)
(105458, 17) (105458,)<jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100)

rf.fit(X_train, y_train)
y_pred = rf.predict(X_test)
train_pred = rf.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Random Forest Classifier: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Random Forest Classifier')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying SMOTE<jupyter_code>from imblearn.over_sampling import SMOTE

y_train.value_counts()<jupyter_output><empty_output><jupyter_text>help(SMOTE())<jupyter_code>smt = SMOTE()
X_train_smt, y_train_smt = smt.fit_sample(X_train, y_train)
np.bincount(y_train_smt)
rf.fit(X_train_smt, y_train_smt)
y_pred = rf.predict(X_test)
train_pred = rf.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Random Forest Classifier with SMOTE: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Random Forest Classifier with SMOTE')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying NearMiss<jupyter_code>from imblearn.under_sampling import NearMiss

y_train.value_counts()
nr = NearMiss()
X_train_nr, y_train_nr = nr.fit_sample(X_train, y_train)
np.bincount(y_train_nr)
rf.fit(X_train_nr, y_train_nr)
y_pred = rf.predict(X_test)
train_pred = rf.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Random Forest Classifier with NearMiss: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Random Forest Classifier with NearMiss')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>from sklearn import tree

dc = tree.DecisionTreeClassifier(criterion='gini')
dc.fit(X_train, y_train)
dc.score(X_train, y_train)
y_pred = dc.predict(X_test)
train_pred = dc.predict(X_train)
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Decision Tree Classifier: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Decision Tree Classifier')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying SMOTE on DT<jupyter_code>from imblearn.over_sampling import SMOTE

y_train.value_counts()<jupyter_output><empty_output><jupyter_text>help(SMOTE()) from sklearn.datasets import make_classificationfrom imblearn.over_sampling import SMOTE  doctest: +NORMALIZE_WHITESPACEX, y = make_classification(n_classes=2, class_sep=2,weights=[0.1, 0.9], n_informative=3, n_redundant=1, flip_y=0,                           n_features=20, n_clusters_per_class=1, n_samples=1000, random_state=10)<jupyter_code>smt = SMOTE()
X_train_smt, y_train_smt = smt.fit_sample(X_train, y_train)
np.bincount(y_train_smt)
dc.fit(X_train_smt, y_train_smt)
y_pred = dc.predict(X_test)
train_pred = dc.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Decision Tree Classifier with SMOTE: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Decision Tree Classifier with SMOTE')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying NearMiss<jupyter_code>from imblearn.under_sampling import NearMiss

y_train.value_counts()
nr = NearMiss()
X_train_nr, y_train_nr = nr.fit_sample(X_train, y_train)
np.bincount(y_train_nr)
dc.fit(X_train_nr, y_train_nr)
y_pred = dc.predict(X_test)
train_pred = dc.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
print("Decision Tree Classifier with NearMiss: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Decision Tree Classifier with NearMiss')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>GradientBoostingClassifier<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier

gb = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1)
gb.fit(X_train, y_train)
y_pred = gb.predict(X_test)
train_pred = gb.predict(X_train)
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score, roc_auc_score,roc_curve

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
precision_score(y_test, y_pred)
f1_score(y_test, y_pred)
print("Gradient Boosting Classifier: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Gradient Boosting Classifier')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying SMOTE on GB<jupyter_code>from imblearn.over_sampling import SMOTE

y_train.value_counts()<jupyter_output><empty_output><jupyter_text>help(SMOTE()) from sklearn.datasets import make_classificationfrom imblearn.over_sampling import SMOTE  doctest: +NORMALIZE_WHITESPACEX, y = make_classification(n_classes=2, class_sep=2,weights=[0.1, 0.9], n_informative=3, n_redundant=1, flip_y=0,                           n_features=20, n_clusters_per_class=1, n_samples=1000, random_state=10)<jupyter_code>smt = SMOTE()
X_train_smt, y_train_smt = smt.fit_sample(X_train, y_train)
np.bincount(y_train_smt)
gb.fit(X_train_smt, y_train_smt)
y_pred = gb.predict(X_test)
train_pred = gb.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
precision_score(y_test, y_pred)
print("Gradient Boosting Classifier with SMOTE: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Gradient Boosting Classifier with SMOTE')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output><jupyter_text>Applying NearMiss<jupyter_code>from imblearn.under_sampling import NearMiss

y_train.value_counts()
nr = NearMiss()
X_train_nr, y_train_nr = nr.fit_sample(X_train, y_train)
np.bincount(y_train_nr)
gb.fit(X_train_nr, y_train_nr)
y_pred = gb.predict(X_test)
train_pred = gb.predict(X_train)
pd.crosstab(y_test, y_pred, rownames=['Actual Result'], colnames=['Predicted Result'])
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, accuracy_score

accuracy_score(y_test, y_pred)
accuracy_score(y_train, train_pred)
confusion_matrix(y_test, y_pred)
recall_score(y_test, y_pred)
recall_score(y_train, train_pred)
precision_score(y_test, y_pred)
print("Gradient Boosting Classifier with NearMiss: ",accuracy_score(y_test, y_pred))
fpr, tpr, thresholds = roc_curve(y_test, y_pred, pos_label=1)
print("Roc/auc: ",roc_auc_score(y_test, y_pred))
# Plotting the roc curve

plt.title('Receiver Operating Characteristic for Gradient Boosting Classifier with NearMiss')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc_score(y_test, y_pred))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14347843198992444, 'fraction_numerical': 0.04154203400503778, 'mean_word_length': 6.860324825986079, 'pattern_counts': {'":': 10, '<': 265, '<?xml version=': 0, '>': 268, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '65', 'dir': 'jupyter-structured-clean-dedup', 'id': '615479', 'license': 'MIT', 'n_tokens_mistral': 20859, 'n_tokens_neox': 19103, 'n_words': 3028, 'path': 'Final_Classification.ipynb', 'repo_name': 'shef4793/U.S-Perm-Visa-Case-status-Classification', 'size': '50841'}