<jupyter_start><jupyter_text>read data<jupyter_code>import os
import sys

data_folder = os.path.abspath(os.path.join(os.pardir, 'data'))
data_file = 'employee_retention_data.csv'
data_folder_file = os.path.join(data_folder, data_file)
raw_data_df = pd.read_csv(data_folder_file)
raw_data_df.head()
raw_data_df.quit_date.isnull().sum() / (len(raw_data_df.quit_date))
# pd.to_datetime(raw_data_df.join_date, format="%m/%d/%Y")
raw_data_df.join_date = pd.to_datetime(raw_data_df.join_date, infer_datetime_format=1)
raw_data_df.quit_date = pd.to_datetime(raw_data_df.quit_date, infer_datetime_format=1)
raw_data_df.head()
raw_data_df.shape<jupyter_output><empty_output><jupyter_text>play with data<jupyter_code>tmp_array = (raw_data_df.employee_id.unique())
len(tmp_array), max(tmp_array), min(tmp_array),
len(raw_data_df.company_id.unique())
len(raw_data_df.dept.unique())
tmp_array = (raw_data_df.seniority.unique())
len(tmp_array), max(tmp_array), min(tmp_array),
tmp_array =  raw_data_df.seniority

num_bins = 50
n, bins, patches = plt.hist(tmp_array, num_bins, facecolor='blue', alpha=0.5)
plt.show()

tmp_array_join_date = (raw_data_df.join_date.unique())
len(tmp_array_join_date), max(tmp_array_join_date), min(tmp_array_join_date),
raw_data_df.head()
raw_data_df['join_day'] = (raw_data_df.join_date - min(tmp_array_join_date)).astype('timedelta64[D]') 

raw_data_df['quit_day'] = (raw_data_df.quit_date - min(tmp_array_join_date)).astype('timedelta64[D]')


# raw_data_df['stay_days'] = raw_data_df['quit_days'] - raw_data_df['join_days'] 
# raw_data_df.quit_day
raw_data_df['if_quit'] = -raw_data_df.quit_date.isnull()
raw_data_df['last_day'] = raw_data_df.quit_day
raw_data_df.head()
last_day = (np.datetime64('2015-12-13')-min(tmp_array_join_date)).astype('timedelta64[D]').astype('float')
na_dict = {'last_day':last_day}
raw_data_df=raw_data_df.fillna(na_dict)
raw_data_df.head()
raw_data_df['stay_days'] = raw_data_df.last_day - raw_data_df.join_day
tmp_array = (raw_data_df.stay_days.unique())
len(tmp_array), max(tmp_array), min(tmp_array),
raw_data_df

tmp_array =  raw_data_df.stay_days

num_bins = 50
n, bins, patches = plt.hist(tmp_array, num_bins, facecolor='blue', alpha=0.5)
plt.show()<jupyter_output><empty_output><jupyter_text>split into 2 groups: quit & not_quit<jupyter_code>raw_data_df.head()
raw_data_df_save = raw_data_df.copy(deep=True)
raw_data_df = raw_data_df_save.drop(['quit_day'], axis=1)
raw_data_df.head()
quit_raw_data_df = raw_data_df[raw_data_df.if_quit == 1]
no_quit_raw_data_df = raw_data_df[raw_data_df.if_quit == 0]
raw_data_df.shape, quit_raw_data_df.shape, no_quit_raw_data_df.shape<jupyter_output><empty_output><jupyter_text>Compare quit vs no_quit<jupyter_code>quit_raw_data_df.stay_days.mean(),  no_quit_raw_data_df.stay_days.mean()
quit_company_mean = quit_raw_data_df.groupby(['company_id'])['stay_days'].mean().values
no_quit_company_mean = no_quit_raw_data_df.groupby(['company_id'])['stay_days'].mean().values
company_id = quit_raw_data_df.groupby(['company_id'])['stay_days'].mean().index.values<jupyter_output><empty_output><jupyter_text>only use quit_df: try to predict the stay_days<jupyter_code>quit_raw_data_df.head()
drop_column_list = [

    'employee_id', 'join_date', 'join_date', 'quit_date',
    'if_quit', 'last_day', '', '',
    '', '', '', '',
]

drop_column_list = [i for i in drop_column_list if i is not '']

quit_raw_data_df = quit_raw_data_df.drop(drop_column_list, axis=1)
quit_raw_data_df.head()
categorical=['company_id','dept']
quit_raw_data_dummy_df=pd.get_dummies(quit_raw_data_df, columns=categorical)
quit_raw_data_dummy_df.head()
tmp_array =  quit_raw_data_dummy_df.stay_days.values
num_bins = 50
n, bins, patches = plt.hist(tmp_array, num_bins, facecolor='blue', alpha=0.5)
plt.show()
tmp_quit_raw_data_dummy_df = quit_raw_data_dummy_df[quit_raw_data_dummy_df.stay_days > 500]
tmp_quit_raw_data_dummy_df.shape
# Select only numeric data and impute missing values as 0
numerics = ['uint8','int16', 'int32', 'int64', 'float16', 'float32', 'float64']
quit_raw_data_x=tmp_quit_raw_data_dummy_df.select_dtypes(include=numerics).drop(['stay_days'],axis=1).fillna(0).values
# quit_raw_data_y= np.log(tmp_quit_raw_data_dummy_df['stay_days'].values)
quit_raw_data_y= (tmp_quit_raw_data_dummy_df['stay_days'].values)

quit_raw_data_x.shape, quit_raw_data_y.shape


tmp_array =  tmp_quit_raw_data_dummy_df.stay_days.values

num_bins = 50
n, bins, patches = plt.hist(tmp_array, num_bins, facecolor='blue', alpha=0.5)
plt.show()
chicago_fair_x = quit_raw_data_x
chicago_fair_y = quit_raw_data_y
#random forest

from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score

cv_groups = KFold(n_splits=5, shuffle=True, random_state=0)

regr_cv_rf = RandomForestRegressor(random_state = 0, n_estimators = 100)

r_2_test_list = []
r_2_train_list = []


for train_index, test_index in cv_groups.split(chicago_fair_x):
    
    # Train the model using the training sets
    regr_cv_rf.fit(chicago_fair_x[train_index], chicago_fair_y[train_index])
    
    # Make predictions using the testing set
    pred_cv_rf_test = regr_cv_rf.predict(chicago_fair_x[test_index])
    
    # Make predictions using the testing set
    pred_cv_rf_train = regr_cv_rf.predict(chicago_fair_x[train_index])
     
    r_2_test = r2_score(chicago_fair_y[test_index], pred_cv_rf_test)
    r_2_test_list += [r_2_test]
    
    r_2_train = r2_score(chicago_fair_y[train_index], pred_cv_rf_train)
    r_2_train_list += [r_2_train]

    print(r_2_test, r_2_train)
    
np.mean(r_2_test_list), np.mean(r_2_train_list)<jupyter_output>0.10678792463002362 0.8768731527244779
0.09451203063617564 0.8788859769801973
0.10952087205394989 0.8763783989853429
0.1254648115312006 0.8747892316469646
0.16672615362093368 0.8732542358000932<jupyter_text>test<jupyter_code>1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12540302053283556, 'fraction_numerical': 0.04310198540641439, 'mean_word_length': 8.252747252747254, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '259441', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2659, 'n_tokens_neox': 2434, 'n_words': 297, 'path': 'others/Data_Challenge1/Hanyu_Shi_Employment_Retention.ipynb', 'repo_name': 'davestanley-cogrhythms/insight-datachallenge1', 'size': '5893'}