<jupyter_start><jupyter_text>reading the files<jupyter_code>train = pd.read_csv('Train.csv')
test = pd.read_csv('Test.csv')
train.shape , test.shape
traintest = train.append(test)
traintest.shape
traintest.head(3)
y = traintest['Attrition_rate']
traintest.drop(['Attrition_rate','Employee_ID'],axis=1,inplace=True)
traintest.head(3)
traintest.shape , y.shape
traintest.nunique(dropna=False)
traintest.isnull().sum()<jupyter_output><empty_output><jupyter_text>for handling missing value in AGE<jupyter_code>traintest.Age.value_counts(dropna=False).head(4)
traintest.Age.describe()
#fig, ax = plt.subplots()
#ax.scatter(x = train['Age'], y = train['Attrition_rate'])
#plt.ylabel('Attrition_rate', fontsize=13)
#plt.xlabel('Age', fontsize=13)
#plt.show()
traintest['Age'].fillna(traintest.groupby(['Gender', 'Education_Level'])['Age'].transform('median'))
traintest.Age.describe()
traintest['Age'].fillna(traintest.groupby(['Gender', 'Time_since_promotion'])['Age'].transform('median'))
traintest.Age.describe()
mean_imputation_df = traintest.copy()
mean_imputation_df['Age_Mean_Filled'] = mean_imputation_df['Age'].fillna(traintest['Age'].mean())
mean_imputation_df.Age_Mean_Filled.describe()
traintest['Age'].fillna(traintest.groupby(['Gender', 'Hometown'])['Age'].transform('median'),inplace=True)
#traintest.Age.value_counts()<jupyter_output><empty_output><jupyter_text>for handling missing value in var4<jupyter_code>traintest.VAR4.value_counts(dropna=False)
traintest.VAR4.fillna(4.0,inplace=True)
traintest.VAR4.value_counts(dropna=False)<jupyter_output><empty_output><jupyter_text>for handling missing value in var2<jupyter_code>traintest.VAR2.value_counts(dropna=False)
traintest.VAR2.fillna(0.0,inplace=True)
traintest.VAR2.value_counts(dropna=False)
traintest.isnull().sum()
traintest['Pay_Scale'].fillna(traintest.Pay_Scale.median(),inplace=True)
traintest['Time_of_service'].fillna(traintest.Time_of_service.mean(),inplace=True)
traintest['Work_Life_balance'].fillna(traintest.Work_Life_balance.mean(),inplace=True)
traintest['time'] = traintest.apply(lambda row: row.Time_of_service - row.Time_since_promotion , axis = 1) 
traintest['age_while_joining'] = traintest.apply(lambda r:r.Age - r.Time_of_service,axis=1)
traintest.isnull().any().sum()
cat_features =  list(traintest.select_dtypes(include=object).columns)
for i in cat_features:
    l = LabelEncoder()
    traintest[i] = l.fit_transform(traintest[i])
traintest.shape
#cat_features =  list(traintest.select_dtypes(include=object).columns)
#traintest = pd.get_dummies(traintest,columns=cat_features,drop_first=True)
traintest.head()
traintest.shape
main_X=traintest.copy()
sc = StandardScaler()
X = sc.fit_transform(traintest)
#X_train = X[:7000,:]
#Y_train = y[:7000]
#X_test = X[7000:,:]

X_train = traintest.iloc[:7000,:]
Y_train = y.iloc[:7000]
X_test = traintest.iloc[7000:,:]

print(X_train.shape,Y_train.shape)
print(X_test.shape)
## Hyper Parameter Optimization

booster=['gbtree','gblinear']
base_score=[0.25,0.5,0.75,1]
n_estimators = [100, 500, 900, 1100, 1500]
max_depth = [3,5, 10,25,50]
learning_rate=[0.05,0.1,0.15,0.20,0.01]
min_child_weight=[0,1,2,4]
subsample =[0.5,0.75,1]

# Define the grid of hyperparameters to search
hyperparameter_grid = {
    'n_estimators': n_estimators,
    'max_depth':max_depth,
    'learning_rate':learning_rate,
    'min_child_weight':min_child_weight,
    'booster':booster,
    'base_score':base_score,
    'subsample':subsample
    }<jupyter_output><empty_output><jupyter_text>XGBRegressor<jupyter_code>%%time
regressor=xgb.XGBRegressor()
# Set up the random search with 4-fold cross validation
random_cv = RandomizedSearchCV(estimator=regressor,
            param_distributions=hyperparameter_grid,
            cv=5, n_iter=50,
            scoring = 'neg_mean_absolute_error',n_jobs = 4,
            verbose = 5, 
            return_train_score = True,
            random_state=42)
random_cv.fit(X_train,Y_train)
random_cv.best_params_
xgbmodel = xgb.XGBRegressor(subsample= 0.5,
 n_estimators=700,
 min_child_weight= 2,
 max_depth= 5,
 learning_rate= 0.2,
 booster='gblinear',
 base_score= 0.25)
xgbmodel.fit(X_train,Y_train)
yxgbreg = xgbmodel.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = yxgbreg
sub.to_csv('xgboost_submission.csv',index=False)
#y_test = np.zeros((test.shape[0],1, 5))
#y_test.shape
kfold = KFold(n_splits=10, shuffle=True, random_state=42)
warnings.filterwarnings('ignore')
scores = []

i=0
for train_index, test_index in kfold.split(X_train):   
    x_train, x_test = X_train[train_index], X_train[test_index]
    y_train, y_test = Y_train[train_index], Y_train[test_index]

    xgb_model = xgb.XGBRegressor(subsample= 0.7,n_estimators=500,
                                 min_child_weight= 2,max_depth= 5,
                                 learning_rate= 0.2,booster='gblinear',
                                 base_score= 0.25)
    
    xgb_model.fit(x_train, y_train)
    y_pred = xgb_model.predict(x_test)
    scores.append(100*max(0,1-np.sqrt(mean_squared_error(y_test,y_pred))))
    y_test = xgb_model.predict(X_test,num_iteration = iter_)
    xgb_model.predict()
    
    
print(scores)
print('mean',np.mean(scores))
print('std',np.std(scores))
n_folds=10 
seed =0
skf = KFold(n_splits=n_folds, shuffle=True, random_state=seed)
X, y = X_train, Y_train
    
preds = np.zeros(test.shape[0])
    
params={'subsample': 0.5,
 'n_estimators': 500,
 'min_child_weight': 2,
 'max_depth': 5,
 'learning_rate': 0.2,
 'booster': 'gblinear',
 'base_score': 0.25}
    
for i, (itr, icv) in enumerate(skf.split(X, y)):
    x_train, x_val = X[itr, :], X[icv, :]
    y_train, y_val = y[itr], y[icv]

    dtrain = xgb.DMatrix(data=x_train, label=y_train, missing=np.nan)
    dval = xgb.DMatrix(data = x_val, label=y_val, missing=np.nan)

    bst = xgb.train(
                params                = params,
                dtrain                = dtrain,
                num_boost_round       = 30_000,
                early_stopping_rounds = 200,
                evals                 = [(dtrain, 'train'), (dval, 'eval')],
                verbose_eval          = 500
            )

    score_, iter_ = bst.best_score, bst.best_iteration
    bst.predict()
    test_preds = bst.predict(xgb.DMatrix(X_train), num_iteration = iter_)
    preds += test_preds
preds=preds / n_folds
    
u=xgb_model.predict(X_test).reshape(3000,1)
u.shape
u=u.reshape(3000,1)
u.shape
x_train, x_test, y_train, y_test = train_test_split( X_train, Y_train, test_size=0.2, random_state=42)
import lightgbm as lgb
hyper_params = {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'regression',
    'metric': 'l2',
    'learning_rate': 0.005,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.7,
    'bagging_freq': 10,
    'verbose': 0,
    "max_depth": 8,
    "num_leaves": 80, 
    "max_bin": 512,
    "num_iterations": 600,
    "n_estimators": 1000
}

gbm = lgb.LGBMRegressor(**hyper_params)

gbm.fit(x_train, y_train,
        eval_set=[(x_test, y_test)],
        eval_metric='l1',
        early_stopping_rounds=100)
y_pr = gbm.predict(x_test)
print(100*max(0,1-np.sqrt(mean_squared_error(y_test,y_pr))))
ylight = gbm.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ylight
sub.to_csv('light_submission.csv',index=False)
#accuracy = 81.24<jupyter_output><empty_output><jupyter_text>Cat boost<jupyter_code>from catboost import CatBoostRegressor
from ml_modules.custom_estimator import Estimator
from ml_modules.custom_fold_generator import FoldScheme

est = Estimator(model=CatBoostRegressor(eval_metric='RMSE', n_estimators=20000, od_type='Iter', 
                    od_wait=200, colsample_bylevel=0.7, max_depth=6, learning_rate=0.1),
              early_stopping_rounds=200, random_state=50, validation_scheme=FoldScheme.KFold,
              eval_metric='rmse', task_type='regression', scoring_metric=rmse, n_splits=5,
              categorical_features_indices=cat_indices)

train_preds = est.fit_transform(x_train, y_train)
cat = CatBoostRegressor(n_estimators = 20000,loss_function = 'RMSE',
                    eval_metric = 'RMSE',od_type='Iter', od_wait=200, colsample_bylevel=0.7, 
                          max_depth=6, learning_rate=0.1)
x_train, x_test, y_train, y_test = train_test_split( X_train, Y_train, test_size=0.2, random_state=42)

cat.fit(x_train,y_train,use_best_model=True,eval_set=(x_test,y_test))
ycat = cat.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ycat
sub.to_csv('cat_submission.csv',index=False)
ens = ylight*0.6 +ycat*0.2 +yxgbreg*0.2
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ens
sub.to_csv('ensemble_submission.csv',index=False)<jupyter_output><empty_output><jupyter_text>XGBRFRegressor<jupyter_code>from xgboost import XGBRFRegressor
regressor=XGBRFRegressor()
random_cv = RandomizedSearchCV(estimator=regressor,
            param_distributions=hyperparameter_grid,
            cv=5, n_iter=60,
            scoring = 'neg_mean_absolute_error',n_jobs = 4,
            verbose = 5, 
            return_train_score = True,
            random_state=42)
random_cv.fit(X_train,Y_train)<jupyter_output><empty_output><jupyter_text>Ridge<jupyter_code>%%time
ridge=Ridge()
parameters={'alpha':[1e-10,1e-8,1e-3,1,5,10,20,30,45,55,100,110,120,130,150,1000,2000,4000,6000,7000,8000,9000]}
ridge_regressor=GridSearchCV(ridge,parameters,scoring='neg_mean_squared_error',cv=10)
ridge_regressor.fit(X_train,Y_train)
ridge_regressor.best_params_
yridge = ridge_regressor.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = yridge
sub.to_csv('ridge_submission.csv',index=False)
#accuracy = 81.28<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>%%time
lasso=Lasso()
parameters={'alpha':[1e-10,1e-8,1e-4,1e-3,1e-2,1,5,10,20,30,35,40,55,100]}
lasso_regressor=GridSearchCV(lasso,parameters,scoring='neg_mean_squared_error',cv=12)

lasso_regressor.fit(X_train,Y_train)
lasso_regressor.best_params_
ylasso = lasso_regressor.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ylasso
sub.to_csv('lasso_submission.csv',index=False)

#accu = 81.24<jupyter_output><empty_output><jupyter_text>ElasticNet<jupyter_code>enet=ElasticNet()
parameters={'alpha':[1e-8,1e-3,1e-2,0,1,2,3,4,5,10,20,30,35,40],
           'l1_ratio':[0.2,0.5,0.8,1],
           'max_iter':[1000,500,1500]}
elasticnet_regressor=GridSearchCV(enet,parameters,
                                  scoring='neg_mean_squared_error',cv=5)

elasticnet_regressor.fit(X_train,Y_train)
elasticnet_regressor.best_params_
ypred = elasticnet_regressor.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ypred
sub.to_csv('elastic_submission.csv',index=False)<jupyter_output><empty_output><jupyter_text>RandomForest<jupyter_code>random_grid = {'n_estimators': [100,200, 400, 600, 800, 1000, 1200, 1400,1800],
               'max_features':['auto', 'sqrt','log2'],
               'max_depth': [7,10, 20, 30, 40, 50, 60, 70, 80, 90, 100, None],
               'min_samples_split':  [2, 5, 10],
               'min_samples_leaf': [1, 2, 4,10],
               'bootstrap':[True, False]}
rf = RandomForestRegressor()
rf_random = RandomizedSearchCV(estimator = rf, 
                               param_distributions = random_grid, 
                               n_iter = 50, cv = 3, verbose=2, 
                               random_state=42,n_jobs = -1)
rf_random.fit(X_train,Y_train)
rf_random.best_params_
'''rf =RandomForestRegressor(n_estimators=800,
                          min_samples_split=10,max_features='log2',
                         min_samples_leaf=10,max_depth=7,
                         bootstrap=True)'''
rf = RandomForestRegressor(n_estimators=100,min_samples_leaf=10,
                          min_samples_split=2,max_features='sqrt',
                          max_depth=7,bootstrap=True)
rf.fit(X_train,Y_train)
yrf = rf.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = yrf
sub.to_csv('randomforest_submission.csv',index=False)


#accuracy = 81.225
#accuracy = 81.20658
ypred= rf_random.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ypred
sub.to_csv('randomforest_submission.csv',index=False)
ensemble = ylasso*0.90 +yridge*.10

sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ensemble
sub.to_csv('ensemble_submission.csv',index=False)

#accuracy = 81.2612<jupyter_output><empty_output><jupyter_text>GradientBoostingRegressor<jupyter_code>%%time

random_grid = {'n_estimators': [90,100,200, 400, 600, 800, 1000, 1200, 1400],
               'loss': ['ls', 'lad', 'huber', 'quantile'],
               'max_depth': [3,7,10, 20, 30, 50, 60, 80, 90, 100, None],
               'min_samples_split':  [2, 5, 10],
               'min_samples_leaf': [1, 2, 4,10],
               'learning_rate': [0.01,0.1,0.2,0.02]
              }
grad = GradientBoostingRegressor()
rf_random = RandomizedSearchCV(estimator = grad, 
                               param_distributions = random_grid, 
                               n_iter = 50, cv = 3, verbose=1, 
                               random_state=42,n_jobs = -1)
rf_random.fit(X_train,Y_train)
rf_random.best_params_
grad = GradientBoostingRegressor(n_estimators=500,min_samples_split=2,
                                min_samples_leaf=10,max_depth=7,
                                loss='ls',learning_rate=0.01,verbose=1)
grad.fit(X_train,Y_train)
ygrad= grad.predict(X_test)
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ygrad
sub.to_csv('grad_submission.csv',index=False)

#80.65<jupyter_output><empty_output><jupyter_text>BayesianRidge<jupyter_code>param = { 'n_iter':[200,400,100,150,500],
          'lambda_1':[1e-06,1e-04,1e-02,1e-01,0],
          'alpha_1': [1e-06,1e-04,1e-02,1e-01,0],
          'lambda_2':[1e-06,1e-04,1e-02,1e-01,0],
          'alpha_2': [1e-06,1e-04,1e-02,1e-01,0]}
bay = BayesianRidge()
rf_random = RandomizedSearchCV(estimator = bay, 
                               param_distributions = param, 
                               n_iter = 100, cv = 10, verbose=0.5, 
                               random_state=42,n_jobs = -1)
rf_random.fit(X_train,Y_train)
print(rf_random.best_params_)
bay = BayesianRidge(n_iter= 100, lambda_2= 1e-04, 
                    lambda_1= 0.1, alpha_2=0.1, alpha_1= 1e-6,verbose=2,normalize=True,compute_score=True)
bay.fit(X_train,Y_train)

#accuarcy default = 81.26082

ybay = bay.predict(X_test)

sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ybay
sub.to_csv('bay_submission.csv',index=False)
ensemble = ybay*0.60 +yridge*.20 + ylasso*.20
 
sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ybay
sub.to_csv('bay_submission.csv',index=False)

from sklearn.neural_network import MLPRegressor
MLP = MLPRegressor()
MLP.fit(X_train,Y_train)
ymlp = MLP.predict(X_test)

sub = pd.DataFrame()
sub['Employee_ID'] = test['Employee_ID']
sub['Attrition_rate'] = ymlp
sub.to_csv('mlp_submission.csv',index=False)

n_folds = 5

from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
def rmsle_cv(model):
    kf = KFold(n_folds, shuffle=True, random_state=42).get_n_splits(train.values)
    rmse= np.sqrt(-cross_val_score(model, X_train, Y_train, 
                                   scoring="neg_mean_squared_error", cv = kf))
    return(rmse)
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, models):
        self.models = models
        
    # we define clones of the original models to fit the data in
    def fit(self, X, y):
        self.models_ = [clone(x) for x in self.models]
        
        # Train cloned base models
        for model in self.models_:
            model.fit(X, y)

        return self
    
    #Now we do the predictions for cloned models and average them
    def predict(self, X):
        predictions = np.column_stack([
            model.predict(X) for model in self.models_
        ])
        return np.mean(predictions, axis=1)   
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
    def __init__(self, base_models, meta_model, n_folds=5):
        self.base_models = base_models
        self.meta_model = meta_model
        self.n_folds = n_folds
   
    # We again fit the data on clones of the original models
    def fit(self, X, y):
        self.base_models_ = [list() for x in self.base_models]
        self.meta_model_ = clone(self.meta_model)
        kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
        
        # Train cloned base models then create out-of-fold predictions
        # that are needed to train the cloned meta-model
        out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))
        for i, model in enumerate(self.base_models):
            for train_index, holdout_index in kfold.split(X, y):
                instance = clone(model)
                self.base_models_[i].append(instance)
                instance.fit(X[train_index], y[train_index])
                y_pred = instance.predict(X[holdout_index])
                out_of_fold_predictions[holdout_index, i] = y_pred
                
        # Now train the cloned  meta-model using the out-of-fold predictions as new feature
        self.meta_model_.fit(out_of_fold_predictions, y)
        return self
   
    #Do the predictions of all base models on the test data and use the averaged predictions as 
    #meta-features for the final prediction which is done by the meta-model
    def predict(self, X):
        meta_features = np.column_stack([
            np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)
            for base_models in self.base_models_ ])
        return self.meta_model_.predict(meta_features)
averaged_models = AveragingModels(models = (ridge ,bay, lasso))

score = rmsle_cv(averaged_models)
print(" Averaged base models score: {:.4f} ({:.4f})\n".format(score.mean(), score.std()))
stacked_averaged_models = StackingAveragedModels(base_models = (bay,ridge),
                                                 meta_model = lasso)

score = rmsle_cv(stacked_averaged_models)
print("Stacking Averaged models score: {:.4f} ({:.4f})".format(score.mean(), score.std()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13666648899312403, 'fraction_numerical': 0.03971003677842332, 'mean_word_length': 4.054418103448276, 'pattern_counts': {'":': 5, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '255370', 'license': 'MIT', 'n_tokens_mistral': 7491, 'n_tokens_neox': 6907, 'n_words': 1034, 'path': 'employee attrition rate/Untitled.ipynb', 'repo_name': 'harsh204016/Hackathon-and-Competitions', 'size': '18761'}