<jupyter_start><jupyter_text>EDA on features  We don't see any null values present in the training or test data<jupyter_code>print(train.isnull().sum())
test.isnull().sum()
test['ROLE_CODE'].unique()<jupyter_output><empty_output><jupyter_text>The columns contain categorical values. A naive bayes, Decision Tree solution as starting point should be taken.<jupyter_code># !pip install klib
# import klib
df = train.drop(columns=['ACTION'])
train_x = train.drop(columns=['ACTION'])
train_y = train['ACTION'] 
test_x = test.drop(columns=['id'])
# test_y = test['ACTION']
train.head()
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test =  train_test_split(train_x, train_y)<jupyter_output><empty_output><jupyter_text>We have our train and test data  Let's go ahead with models, DT and naive bayes to be applied first<jupyter_code># Logistic Regression
# Not suitable at all due to categorical variables

from sklearn import linear_model as lm
from sklearn.metrics import accuracy_score
model = lm.LogisticRegression()
model.fit(X_train, y_train)
predictions = model.predict_proba(test_x)
# model.score(X_test, y_test)
# accuracy_score(predictions, y_test)

# 0.50 accuracy
test.drop(columns=['id']).shape
print(f"{X_train.shape}, {X_test.shape}, {test.drop(columns=['id']).shape}")
test_x = test.drop(columns=['id'])
X_train.reset_index(drop=True, inplace=True)
y_train.reset_index(drop=True, inplace=True)
# Naive Bayes doesn't work here due to an extra category in test_data

from sklearn import naive_bayes
model = naive_bayes.CategoricalNB()
# clf = model.fit(X_train, y_train)
# print(f'{clf.score(X_test,y_test)}')
# predictions = clf.predict(test_x)
# Decision Tree
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
model = DecisionTreeClassifier(max_depth=30)
clf = model.fit(X_train, y_train)
print(f'{clf.score(X_test,y_test)}')
predictions = clf.predict(test_x)

# 0.679 Score on test dataset
# Random Forest
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(n_estimators = 300)
clf = model.fit(X_train, y_train)
print(f'{clf.score(X_test, y_test)}')
predictions = clf.predict_proba(test_x)

## Score: 0.67333 on Binary Classification 
## Score: 0.85201 on Probabilitistic outputs<jupyter_output>0.946783839863298<jupyter_text>Catboost implementation Catboost is a boosting algorithm. It is better than lightGBM, XGBoost on various tasks or at least on par with them. It's particularly useful here because of it's ability to work with categorical data.  Catboost tutorial [link](https://github.com/catboost/tutorials/blob/master/classification/classification_tutorial.ipynb)<jupyter_code>os.chdir('/kaggle/working')
os.makedirs('/kaggle/working/amazon')
os.listdir('.')
from catboost.utils import create_cd
feature_names = dict()
for column, name in enumerate(train):
    if column == 0:
        continue
    feature_names[column - 1] = name
dataset_dir = './amazon'   
create_cd(
    label=0, 
    cat_features=list(range(1, train.columns.shape[0])),
    feature_names=feature_names,
    output_path=os.path.join(dataset_dir, 'train.cd')
)
X = train.drop(columns=['ACTION'])
y = train.ACTION
cat_features = list(range(0, X.shape[1]))
print(cat_features)
pool1 = Pool(data=X, label=y, cat_features=cat_features)
pool2 = Pool(
    data=os.path.join('/kaggle/input/amazon-employee-access-challenge/', 'train.csv'), 
    delimiter=',', 
    column_description=os.path.join(dataset_dir, 'train.cd'),
    has_header=True
)
pool3 = Pool(data=X, cat_features=cat_features)

# Fastest way to create a Pool is to create it from numpy matrix.
# This way should be used if you want fast predictions
# or fastest way to load the data in python.

X_prepared = X.values.astype(str).astype(object)
# For FeaturesData class categorial features must have type str

pool4 = Pool(
    data=FeaturesData(
        cat_feature_data=X_prepared,
        cat_feature_names=list(X)
    ),
    label=y.values
)

print('Dataset shape')
print('dataset 1:' + str(pool1.shape) +
      '\ndataset 2:' + str(pool2.shape) + 
      '\ndataset 3:' + str(pool3.shape) +
      '\ndataset 4: ' + str(pool4.shape))

print('\n')
print('Column names')
print('dataset 1:')
print(pool1.get_feature_names()) 
print('\ndataset 2:')
print(pool2.get_feature_names())
print('\ndataset 3:')
print(pool3.get_feature_names())
print('\ndataset 4:')
print(pool4.get_feature_names())
# train, test split

from sklearn.model_selection import train_test_split
X_train, X_validation, y_train, y_validation = train_test_split(X, y, train_size=0.7, random_state=1234)
from catboost import CatBoostClassifier
model = CatBoostClassifier(
    iterations=3000,
    random_seed=1,
    learning_rate=0.03,
    loss_function = 'CrossEntropy'
#     custom_loss=['AUC', 'Accuracy']
)
model.fit(
    X_train, y_train,
    cat_features=cat_features,
    eval_set=(X_validation, y_validation),
    verbose=50,
#     plot=True
)
model.get_feature_importance(prettified=True)
# Cross Validation
from catboost import cv

params = {}
params['loss_function'] = 'Logloss'
params['iterations'] = 93
params['custom_loss'] = 'AUC'
params['random_seed'] = 63
params['learning_rate'] = 0.5

cv_data = cv(
    params = params,
    pool = Pool(X, label=y, cat_features=cat_features),
    fold_count=5,
    shuffle=True,
    partition_random_seed=0,
#     plot=True,
    stratified=False,
    verbose=False
)
cv_data
best_value = np.min(cv_data['test-Logloss-mean'])
best_iter = np.argmin(cv_data['test-Logloss-mean'])

print('Best validation Logloss score, not stratified: {:.4f}±{:.4f} on step {}'.format(
    best_value,
    cv_data['test-Logloss-std'][best_iter],
    best_iter)
)
test = pd.read_csv('/kaggle/input/amazon-employee-access-challenge/test.csv')
predictions = model.predict_proba(test_x)
np.unique(predictions, return_counts= True)
predictions = predictions[:,1]
os.chdir('/kaggle/working/')
os.curdir
sol = pd.DataFrame(predictions)


sol = sol.rename(columns={0:'Action'})
sol.index = range(1, 58922,1)
sol = sol.rename_axis('Id')
sol.to_csv('submission.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1300653594771242, 'fraction_numerical': 0.016666666666666666, 'mean_word_length': 5.646036916395222, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '460814', 'license': 'Apache-2.0', 'n_tokens_mistral': 2231, 'n_tokens_neox': 2138, 'n_words': 469, 'path': 'amazon-prediction.ipynb', 'repo_name': 'spoonia1/amzn-accss-challenge', 'size': '6120'}