<jupyter_start><jupyter_text>--- Model 1: Linear Learner with `precision_at_target_recall` model selection criteria train<jupyter_code>from sagemaker import LinearLearner

# specify an output path
prefix = 'LinearLearner'
output_path = 's3://{}/{}'.format(bucket, prefix)

# instantiate LinearLearner
linear_recall = LinearLearner(role=role,
                              train_instance_count=1, 
                              train_instance_type='ml.c4.xlarge',
                              predictor_type='binary_classifier',
                              output_path=output_path,
                              sagemaker_session=sagemaker_session,
                              epochs=15,
                              binary_classifier_model_selection_criteria='precision_at_target_recall', # target recall
                              target_recall=0.9) # 90% recall
train = mailout_joint_dataset_imputed_df[mailout_joint_dataset_imputed_df['dataset']==1]
train_labels = train['RESPONSE']
train = train.drop('dataset',axis='columns') 
train = train.drop('RESPONSE',axis='columns') 

formatted_train_data = linear_recall.record_set(train.to_numpy().astype('float32'), labels=train_labels.to_numpy().astype('float32') )

#del train, train_labels
%%time 
# train the estimator on formatted training data
linear_recall.fit(formatted_train_data)

%%time 
# deploy and create a predictor
recall_predictor = linear_recall.deploy(initial_instance_count=1, instance_type='ml.t2.medium')

print('Metrics for tuned (recall), LinearLearner.\n')

# get metrics for tuned predictor
metrics = evaluate(recall_predictor, 
                   train.to_numpy().astype('float32'), 
                   train_labels.to_numpy().astype('int'), 
                   verbose=True)<jupyter_output>Metrics for tuned (recall), LinearLearner.

prediction (col)    0.0   1.0
actual (row)                 
0                 25645  9013
1                    42   394

Recall:     0.904
Precision:  0.042
Accuracy:   0.742<jupyter_text>Predict test cases<jupyter_code>test = mailout_joint_dataset_imputed_df[mailout_joint_dataset_imputed_df['dataset']==0]
 
test = test.drop('dataset',axis='columns') 
test = test.drop('RESPONSE',axis='columns') 
test
prediction_batches = [recall_predictor.predict(batch) for batch in np.array_split(test.to_numpy().astype('float32'), 100)]

# LinearLearner produces a `predicted_label` for each data point in a batch
# get the 'predicted_label' for every point in a batch
test_preds = np.concatenate([np.array([x.label['predicted_label'].float32_tensor.values[0] for x in batch]) 
                             for batch in prediction_batches])

llearner_recall = pd.DataFrame({'LNR':test.index ,'RESPONSE':  test_preds })
llearner_recall.to_csv("../submissions/llearner_recall.csv",sep=",", index=False)
# delete the predictor endpoint 
delete_endpoint(recall_predictor)<jupyter_output>Deleted linear-learner-2020-06-22-17-20-25-487<jupyter_text>TEST SCORE on Kaggle: `0.67178`Evaluated on `Kaggle` ---  Model 2:  Linear Learner with `precision_at_target_recall` model selection criteria **with BALANCED weights matrix**We want the algorithm to choose a weight so that errors in classifying negative vs. positive examples have equal impact on training loss, specify balanced."<jupyter_code>linear_balanced = LinearLearner(role=role,
                                train_instance_count=1, 
                                train_instance_type='ml.c4.xlarge',
                                predictor_type='binary_classifier',
                                output_path=output_path,
                                sagemaker_session=sagemaker_session,
                                epochs=15,
                                binary_classifier_model_selection_criteria='precision_at_target_recall', # target recall
                                target_recall=0.9,
                                positive_example_weight_mult='balanced')
%%time 
# train the estimator on formatted training data
linear_balanced.fit(formatted_train_data)

%%time 
# deploy and create a predictor
linear_balanced = linear_balanced.deploy(initial_instance_count=1, instance_type='ml.t2.medium')

print('Metrics for tuned (recall), LinearLearner.\n')

# get metrics for tuned predictor
metrics = evaluate(linear_balanced, 
                   train.to_numpy().astype('float32'), 
                   train_labels.to_numpy().astype('int'), 
                   verbose=True)

prediction_batches = [linear_balanced.predict(batch) for batch in np.array_split(test.to_numpy().astype('float32'), 100)]

# LinearLearner produces a `predicted_label` for each data point in a batch
# get the 'predicted_label' for every point in a batch
test_preds = np.concatenate([np.array([x.label['predicted_label'].float32_tensor.values[0] for x in batch]) 
                             for batch in prediction_batches])

llearner_recall = pd.DataFrame({'LNR':test.index ,'RESPONSE':  test_preds })
llearner_recall.to_csv("../submissions/llearner_recall_balanced.csv",sep=",", index=False)<jupyter_output><empty_output><jupyter_text>Kaggle score: `0.62746`<jupyter_code># delete the predictor endpoint 
delete_endpoint(linear_balanced)<jupyter_output>Deleted linear-learner-2020-06-22-17-34-32-355
{'fraction_non_alphanumeric': 0.09979091427485269, 'fraction_numerical': 0.023759741494012544, 'mean_word_length': 3.1728786677240284, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '101207', 'license': 'MIT', 'n_tokens_mistral': 1719, 'n_tokens_neox': 1557, 'n_words': 304, 'path': 'Supervised/01_LinearLerner.ipynb', 'repo_name': 'vaxherra/Arvato-Customer-Segmentation', 'size': '5261'}