<jupyter_start><jupyter_text>Split<jupyter_code>X = pd.read_csv('../data/train_input.csv')
y = pd.read_csv('../data/train_output.csv')
X_test = pd.read_csv('../data/test_input.csv')
train_X, test_X, train_y, test_y = split(X["conversation"], y["category"])
unholy_train_X, unholy_train_y = get_unholy_trinity(train_X, train_y)<jupyter_output><empty_output><jupyter_text>First stage<jupyter_code>first_stage = LinearSVC(loss='hinge', dual=False)
first_stage = train(first_stage, train_X, train_y)
first_stage_prediction = first_stage.predict(test_X)
score(first_stage_prediction, test_y)<jupyter_output>0.964
             precision    recall  f1-score   support

     hockey       0.98      0.98      0.98      4170
     movies       0.98      0.99      0.99      4510
        nba       0.99      0.97      0.98      3696
       news       0.93      0.90      0.92      4256
        nfl       0.98      0.98      0.98      3978
   politics       0.93      0.94      0.94      3943
     soccer       0.99      0.99      0.99      4299
  worldnews       0.94      0.96      0.95      4148

avg / total       0.96      0.96      0.96     33000

[[4085   15   19    5   25    1   16    4]
 [   2 4472    1   11    0    6    4   14]
 [  40   18 3573    8   32    3   21    1]
 [   3   21    3 3846   15  191    2  175]
 [  27    5   17    9 3906    4    8    2]
 [   2    1    1  179    1 3715    1   43]
 [  14    7    6    6    9    2 4242   13]
 [   3    7    0   86    0   67   12 3973]]<jupyter_text>Second stage<jupyter_code>from sklearn.svm import SVC
second_stage = SVC(kernel='poly', degree=1)
second_stage = train(second_stage, unholy_train_X, unholy_train_y)<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>def merge(X, original_prediction, second_classifier):
    prediction = original_prediction.copy()
    n_documents = len(X)
    indices = [i for i in range(n_documents)
               if prediction[i] in unholy_trinity]
    unholy = [X[X.index[i]] for i in indices]
    unholy_predictions = second_classifier.predict(unholy)
    for i, j in enumerate(indices):
        prediction[j] = unholy_predictions[i]
    return prediction
prediction = merge(test_X, first_stage_prediction, second_stage)
score(prediction, test_y)<jupyter_output><empty_output><jupyter_text>Output<jupyter_code>result = pd.DataFrame({'category' : predicted2})
result.to_csv('../data/test_predict.csv', index = True, header = True, index_label = 'id')
clean_testinput.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10044017607042817, 'fraction_numerical': 0.09563825530212085, 'mean_word_length': 2.3829499323410013, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '333686', 'license': 'MIT', 'n_tokens_mistral': 1042, 'n_tokens_neox': 938, 'n_words': 202, 'path': 'linearSVM/linearSVM-The Unholy Trinity.ipynb', 'repo_name': 'anassinator/reddit-text-classifier', 'size': '2499'}