<jupyter_start><jupyter_text>Things to do to improve performance.So the first submission didn't score as well as the train set, not surprising!Lets try and;1. Extract more of the text from the html, the body and title aren't enough it seems.2. Over/undersample and see if I improve performance with either strategy.<jupyter_code>import numpy as np
import pandas as pd
import os
from bs4 import BeautifulSoup
import nltk
from nltk import wordpunct_tokenize
from nltk.stem.snowball import EnglishStemmer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.pipeline import make_pipeline, make_union
from tpot.builtins import StackingEstimator
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import FunctionTransformer
from sklearn.model_selection import train_test_split
import tldextract
from copy import copy

data_dir = "../data/2018-08-10_AV_Innoplexus/"
source_df = pd.read_csv(data_dir+'train_df_all_tokens.csv')
source_df.sample(10)<jupyter_output><empty_output><jupyter_text>Lets transform! And then train.<jupyter_code>vectorizer = TfidfVectorizer(input='content', analyzer='word')
svd = TruncatedSVD(n_components=500, n_iter=5, random_state=27)
word_vector_array = vectorizer.fit_transform(source_df['all_tokens'])
svd_array = svd.fit_transform(word_vector_array)
vector_features = vectorizer.get_feature_names()
eigen_features = [vector_features[i] for i in svd.components_[0].argsort()[::-1]][:500]

train_df = pd.DataFrame(svd_array,columns=eigen_features)
train_df['Tag'] = source_df['Tag']
    
tags = train_df['Tag'].unique().tolist()
tags.sort()

tag_dict = {key: value for (key, value) in zip(tags,range(len(tags)))}
tag_dict_inverted = {v: k for k, v in tag_dict.items()}

train_df['Tag_encoded'] = train_df['Tag'].map(tag_dict)
train_df = train_df.drop('Tag',axis=1)
train_df.sample(10)
exported_pipeline = make_pipeline(
    make_union(
        FunctionTransformer(copy),
        StackingEstimator(estimator=LogisticRegression(C=0.001, dual=True, penalty="l2"))
    ),
    ExtraTreesClassifier(
        bootstrap=False, criterion="entropy",
        max_features=0.35000000000000003, min_samples_leaf=1,
        min_samples_split=2, n_estimators=100, verbose=2,
        n_jobs=2
    )
)
x_cols = [x for x in train_df.columns if x != "Tag_encoded"]
X_train, X_test, y_train, y_test = train_test_split(
    train_df[x_cols],
    train_df['Tag_encoded'],
    test_size=0.33
)
print("Fitting the model...")
exported_pipeline.fit(X_train, y_train)
print("Done!")
#Sanity qual check
from sklearn.metrics import confusion_matrix, f1_score
preds = exported_pipeline.predict(X_test)
conf_mat = confusion_matrix(y_test, preds)
conf_mat
model_f1 = f1_score(y_test, preds, average='weighted')
model_f1
test_df = pd.read_csv(data_dir+"test_df_all_tokens.csv")
test_vector_arr = vectorizer.transform(test_df['all_tokens'])
test_svd_array = svd.transform(test_vector_arr)
test_df_predict = pd.DataFrame(test_svd_array,columns=eigen_features)
test_df_predict.head()
submission_preds = exported_pipeline.predict(test_df_predict)
submission_df = pd.DataFrame(data=submission_preds,columns=['Tag'],index=test_df_predict.index)
submission_df['Tag'] = submission_df['Tag'].map(tag_dict_inverted)
submission_df['Webpage_id'] = test_df['Webpage_id'].values.tolist()
col_order = ["Webpage_id","Tag"]
submission_df = submission_df[col_order]
submission_df.to_csv('submission_02.csv',index=False)
submission_df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10715272024302679, 'fraction_numerical': 0.018227009113504555, 'mean_word_length': 7.269406392694064, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '203787', 'license': 'MIT', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1255, 'n_words': 249, 'path': '2018-08-10_AV_Innoplexus/08. Submission 02 - Training and Optimization.ipynb', 'repo_name': 'DataAndDeadlifts/Hackathons', 'size': '3621'}