<jupyter_start><jupyter_text>Predict tags on StackOverflow with linear models<jupyter_code>import nltk
nltk.download('stopwords')
nltk.download('punkt')
from nltk.corpus import stopwords
from ast import literal_eval
import pandas as pd
import numpy as np
def read_data(filename):
    data = pd.read_csv(filename, sep='\t')
    data['tags'] = data['tags'].apply(literal_eval)
    return data
train = read_data('data/train.tsv')
validation = read_data('data/validation.tsv')
test = pd.read_csv('data/test.tsv', sep='\t')
train.head()
X_train, y_train = train['title'].values, train['tags'].values
X_val, y_val = validation['title'].values, validation['tags'].values
X_test = test['title'].values
import re
from nltk.tokenize import word_tokenize
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
j = 'BOBBY@FIRMINO c# c++ '
j.lower()
re.sub(REPLACE_BY_SPACE_RE," ",j)
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-z +#_]')
STOPWORDS = set(stopwords.words('english'))

def text_prepare(text):

    text = text.lower()
    text = re.sub(REPLACE_BY_SPACE_RE," ",text)
    text = re.sub(BAD_SYMBOLS_RE,"",text)
    text = [i for i in text.split() if i not in STOPWORDS]
    return " ".join(text)
text_prepare(j)
X_train = [text_prepare(x) for x in X_train]
X_val = [text_prepare(x) for x in X_val]
X_test = [text_prepare(x) for x in X_test]
import itertools
import nltk
tags_counts = {}
y_train_list=list(itertools.chain.from_iterable(y_train.tolist()))
words_counts = {}
def most_common(X):
    X_train_join = " ".join(X)
    X_train_tokens = X_train_join.split()
    dist = nltk.FreqDist(X_train_tokens)
    return dist.most_common(5000)
jj= most_common(X_train)
words_counts = dict(most_common(X_train))
tags_counts = dict(most_common(y_train_list))
# #sorted(tags_counts.keys())
# 'c #' in words_counts
# X_train_join = " ".join(y_train_list)
# X_train_tokens = nltk.word_tokenize(X_train_join)
# dist = nltk.FreqDist(X_train_join)
# #dist.most_common(5000)
# X_train_tokens
words_counts
most_common_tags = sorted(tags_counts.items(), key=lambda x: x[1], reverse=True)[:3]
most_common_words = sorted(words_counts.items(), key=lambda x: x[1], reverse=True)[:3]
most_common_tags
from sklearn.feature_extraction.text import CountVectorizer
a=list(words_counts.keys())
DICT_SIZE = 5000
WORDS_TO_INDEX = {k: v for v, k in enumerate(a)}
ALL_WORDS = WORDS_TO_INDEX.keys()

def my_bag_of_words(text, words_to_index, dict_size):

    #result_vector = np.zeros(dict_size)
    text = [text]
    vectorizer = CountVectorizer(vocabulary = words_to_index,max_features = dict_size)
    vector = vectorizer.transform(text)
    result_vector=vector.toarray()
    return result_vector


from scipy import sparse as sp_sparse
X_train_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_train])
X_val_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_val])
X_test_mybag = sp_sparse.vstack([sp_sparse.csr_matrix(my_bag_of_words(text, WORDS_TO_INDEX, DICT_SIZE)) for text in X_test])
print('X_train shape ', X_train_mybag.shape)
print('X_val shape ', X_val_mybag.shape)
print('X_test shape ', X_test_mybag.shape)
row = X_train_mybag[10].toarray()[0]
non_zero_elements_count = np.count_nonzero(row)
from sklearn.feature_extraction.text import TfidfVectorizer
def tfidf_features(X_train, X_val, X_test):

    tfidf_vectorizer = TfidfVectorizer(min_df=5,max_df = 0.9,ngram_range=(1, 2),token_pattern = '(\S+)')
    tfidf_vectorizer.fit(X_train)
    X_train = tfidf_vectorizer.transform(X_train)
    X_val = tfidf_vectorizer.transform(X_val)
    X_test = tfidf_vectorizer.transform(X_test)
    return X_train, X_val, X_test, tfidf_vectorizer.vocabulary_
X_train_tfidf, X_val_tfidf, X_test_tfidf, tfidf_vocab = tfidf_features(X_train, X_val, X_test)
tfidf_reversed_vocab = {i:word for word,i in tfidf_vocab.items()}
'c++' in tfidf_vocab
'c#' in tfidf_vocab<jupyter_output><empty_output><jupyter_text>MultiLabel classifier<jupyter_code>from sklearn.preprocessing import MultiLabelBinarizer
mlb = MultiLabelBinarizer(classes=sorted(tags_counts.keys()))
y_train = mlb.fit_transform(y_train)
y_val = mlb.fit_transform(y_val)
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import LogisticRegression, RidgeClassifier
def train_classifier(X_train, y_train):

    model = OneVsRestClassifier(LogisticRegression(penalty='l1', C = 1)).fit(X_train, y_train)
    return model
classifier_mybag = train_classifier(X_train_mybag, y_train)
classifier_tfidf = train_classifier(X_train_tfidf, y_train)
sorted_coef_index = classifier_tfidf.coef_[0].argsort()
sorted_coef_index.shape
y_val_predicted_labels_mybag = classifier_mybag.predict(X_val_mybag)
y_val_predicted_scores_mybag = classifier_mybag.decision_function(X_val_mybag)

y_val_predicted_labels_tfidf = classifier_tfidf.predict(X_val_tfidf)
y_val_predicted_scores_tfidf = classifier_tfidf.decision_function(X_val_tfidf)
y_val_pred_inversed = mlb.inverse_transform(y_val_predicted_labels_tfidf)
y_val_inversed = mlb.inverse_transform(y_val)
for i in range(10):
    print('Title:\t{}\nTrue labels:\t{}\nPredicted labels:\t{}\n\n'.format(
        X_val[i],
        ','.join(y_val_inversed[i]),
        ','.join(y_val_pred_inversed[i])
    ))
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score 
from sklearn.metrics import average_precision_score
from sklearn.metrics import recall_score
def print_evaluation_scores(y_val, predicted):
    x = accuracy_score(y_val, predicted)
    y_macro = f1_score(y_val, predicted,average='macro')
    y_micro = f1_score(y_val, predicted,average='micro')
    y_weighted = f1_score(y_val, predicted,average='weighted')
    z_macro = average_precision_score(y_val, predicted,average='macro')
    z_micro = average_precision_score(y_val, predicted,average='micro')
    z_weighted = average_precision_score(y_val, predicted,average='weighted')
    return print(x,y_macro,y_micro,y_weighted,z_macro,z_micro,z_weighted)
print('Bag-of-words')
print_evaluation_scores(y_val, y_val_predicted_labels_mybag)
print('Tfidf')
print_evaluation_scores(y_val, y_val_predicted_labels_tfidf)
from metrics import roc_auc
%matplotlib inline
n_classes = len(tags_counts)
roc_auc(y_val, y_val_predicted_scores_mybag, n_classes)
n_classes = len(tags_counts)
roc_auc(y_val, y_val_predicted_scores_tfidf, n_classes)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1185480165415837, 'fraction_numerical': 0.0053606984224230355, 'mean_word_length': 7.121890547263682, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '459500', 'license': 'MIT', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2524, 'n_words': 413, 'path': 'StackOverflow-MultilabelClassification.ipynb', 'repo_name': 'supersuyash/project', 'size': '6529'}