<jupyter_start><jupyter_text>SGD Classifier for 500 tags Analysis model<jupyter_code>import pandas as pd
import numpy as np
import re
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from wordcloud import WordCloud
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem.snowball import SnowballStemmer
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsRestClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import f1_score,precision_score,recall_score
%%time
## sample_500k is a sample from main dataset 
row_data = pd.read_csv("sample_500k.csv")
row_data.head()
sample_data = row_data.sample(100000)
# To drop duplicate rows based on title, body, and tags
sample_data = sample_data.drop_duplicates(['Title', 'Body', 'Tags'])
stop_words = {'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn', "hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn', "mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", 'won', "won't", 'wouldn', "wouldn't"}
import nltk
nltk.download('punkt')
def text_preprocess(sample_data):
    preprocessed_data_list=[]
    questions_with_code=0
    len_before=0
    len_after=0
    counter=0
    preprocess_data = pd.DataFrame(columns=['question','code','tags','word_count_before','word_count_after','is_code'])
    # Stopwords are {"the","a","also",etc} they did not contribute any thing then we will delete them
    #stop_words = set(stopwords.words('english'))

    # Stemmer is converting like: eating --> eat or running --> run
    stemmer = SnowballStemmer("english")
    
    for row in tqdm(sample_data.iterrows()):
        
        # Extract title and repeat it 3 times, question, and tags from each row
        title, question, tags = 3*(' '+row[1][2]), row[1][3], row[1][4]
        
        code_flag = 0
        
        # Check if there is any code in the question then make code flag 1
        # Increase question  with code count to see how many question we have with code
        if '<code>' in question:
            questions_with_code += 1
            code_flag = 1
            
        # Add title and question length to see how much we have changed it
        question_before_length = len(question)+len(title)
        len_before += question_before_length
        
        # To extract all code from <code></code> tags and convert it into string
        code = str(re.findall(r'<code>(.*?)</code>', question, flags=re.DOTALL))
        
        # To replace <code>...</code> with space " "
        question=re.sub('<code>(.*?)</code>', '', question, flags=re.MULTILINE|re.DOTALL)
        
        # To remove all HTML tags from questions
        html_cleaner = re.compile('<.*?>')
        question = re.sub(html_cleaner, ' ', str(question.encode("utf-8")))
        
        title=title.encode('utf-8')
        
        # Concatinate title and question
        question=str(title)+" "+str(question)
        
        # To remove all characters instead of alphabets
        question=re.sub(r'[^A-Za-z]+',' ',question)
        
        # First, we convert all text to lowercase and Word tokenize will convert 
        # "This is data preprocessing" --> ["This","is","data","preprocessing"]
        words=word_tokenize(str(question.lower()))
        
        # Removing all single letter and stopwords from question except letter 'c' because c is a prog. language
        # After removing all single words join then with space
        question= ' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))
        
        len_after += len(question)
        
        # Append new datapoint to the dataframe
        preprocess_data.loc[len(preprocess_data.index)] = [question,code,tags,question_before_length,len(question),code_flag]
        counter += 1 
        
        # END OF THE FOR LOOP
    
    # To print some insights 
    print( "Avg. length of questions(Title+Body) before processing: ",(len_before*1.0)/counter)
    print( "Avg. length of questions(Title+Body) after processing: ",(len_after*1.0)/counter)
    print ("Percent of questions containing code: %d"%((questions_with_code*100.0)/counter)) 
    return preprocess_data
start = time.time()
preprocess_data = text_preprocess(sample_data)
preprocess_data.to_csv("preprocessed_3title_100k.csv")
print("Time it takes to run this :",time.time()-start)
preprocess_data.head(2)
def text_splitter(text):
    return text.split()
# binary='true' will give a binary vectorizer
tag_vectorizer = CountVectorizer(tokenizer = text_splitter, binary=True)
multi_label_y = tag_vectorizer.fit_transform(preprocess_data['tags'].values.astype(str))
# make sum column wise
tag_column_sum = multi_label_y.sum(axis=0).tolist()[0]
# To select n number of top tags
def select_top_tags(n):
    
    # To get sotred list (means: tags appear in maximum number of questions come first)
    # top 10: [3711, 15246, 22934, 15324, 1054, 15713, 3720, 24481, 14905, 1897]
    sorted_tags = sorted(range(len(tag_column_sum)), key=lambda i: tag_column_sum[i], reverse=True)
    
    # With this line of code we get tags in our columns which are come in most of the questions
    # we will get shape: (999999, n)
    multi_label_n_y = multi_label_y[:,sorted_tags[:n]]
    return multi_label_n_y
# 
def questions_covered_fn(n):
    multi_label_n_y = select_top_tags(n)
    
    # This line will give us row wise sum of each row  [[1, 2],           [[3],
    #                                                   [4, 3]]     to     [7]]
    row_sum_array = multi_label_n_y.sum(axis=1)

    # Counts the number of non-zero values in the array
    return (np.count_nonzero(row_sum_array==0))
# With this code we checking how much percent questions are explained by how many tags
# Here we are starting from 500 because we think top 500 are most important tags we can't skip them
questions_covered=[]
total_tags=multi_label_y.shape[1]
total_qs=preprocess_data.shape[0]
for i in range(500, total_tags, 100):
    questions_covered.append(np.round(((total_qs-questions_covered_fn(i))/total_qs)*100,3))
fig, ax = plt.subplots()
ax.plot(questions_covered)
xlabel = list(500+np.array(range(-50,450,50))*50)
ax.set_xticklabels(xlabel)
plt.xlabel("Number of tags")
plt.ylabel("Percentage of Questions coverd partially")
plt.grid()
plt.show()
# you can choose any number of tags based on your computing power, minimun is 50(it covers 90% of the tags)
print("with ",5500,"tags we are covering ",questions_covered[50],"% of questions")
multi_label_n_y = select_top_tags(500)
print("number of questions that are not covered :", questions_covered_fn(5500),"out of ", total_qs)
print("Number of tags in sample :", multi_label_y.shape[1])
print("number of tags taken :", multi_label_n_y.shape[1],"-->",round((multi_label_n_y.shape[1]/multi_label_y.shape[1]),3)*100,"%")
total_size=preprocess_data.shape[0]
train_size=int(0.80*total_size)

x_train=preprocess_data.head(train_size)
x_test=preprocess_data.tail(total_size - train_size)

y_train = multi_label_n_y[0:train_size,:]
y_test = multi_label_n_y[train_size:total_size,:]
def text_splitter(text):
    return text.split()
# To get new features with tfidf technique get 200000 features with upto 3-grams
vectorizer = TfidfVectorizer(min_df=0.00009, max_features=200000, smooth_idf=True, norm="l2", tokenizer = text_splitter, sublinear_tf=False, ngram_range=(1,3))
# Apply this vectorizer only on question data column
x_train_multi_label = vectorizer.fit_transform(x_train['question'])
x_test_multi_label = vectorizer.transform(x_test['question'])
# Now check data shapes after featurization
print("Dimensions of train data X:",x_train_multi_label.shape, "Y :",y_train.shape)
print("Dimensions of test data X:",x_test_multi_label.shape,"Y:",y_test.shape)
from joblib import dump
dump(vectorizer, 'stackoverflow_tfidf_vectorizer_3title_500k.pkl')
classifier = OneVsRestClassifier(SGDClassifier(loss='log', alpha=0.00001, penalty='l1'), n_jobs=-1)
start = time.time()
classifier.fit(x_train_multi_label, y_train)
print("Time it takes to run this :",time.time()-start)
dump(classifier, 'stackoverflow_model_3title_100k.pkl')
predictions = classifier.predict(x_test_multi_label)
print("accuracy :",metrics.accuracy_score(y_test,predictions))
print("macro f1 score :",metrics.f1_score(y_test, predictions, average = 'macro'))
print("micro f1 scoore :",metrics.f1_score(y_test, predictions, average = 'micro'))
print("hamming loss :",metrics.hamming_loss(y_test,predictions))
report = metrics.classification_report(y_test, predictions, output_dict=True)
report_df = pd.DataFrame(report).transpose()
report_df.head()
report_df.to_csv("report_3title_100k.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1498879251534938, 'fraction_numerical': 0.021342949030308936, 'mean_word_length': 4.647771051183269, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '482115', 'license': 'MIT', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3377, 'n_words': 931, 'path': 'Models and Tags Analysis notebooks/SGDClassifier with logos for 3title 500tags analysis.ipynb', 'repo_name': 'Pushpendra9350/Stackoverflow-Tag-Predictor', 'size': '10261'}