<jupyter_start><jupyter_text>Create new 'text' and 'end' columns in unlabeled and labeled datasets<jupyter_code>import pandas as pd

LABELED_FILE_NAME = "redo-labeled-run3.csv"
UNLABELED_FILE_NAME = "unlabeled-run3.csv"
OUTPUT_CSV_FILE_PATH = "results_full.csv"
FILES_TO_TAG = [LABELED_FILE_NAME, UNLABELED_FILE_NAME]
SELECTED_COLUMNS = {
    "TITLE": "Title",
    "UPLOADER": "UploaderUsername",
    "DESCRIPTION": "Description"
}
TEXT_COL_NAME = "text"
END_COL_NAME = "end"
for file_name in FILES_TO_TAG:
    preproc_labeled_df = pd.read_csv(file_name, encoding='utf-8')
    for index, row in preproc_labeled_df.iterrows():
        text_col_initial_text = ""
        for key, value in SELECTED_COLUMNS.items():
            text_in_cell = str(preproc_labeled_df.loc[index, value])
            text_col_initial_text += key + ": " + text_in_cell + "\n"
        preproc_labeled_df.at[index, TEXT_COL_NAME] = text_col_initial_text
        preproc_labeled_df.at[index, END_COL_NAME] = 1
    preproc_labeled_df.to_csv(file_name, index=False, encoding='utf-8-sig')<jupyter_output><empty_output><jupyter_text>Download packages, split test and training data<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
from sklearn.svm import SVC, LinearSVC
from sklearn.metrics import classification_report
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import RFE
from sklearn.utils import class_weight
from sklearn import metrics
from sklearn.model_selection import train_test_split
import numpy as np
from scipy.sparse import csr_matrix
import spacy
from spacy.lang.en import English
from spacy.lang.en.stop_words import STOP_WORDS
import sys
import os
import warnings
import pandas as pd
import json
import string
import re
import nltk
import random
import matplotlib.pyplot as plt
from nltk.tokenize import RegexpTokenizer
from scipy.signal import savgol_filter
from kneed import KneeLocator
warnings.filterwarnings("ignore")
from kneed import DataGenerator, KneeLocator
df = pd.read_csv(LABELED_FILE_NAME, encoding='utf-8')
print(df.shape)
#df = df[df['message'].notna()]

# df['perception'] = df.apply(lambda row: 0 if row['reciever_perception'] == 'Truth' else 1, axis=1)
# train = df[(df['train_test'] == 'Train') | (df['train_test'] == 'Validation')]
# test = df[df['train_test'] == 'Test']
train, test = train_test_split(df, test_size=0.2)
print(len(test))<jupyter_output>(423, 17)
85<jupyter_text>Bag of words and test only logreg<jupyter_code>nlp = English()

def is_number(tok):
    try:
        float(tok)
        return True
    except ValueError:
        return False

def spacy_tokenizer(text):
    return [tok.text if not is_number(tok.text) else '_NUM_' for tok in nlp(text)]

vectorizer = CountVectorizer(tokenizer=spacy_tokenizer, stop_words=STOP_WORDS, strip_accents='unicode')
corpus = list(train['text'].str.lower())
X_train = vectorizer.fit_transform(corpus)

vectorizer2 = CountVectorizer(tokenizer=spacy_tokenizer, vocabulary=vectorizer.vocabulary_, stop_words=STOP_WORDS, strip_accents='unicode')
X_test = vectorizer2.fit_transform(list(test['text'].str.lower()))
X_train = csr_matrix(X_train)
y_train = train['IsCollab']
X_test = csr_matrix(X_test)
y_test = test['IsCollab']
logreg = LogisticRegression(class_weight = 'balanced', max_iter=1000)
logreg.fit(X_train, y_train)
y_pred = logreg.predict(X_test)
print(classification_report(y_test, y_pred, digits=3))<jupyter_output>precision    recall  f1-score   support

           0      0.804     0.771     0.787        48
           1      0.718     0.757     0.737        37

    accuracy                          0.765        85
   macro avg      0.761     0.764     0.762        85
weighted avg      0.767     0.765     0.765        85<jupyter_text>Extract tfidf features and test a lot of classifiers<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
def tfidf_features(df):
    """
        X_train, X_test — samples        
        return TF-IDF vectorized representation of each sample and vocabulary
    """
    # Create TF-IDF vectorizer with a proper parameters choice
    # Fit the vectorizer on the train set
    # Transform the train, test set and return the result
    
    
    tfidf_vectorizer = TfidfVectorizer(ngram_range=(1,2), max_df=0.9, min_df=5, token_pattern='(\S+)')

    tfidfs = tfidf_vectorizer.fit_transform(df['text'])
    df_tfidf = pd.DataFrame(tfidfs.toarray(), columns=tfidf_vectorizer.get_feature_names())
    return df_tfidf

#df1 = 
df_tfidf = tfidf_features(df)
#print(tfidf_reversed_vocab)
#for i in range(df_tfidf.shape[1]):
#  df['tfidf_{}'.format(i)] = df_tfidf[:, i].toarray()[0]

import csv
print(df_tfidf.shape)
#tf_idf_matrix = df_tfidf.todense()
#tf_idf_df = pd.DataFrame(tf_idf_matrix)
 
#tf_idf_df.columns = tfidf_reversed_vocab
data_top = df_tfidf.head() 
    
# display 
print(data_top)
df_withtfidf = pd.concat([df, df_tfidf], axis=1)

data_top = df_withtfidf.head() 
    
# display 
print(data_top)<jupyter_output>(423, 3055)
   "hi"  "hi" on...   #3         &  & cape  & film  & film.  & georgie?  \
0   0.0         0.0  0.0  0.000000     0.0     0.0      0.0         0.0   
1   0.0         0.0  0.0  0.082391     0.0     0.0      0.0         0.0   
2   0.0         0.0  0.0  0.000000     0.0     0.0      0.0         0.0   
3   0.0         0.0  0.0  0.000000     0.0     0.0      0.0         0.0   
4   0.0         0.0  0.0  0.000000     0.0     0.0      0.0         0.0   

   & jimmy  & my  & nz    & sony  & the  & vv  'ask  'ask ben'  'ben  \
0      0.0   0.0   0.0  0.000000    0.0   0.0   0.0        0.0   0.0   
1      0.0   0.0   0.0  0.083265    0.0   0.0   0.0        0.0   0.0   
2      0.0   0.0   0.0  0.000000    0.0   0.0   0.0        0.0   0.0   
3      0.0   0.0   0.0  0.000000    0.0   0.0   0.0        0.0   0.0   
4      0.0   0.0   0.0  0.000000    0.0   0.0   0.0        0.0   0.0   

   'ben brown'  'feel  'feel good  'like'    (funky  (funky jazz  \
0          0.0    0.0         0.0   [...]<jupyter_text>All preds<jupyter_code>main_df = df_withtfidf
feature_start, num_ftrs = 17,1496
print(df_withtfidf.shape)
all_classifiers = {}
all_scalers = {}
pd.set_option("display.max_rows", None, "display.max_columns", None)
for feature in ['IsCollab','IsGaming']:
  print('='*80)
  print('For {}:'.format(feature))
  print('Before: {}'.format(len(main_df)), end=' ')
  df = main_df[main_df[feature].notna()]
  print('After: {}'.format(len(df)))
  print('\nClass distribution:')
  print(df[feature].value_counts())
  print()
  vc = dict(df[feature].value_counts())
  minority_class = min(vc, key=vc.get)
  X = df.iloc[:, feature_start:feature_start+num_ftrs]
  y = df[feature]
  X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
  scaler = StandardScaler()
  X_train = scaler.fit_transform(X_train)
  X_test = scaler.transform(X_test)

  classifiers = {'logreg': LogisticRegression(class_weight='balanced'), 
               'knn': KNeighborsClassifier(), 
               'gaussianNB': GaussianNB(),
               'bernoulliNB': BernoulliNB(),
               'adaboost': AdaBoostClassifier(), 
               'grad-boost': GradientBoostingClassifier(),
               'dec-tree': DecisionTreeClassifier(), 
               'linear-svc': LinearSVC(class_weight='balanced'), 
               'c-svc': SVC(class_weight='balanced'), 
               'lda': LinearDiscriminantAnalysis()}
  report = []
  all_classifiers[feature] = classifiers
  all_scalers[feature] = scaler

  

  for method, clf in classifiers.items():
    clf.fit(X_train, y_train)
    y_pred = clf.predict(X_test)
    report.append([method, 
                 metrics.accuracy_score(y_test, y_pred), 
                 metrics.f1_score(y_test, y_pred),
                 metrics.precision_score(y_test, y_pred),
                 metrics.recall_score(y_test, y_pred), 
                 metrics.f1_score(y_test, y_pred, average='macro'),
                 classification_report(y_test, y_pred, output_dict=True)[str(minority_class)]['f1-score']])
    
    # print('-'*60)
    # print('{:^60}'.format(method))
    # print('-'*60)
    # print(classification_report(y_test, y_pred, digits=3))
    # print()
    
  report = pd.DataFrame(report, columns = ['method', 'accuracy', 'f1', 'precision', 'recall', 'macro-f1', 'minority-f1'])
  print(report)
  print('='*80, '\n\n')

best_classifiers = {'IsCollab': 'knn', 
                    'IsGaming': 'knn'}<jupyter_output><empty_output><jupyter_text>Predicting for new data: generate tfidf features again<jupyter_code>#predicting for new data
#1. generate tfidf features again for your unlabeled dataset
unlabeled_df = pd.read_csv(UNLABELED_FILE_NAME, encoding='UTF-8')
print(unlabeled_df.shape)
unlabeled_df_tfidf = tfidf_features(unlabeled_df)
import csv
print(unlabeled_df_tfidf.shape)
#tf_idf_matrix = df_tfidf.todense()
#tf_idf_df = pd.DataFrame(tf_idf_matrix)
 
#tf_idf_df.columns = tfidf_reversed_vocab

## keep only those tfidf features we saw last time
intersect = list(set(df_tfidf.columns).intersection(unlabeled_df_tfidf.columns))
unlabeled_df_tfidf = unlabeled_df_tfidf[intersect]
unlabeled_df_withtfidf = pd.concat([unlabeled_df, unlabeled_df_tfidf], axis=1)

# X = np.random.rand(15783, num_ftrs)
##some features are missing
df_newdata = unlabeled_df_withtfidf.reindex(labels=df_withtfidf.columns,axis=1)
df_newdata =df_newdata.fillna(0)
print(df_newdata.shape)
print(df_newdata.columns)<jupyter_output>(19894, 13)<jupyter_text>Predicting for new data: run trained classifiers on new data<jupyter_code>feature_start, num_ftrs = 16,1496
print("Running prediction for new data")
X = df_newdata.iloc[:, feature_start:feature_start+num_ftrs]
print(len(X))
print(df_newdata.shape)
for feature in ['IsCollab','IsGaming']:
    X_scaled = all_scalers[feature].transform(X)
    y = all_classifiers[feature][best_classifiers[feature]].predict(X_scaled)
    unlabeled_df['predicted_' + feature] = y
#print the file somewhere
unlabeled_df.to_csv(OUTPUT_CSV_FILE_PATH, index=False, encoding='utf-8-sig')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11657784545108489, 'fraction_numerical': 0.03759040730871717, 'mean_word_length': 3.577090592334495, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '557983', 'license': 'MIT', 'n_tokens_mistral': 4083, 'n_tokens_neox': 3861, 'n_words': 775, 'path': 'collab_labelling/machine_learning/machine_learning.ipynb', 'repo_name': 'Yoshi275/ism-youtube-scraper', 'size': '10508'}