<jupyter_start><jupyter_text>TF-IDF $n_{\mathbb{d}\mathbb{w}}$ - term frequency - the number of times that word/term $\mathbb{w}$ occurs in document $\mathbb{d}$ divided by the number of all words in the document;$N_{\mathbb{w}}$ - the number of documents containing the term $\mathbb{w}$;$N$ - total number of documents; $p(\mathbb{w}, \mathbb{d}) = N_{\mathbb{w}} / N$ - probabilty of appearence of the term $\mathbb{w}$ in any document $\mathbb{d}$ $P(\mathbb{w}, \mathbb{d}, n_{\mathbb{d}\mathbb{w}}) = (N_{\mathbb{w}} / N)^{n_{\mathbb{d}\mathbb{w}}}$ - probability that the given term $\mathbb{w}$ appears $n_{\mathbb{d}\mathbb{w}}$ times in document $\mathbb{d}$$-\log{P(\mathbb{w}, \mathbb{d}, n_{\mathbb{d}\mathbb{w}})} = n_{\mathbb{d}\mathbb{w}} \cdot \log{(N / N_{\mathbb{w}})} = TF(\mathbb{w}, \mathbb{d}) \cdot IDF(\mathbb{w})$$TF(\mathbb{w}, \mathbb{d}) = n_{\mathbb{d}\mathbb{w}}$ - term frequency;$IDF(\mathbb{w}) = \log{(N /N_{\mathbb{w}})}$ - inverted document frequency;  Some parameters of TfidfVectorizer input : string {‘filename’, ‘file’, ‘content’} lowercase : boolean, default True preprocessor : callable or None (default) tokenizer : callable or None (default) stop_words : string {‘english’}, list, or None (default) ngram_range : tuple (min_n, max_n) max_df : float in range [0.0, 1.0] or int, default=1.0 min_df : float in range [0.0, 1.0] or int, default=1 max_features : int or None, default=None<jupyter_code>from sklearn.datasets import fetch_20newsgroups

categories = ['alt.atheism', 'talk.religion.misc',
              'comp.graphics', 'sci.space']
newsgroups_train = fetch_20newsgroups(subset='train',
                                      categories=categories)
newsgroups_train.data[:5]
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
vectorizer = TfidfVectorizer(lowercase=False)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
vectorizer.get_feature_names()[:10]
vectorizer = TfidfVectorizer(min_df=0.8)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
vectorizer.get_feature_names()
vectorizer = TfidfVectorizer(min_df=0.01, max_df=0.8)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
vectorizer = TfidfVectorizer(ngram_range=(1, 3), min_df=0.03, max_df=0.9)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

stopWords = set(stopwords.words('english'))
wnl = nltk.WordNetLemmatizer()
def preproc1(text):
    return ' '.join([wnl.lemmatize(word) for word in word_tokenize(text.lower()) if word not in stopWords])

st = "The saddest aspect of life right now is that science gathers knowledge faster than society gathers wisdom."
preproc1(st)
vectorizer = TfidfVectorizer(preprocessor=preproc1)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
#vectorizer = TfidfVectorizer(preprocessor=preproc1, ngram_range=(1, 3), max_df=0.5, max_features=1000)
vectorizer = TfidfVectorizer(preprocessor=preproc1, min_df = 0.001, max_df=0.9)
vectors = vectorizer.fit_transform(newsgroups_train.data)
vectors.shape
vectorizer.get_feature_names()[::100]
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn.linear_model import SGDClassifier
dense_vectors = vectors.todense()
X_train, X_test, y_train, y_test= train_test_split(dense_vectors, newsgroups_train.target, test_size=0.2, random_state=0)
y_train.shape, y_test.shape
from sklearn.metrics import accuracy_score

svc = svm.SVC()
svc.fit(X_train, y_train)
accuracy_score(y_test, svc.predict(X_test))
sgd = SGDClassifier()
sgd.fit(X_train, y_train)
accuracy_score(y_test, sgd.predict(X_test))<jupyter_output><empty_output><jupyter_text>Custom TF-IDF<jupyter_code># just for practicing with unigrams 
import math
from collections import Counter

class MyTfIdf():
  
  def __init__(self, preprocessor=None, min_df = 0.001, max_df = 0.9):
    self.preproc_func = preprocessor
    self.vocab = []
    self.idf = {}
    self.term2id = {}
    self.min_df = min_df
    self.max_df = max_df

  def __prepoc_data(self, data):
    preproc_data = data
    if self.preproc_func is not None:
      preproc_data = list(map(self.preproc_func, preproc_data))
    preproc_data = list(map(lambda x: x.split(), preproc_data))
    return preproc_data


  def fit(self, data):
    preproc_data = self.__prepoc_data(data)
    
    vocab_set = set()
    for doc in preproc_data:
      vocab_set.update(doc)

    self.vocab = sorted(list(vocab_set))
    df = dict(zip(self.vocab, [0]*len(self.vocab)))
    
    for doc in preproc_data:
      doc_vocab_set = set(doc)
      for term in doc_vocab_set:
        if term in df.keys():
          df[term] += 1
    
    for k, v in df.items():
      df[k] = v / len(preproc_data)

    df = {k: v for k, v in df.items() if v > self.min_df and v < self.max_df}
    
    for k, v in df.items():
      self.idf[k] = math.log(1 / v)
    
    for i, val in enumerate(self.idf.keys()):
      self.term2id[val] = i


  def fit_transform(self, data):
    self.fit(data)
    preproc_data = self.__prepoc_data(data)
    results = []
    for doc in preproc_data:
      doc_vector = [0]*len(self.idf.keys())

      term_cnts = Counter(doc)
      for term in term_cnts.keys():
        if term in self.term2id.keys():
          tf = term_cnts[term]/len(doc)
          doc_vector[self.term2id[term]] = tf * self.idf[term] 
      
      results.append(doc_vector)

    return results #self.idf #preproc_data#self.idf 
my_vectorizer = MyTfIdf(preprocessor=preproc1)
my_vectors = my_vectorizer.fit_transform(newsgroups_train.data)
list(filter(lambda x: x!=0, my_vectors[0]))
X_train, X_test, y_train, y_test= train_test_split(my_vectors, newsgroups_train.target, test_size=0.2, random_state=0)
y_train.shape, y_test.shape
from sklearn.metrics import accuracy_score

svc = svm.SVC()
svc.fit(X_train, y_train)
accuracy_score(y_test, svc.predict(X_test))
sgd = SGDClassifier()
sgd.fit(X_train, y_train)
accuracy_score(y_test, sgd.predict(X_test))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1338745622413244, 'fraction_numerical': 0.012257242916268705, 'mean_word_length': 4.743144424131627, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '487861', 'license': 'MIT', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2251, 'n_words': 488, 'path': 'tf_idf_details/TF_IDF_details.ipynb', 'repo_name': 'evilden/nlp-scripts', 'size': '6282'}