<jupyter_start><jupyter_text>References - [https://medium.com/analytics-vidhya/nlp-text-encoding-a-beginners-guide-fa332d715854:~:text=Text%20encoding%20is%20a%20process,out%20the%20context%20of%20sentences.]article is severly flawed  Implementation of traditional text encoding techniques<jupyter_code>document_corpus = ["this is a good phone phone",
                  "this is a bad mobile mobile",
                  "she is a good good cat",
                  "he has a bad temper temper",
                  "this mobile phone phone is not good good"]<jupyter_output><empty_output><jupyter_text>Creating corpus<jupyter_code>corpus = list(set(" ".join(document_corpus).split(" ")))
print(corpus)<jupyter_output>['she', 'good', 'bad', 'temper', 'this', 'mobile', 'is', 'phone', 'cat', 'has', 'a', 'not', 'he']<jupyter_text>Bag of wordsUse whole corpus matrix to represent each sentence in form of 1 & 0  Binary Bag of words<jupyter_code>all_encodings = []
for sentence in document_corpus:
    encoding = []
    for word in corpus:
        if word in sentence.split(" "):
            encoding.append(1)
        else:
            encoding.append(0)
    all_encodings.append(encoding)
all_encodings<jupyter_output><empty_output><jupyter_text>Count based Bag of words (CountVectorizer)<jupyter_code>all_encodings = []
for sentence in document_corpus:
    encoding = []
    for word in corpus:
        if word in sentence.split(" "):
            encoding.append(sentence.split(" ").count(word))
        else:
            encoding.append(0)
    all_encodings.append(encoding)
all_encodings<jupyter_output><empty_output><jupyter_text>TF IDF<jupyter_code>import numpy as np
def tf(t,d):
    return d.count(t)/len(set(d.split(" ")))
def idf(t,d,D):
    number_of_tokens = len(corpus)
    number_of_documents_with_t_in_them = sum([1 if t in d else 0 for d in D])
    df = number_of_documents_with_t_in_them/number_of_tokens
    idf = 1/df
    return np.log(idf)
def tf_idf(t,d,D):
    return tf(t,d)*idf(t,d,D)
all_encodings = []
for sentence in document_corpus:
    encoding = []
    for word in corpus:
        if word in sentence.split(" "):
            encoding.append(tf_idf(word,sentence,document_corpus))
        else:
            encoding.append(0)
    all_encodings.append(encoding)
print(all_encodings)<jupyter_output>[[0, 0.2932674137586854, 0, 0, 0.2932674137586854, 0, 0.4714619985366585, 0.7487208707606365, 0, 0, 0.23573099926832924, 0, 0], [0, 0, 0.37436043538031827, 0, 0.2932674137586854, 0.7487208707606365, 0.4714619985366585, 0, 0, 0, 0.4714619985366585, 0, 0], [0.5129898714923073, 0.5865348275173708, 0, 0, 0, 0, 0.23573099926832924, 0, 0.5129898714923073, 0, 0.4714619985366585, 0, 0], [0, 0, 0.37436043538031827, 1.0259797429846147, 0, 0, 0, 0, 0, 0.5129898714923073, 0.7071929978049877, 0, 0.37436043538031827], [0, 0.4887790229311423, 0, 0, 0.24438951146557114, 0.31196702948359856, 0.3928849987805487, 0.6239340589671971, 0, 0, 0, 0.42749155957692275, 0]]
{'fraction_non_alphanumeric': 0.12878278412911903, 'fraction_numerical': 0.175857431069267, 'mean_word_length': 4.120481927710843, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '164427', 'license': 'MIT', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1126, 'n_words': 236, 'path': 'traditional text encoding techniques.ipynb', 'repo_name': 'sonwanesuresh95/traditional-text-enc', 'size': '2974'}