<jupyter_start><jupyter_text>Discrete Word Representations<jupyter_code>## Based on material at:
# - https://github.com/biplav-s/course-nl/blob/master/l6-probabilistic/Explorations%20in%20Vector%20Representation.ipynb
# - https://github.com/biplav-s/course-nl/blob/master/l7-language/code/Context%20with%20TF-IDF.ipynb
# A function that will encode a-z, space and '.'
def do_integer_encoding(data):
    # define universe of possible input values
    alphabet = 'abcdefghijklmnopqrstuvwxyz .'
    
    # define a mapping of chars to integers
    char_to_int = dict((c, i) for i, c in enumerate(alphabet))
    int_to_char = dict((i, c) for i, c in enumerate(alphabet))
    
    # integer encoded input data
    integer_encoded = [char_to_int[char] for char in data]
    char_decoded = [int_to_char[integ] for integ in integer_encoded]
    
    return integer_encoded, char_decoded
# See with a sample string
small_data = "hello world"
enc, dec = do_integer_encoding(small_data)
print ("data = " + small_data + ", enc = " + str(enc))
print ("decoded data: " +  "dec = " + "".join(dec))
# See with a sample string with out of scope alphabets. Gives error.
small_data = "hello world 12"
enc = do_integer_encoding(small_data)
print ("data = " + small_data + ", enc = " + str(enc))
# Will loose information if we force with out-of-scope alphabets.
# Example - lowecases and uppercase
medium_data = "This is an important document. It contains the contract governing \
your deposit relationship with the Bank and required legal \
disclosures. Please have it translated. "

enc, dec = do_integer_encoding(medium_data.lower())

print ("data = " + medium_data + "\n enc = " + str(enc))
print ("decoded data: " +  "dec = " + "".join(dec))
# Scikit has support for label encoding
# https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html
from sklearn.preprocessing import LabelEncoder

# But need an array of words as input
# We use spacy to tokenize
# Import more
import spacy
nlp = spacy.load('en_core_web_sm')

import numpy as np
# Tokenize larger data for ease

doc = nlp(medium_data)
values = np.array([token.text for token in doc])
print (values)
# Now we can ask for encoding
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
print(integer_encoded)
# And decode
inverted = label_encoder.inverse_transform(integer_encoded)
print(inverted)
# And recreating original string is trivial
print (" ".join(inverted))<jupyter_output>This is an important document . It contains the contract governing your deposit relationship with the Bank and required legal disclosures . Please have it translated .<jupyter_text>Contextual Word Representation<jupyter_code># Based on sci-kit documentation
from sklearn.feature_extraction.text import CountVectorizer
corpus = [
'An alpha document.',
'A beta document.',
'Guten Morgen!',
'Gamma manuscript is old.',
'Whither my document?',
]
# Single word representation
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)
print(vectorizer.get_feature_names())
print(X.toarray())

# Notice document has one dimension although 3 occurences
# N-gram representation (2- and 3-; word based)

vectorizer2 = CountVectorizer(analyzer='word', ngram_range=(2, 3))
X2 = vectorizer2.fit_transform(corpus)

print(vectorizer2.get_feature_names())
print(X2.toarray())
# N-gram representation (2- and 3-; char based)
vectorizer3 = CountVectorizer(analyzer='char', ngram_range=(2,2))
X3 = vectorizer3.fit_transform(corpus)

print(vectorizer3.get_feature_names())
print(X3.toarray())<jupyter_output>[' a', ' b', ' d', ' i', ' m', ' o', 'a ', 'al', 'am', 'an', 'be', 'cr', 'cu', 'd.', 'do', 'en', 'er', 'et', 'ga', 'ge', 'gu', 'ha', 'he', 'hi', 'ip', 'is', 'it', 'ld', 'lp', 'ma', 'me', 'mm', 'mo', 'my', 'n ', 'n!', 'nt', 'nu', 'oc', 'ol', 'or', 'ph', 'pt', 'r ', 'rg', 'ri', 's ', 'sc', 't ', 't.', 't?', 'ta', 'te', 'th', 'um', 'us', 'ut', 'wh', 'y ']
[[1 0 1 0 0 0 1 1 0 1 0 0 1 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0
  1 0 1 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0]
 [0 1 1 0 0 0 2 0 0 0 1 0 1 0 1 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0
  1 0 1 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 1 0 0 0 0]
 [0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 2 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1
  0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 0 0]
 [0 0 0 1 1 1 1 0 1 1 0 1 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 1 0 2 0 1 0 0 0 0
  0 1 0 1 0 0 1 0 0 1 1 1 1 0 0 0 0 0 0 1 0 0 0]
 [0 0 1 0 1 0 0 0 0 0 0 0 1 0 1 1 1 0 0 0 0 0 1 1 0 0 1 0 0 0 1 0 0 1 0 0
  1 0 1 0 0 0 0 1 0 0 0 0 0 0 1 0 0 1 1 0 0 1 1]]<jupyter_text>Contextual Representation Using IDF<jupyter_code># https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html
from sklearn.feature_extraction.text import TfidfVectorizer
# TFIDR Vectorizer gives value based on Inverse Document Frequency, i.e., relative
# occurence of words in the documents. Hence, context is by word frequency.

vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(corpus)

print(vectorizer.get_feature_names())
print(X.toarray())
# We can use relative word occurence (similarity) to measure similarity between documents
from sklearn.metrics.pairwise import cosine_similarity
for i in range(1, len(corpus)):
    print ("similarity of doc-1 (" + 
           corpus[0] + ") with doc-" + 
           str(i+1) + " (" + corpus[i] + ") is = "  + 
           str(cosine_similarity (X[0], X[i])))<jupyter_output>similarity of doc-1 (An alpha document.) with doc-2 (A beta document.) is = [[0.23815688]]
similarity of doc-1 (An alpha document.) with doc-3 (Guten Morgen!) is = [[0.]]
similarity of doc-1 (An alpha document.) with doc-4 (Gamma manuscript is old.) is = [[0.]]
similarity of doc-1 (An alpha document.) with doc-5 (Whither my document?) is = [[0.18317794]]
{'fraction_non_alphanumeric': 0.12678971882007936, 'fraction_numerical': 0.062273589787821285, 'mean_word_length': 3.868178001679261, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '566270', 'license': 'MIT', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2032, 'n_words': 810, 'path': 'sample-code/l23-textrepresent/word-representation.ipynb', 'repo_name': 'biplav-s/course-d2d-ai', 'size': '5797'}