<jupyter_start><jupyter_text>Functions<jupyter_code>import pandas as pd 
import numpy as np
import matplotlib.pyplot as plt
def is_there_unk(test_corpus, ref_freq_dict):
  # reference_frequency is a dictionary that contains our vocabulary
  # test_corpus is a testset we want to check whether it has unknown word
  # test_corpus is a list of list of words
  at_least_one_unk =[word for sublist in test_corpus for word in sublist if word not in ref_freq_dict]
  if  at_least_one_unk:
    return True
  return False
def remove_rare(corpus, freq, num, min_sentence_len):
  #corpus is a list of list of rare_tokens
  # freq is the frequency of all trainset words
  remove_ids = []
  for i in range(len(corpus)):
    tmp = []
    tmp = [w for w in corpus[i] if freq[w] > num]
    corpus[i] = tmp
    if len(tmp) < min_sentence_len:
      remove_ids.append(i)
  for id in sorted(remove_ids, reverse=True):
    #you need to delete them in reverse order so that you don't 
    # throw off the subsequent indexes.
    del corpus[id]
  # we don't return anything since inputs are received by address

def find_freq(input,num = 1):
  # input is a list of list of tokens
  from collections import Counter
  # Flatten a 2D list in an elegent, yet not very efficient, way
  flat_list = [item for sublist in input for item in sublist]
  # compute global word frequency
  c = Counter(flat_list)
  # return frequency dict
  return c

def preprocessing(input, is_list = False, remove_punc = True):
  #remove_rare: remove low frequency words
  import re
  from string import punctuation
  if is_list:
    sentences = []
    for sent in input:
      if remove_punc:
        sent = re.sub(f'[{punctuation}؟،٪×÷»«]+', '', sent)
      res = sent.strip().split()
      if len(res)>3:
        sentences.append(res)
    return sentences

  if remove_punc:
    input = re.sub(f'[{punctuation}؟،٪×÷»«]+', '', input) 
  res = input.strip().split()
  return res if len(res)>3 else -1

def load_data(address, process = False):
    # return a list of unpreprocessed strings
    file = open(address, 'r',encoding="utf-8")
    sentences = []
    if process:
      while True:
        line =  preprocessing(file.readline())
        if not line == -1:
          sentences.append(line)
        if not line:
          break
      file.close()
      return sentences

    while True:
      line = file.readline()
      sentences.append(line)
      if not line:
        break
    file.close()
    return sentences
def accum_ngram(corp , n_gram = 5):
    
    # Python dictionary has dynamic hashing and can access each key's value in O(1) time.
    # It, however, needs O(n) space to store n strings. "Trie" is an attempt to reduce 
    # this space complexity
    
    # accumulated n-grams 
    # by accumulated I mean we compute all n-grams from 1 up to n
    # Result will be stored and returned as a trie, implemented with python dictionary
    
    n_grams = {}
    n_grams['#'] = 0 
    for stc in corp:
        n = len(stc)
        n_grams['#'] += n # update total words count
        for token_idx in range(n):
            dict_ptr = n_grams
            for next_token_idx in range(token_idx,n):
                if next_token_idx - token_idx >= n_gram:
                    break
                #print(f'token_idx:{token_idx}, next_token_idx:{next_token_idx},the word:{stc[next_token_idx]}')
                if not stc[next_token_idx] in dict_ptr:
                    dict_ptr[stc[next_token_idx]] = {'#':0}
                    
                dict_ptr[stc[next_token_idx]]['#'] = dict_ptr[stc[next_token_idx]]['#'] + 1
                dict_ptr = dict_ptr[stc[next_token_idx]]
                
            
    return n_grams
# Here (in $$$ tagged line) I am using encouraged EAFP style
# https://docs.python.org/3.6/glossary.html#term-eafp
# https://stackoverflow.com/questions/1835756/using-try-vs-if-in-python

def count_finder(sentence, n_grams_dict, j, i,mu,k):
    # find both numerator and denominator 
    numerat ,denominat= k,mu
    ptr_dict = n_grams_dict
    flag = 1
    for id in range(j,i):
      # $$$
      try: 
        ptr_dict = ptr_dict[sentence[id]]
      except:
        # if we didn't have a key = sentence[id] in our tire
        # it means we don't have both numerator and denominator n_grams 
        flag = 0
        break

    if flag:
      denominat += ptr_dict['#']

    # Even if we have the n-gram associated with denominator
    # chances are that we don't have n-gram associated with numerator
    # since numerator's n-gram is longer that denominator's
    # following try-except statement is to check this
    try:
      tmp = ptr_dict[sentence[i]]['#']
      numerat += tmp
    except:
      pass

    return numerat ,denominat

def occurance_drichlet_probability(sentence, n_grams_dict, n_gram, mu,Pbg):
  # n_gram: numeber of previous words each conditional probability depends on
  # sentence: list of tokens

  # mu = k * v , Pbg= 1/v
  k = mu* Pbg
  
  # logic for finding uni-grams is slightly different than other grams
  # that's why we are handling uni-gram seperately
  if n_gram ==1:
      prob = 1
      for token in sentence:
          prob *= (n_grams_dict[token]['#']+k)
      prob *= (1/((n_grams_dict['#']+mu)**len(sentence)))
      return prob

  # here we handle x-grams: x>=2    
  # handeling the first fraction separately
  # # put the following line into try except, later :) 
  # prob = (n_grams_dict[sentence[0]]['#']+k)/(n_grams_dict['#']+mu) 
  first_num = 0
  first_deno = 0
  try:
    prob = (n_grams_dict[sentence[0]]['#']+k)/(n_grams_dict['#']+mu)
  except:
    prob = k/mu

  
  for i in range(1,len(sentence)):
      j = i - (n_gram -1) if (i-n_gram>=0) else 0
      numerator, denominator =count_finder(sentence,n_grams_dict,j,i,mu,k)
      prob *= (numerator/denominator)
      
  return prob
def dfs(ptr_dict, current_str, remained_depth, counts_ls, terms_ls):
    if remained_depth == 1:
        keys = list(ptr_dict.keys())
        keys.pop(0) # first key is '#'
        for key in keys:
            s = current_str + ' ' + key # we gotta remove first ' '
            terms_ls.append(s[1:])
            counts_ls.append(ptr_dict[key]['#'])
    else:
        keys = list(ptr_dict.keys())
        keys.pop(0)
        for key in keys:
            dfs(ptr_dict= ptr_dict[key],current_str = current_str+' '+key,
                remained_depth =remained_depth-1,counts_ls = counts_ls, terms_ls=terms_ls)
        
# Visualize different n_grams with tables

def visual(n_grams_dict, n_gram ):
    # n_gram dicts contains all n grams in a trie data structure
    # we convert trie structure into pandas df
    import pandas as pd
    counts = []
    terms = []
    dfs(n_grams_dict, current_str = '', remained_depth = n_gram,
            counts_ls = counts, terms_ls = terms)
    
    # tabularizing
    n  = sum(counts)
    size = len(terms)
    df = pd.DataFrame({'terms':terms})
    df['counts'] = counts
    df.sort_values(by=['counts'], inplace=True, ascending=False,ignore_index=True)
    df['freq'] = df['counts'].apply(lambda x: x/n)
    df['freq_index'] = df['freq'] * [i for i in range(1,size+1)]
    return df
    
    
            
def tokensIDs_to_words(tokens_ids, look_up_dict):
  n = len(tokens_ids) ; found = 0
  words = ['' for _ in range(n)]
  for word, index in look_up_dict.items():
    i = -1
    for token_id in tokens_ids:
      i +=1
      if token_id == index:
        words[i] = word
        found = found+1
        if found ==n:
          return words<jupyter_output><empty_output><jupyter_text>Driver code  Loading/Exploring/Preprocessing data<jupyter_code>two_blank_sent = ['چون مشک سیه بود مرا هر دو',
                   "دوست در خانه و ما گرد",]

one_blank_sent= [ "شب است و شمع و شراب و",
                 "ازین زشت تر در جهان رنگ",
                 "زانک نفس آشفته تر گردد از",
                 "گر خورد سوگند هم آن"]



# downloading data
!gdown --id 1qCVYpb67RuzUbyrJ3w-ohtCf-tzZKTal
!gdown --id 1dW5SkCYIFbXmNe3xKv4EhrLPDPlXyIDy
train = load_data('/content/train.txt')
test = load_data('/content/test.txt')
# 2. Exploring
import pandas as pd 

train_df = pd.DataFrame({'lines':train})
test_df = pd.DataFrame({'lines':test})

train_df['len'] = train_df['lines'].apply(lambda x :len(x))
train_df['white_space_divide_by_len'] = train_df['lines'].apply(lambda x : (x.count('\n') + x.count(' ') + x.count('\t')) / len(x) if len(x)>0 else 0 )

test_df['len'] = test_df['lines'].apply(lambda x :len(x))
test_df['white_space_divide_by_len'] = test_df['lines'].apply(lambda x : (x.count('\n') + x.count(' ') + x.count('\t')) / len(x) if len(x)>0 else 0 )
train_df.sample(20)
test_df.sample(20)
# Preprocessing data
processed_train = preprocessing(train, is_list = True, remove_punc= True)
processed_test = preprocessing(test, is_list =True, remove_punc= True)
processed_removed_train = processed_train.copy()
processed_removed_test = processed_test.copy()
freq_dict = find_freq(processed_train)
#we'll implement a 3-gram Neural Nets model,hence; set min_sentence_len =3
remove_rare(processed_removed_train,freq_dict,num = 1,min_sentence_len =3)
remove_rare(processed_removed_test,freq_dict,num = 1,min_sentence_len = 3)
print(f'Train-> Sentences:{len(processed_train)}, Unique words:{len(freq_dict)}, Sum of word freq:{sum(freq_dict.values())}')
print(f'Test-> Sentences:{len(processed_test)}')
print(f'Is there unknown word in test set after preprocessings and removings: {is_there_unk(processed_removed_test,freq_dict)}')<jupyter_output>Is there unknown word in test set after preprocessings and removings: False<jupyter_text>Generating n-grams by trie and pandas.dfGenerating accumulated n-grams and specific n-grams (for a specific numeber, like 3) for both train and test datasets and for both only-preprocesed and preprocessed_and_removed_rare versions  generating for train<jupyter_code>accum_3_gram_p_train = accum_ngram(processed_train, n_gram = 3)
accum_3_gram_pr_train = accum_ngram(processed_removed_train, n_gram = 3)
accum_3_gram_p_train['#']
p_train_tri_grams_df = visual(n_grams_dict=accum_3_gram_p_train,n_gram=3)  
p_train_tri_grams_df.head(10)
pr_train_tri_grams_df = visual(n_grams_dict=accum_3_gram_pr_train,n_gram=3)  
pr_train_tri_grams_df.head(10)<jupyter_output><empty_output><jupyter_text>generating for test<jupyter_code>accum_3_gram_pr_test = accum_ngram(processed_removed_test, n_gram = 3)
accum_3_gram_pr_test['#']

pr_test_tri_grams_df = visual(n_grams_dict = accum_3_gram_pr_test, n_gram=3)  
pr_test_tri_grams_df.head(10)<jupyter_output><empty_output><jupyter_text>Question 1 (Sentence Completion)  Filling sentences with one blank<jupyter_code>one_blank_sent[0]
27**(1/3)
m = 0.01
pbg = (1/len(freq_dict))
p= occurance_drichlet_probability(one_blank_sent[0].split(),accum_3_gram_pr_train,3,m,pbg )
(1/p)**(1/len(one_blank_sent[0].split()))
# vocabulary is freq_dict.keys()
best_words_one = []
for sent in one_blank_sent:
  s = sent.split()
  highest_pro = 0
  best_word = ''
  for word in freq_dict.keys():
    t = s + [word]
    p = occurance_drichlet_probability(t,accum_3_gram_pr_train,3,m,pbg )
    if p > highest_pro:
      highest_pro = p
      best_word = word
  best_words_one.append(best_word)

best_words_one<jupyter_output><empty_output><jupyter_text>Predicting two consequtive gaps When we have multiple words to predict, we need a decode strategy.Some decoding strategies are, but not limited to:+ Greedy Search+ Beam Search+ Top-k sampling+ Top-p sampling Reference: Dr. <NAME> on Generation based Conversational AI Lecture / DARMSTADT University / UKP For the sake of simplicity, I am gonna use "Greedy Search"<jupyter_code># vocabulary is freq_dict.keys()
first_prediction = []
for sent in two_blank_sent:
  s = sent.split()
  highest_pro = 0
  best_word = ''
  for word in freq_dict.keys():
    t = s + [word]
    p = occurance_drichlet_probability(t,accum_3_gram_pr_train,3,m,pbg )
    if p > highest_pro:
      highest_pro = p
      best_word = word
  first_prediction.append(best_word)
first_prediction
two_blank_sent[1].split() + [first_prediction[1]]
second_prediciton = []
i = -1
for sent in two_blank_sent:
  i+=1
  s = sent.split()
  s = s + [first_prediction[i]]
  highest_pro = 0
  best_word = ''
  for word in freq_dict.keys():
    t = s + [word]
    p = occurance_drichlet_probability(t,accum_3_gram_pr_train,3,m,pbg )
    if p > highest_pro:
      highest_pro = p
      best_word = word
  second_prediciton.append(best_word)
second_prediciton<jupyter_output><empty_output><jupyter_text>Question 2<jupyter_code># tokenized&preprocessed&removed_rare of testset= processed_removed_test
PP_sum = 0
total= len(processed_test)
m = 0.01
pbg = (1/len(freq_dict))
for sent in processed_removed_test:
  p= occurance_drichlet_probability(sent,accum_3_gram_p_train,3,m,pbg)
  PP = (1/p)**(1/len(sent))
  PP_sum += PP

PP_avg = PP_sum/total
print(f'Average Perplexity on testset: {round(PP_avg,2)}')<jupyter_output>Average Perplexity on testset: 335349.57<jupyter_text>Question 3  Question 3 manipulating data set to fed the model<jupyter_code>#!pip install tensorflow-gpu
import tensorflow as tf
tf.__version__
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
with tf.compat.v1.Session() as sess:
  devices = sess.list_devices()
  print(devices)
gpus = tf.config.list_physical_devices('GPU')
tf.config.set_logical_device_configuration(
        gpus[0],
        [tf.config.LogicalDeviceConfiguration(memory_limit=15240)])
# connecting to the drive
from google.colab import drive
drive.mount('/content/drive')
path = '/content/drive/MyDrive/chatBot_momtazi'
# pr is an abbreviation of Preprocessed and Removed its rare words
pr_train_sentences = pr_train_tri_grams_df['terms'].tolist()
pr_test_senteces = pr_test_tri_grams_df['terms'].tolist()
pr_train_sentences[:3]
# indexing words 
from keras.preprocessing.text import Tokenizer
tokenizer =  Tokenizer()
tokenizer.fit_on_texts(pr_train_sentences)
train_tokenized_sentc = tokenizer.texts_to_sequences(pr_train_sentences)
test_tokenized_sentc = tokenizer.texts_to_sequences(pr_test_senteces)
#tokenizer.word_index
## note: keys are word. Farsi font make the print reverse 
# {'و': 1,
#  'از': 2,
#  'به': 3,
#  'که': 4,
#  'در': 5,
#  'تو': 6,
#  'را': 7,
#  . . .   . . . 
# test purpose
last_key = list(tokenizer.word_index)[-1]
print(last_key)
print(tokenizer.word_index[last_key])
vocab_size = len(tokenizer.word_index) + 1
vocab_size
# Manipulating data to make it suitable as our models inputs
import numpy as np
train_vec = np.array(train_tokenized_sentc)
test_vec = np.array(test_tokenized_sentc)
print(test_vec.shape)
train_vec.shape
x_train , y_train = train_vec[:,:-1] , train_vec[:,-1] 
x_test , y_test = test_vec[:,:-1] , test_vec[:,-1] 
batch_s = 1000
y_train[70000]
from tensorflow.keras.utils import to_categorical

def generator(x,y, batch_size):
#Ref: http://www.jessicayung.com/using-generators-in-python-to-train-machine-learning-models/
    num_samples = len(x)
    while True: # Loop forever so the generator never terminates

        for offset in range(0, num_samples, batch_size):
            # Get the samples you'll use in this batch
            x_batch_samples = x[offset:offset+batch_size]
            y_batch_samples = y[offset:offset+batch_size]
            y_batch_samples = to_categorical(y_batch_samples, num_classes= vocab_size)
            # The generator-y part: yield the next training batch            
            yield x_batch_samples, y_batch_samples


""" DEVELOPMENT NOTE 5: """
train_generator=generator(x_train, y_train, batch_size=batch_s)
#validation_generator=generator(x_test,y_test, batch_size=batch_s)
# generate one-hot representation of all y values
# need this representation to make "cross-entropy" validation method possible
from tensorflow.keras.utils import to_categorical
y_test = to_categorical(y_test, num_classes= vocab_size)
val_tuple = (x_test,y_test)<jupyter_output><empty_output><jupyter_text>Defining the model<jupyter_code>import keras
import matplotlib.pyplot as plt
#!pip install IPython
from IPython.display import clear_output as cls

class PlotLosses(keras.callbacks.Callback):

  def on_train_begin(self,logs={}):
    self.i = 0
    self.x = []
    self.losses= []
    self.val_losses = []
    self.fig = plt.figure()
    self.logs = []

  def on_epoch_end(self, epoch, logs={}):
    self.logs.append(logs)
    self.x.append(self.i)
    self.losses.append(logs.get('loss'))
    self.val_losses.append(logs.get('val_loss'))
    self.i +=1

    cls(wait=True)
    plt.plot(self.x, self.losses, label= 'loss')
    plt.plot(self.x, self.val_losses, label= 'val_loss')
    plt.legend()
    plt.show()

plot_losses = PlotLosses()
window_size = 2
seq_length = window_size +1
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Embedding, Flatten

#denining the model
model = Sequential()
model.add(Embedding(vocab_size,50,input_length=window_size, name='Embedding-layers'))
model.add(Flatten())
model.add(Dense(vocab_size//2,activation='relu',name='hidden-layer'))
model.add(Dense(vocab_size,activation='softmax',name='output-layer'))

# print model structure summary
print(model.summary())
train_vec.shape[0]//50
import logging 
logging.basicConfig(filename='trainning.log',level=logging.DEBUG,
                    format='%(asctime)s:%(levelname)s:%(message)s')<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># trainning the model:
from tensorflow.keras.callbacks import ModelCheckpoint

# define checkpoints
checkpoint = ModelCheckpoint('/content/drive/MyDrive/chatBot_momtazi/model2-{epoch:03d}--{val_accuracy:03f}.h5',
                             verbose=1, monitor='val_loss',save_best_only=True,
                             save_weights_only=True, mode='min')

# complie the model
model.compile(loss= 'categorical_crossentropy', optimizer='adam',metrics=['accuracy'])

# train the model
model.fit(train_generator,batch_size = batch_s,
                    steps_per_epoch=(train_vec.shape[0]//batch_s),
          epochs=10, validation_data= val_tuple, 
          callbacks= [checkpoint, plot_losses])
# # Fit model using generator
# model.fit_generator(train_generator, 
#                     steps_per_epoch=len(test_vec.shape[0]//batch_s), 
#                     validation_data=validation_generator,
#                     nb_val_samples=len(validation_samples), nb_epoch=100)<jupyter_output><empty_output><jupyter_text>**  **Note: Above training process left incompleted due to google colab run time limitation**  **  Flush GPUOccasionally we need to manually free up GPU!<jupyter_code># # flush GPU
# # https://stackoverflow.com/questions/39758094/clearing-tensorflow-gpu-memory-after-model-execution
# !pip install numba 
# from numba import cuda 
# device = cuda.get_current_device()
# device.reset()<jupyter_output><empty_output><jupyter_text>Making Predection<jupyter_code># Loading the model
best_model_path = '/content/drive/MyDrive/chatBot_momtazi/model2-002--0.072106.h5'
model.load_weights(best_model_path)<jupyter_output><empty_output><jupyter_text>Predicting one gaps This part does not have any special logic. It is just plug and play!<jupyter_code># making prediction
one_blank_sent_tokenized =  tokenizer.texts_to_sequences(one_blank_sent)
two_blank_sent_tokenized = tokenizer.texts_to_sequences(two_blank_sent)
one_blank_sent_tokenized = np.array([sent[-2:] for sent in one_blank_sent_tokenized])
one_blank_sent_tokenized
one_blank_pridicted = model.predict(one_blank_sent_tokenized)
one_blank_pridicted =np.argmax(one_blank_pridicted,axis=1)
one_blank_pridicted
tokensIDs_to_words(one_blank_pridicted,tokenizer.word_index)<jupyter_output><empty_output><jupyter_text>Predicting two gaps<jupyter_code>two_blank_sent_tokenized
two_blank_sent_tokenized = np.array([sent[-2:] for sent in two_blank_sent_tokenized])
two_blank_sent_tokenized
two_blank_pridicted = model.predict(two_blank_sent_tokenized)
two_blank_pridicted =np.argmax(two_blank_pridicted,axis=1)
two_blank_pridicted
tokensIDs_to_words(two_blank_pridicted,tokenizer.word_index)

second_gap = np.array([[35,7], [133,1]])
second_gap
second_gap_pridicted = model.predict(second_gap)
second_gap_pridicted =np.argmax(second_gap_pridicted,axis=1)
second_gap_pridicted
tokensIDs_to_words(second_gap_pridicted,tokenizer.word_index)<jupyter_output><empty_output><jupyter_text>Further Developments notes:1. tokensIDs_to_words can be optimized by bi-directional dictionary2. Write a general "sentence occurence probability calculator" function which receive smoothing name as an arg3. If the sentence is too long, we might encounter "over/under flow" or "cancelation error" in "occurence_probablity" function. (use log function to prevent this)4. I have used gived TEST SET for "development" (aka "evaluation") of model. Better to split train to "train" and "eval" and leave Test Set a side for final testing. Anyways, It was not the concern of this notebook. 5. When I used the already commented generator for validation set, My model didn't get out of validation loss calculation. I Guess it was due to using infinite "while" loop. It seems that "steps_per_iteration" helps in not calling train set generator infinitly, but since we don't have such a variable for validaiton generation, our model calls validaiton generator utill it exaust, whihc in our case validation generator does not exaust (finish) at all. 6. How to log training process and know exatcly what the hell is happening ?   Manage Resources<jupyter_code>whos<jupyter_output>Variable                         Type          Data/Info
--------------------------------------------------------
Dense                            type          <class 'keras.layers.core.dense.Dense'>
Embedding                        type          <class 'keras.layers.embeddings.Embedding'>
Flatten                          type          <class 'keras.layers.core.flatten.Flatten'>
PlotLosses                       type          <class '__main__.PlotLosses'>
Sequential                       type          <class 'keras.engine.sequential.Sequential'>
Tokenizer                        type          <class 'keras_preprocessing.text.Tokenizer'>
accum_3_gram_p_train             dict          n=56353
accum_3_gram_pr_test             dict          n=435
accum_3_gram_pr_train            dict          n=27832
accum_ngram                      function      <function accum_ngram at 0x7f7c0f809200>
batch_s                          int           1000
best_model_path                  str           /conte[...]<jupyter_text>**Thanks for your time and attention. Please let me know my codes Weeknesses and ways to cover them**<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10677943702920577, 'fraction_numerical': 0.01171754548257786, 'mean_word_length': 3.7945089757127772, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 83, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '268445', 'license': 'MIT', 'n_tokens_mistral': 8094, 'n_tokens_neox': 7694, 'n_words': 1875, 'path': 'W2/answers/HW2_MA.ipynb', 'repo_name': 'MateAnderson/ChatBot', 'size': '22703'}