<jupyter_start><jupyter_text><jupyter_code># AUTOCORRECT
"""
https://repository.lib.fit.edu/bitstream/handle/11141/682/ZHANG-THESIS.pdf?isAllowed=y&sequence=1
https://arxiv.org/abs/1706.07786
http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf
 
"""

# ASSUMPTIONS
"""
- The first word in the input sentence is correct
- The dataset is not exhaustive
- (without bias) If a given word exists in the dictionary, it is assumed to be correct
- (with bias) the context of the sentence is checked for with more weightage to bigram and trigram for autocorrecting a word in input already existing in the dictionary
- punctuation is not considered
"""
!apt install enchant
!pip install pyenchant
import re
import random
from collections import Counter
import pandas as pd
import glob
import numpy as np
import string
import enchant
import tensorflow as tf
import nltk
nltk.download('averaged_perceptron_tagger')
from nltk.tag import pos_tag
ins = 1
delete = 1
sub = 1
trans = 1
threshold = 2
# LOAD WORDS
def words(text):
  return re.findall(r'\w+', text.lower())

word_list = Counter(words(open('wordlist.txt').read()))
n =sum (word_list.values())
# LEVENSHTEIN EDIT DISTANCE CRUDE ALGO
def edit_dis_algo(s1, s2):
  l = [1001]*(len(s2)+1)
  dp = [l]*(len(s1)+1)
 
  for j in range(len(s2)+1):
    dp[0][j] = j
  for i in range(len(s1)+1):
    dp[i][0] = i

  
  for i in range(len(s1)+1):
    for j in range(len(s2)+1):
      if i==0 or j==0:
        continue
      if s1[i-1] == s2[j-1]:
        dp[i][j] = dp[i-1][j-1]
      else:
        dp[i][j] = min(dp[i-1][j-1] + rep, dp[i-1][j] + delete, dp[i][j-1] + ins)
        if i>2 and j>2 and s1[i-1] == s2[j-2] and s1[i-2] == s1[j-1]:
          dp[i][j] = min(dp[i][j], dp[i-3][j-3] + trans)
  
  return dp[len(s1)][len(s2)]

# LEVENSHTEIN - EDIT DISTANCE FOR THE LM
def edit(s):
  edited_list = set()
  alpha = 'abcdefghijklmnopqrstuvwxyz'
  l = set()
  print(s)
  l.add(s)
  for iter in range(threshold):
    for word in l:
      split_word = [(word[:i], word[i:])   for i in range(len(word) + 1)]
      deletes    = [i + j[1:]               for i, j in split_word if j]
      transposes = [i + j[1] + j[0] + j[2:] for i, j in split_word if len(j)>1]
      replaces   = [i + c + j[1:]           for i, j in split_word if j for c in alpha]
      inserts    = [i + c + j               for i, j in split_word for c in alpha]
  
      l = set(deletes + transposes + replaces + inserts)
      #print(l)

    for ss in l:
      if d.check(ss):
        edited_list.add(ss)
  return edited_list<jupyter_output><empty_output><jupyter_text>EDIT DISTANCE<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')<jupyter_output>Mounted at /content/gdrive<jupyter_text>USING THE IMDB DATASET<jupyter_code># IMDB DATASET
#def IMDB_to_csv(directory):    
#   data = pd.DataFrame()
    
#   for filename in glob.glob(str(directory)+'/neg/*.txt'):
#     with open(filename, 'r',  encoding="utf8") as f:
#       content = f.readlines()
#       content_table = pd.DataFrame({'id':filename.split('_')[0].split('/')[-1],'rating':filename.split('_')[1].split('.')[0],'pol':'neg', 'text':content})
#       data = data.append(content_table)
        
#   for filename in glob.glob(str(directory)+'/pos/*.txt'):
#     with open(filename, 'r',  encoding="utf8") as f:
#       content = f.readlines()
#       content_table = pd.DataFrame({'id':filename.split('_')[0].split('/')[-1],'rating':filename.split('_')[1].split('.')[0],'pol':'pos', 'text':content})
#     data = data.append(content_table)
#     data = data.sort_values(['pol','id'])
#     data = data.reset_index(drop=True)
#     #data['rating_norm'] = (data['rating'] - data['rating'].min())/( data['rating'].max() - data['rating'].min() )

#   return(data)

# dir ='/content/drive/MyDrive/aclImdb_v1.tar.gz'
# imdb_data = IMDB_to_csv(dir)
# print(imdb_data)
# imdb_data.columns = ['id', 'dataset', 'text', 'pol','file']
# imdb_data.head()
# data1 = imdb_data['text'].split('.')
# train_data_sent = pd.DataFrame()

# for index in imdb_data.index:
#     data_row = imdb_data.iloc[index,:]

#     for sent_id in range(0,len(data_row['text'].split('.'))-1):
#         sentence = data_row['text'].split('.')[sent_id]
#         sentence_row = pd.DataFrame({
#                                      'id':data_row['id'],
#                                      'pol':data_row['pol'],
#                                      'sent_id':sent_id,
#                                      'sentence':sentence}, index = [index]) 
#         train_data_sent = train_data_sent.append(sentence_row)

# train_data_sent.head()
# train_data_sent['sentence_clean'] = train_data_sent['sentence'].str.replace('[{}]'.format(string.punctuation), '')
# train_data_sent['sentence_clean'] = train_data_sent['sentence_clean'].str.lower()

# train_data_sent['sentence_clean'] = '<s> ' + train_data_sent['sentence_clean']
# train_data_sent['sentence_clean'] = train_data_sent['sentence_clean'] + ' </s>'

# train_data_sent.head()
# text = train_data_sent['sentence_clean']
# text_list = " ".join(map(str, text))
# print(text_list[:200])

# LOAD DATASET AND DICT
d = enchant.Dict("en_US") 
dataset = (open('wordlist.txt').read())
print(dataset[:100])
#PREPROCESS THE DATA
def preprocess(data):
  temp = data.replace('[{}]'.format(string.punctuation), '')
  temp_list = temp.split('.')
  for i,sent in enumerate(temp_list):
    words = sent.split(' ')
    for j in range(1,len(words)):
      if words[j]!='' and words[j][0].isupper():
        words[j] = '<unk>'
    temp_list[i] = " ".join(words).lower()

  temp_list[:] = ['<s>' + s + '</s>' for s in temp_list]
  return temp_list

# PREPROCESSING THE INPUT
def input_process(inp):
  inp = re.sub(r'[^\w\s]','',inp)
  temp_list = inp.split(' ')
  v = len(Counter(temp_list))
  wordlist = []
  wordlist = ['<s>'] + temp_list + ['</s>']
  return wordlist, v

# FUNCTIONS FOR N-GRAM WITH STUPID BACKOFF/ LAPLACIAN SMOOTHING
backoff = 0.4
alpha = 1.5
smoothing = {'backoff':1, 'laplacian':2}

def unigram(st,case, v):
  prob =  len(re.findall(' ' + st + ' ', data))
  if case == 1:
    return prob/n
  if case == 2:
    return (prob + alpha)/n + v*alpha

def bigram(st1, st2, case, v):
  len1 = len(re.findall(' ' + st1 + ' '+ st2 + ' ', data))
  #print(st1,'llllll')
  len2 = len(re.findall(' ' + st1 + ' ', data))
  if len2 == 0:
    return 0
  prob = len1/len2
  #print('bigram: ', prob)
  if case == 1:
    return prob if prob >0 else backoff*unigram(st2, case, v)
  if case == 2:
    return (len1 + alpha/len2 + v*alpha)

def trigram(st1, st2, st3, case , v):
  len1 = len(re.findall(' '+ st1 + ' ' + st2 + ' ' + st3+ ' ',data))
  len2 = len(re.findall(' ' + st1 + ' '+ st2 +' ', data))
  if len2 == 0:
    return np.square(backoff)*unigram(st3, case, v)
  prob = len1/len2
  #print('trigram: ', prob)
  if case == 1:
    return prob if prob>0 else  backoff*bigram(st2, st3,case ,v)
  if case == 2:
    return (len1 + alpha/ len2 + v*alpha)

# TRAIN INTERPOLATION COEFFICIENTS
x1 = tf.Variable(initial_value=0, name='x1', trainable=True, dtype=tf.float32)
x2 = tf.Variable(initial_value=0, name='x2', trainable=True, dtype=tf.float32)

def loss(sent):
  return (perplexity(sent,x1,x2,1-x1-x2) - 1)** 2
  
def train(train_data):
  x1.assign(0.33)
  x2.assign(0.33)
  opt = tf.keras.optimizers.Adadelta(learning_rate = 0.01)

  for sent in train_data:
    with tf.GradientTape(persistent=False) as tape:
      loss_fn = loss(sent)
    vars = [x1, x2]
    grads = tape.gradient(loss_fn, vars)
    opt.apply_gradients(zip(grads, vars))

    # ERROR
    # err = np.abs(x.numpy() - x_init)
    # print("Iteration: %d Val: %f Error: %f Grad: %f" % (
    #         opt.iterations.numpy(), 
    #         x.numpy(), 
    #         err, 
    #         grads[0].numpy()))

    print('hi')
  return x1, x2, 1-x1-x2
# INTERPOLATION TRAIN FUNCTION FOR 3 WORDS
def interp_train(str_list, x1,x2,x3,v = 3):
  case = smoothing['backoff']
  prob = x1*unigram(str_list[2], case, v) + x2*bigram(str_list[2], str_list[1], case, v) + x3*trigram(str_list[2], str_list[1], str_list[0], case, v)
  print('prob:', prob)
  return prob

# FUNCTION TO CALCULATE PERPLEXITY AND CROSS ENTROPY OF LM
def perplexity(text, x1, x2, x3):
  words = text.split(' ')
  words= list(filter(lambda x: x!= ' ' or x!='', words))
  words = ['<s>'] + words + ['</s>']
  n, v = len(words), len(Counter(words))
  score = 0
  entr =0
  for i in range(2,len(words)):
    str_list = [words[i-2], words[i-1], words[i]]
    val = interp_train(str_list, x1, x2, x3 , v)
    score += val
    #entr += val*np.log(val)
  #score = np.exp(score)*100000
  print('score:',score)
  return score*10000
# FUNCTION TO GENERATE TRAINING DATASET
def generate_train_data(data_len, word_len):
  train_data = list()

  for i in range(data_len):
    length = 0
    l = len(clean_list)
    while length<4:
      ind = random.randint(0,l-1)
      sent_list = clean_list[ind].split(' ')
      length = len(sent_list)

    word_ind = random.randint(0, len(sent_list)-3)
    temp_list = []
    for j in range(word_ind, word_ind + word_len):
      temp_list.append(sent_list[j])
    train_str = " ".join(temp_list)
    train_data.append(train_str)
  print(train_data)
  return train_data

mock_params = [[0.4, 0.6, 0.0], [0.15, 0.35, 0.5], [0.25, 0.35, 0.4]]
cases = len(mock_params)
 
# APPLYING INTERPOLATION
def interpolation(*args, **kwargs):
  c,v = kwargs['c'], kwargs['v']
  x1,x2,x3 = params[c-1]
  str_list =[arg for arg in args]
  case = smoothing['backoff']
  prob = x1*unigram(str_list[2], case, v) + x2*bigram(str_list[2], str_list[1], case, v) + x3*trigram(str_list[2], str_list[1], str_list[0], case, v)
  return prob
# N-GRAM FUNCTION FOR CALLING INTERPOLATION FOR DIFFERENT CASES
def ngram(str1, str2, str3, v):
  if d.check(str3):
    return str3
  possible_words = list(edit(str3))
  prob = 0
  ans = -1
  pred_word = ''
  # ASSUMPTION IS THAT FIRST WORD IF EXISTS IN THE DICTIONARY IS CORRECT
  for word in possible_words:
    if str1 == ' ' and not d.check(str3):
      prob = interpolation(str1, str2, word, c =1, v = v)
  # elif str1 != ' ' and d.check(str3):
  #   prob = interpolation(str1, str2, word, c =2, v=v)
    elif str1 != ' ' and not d.check(str3):
      prob = interpolation(str1, str2, word, c =3, v=v)
    if prob > ans:
      ans = prob
      pred_word = word
    
  return pred_word
# SLIDING WINDOW ALGORITHM
def autocorrect(inp_list,sentence,v):
  corrected_list = []
  postag = pos_tag(sentence.split())
  prev2, prev1, cur = '', '', '<s>'
  for i in range(1,len(inp_list)-1):
    word = inp_list[i]
    #print(word, '----')
    if i<len(inp_list)-1 and postag[i-1][1] == 'NNP':
      corrected_list.append(word)
      cur = '<unk>'
      continue
    # if not word_list[word] and d.check(word):
    #   word = '<unk>'
    if i<2:
      prev1 = cur
      cur = word
      prev2 = ''
    else:
      if i==2:
        prev2 = prev1
      else:
        prev2 = corrected_list[i-2-1]

      prev1 = corrected_list[i-1-1]
      cur = word
    # print(prev2, '11')
    # print(prev1, '22')
    # print(cur, '33')
    new_word = ngram(prev2, prev1, cur, v)
    print(new_word, ' new-word')
    if new_word != ' ':
      corrected_list.append(new_word)

  return " ".join(corrected_list)
clean_list = preprocess(dataset)
data = " ".join(map(str, clean_list))
print(data[:1000])
# TRAIN INTERPOLATION PARAMETERS
train_data = generate_train_data(3, 3)
x1, x2, x3 = train(train_data)
x1_new = x1.numpy()
x2_new = x2.numpy()
x3_new = x3.numpy()
params = [x1_new, x2_new, x2_new]
# MAIN
input_str = 'How aee yuu'
input_list,v = input_process(input_str)
#print(input_list)
output = autocorrect(input_list, input_str, v)
output
d.check('arr')<jupyter_output><empty_output><jupyter_text>FLASK CODE<jupyter_code>!pip install flask-ngrok
!pip install flask
%mkdir templates -p
%mkdir static/css -p
## HTML CODE
%%writefile templates/index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8"> 
<title>AUTOCORRECT</title>
<link href= 'style.css' rel='stylesheet' type='text/css'>
</head>
<body>
<div class="text-input">
<form method = "POST" action = "{{url_for('index')}}">
  <input type="text" id="input1" name = "input" placeholder="Try typing something in here!">
  <label for="input1">Your input</label>
  <input type = "submit" id = "submit" value = "Go!">
</form>
</div>
<div class="text-input">
  <input type="text" id="output1" placeholder="Your output will be shown here!" value ="{{ output_str }}">
</div>
</body>
</html>
## CSS CODE
%%writefile static/style.css
@import  url(https://fonts.googleapis.com/css?family=Montserrat);

body{
  display: flex;
  flex-direction: column;
  align-items: center;
  justify-content: center;
  height: 100vh;
  font-family: Montserrat;
  background: #313E50;
}

.text-input{
  
  position: relative;
  margin-top: 50px;
  
  input[type="text"]{
    display: inline-block;
    width: 500px;
    height: 40px;
    box-sizing: border-box;
    outline: none;
    border: 1px solid lightgray;
    border-radius: 3px;
    padding: 10px 10px 10px 100px;
    transition: all 0.1s ease-out;
  }
  
  input[type="text"] + label{
    position: absolute;
    top: 0;
    left: 0;
    bottom: 0;
    height: 40px;
    line-height: 40px;
    color: white;
    border-radius: 3px 0 0 3px;
    padding: 0 20px;
    background: #E03616;
    transform: translateZ(0) translateX(0);
    transition: all 0.3s ease-in;
    transition-delay: 0.2s;
  }
  
  input[type="text"]:focus + label{
    transform: translateY(-120%) translateX(0%);
    border-radius: 3px;
    transition: all 0.1s ease-out;
  }
  
  input[type="text"]:focus{
    padding: 10px;
    transition: all 0.3s ease-out;
    transition-delay: 0.2s;
  }
}
## FLASK BACKEND CODE
from flask_ngrok import run_with_ngrok
from flask import Flask, render_template, redirect, url_for, request

app = Flask(__name__, static_folder = 'static')
run_with_ngrok(app) 

@app.route('/', methods = ['GET','POST'])
def index():
  output_str = 'TEST'
  if request.method == 'POST':
    input = request.form['input']
    input_list,v = input_process(input)
    output = autocorrect(input_list, input, v)
    output_str = output

  return render_template('index.html', output_str = output_str)

if __name__ == "__main__":
  app.run()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16401468788249693, 'fraction_numerical': 0.027539779681762546, 'mean_word_length': 3.079611650485437, 'pattern_counts': {'":': 1, '<': 53, '<?xml version=': 0, '>': 56, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '383767', 'license': 'MIT', 'n_tokens_mistral': 5919, 'n_tokens_neox': 5634, 'n_words': 1263, 'path': 'autocorrect.ipynb', 'repo_name': 'smit25/Autocorrect-using-Core-NLP', 'size': '14706'}