<jupyter_start><jupyter_text>Libraries and mounting google drive<jupyter_code>from numpy import argmax
from google.colab import drive
from keras.utils.vis_utils import plot_model
from keras.models import Model
from keras.layers import Input
from keras.layers import Dropout
from keras.layers import add
from keras.layers import LSTM
from keras.layers import Dense
from keras.layers import Embedding
from keras.layers import RepeatVector
from keras.layers import TimeDistributed
from keras.callbacks import ModelCheckpoint
from numpy import random
from numpy import array
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.preprocessing.text import Tokenizer
from keras import optimizers
from keras.models import load_model
from numpy import asarray
from numpy import zeros
from keras.callbacks import TensorBoard
import keras.optimizers as optimizer
import numpy as np
import pandas as pd 
import html
import re
import pickle

drive.mount('/content/drive/')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /content/drive/<jupyter_text>kaggle download<jupyter_code>Questions=pd.read_csv('Questions.csv',encoding = 'iso-8859-1')
Answers=pd.read_csv('Answers.csv',encoding = 'iso-8859-1')
Questions = Questions.sort_values(['Id'])[['Id','Score','Body','Title']]
Answers = Answers.sort_values(['ParentId','Score'])[['ParentId','Score','Body']]<jupyter_output><empty_output><jupyter_text>Pre processing<jupyter_code># This is done to compile a dict of title question, answer body and answer code 
Questionbank=dict()
cleanr = re.compile('(?i)<(?!code|/code).*?>')
for  index, row in Questions.iterrows():
  Questionbank[row['Id']]={'title':row['Title'],'body':re.sub('(?i)<(?!code|/code).*?>','',row['Body']),'result':list()}
for index, row in Answers.iterrows():
  if row['Score'] != 0:
    Questionbank[row['ParentId']]['result'].append({'score':row['Score'],'body':re.sub('(?i)<(?!code|/code).*?>','',row['Body']) })

#Questions are parsed into code and Title of the question and answer code for predicting the title 

codes=list()
for i in Questionbank:
  for j in Questionbank[i]['result']:
    code_list = re.findall(r'<code.*?>(.+?)</code>', j['body'],flags=re.DOTALL)
    code_list = [html.unescape(x) for x in code_list if len(x) > 50 and len(x) < 1000 ]
    if len(code_list) > 0 and j['score']>0 :
      for code_ls in code_list:
        codes.append({'title':Questionbank[i]['title'],'score':j['score'],'code':replace(code_ls)})<jupyter_output><empty_output><jupyter_text>**reward is taken as the score of the answer so we compile this into 3 main array as below**<jupyter_code>code_list = list()
score_list = list()
title_list = list()
temp_code_list=list()
temp_score_list=list()
temp_title_list=list()
for i in range(len(codes)):
  title_list.append("(start) "+codes[i]['title']+ " (stop)")
  score_list.append(codes[i]['score'])
  code_list.append(codes[i]['code'])
token_title = token_create(title_list)
token_code = token_create(code_list,' ')
temp_seq_code = token_code.texts_to_sequences(code_list)
temp_seq_title= token_title.texts_to_sequences(title_list)
for i in range(len(codes)):
	if len(temp_seq_code[i]) > 5 and len(temp_seq_title[i]) > 2 :
		temp_code_list.append(code_list[i])
		temp_score_list.append(score_list[i])
		temp_title_list.append(title_list[i])
code_list = temp_code_list
title_list = temp_title_list
score_list = temp_score_list
#save the initial Questionbank array
!rm dict_pkl.pkl
pickle.dump(Questionbank, open("dict_pkl.pkl", "wb"))  
training_dict = {
    'code':code_list,
    'score':score_list,
    'title':title_list,
    'token_title':token_title,
    'token_code':token_code
}

!rm train_dict_pkl.pkl
pickle.dump(training_dict, open("train_dict_pkl.pkl", "wb"))  # save it into a file
!cp train_dict_pkl.pkl drive/My\ Drive/colabapp/codi<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def token_create(doc,avoid='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~ '):
  tokenizer = Tokenizer(num_words=5000,filters=avoid)
  tokenizer.fit_on_texts(doc)
  return tokenizer
def embedding_dict(vocab_size,token):
	embeddings_index = dict()
	f = open('drive/My Drive/colabapp/codi/glove.6B.100d.txt')
	for line in f:
		values = line.split()
		word = values[0]
		coefs = asarray(values[1:], dtype='float32')
		embeddings_index[word] = coefs
	f.close()
	embedding_matrix = zeros((vocab_size, 100))
	for word, i in token.word_index.items():
		if i < vocab_size:
			embedding_vector = embeddings_index.get(word)
			if embedding_vector is not None:
				embedding_matrix[i] = embedding_vector
	return embedding_matrix
def create_sequences(token_input,token_codes,doc_title,doc_code,max_inp,max_code,vocab_size_out):
	X1, X2, y = list(), list(), list()
	seq_inp = token_input.texts_to_sequences([doc_code])[0]
	seq_out = token_codes.texts_to_sequences([doc_title])[0]
	for i in range(1,len(seq_out)):
		in_seq,out_seq=seq_out[:i],seq_out[i]
		in_seq_2 = pad_sequences([in_seq], maxlen=max_code)[0]
		in_seq = pad_sequences([seq_inp], maxlen=max_inp)[0]
		# encode output sequence
		out_seq = to_categorical([out_seq], num_classes=vocab_size_out)[0]
		X1.append(in_seq)
		X2.append(in_seq_2)
		y.append(out_seq)
	return array(X1), array(X2), array(y)
def seq_gen(mini,maxi,vocab_size_out):
  while 1:
    i = random.randint(mini,maxi)
    x1,x2,y = create_sequences(token_title,token_code,title_list[i],code_list[i],max_len_input,max_len_output,5000)
    # outputs request,probable answer,output
    if x1.shape != (0,):
      yield ([x2,x1],y)
def define_model(matrix,token_input,token_output,max_len_input,max_len_output):
	question_input = Input(shape=(max_len_input,))
	embeding = Embedding(5000, 100,weights=[matrix],trainable=False)(question_input)
	fe1 = Dropout(0.2)(embeding)
	fe2 = LSTM(1024,return_sequences=True)(fe1)
	fe2 = LSTM(1024,return_sequences=True)(fe2)
	fe2 = LSTM(1024,return_sequences=False)(fe2)
	# sequence model
	inputs2 = Input(shape=(max_len_output,))
	se1 = Embedding(5000, 256, mask_zero=True)(inputs2)
	se2 = Dropout(0.2)(se1)
	se3 = LSTM(1024,return_sequences=True)(se2)
	se3 = LSTM(1024,return_sequences=True)(se3)
	se3 = LSTM(1025,return_sequences=False)(se3)
	# decoder model
	decoder1 = add([fe2, se3])
	decoder2 = Dense(1024, activation='relu')(decoder1)
	decoder3 = Dense(1024, activation='relu')(decoder2)
	decoder4 = Dense(1024, activation='relu')(decoder3)
	outputs = Dense(5000, activation='softmax')(decoder4)
	model = Model(inputs=[question_input, inputs2], outputs=outputs)
	model.compile(loss='categorical_crossentropy', optimizer='adam')
	return model
import re
def replace(text):
  rep = {'>>>': "",
         '\n': " (newline) ",
         '+':' ',
         '.':' . ',
         '-':' ',
         '/':' ',
         '*':' ',
         '(':' ( ',
         ')':' ) ',
         '[':' [ ',
         ']':' ] ',
        ':':' : ',
        ',':' , '} # define desired replacements here
  rep = dict((re.escape(k), v) for k, v in rep.items())
  pattern = re.compile("|".join(rep.keys()))
  text = pattern.sub(lambda m: rep[re.escape(m.group(0))], text)
  text = re.sub('\d+',' num ',text)
  text = re.sub('\'(.*?)\'',' string ',text)
  text = re.sub('\"(.*?)\"',' string ',text)
  return text
import numpy as np
def random_func(gamma):
	while 1:
		pairs = next(source)
		for i in range(9):
			pair2 = next(source)
			pairs = [[np.concatenate((pairs[0][0], pair2[0][0]), axis=0),np.concatenate((pairs[0][1], pair2[0][1]), axis=0)],np.concatenate((pairs[1], pair2[1]), axis=0)]
		output_org = pairs[1]
		prev_max=0
		output_pred = model2.predict(pairs[0])
		for i in range(len(output_pred)-1,0,-1):
			reward = 1
			arg_org = argmax(output_org[i])
			arg_pred = argmax(output_pred[i])
			temp_max=np.max(output_pred[i])
			if (arg_org != arg_pred):
				reward=0
			if reward == 0:
				output_pred[i][arg_pred] = reward
				output_pred[i][arg_org] = output_org[i][arg_org] 
			else:
				output_pred[i][arg_pred] = reward + gamma*prev_max
			prev_max=temp_max
		yield [pairs[0],output_pred]<jupyter_output><empty_output><jupyter_text>Import from saved files<jupyter_code>import pickle
Questionbank = pickle.load(open('drive/My Drive/colabapp/codi/dict_pkl.pkl', 'rb'))
import pickle
training_dict = pickle.load(open('drive/My Drive/colabapp/codi/train_dict_pkl.pkl', 'rb'))<jupyter_output><empty_output><jupyter_text>Initialize for training(both initial and continued)<jupyter_code>code_list,score_list,title_list,token_title,token_code = training_dict['code'],training_dict['score'],training_dict['title'],training_dict['token_title'],training_dict['token_code']
max_len_input = max([len(x) for x in token_title.texts_to_sequences(title_list)])
max_len_output = max([len(x) for x in token_code.texts_to_sequences(code_list)])
print(max_len_input)
print(max_len_output)<jupyter_output>40
780<jupyter_text>Initilize Model for training first time<jupyter_code>model2 = define_model(embedding_weight,token_title,token_code,max_len_output,max_len_input)
model2.summary()
optimizer_1 = optimizer.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model2.compile(loss='categorical_crossentropy', optimizer=optimizer_1, metrics=[ 'accuracy'])
embedding_weight=embedding_dict(5000,token_title)<jupyter_output><empty_output><jupyter_text>For pretrained model<jupyter_code>model2 = load_model('drive/My Drive/colabapp/codi/model3.h5')
tbCallBack = TensorBoard(log_dir='./log',
                         write_graph=True,
                         write_grads=True,
                         batch_size=100,
                         write_images=True)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>#@title Default title text
for j in range(10,0,-1):
	source= seq_gen(0,2000*(j+1),5000)
	source2= seq_gen(2000*(j+1),2000*(j+2),5000)
	next(random_func(0.9))	
	print('\n---------dataset({}-{}) and ({}-{}) validation --------------------'.format(2000*j,2000*(j+1),2000*(j+1),2000*(j+2)))
	model2.fit_generator(random_func(0.9),steps_per_epoch=50, epochs=5,validation_data=source2, validation_steps=5,callbacks=[tbCallBack])
	model2.save("model3.h5")
	!cp 'model3.h5' 'drive/My Drive/colabapp/codi/'<jupyter_output>---------dataset(20000-22000) and (22000-24000) validation --------------------
Epoch 1/5
50/50 [==============================] - 522s 10s/step - loss: 8.5415 - acc: 0.2542 - val_loss: 5.1901 - val_acc: 0.1429
Epoch 2/5
50/50 [==============================] - 506s 10s/step - loss: 8.2465 - acc: 0.2624 - val_loss: 3.8943 - val_acc: 0.1852
Epoch 3/5
50/50 [==============================] - 508s 10s/step - loss: 8.2147 - acc: 0.2496 - val_loss: 4.4293 - val_acc: 0.2500
Epoch 4/5
50/50 [==============================] - 514s 10s/step - loss: 8.2686 - acc: 0.2475 - val_loss: 5.2695 - val_acc: 0.1765
Epoch 5/5
50/50 [==============================] - 494s 10s/step - loss: 8.0577 - acc: 0.2549 - val_loss: 4.0033 - val_acc: 0.2727

---------dataset(18000-20000) and (20000-22000) validation --------------------
Epoch 1/5
37/50 [=====================>........] - ETA: 2:11 - loss: 7.6851 - acc: 0.2791<jupyter_text>logging<jupyter_code>!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
!unzip ngrok-stable-linux-amd64.zip
LOG_DIR = './log'
get_ipython().system_raw(
    'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'
    .format(LOG_DIR)
)
get_ipython().system_raw('./ngrok http 6006 &')
! curl -s http://localhost:4040/api/tunnels | python3 -c \
    "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"<jupyter_output>http://79ed92ee.ngrok.io
{'fraction_non_alphanumeric': 0.17840258770838516, 'fraction_numerical': 0.04802189599402837, 'mean_word_length': 5.539045553145336, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 56, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '408420', 'license': 'MIT', 'n_tokens_mistral': 4960, 'n_tokens_neox': 4514, 'n_words': 764, 'path': 'codi.ipynb', 'repo_name': 'Gauthams1/codi', 'size': '12057'}