<jupyter_start><jupyter_text>Question GenerationPipeline description:1. Load data (**SQUAD**):    * Read JSON format    * Rearrange to the required format    * Explore and clean the data2. Data Processing:    * Word/Sentence Tokenize    * Named Entity Recognition3. Model Ready4. Train and save the model5. Evaluate the model for question generation<jupyter_code>import pandas as pd
import json
import numpy as np
data = pd.read_json(r'dev-v2.0.json')
data = pd.DataFrame(data)
def squad_json_to_dataframe_train(input_file_path, record_path = ['data','paragraphs','qas','answers'],
                           verbose = 1):
    """
    input_file_path: path to the squad json file.
    record_path: path to deepest level in json file default value is
    ['data','paragraphs','qas','answers']
    verbose: 0 to suppress it default is 1
    """
    if verbose:
        print("Reading the json file")    
    file = json.loads(open(input_file_path).read())
    if verbose:
        print("processing...")
    # parsing different level's in the json file
    js = pd.io.json.json_normalize(file , record_path )
    m = pd.io.json.json_normalize(file, record_path[:-1] )
    r = pd.io.json.json_normalize(file,record_path[:-2])
    
    #combining it into single dataframe
    idx = np.repeat(r['context'].values, r.qas.str.len())
    ndx  = np.repeat(m['id'].values,m['answers'].str.len())
    m['context'] = idx
    js['q_idx'] = ndx
    main = pd.concat([ m[['id','question','context']].set_index('id'),js.set_index('q_idx')],1,sort=False).reset_index()
    main['c_id'] = main['context'].factorize()[0]
    if verbose:
        print("shape of the dataframe is {}".format(main.shape))
        print("Done")
    return main
input_file_path = 'train-v2.0.json'
record_path = ['data','paragraphs','qas','answers']
train = squad_json_to_dataframe_train(input_file_path=input_file_path,record_path=record_path)
train
train = train.dropna()
! pip install -U tokenizers
import nltk
!nltk.download()
nltk.download('punkt')
train_cont = train['context']
train_cont[4]
train_len = len(train_cont)
from nltk.tokenize import sent_tokenize, word_tokenize


for i in range(train_len):
  # words = word_tokenize(train_cont[i])
  words = sent_tokenize(train_cont[i])
#words = word_tokenize(train_cont[1])

# print(phrases)
print(words)
words[]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1389124893797791, 'fraction_numerical': 0.0076465590484282074, 'mean_word_length': 4.244988864142539, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '552313', 'license': 'MIT', 'n_tokens_mistral': 809, 'n_tokens_neox': 788, 'n_words': 181, 'path': 'Merida/QuestionGeneration/QuestionGeneration.ipynb', 'repo_name': 'rahulmadanraju/Semantic-Search-Engine', 'size': '2354'}