<jupyter_start><jupyter_text>Importing packages<jupyter_code>import warnings
warnings.filterwarnings('ignore')

import json
import sys
import os
import pandas as pd<jupyter_output><empty_output><jupyter_text>Loading RAW DATASET<jupyter_code># DATA_DIR = "../../dataset/new-prepared-data/qa/"

DATA_FILES = "../../dataset/new-prepared-data/qa/full_qa.json"<jupyter_output><empty_output><jupyter_text>Parsing XML to Data Frame with necessary tags<jupyter_code>data_column_names = ["Answer"]

TXT_GEN_SIZE = 0
Answers = []

with open(DATA_FILES) as f:
    tree = json.loads(f.read())

for each_data in tree['data']:
    for each_para in each_data['paragraphs']:
        Answers.append(each_para['context'])
        TXT_GEN_SIZE +=1

TXT_GEN_DATAFRAME = pd.DataFrame(Answers, columns = data_column_names)<jupyter_output><empty_output><jupyter_text>Spliting Data Frames to TRAIN(80%), DEV(10%), TEST(10%)<jupyter_code>TRAIN_TXT_GEN_DATAFRAME = TXT_GEN_DATAFRAME.sample(frac=0.8,random_state=200)

temp_df = TXT_GEN_DATAFRAME.drop(TRAIN_TXT_GEN_DATAFRAME.index)
DEV_TXT_GEN_DATAFRAME = temp_df.sample(frac=0.5,random_state=200)

TEST_TXT_GEN_DATAFRAME = temp_df.drop(DEV_TXT_GEN_DATAFRAME.index)

TRAIN_TXT_GEN_DATAFRAME = TRAIN_TXT_GEN_DATAFRAME.reset_index(drop=True)
DEV_TXT_GEN_DATAFRAME = DEV_TXT_GEN_DATAFRAME.reset_index(drop=True)
TEST_TXT_GEN_DATAFRAME = TEST_TXT_GEN_DATAFRAME.reset_index(drop=True)

print("  Train datasize (80%)     :", len(TRAIN_TXT_GEN_DATAFRAME), 
      "\n+ Validation datasize (10%): ",len(DEV_TXT_GEN_DATAFRAME),
      "\n+ Test datasize (10%)      : ", len(TEST_TXT_GEN_DATAFRAME), 
      "\n", "-"*32, "\n", 
      " Total QA size            :", TXT_GEN_SIZE)<jupyter_output>Train datasize (80%)     : 583 
+ Validation datasize (10%):  73 
+ Test datasize (10%)      :  73 
 -------------------------------- 
  Total QA size            : 729<jupyter_text>Added $\lt$BOS$\gt$ and $\lt$EOS$\gt$ tags<jupyter_code>TRAIN_TXT_GEN_DATAFRAME['Answer'][0]
def dataFrame_to_text_generate(DataFrame_name, OutPut_json_file_name):
    with open(OutPut_json_file_name, 'w') as outfile:
        for index in range(len(DataFrame_name)):
            temp_answer = DataFrame_name.Answer[index]
            if temp_answer:
                temp_answer = "<BOS>"+temp_answer+"<EOS>\n"
#             for line in temp_answer:
#                 line = line.strip()
#                 if line:
#                     temp_line = "<BOS>"+line+"<EOS>\n"
                outfile.write(temp_answer)<jupyter_output><empty_output><jupyter_text>Saving data to file<jupyter_code>PREPARED_DATA_PATH ='../../dataset/new-prepared-data/text-generation/'

TRAIN_TXT_GEN_TXT = PREPARED_DATA_PATH + 'train_text-generation.txt'
DEV_TXT_GEN_TXT = PREPARED_DATA_PATH + 'dev_text-generation.txt'
TEST_TXT_GEN_TXT = PREPARED_DATA_PATH + 'test_text-generation.txt'

dataFrame_to_text_generate(TRAIN_TXT_GEN_DATAFRAME, TRAIN_TXT_GEN_TXT)
dataFrame_to_text_generate(DEV_TXT_GEN_DATAFRAME, DEV_TXT_GEN_TXT)
dataFrame_to_text_generate(TEST_TXT_GEN_DATAFRAME, TEST_TXT_GEN_TXT)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1282798833819242, 'fraction_numerical': 0.013929381276320051, 'mean_word_length': 4.624772313296903, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '658644', 'license': 'MIT', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1156, 'n_words': 160, 'path': 'dataset/data-prepare-script/new-data-prepare-script_text-generation.ipynb', 'repo_name': 'uabinf/nlp-group-project-fall-2020-deepbiocomp', 'size': '3087'}