<jupyter_start><jupyter_text>FinBERT QA  Data preprocessing1. Loads and cleans the raw data2. Prepares the data for the retriever3. Pre-processes and tokenizes the raw cleaned data4. Creates vocabulary from the corpus<jupyter_code>!git clone https://github.com/sparsh9012/FinBERT-QA
%cd FinBERT-QA
from src.utils import *
from src.process_data import *
# document dataset
collection = load_answers_to_df("data/raw/FiQA_train_doc_final.tsv")
collection.head()
# question dataset
queries = load_questions_to_df("data/raw/FiQA_train_question_final.tsv")
queries.head()
# question to document mapping
qid_docid = load_qid_docid_to_df("data/raw/FiQA_train_question_doc_final.tsv")
qid_rel = label_to_dict(qid_docid)
qid_docid.head()
# Cleaning data
empty_docs, empty_id = get_empty_docs(collection)
# Remove empty answers from collection of answers
collection_cleaned = collection.drop(empty_id)
# Remove empty answers from qa pairs
qid_docid = qid_docid[~qid_docid['docid'].isin(empty_docs)]
# Write collection df to file
save_tsv("retriever/collection_cleaned.tsv", collection_cleaned)

# Convert collection df to JSON file for document indexer
collection_to_json("retriever/collection_json/docs.json", "retriever/collection_cleaned.tsv")
# process questions
processed_questions = process_questions(queries)
processed_questions.head()
# process answers
processed_answers = process_answers(collection_cleaned)
processed_answers.head()
# statistics
avg_ans_count = processed_answers['ans_len'].mean()
avg_q_count = processed_questions['q_len'].mean()

print("Average answer length: {}".format(round(avg_ans_count)))
print("Average question length: {}".format(round(avg_q_count)))

print("Total answers: {}".format(len(processed_answers)))
print("Number of answers with length greater than 512: {}".format(len(processed_answers[processed_answers['ans_len'] > 512])))
# Create vocabulary
word2index, word2count = create_vocab(processed_answers, processed_questions)
print("Vocab size: {}".format(len(word2index)))
print("Top {} common words: {}".format(35, Counter(word2count).most_common(35)))

qid_to_text, docid_to_text = id_to_text(collection, queries)
qid_to_tokenized_text, docid_to_tokenized_text = id_to_tokenized_text(processed_answers, processed_questions)

# Save objects to pickle
save_pickle("data/qa_lstm_tokenizer/word2index.pickle", word2index)
save_pickle("data/qa_lstm_tokenizer/word2count.pickle", word2count)

# id map to raw text
save_pickle("data/id_to_text/qid_to_text.pickle", qid_to_text)
save_pickle("data/id_to_text/docid_to_text.pickle", docid_to_text)

# id map to tokenized text
save_pickle("data/qa_lstm_tokenizer/qid_to_tokenized_text.pickle", qid_to_tokenized_text)
save_pickle("data/qa_lstm_tokenizer/docid_to_tokenized_text.pickle", docid_to_tokenized_text)<jupyter_output>Vocab size: 85034
Top 35 common words: [('the', 371203), ('to', 233559), ('a', 201620), ('you', 166702), ('and', 163066), ('of', 157574), ('is', 129894), ('in', 120019), ('that', 111416), ('for', 89366), ('it', 83822), ('i', 74100), ('your', 68153), ('are', 67255), ('if', 60689), ('be', 59266), ('on', 58382), ('have', 55754), ('as', 50088), ('this', 49868), ('not', 49227), ('or', 46080), ('with', 45894), ('they', 44485), ('but', 41690), ('can', 38863), ('will', 36865), ('at', 35548), ('an', 31392), ('money', 31003), ('so', 29980), ('$', 29096), ('would', 28750), ('from', 28582), ('more', 27378)]<jupyter_text>**FinBERT-QA**<jupyter_code>!git clone https://github.com/sparsh9012/FinBERT-QA
%cd FinBERT-QA
from src.utils import *
!pip install transformers
import torch
import pickle
import csv
import regex as re
import pandas as pd
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using device:', device)
torch.backends.cudnn.deterministic = True
torch.manual_seed(1234)
# Collection of answers - docid, text
collection = pd.read_csv("retriever/collection_cleaned.tsv", sep="\t", header=None)
collection = collection.rename(columns={0: 'docid', 1: 'doc'})
# Questions - qid, text
query_df = pd.read_csv("data/raw/FiQA_train_question_final.tsv", sep="\t")
queries = query_df[['qid', 'question']]

# List of empty docs
empty_docs = load_pickle('data/id_to_text/empty_docs.pickle')

# docid to text mapping
docid_to_text = load_pickle('data/id_to_text/docid_to_text.pickle')
# qid to text mapping
qid_to_text = load_pickle('data/id_to_text/qid_to_text.pickle')
# Load and process dataset
dataset = pd.read_csv("data/raw/FiQA_train_question_doc_final.tsv", sep="\t")
dataset = dataset[["qid", "docid"]]
dataset = dataset[~dataset['docid'].isin(empty_docs)]
dataset['question'] = dataset['qid'].apply(lambda x: qid_to_text[x])
dataset['answer'] = dataset['docid'].apply(lambda x: docid_to_text[x])
dataset.head()
def add_ques_token(string):
    question = string + " [SEP] "
    return question
# Concatenate question and answer with a separator
dataset['question'] = dataset['question'].apply(add_ques_token)
dataset['seq'] = dataset['question'] + dataset['answer']
dataset = dataset[['seq']]

dataset.at[17081, "seq"]
# Write data to file
dataset.to_csv('data/data.txt',index=False,header=False, sep="\t", quoting=csv.QUOTE_NONE)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15045270660759005, 'fraction_numerical': 0.0446927374301676, 'mean_word_length': 7.844974446337308, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '375778', 'license': 'Apache-2.0', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1825, 'n_words': 384, 'path': '_notebooks/2022-01-29-finbert.ipynb', 'repo_name': 'recohut/notebook', 'size': '5191'}