<jupyter_start><jupyter_text>IntroductionIn this notebook we demonstrate the use of **Word Embeddings (Word2Vec)** weighting technique into Information Retrieval to make trace link recovery between Test Cases and Bug Reports.We model our study as follows:* Each bug report title, summary and description compose a single query.* We use each test case content as an entire document that must be returned to the query made  Import Libraries<jupyter_code>from mod_finder_util import mod_finder_util
mod_finder_util.add_modules_origin_search_path()

import pandas as pd

from modules.models_runner.tc_br_models_runner import TC_BR_Runner
from modules.models_runner.tc_br_models_runner import TC_BR_Models_Hyperp
from modules.utils import aux_functions
from modules.utils import firefox_dataset_p2 as fd
from modules.utils import tokenizers as tok

from modules.models.wordvec import WordVec_BasedModel

from IPython.display import display

import warnings; warnings.simplefilter('ignore')<jupyter_output><empty_output><jupyter_text>Load Datasets<jupyter_code>tcs = [x for x in range(37,59)]
orc = fd.Tc_BR_Oracles.read_oracle_expert_df()
orc_subset = orc[orc.index.isin(tcs)]
#aux_functions.highlight_df(orc_subset)
tcs = [13,37,60]
brs = [1267501]

testcases = fd.Datasets.read_testcases_df()
testcases = testcases[testcases.TC_Number.isin(tcs)]
bugreports = fd.Datasets.read_selected_bugreports_df()
bugreports = bugreports[bugreports.Bug_Number.isin(brs)]

print('tc.shape: {}'.format(testcases.shape))
print('br.shape: {}'.format(bugreports.shape))<jupyter_output>TestCases.shape: (195, 12)
SelectedBugReports.shape: (91, 18)
tc.shape: (3, 12)
br.shape: (1, 18)<jupyter_text>Running WordVector Model<jupyter_code>corpus = testcases.tc_desc
query = bugreports.br_desc
test_cases_names = testcases.tc_name
bug_reports_names = bugreports.br_name

wv_hyperp = TC_BR_Models_Hyperp.get_w2v_model_hyperp()
wv_model = WordVec_BasedModel(**wv_hyperp)
wv_model.set_name('WV_Model_TC_BR')
wv_model.recover_links(corpus, query, test_cases_names, bug_reports_names)
wv_model.get_sim_matrix().shape
sim_matrix = wv_model.get_sim_matrix()
aux_functions.highlight_df(sim_matrix)
df_tcs = pd.DataFrame([tc.vector for tc in wv_model.tc_docs])
df_tcs.index = test_cases_names
print(df_tcs.shape)
df_tcs.iloc[0:5,0:4]
df_brs = pd.DataFrame([wv_model.br_docs[0].vector])
df_brs.index = bug_reports_names
print(df_brs.shape)
df_brs.iloc[0:5,0:4]<jupyter_output>(1, 300)<jupyter_text>Word Vector Pseudo-Example<jupyter_code>br_doc = wv_model.br_docs[0]
br_tokens = [token for token in br_doc]

df_w_emb = pd.DataFrame([br_doc[i].vector for i in range(len(br_doc))])
df_w_emb.index = br_tokens
df_w_emb.iloc[71:76,0:4]<jupyter_output><empty_output><jupyter_text>Bug Report<jupyter_code>br_doc = wv_model.br_docs[0]
br_tokens = [token for token in br_doc]

df_w_emb = pd.DataFrame([br_doc[i].vector for i in range(len(br_doc))])
df_w_emb.index = br_tokens
df_w_emb.iloc[24:27,0:4]<jupyter_output><empty_output><jupyter_text>Test Case 13<jupyter_code>tc_doc = wv_model.tc_docs[0]
tc_tokens = [token for token in tc_doc]

df_w_emb = pd.DataFrame([tc_doc[i].vector for i in range(len(tc_doc))])
df_w_emb.index = tc_tokens
df_w_emb.iloc[30:33,0:4]<jupyter_output><empty_output><jupyter_text>Test Case 37<jupyter_code>tc_doc = wv_model.tc_docs[1]
tc_tokens = [token for token in tc_doc]

df_w_emb = pd.DataFrame([tc_doc[i].vector for i in range(len(tc_doc))])
df_w_emb.index = tc_tokens
df_w_emb.iloc[35:38,0:4]<jupyter_output><empty_output><jupyter_text>Test Case 60<jupyter_code>tc_doc = wv_model.tc_docs[2]
tc_tokens = [token for token in tc_doc]

df_w_emb = pd.DataFrame([tc_doc[i].vector for i in range(len(tc_doc))])
df_w_emb.index = tc_tokens
df_w_emb.iloc[30:33,0:4]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10969793322734499, 'fraction_numerical': 0.023582405935347113, 'mean_word_length': 8.162621359223301, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '41287', 'license': 'MIT', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1478, 'n_words': 272, 'path': 'notebooks/firefox_p2/tc_br_tracing/wordvec-model.ipynb', 'repo_name': 'guilhermemg/trace-links-tc-br', 'size': '3774'}