<jupyter_start><jupyter_text>NLTK Preprocessing<jupyter_code># export
# Imports
import pandas as pd
import random
import sentencepiece as sp

from ds4se.mgmnt.prep.traceability import *
from fastprogress.fastprogress import master_bar
from pathlib import Path
#hide
from nbdev.showdoc import *
# export
def get_nltk_docs(df, src_fns, trgt_fns):
    src_docs, trgt_docs = [], []
    
    for src, trgt in zip(src_fns, trgt_fns):
        src_docs.append(df.loc[df.file == src].text.values[0])
        trgt_docs.append(df.loc[df.file == trgt].text.values[0])
    
    return src_docs, trgt_docs
lang = 'english'
path = Path('../benchmarking/traceability/testbeds'); list(path.glob("*"))
links_df = get_gt_links(path/'groundtruth'/lang, lang)
links_df.head()
req_df = pd.read_csv(path/'nltk/[libest-pre-req].csv', header = None, names = ['file', 'text'], delimiter = ' ')
src_df = pd.read_csv(path/'nltk/[libest-pre-src].csv', header = None, names = ['file', 'text'], delimiter = ' ')
tc_df = pd.read_csv(path/'nltk/[libest-pre-tc].csv', header = None, names = ['file', 'text'], delimiter = ' ')
nltk_df = pd.concat([req_df, src_df, tc_df]).reset_index(drop = True)
nltk_df['file'] = nltk_df['file'].apply(lambda x: str(Path(x).name).replace('-pre', ''))
nltk_df.head()
libest_links = links_df.loc[links_df['sys'] == 'libest'].copy()
src_docs, trgt_docs = get_nltk_docs(nltk_df, libest_links['src_file'].values, libest_links['trgt_file'].values)
libest_links['src_doc'] = src_docs
libest_links['trgt_doc'] = trgt_docs
libest_links.head()
from nbdev.export import notebook2script
notebook2script()<jupyter_output>Converted 00_mgmnt.prep.i.ipynb.
Converted 01_exp.i.ipynb.
Converted 02_mgmnt.db.mongo.ipynb.
Converted 03_repr.i.ipynb.
Converted 04_mining.ir.model.ipynb.
Converted 05_mining.ir.i.ipynb.
Converted 06_benchmark.traceability.ipynb.
Converted 07_repr.roberta.train.ipynb.
Converted 08_exp.info.ipynb.
Converted 09_desc.stats.ipynb.
Converted 10_vis.ipynb.
Converted 11_mgmnt.prep.conv.ipynb.
Converted 12_repr.roberta.eval.ipynb.
Converted 14_mgmnt.prep.bpe.ipynb.
Converted 15_desc.metrics.se.ipynb.
Converted 16_repr.word2vec.train.ipynb.
Converted 17_repr.doc2vec.train.ipynb.
Converted 18_repr.doc2vec.eval.ipynb.
Converted 19_repr.word2vec.eval.ipynb.
Converted 20_benchmark.codegen.ipynb.
Converted 21_inf.i.ipynb.
Converted 22_inf.bayesian.ipynb.
Converted 23_inf.causal.ipynb.
Converted 24_mgmnt.corpus.ipynb.
Converted 25_mgmnt.prep.traceability.ipynb.
Converted 26_mgmnt.prep.nltk.ipynb.
Converted aa_blog.example.ipynb.
Converted ab_templates.example.ipynb.
Converted ac_emp.eval.pp1.rq1.ip[...]
{'fraction_non_alphanumeric': 0.14389590508993494, 'fraction_numerical': 0.024110218140068886, 'mean_word_length': 5.454320987654321, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '258170', 'license': 'Apache-2.0', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1075, 'n_words': 167, 'path': 'nbs/0.6_mgmnt.prep.nltk.ipynb', 'repo_name': 'rmclanton/ds4se', 'size': '2613'}