<jupyter_start><jupyter_text>Config<jupyter_code>qlang = "en"
dlang = "en"
root_dir = "/work/ogalolu/datasets"
year = "2002"
num_negative = 32
doc_ids, documents, query_ids, queries, relass = load_clef(qlang, dlang, year)
print("The number total documents:", len(documents))<jupyter_output>The number total documents: 169477<jupyter_text>Initial Checks<jupyter_code>s1 = set(relass.keys())
s2 = set(query_ids)
print("There are no relevance assesment for the following queries:", s2 - s1)
ids = list(chain.from_iterable(list(relass.values())))
print("Number of available query-document pairs:", len(ids))<jupyter_output>Number of available query-document pairs: 821<jupyter_text>Filter Documents<jupyter_code># Some documents are very short
exclude_set = set()
for i, doc in enumerate(documents):
    if len(doc) < 100:
        exclude_set.add(i)
print("Number of documents that are excluded from dataset building:", len(exclude_set))<jupyter_output>Number of documents that are excluded from dataset building: 2765<jupyter_text>Obtain Training Instances<jupyter_code>def find_negative(d_ind_list: list, num_negative: int) -> tuple:    
    """ Find negative examples by excluding the given list of document indices."""
    all_set = set(range(len(documents)))
    # To ensure that relevant documents won't appear in the set of negative examples
    available_set = all_set - set(d_ind_list) - exclude_set
    neg_ind_list = random.choices(tuple(available_set), k=num_negative)  
    assert not any(x in neg_ind_list for x in d_ind_list)
    negative_tuple = [documents[idx] for idx in neg_ind_list]
    
    return negative_tuple
instance_list = list()
for q_ind, d_ind in tqdm.tqdm(relass.items()):
    temp_query = [queries[query_ids.index(q_ind)]]
    temp_documents = [documents[doc_ids.index(idx)] for idx in d_ind]
    temp_list = list(product(temp_query, temp_documents))  # list of tuples
    
    # if negative examples to be added, then for each training instance that has the same query, create a 
    # different set of negative examples, the number of negative examples is set at the beginning
    
    if num_negative is not None:
        d_ind_list = [doc_ids.index(idx) for idx in d_ind]  
        for q in temp_list:
            temp_negatives = find_negative(d_ind_list, num_negative)
            temp_tuple = (*q, *temp_negatives)
            instance_list.append(temp_tuple)
    else:
        instance_list.extend(temp_list)

assert len(instance_list) == len(ids)
instance_dic = defaultdict(list)
for q, d, *negatives in instance_list:
    # For the naming convengtion, "article" is used, whereas article_1 and article_2 are reserved for query and document
    instance_dic["article_1"].append(q)
    instance_dic["article_2"].append(d)
    
    # If negatives to be added    
    if num_negative is not None:
        for idx, neg in enumerate(negatives, 3):
            instance_dic[f"article_{idx}"].append(neg)<jupyter_output><empty_output><jupyter_text>Create Dataset<jupyter_code>dataset = Dataset.from_dict(instance_dic)
print(dataset[0]["article_18"])
dataset = dataset.train_test_split(test_size=0.1)
print(dataset)
path = os.path.join(root_dir, f"clef_{year}_{num_negative}")
dataset.save_to_disk(path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.099049371358479, 'fraction_numerical': 0.011039558417663294, 'mean_word_length': 4.835420393559929, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '320394', 'license': 'Apache-2.0', 'n_tokens_mistral': 1094, 'n_tokens_neox': 1056, 'n_words': 283, 'path': 'notebooks/clef_finetuning_data.ipynb', 'repo_name': 'ogal93/pre-training-multilingual-document-encoders', 'size': '3261'}