<jupyter_start><jupyter_text>Parsing the Ground Truth<jupyter_code># Build the dictionary of all plagiarism pairings.
pDict = {}
# Build a list of plagiarism docs to include in this subset.
pInc = []
f = open("articles_10000.truth", 'rU')
for line in f:
    if line[-1] == '\n':
        line = line[0:-1]
        print('line is', line)
    print('second time', line)
    docs = line.split(" ")
    print('docs is', docs)
for line in f:
    if line[-1] == '\n':
        line = line[0:-1]
    docs = line.split(" ")
    #Mapping the last two documents to each other 
    pDict[docs[0]] = docs[1]
    pDict[docs[1]] = docs[0]
    
    if len(pInc) < (2*numPs):
        pInc.append(docs[0])
        pInc.append(docs[1])
print("Plagiarisms to grab", str(pInc))
len(pInc)<jupyter_output><empty_output><jupyter_text>Grab the Plagiarism examples<jupyter_code>f = open("articles_10000.train", "rU")
outLines = []
#non plagiarised articles 
nonPs = numDocs - (numPs*2)
for line in f:
    m = re.search(r't\d+', line)
    docID = m.group() 
    
    #if ID in plagiarsm subset group, add here
    if docID in pInc:
        outLines.append(line)
    elif nonPs > 0 and not docID in pDict:
        outLines.append(line)
        nonPs = nonPs - 1
    
    if len(outLines) == numDocs:
        break  
f.close()
#Retrieve integer document ID from beginning of the line 
def nameToInt(s):
    m = re.search(r'\d+', s)
    return int(m.group())
outLines = sorted(outLines, key=nameToInt)
#write out the lines
filename = "articles_" + str(numDocs) + ".train"
print("Writing dataset to ", filename)
f = open(filename, "w")
f.writelines(outLines)
f.close()
# Write out the ground truth.
filename = "articles_" + str(numDocs) + ".truth"
print("Writing truth to " + filename)
f = open(filename, "w")

for i in range(0, len(pInc), 2):
  # Add the pair to the truth table.
  f.write(pInc[i] + " " + pInc[i + 1] + "\n")
    
f.close()<jupyter_output>Writing truth to articles_1000.truth
{'fraction_non_alphanumeric': 0.14737378888322283, 'fraction_numerical': 0.016828148903620603, 'mean_word_length': 3.237580993520518, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '174299', 'license': 'MIT', 'n_tokens_mistral': 765, 'n_tokens_neox': 714, 'n_words': 204, 'path': 'data/createSubset.ipynb', 'repo_name': 'anamikasen/MinHash', 'size': '1961'}