<jupyter_start><jupyter_text>Imports<jupyter_code>import pandas as pd
import json<jupyter_output><empty_output><jupyter_text>Settings to enable real-time output from a shell command<jupyter_code>from subprocess import Popen, PIPE, STDOUT
from IPython.core.magic import register_line_magic

@register_line_magic
def runrealcmd(command):
    process = Popen(command, stdout=PIPE, shell=True, stderr=STDOUT, bufsize=1, close_fds=True)
    for line in iter(process.stdout.readline, b''):
        print(line.rstrip().decode('utf-8'))
    process.stdout.close()
    process.wait()<jupyter_output><empty_output><jupyter_text>1. Self Join  1.a Preview the input file<jupyter_code>df = pd.read_csv('../data/dblp_papers_test.csv', sep=';', header=None, names=['Paper_ID', 'Title'])
df.head()<jupyter_output><empty_output><jupyter_text>1.b Load and optionally edit the config file<jupyter_code>config_file_example = '../config.json.example'
config_file = '../config.json'
input_file = '../data/dblp_papers_test.csv'
output_file = '../data/output/dblp_papers_selfjoin_out.csv'
log_file = '../data/output/log.txt'
params = json.load(open(config_file_example))
params
params['join_type'] = 'threshold'
params['threshold'] = '0.33'
params['input_file'] = input_file
del(params['query_file'])
params['output_file'] = output_file
params['log_file'] = log_file
params['set_column'] = '1'
params['tokens_column'] = '2'
params
json.dump(params, open(config_file, 'w'), indent=4)<jupyter_output><empty_output><jupyter_text>1.c Execute the join operation<jupyter_code>%runrealcmd java -jar ../target/simjoin-0.0.1-SNAPSHOT-jar-with-dependencies.jar $config_file<jupyter_output>Finished reading file. Lines read: 18. Lines skipped due to errors: 0. Num of sets: 18. Elements per set: 6.222222222222222
Read time: 0.002008185 sec.
Transform time: 0.011628722 sec.
Collection size: 18
Join time: 0.017142176 sec.
Number of matches: 11<jupyter_text>1.d Load the results<jupyter_code>out1 = pd.read_csv(output_file, header=None, names=['Paper_ID_1', 'Paper_ID_2', 'Similarity'])
out1['Title_1'] = df.set_index('Paper_ID').Title[out1.Paper_ID_1].reset_index(drop=True)
out1['Title_2'] = df.set_index('Paper_ID').Title[out1.Paper_ID_2].reset_index(drop=True)
out1.sort_values('Similarity', ascending=False).head(11)<jupyter_output><empty_output><jupyter_text>2. Self Top K  2.a Load and optionally edit the config file<jupyter_code>output_file = '../data/output/dblp_papers_selfclosestpairs_out.csv'

params['join_type'] = 'topk'
params['k'] = '5'
params['input_file'] = input_file
params['output_file'] = output_file
params['log_file'] = log_file
params['set_column'] = '1'
params['tokens_column'] = '2'
json.dump(params, open(config_file, 'w'), indent=4)<jupyter_output><empty_output><jupyter_text>2.b Execute the join operation<jupyter_code>%runrealcmd java -jar ../target/simjoin-0.0.1-SNAPSHOT-jar-with-dependencies.jar $config_file<jupyter_output>Finished reading file. Lines read: 18. Lines skipped due to errors: 0. Num of sets: 18. Elements per set: 6.222222222222222
Read time: 0.003324423 sec.
Transform time: 0.016882793 sec.
Collection size: 18
Join time: 0.020287217 sec.
Number of matches: 5<jupyter_text>2.c Load the results<jupyter_code>out2 = pd.read_csv(output_file, header=None, names=['Paper_ID_1', 'Paper_ID_2', 'Similarity'])
out2['Title_1'] = df.set_index('Paper_ID').Title[out2.Paper_ID_1].reset_index(drop=True)
out2['Title_2'] = df.set_index('Paper_ID').Title[out2.Paper_ID_2].reset_index(drop=True)
out2.sort_values('Similarity', ascending=False).head()
out1.sort_values("Similarity", ascending=False).head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14729321242099477, 'fraction_numerical': 0.04534212695795548, 'mean_word_length': 7.329519450800915, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '177361', 'license': 'Apache-2.0', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1337, 'n_words': 245, 'path': 'notebooks/Example-02-Papers_with_Similar_Titles.ipynb', 'repo_name': 'smartdatalake/simjoin', 'size': '3639'}