<jupyter_start><jupyter_text>1. Parameters<jupyter_code># Defaults
cases_dir = 'cases/unset'
reference_file = 'references/NC_045512.gbk.gz'
input_files_all = 'input/input-files.tsv'
iterations = 3
mincov = 10
ncores = 32
number_samples = 10
build_tree = False
sample_batch_size=2000
# Parameters
cases_dir = "cases/case-10000-batch-10000"
iterations = 3
number_samples = 10000
sample_batch_size = 10000
build_tree = False

from pathlib import Path
from shutil import rmtree
from os import makedirs
import imp
fp, pathname, description = imp.find_module('gdi_benchmark', ['../../lib'])
gdi_benchmark = imp.load_module('gdi_benchmark', fp, pathname, description)

cases_dir_path = Path(cases_dir)

if cases_dir_path.exists():
    rmtree(cases_dir_path)
    
if not cases_dir_path.exists():
    makedirs(cases_dir_path)

input_files_all = Path(input_files_all)
reference_file = Path(reference_file)

case_name = str(cases_dir_path.name)
reference_name = reference_file.name.split('.')[0]

cases_input = cases_dir_path / 'input-files-case.tsv'
index_path = cases_dir_path / 'index'
benchmark_path = cases_dir_path / 'index-info.tsv'
output_tree = cases_dir_path / 'tree.tre'<jupyter_output><empty_output><jupyter_text>2. Create subset input<jupyter_code>import pandas as pd

all_input_df = pd.read_csv(input_files_all, sep='\t')
all_input_total = len(all_input_df)

subset_input_df = all_input_df.head(number_samples)
subset_input_total = len(subset_input_df)

subset_input_df.to_csv(cases_input, sep='\t', index=False)

print(f'Wrote {subset_input_total}/{all_input_total} samples to {cases_input}')<jupyter_output>Wrote 10000/100000 samples to cases/case-10000-batch-10000/input-files-case.tsv<jupyter_text>2. Index genomes<jupyter_code>!gdi --version<jupyter_output>gdi, version 0.4.0.dev1<jupyter_text>2.1. Index reads<jupyter_code>results_handler = gdi_benchmark.BenchmarkResultsHandler(name=case_name)
benchmarker = gdi_benchmark.IndexBenchmarker(benchmark_results_handler=results_handler,
                                             index_path=index_path, input_files_file=cases_input,
                                             reference_file=reference_file, mincov=mincov,
                                             build_tree=build_tree,
                                             ncores=ncores,
                                             sample_batch_size=sample_batch_size)

benchmark_df = benchmarker.benchmark(iterations=iterations)
benchmark_df
benchmark_df.to_csv(benchmark_path, sep='\t', index=False)<jupyter_output><empty_output><jupyter_text>3. Export trees<jupyter_code>if build_tree:
    !gdi --project-dir {index_path} export tree {reference_name} > {output_tree}
    print(f'Wrote tree to {output_tree}')
else:
    print(f'build_tree={build_tree} so no tree to export')<jupyter_output>build_tree=False so no tree to export
{'fraction_non_alphanumeric': 0.09961417046650298, 'fraction_numerical': 0.024552788495264818, 'mean_word_length': 4.411764705882353, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '105809', 'license': 'Apache-2.0', 'n_tokens_mistral': 1028, 'n_tokens_neox': 967, 'n_words': 160, 'path': 'evaluations/sars-cov-2/3-index-genomes.case-10000-batch-10000.ipynb', 'repo_name': 'apetkau/genomics-data-index-evaluation', 'size': '2851'}