<commit_before>#!/usr/bin/python

import argparse
import os
import sys
import subprocess
import logging
import shutil
import time
import signal
import multiprocessing as mp
from multiprocessing import Process

def get_contigs_list(reference):
  with open("%s.fai" % (reference)) as fai_file:
    contigs = [line.strip().split()[0] for line in fai_file.readlines()]
  return contigs

my_dir = os.path.dirname(os.path.realpath(__file__))

default_vcf2diploid = os.path.join(my_dir, "target/build/vcf2diploid.jar")
default_randvcf2vcf = os.path.join(my_dir, "target/build/randvcf2vcf.jar")
default_randdgv2vcf = os.path.join(my_dir, "target/build/randdgv2vcf.jar")
default_liftover    = os.path.join(my_dir, "target/build/fastq_liftover.jar")
default_vcfstats    = os.path.join(my_dir, "target/build/vcfstats.jar")

require_randvcf2vcf = not os.path.isfile(default_randvcf2vcf)
require_randdgv2vcf = not os.path.isfile(default_randdgv2vcf)
require_vcf2diploid = not os.path.isfile(default_vcf2diploid)
require_liftover    = not os.path.isfile(default_liftover)
require_vcfstats    = not os.path.isfile(default_vcfstats)

if not os.path.isfile(default_randvcf2vcf): default_randvcf2vcf = None
if not os.path.isfile(default_randdgv2vcf): default_randdgv2vcf = None
if not os.path.isfile(default_vcf2diploid): default_vcf2diploid = None
if not os.path.isfile(default_liftover):    default_liftover    = None
if not os.path.isfile(default_vcfstats):    default_vcfstats    = None

main_parser = argparse.ArgumentParser(description="VarSim: An accurate reads simulator", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
main_parser.add_argument("--out_dir", metavar="Out directory", help="Output directory", required=False, default="out")
main_parser.add_argument("--work_dir", metavar="Work directory", help="Work directory", required=False, default="work")
main_parser.add_argument("--log_dir", metavar="Log directory", help="Directory to log to", required=False, default="log")
main_parser.add_argument("--reference", metavar="Reference", help="Reference file", required=True, type=file)
main_parser.add_argument("--seed", metavar="seed", help="Random number seed", type=int, default=0)
main_parser.add_argument("--sex", metavar="Sex", help="Sex of the person (male/female)", required=False, type=str, choices=["male", "female"], default="male")
main_parser.add_argument("--id", metavar="id", help="Sample ID", required=True)
main_parser.add_argument("--simulator", metavar="simulator", help="Read simulator", required=False, type=str, choices=["art", "dwgsim"], default="art")
main_parser.add_argument("--vcf2diploid_jar", metavar="vcf2diploid_jar", help="vcf2diploid jar", type=file, default=default_vcf2diploid, required=require_vcf2diploid)
main_parser.add_argument("--read_length",    metavar="read_length",    help="Length of reads", default=100, type=int)
main_parser.add_argument("--nlanes",         metavar="nlanes",         help="Number of lanes to generate", default=1, type=int)
main_parser.add_argument("--total_coverage", metavar="total_coverage", help="Total coverage", default=1.0, type=float)
main_parser.add_argument("--mean_fragment_size", metavar="mean_fragment_size", help="Mean fragment size", default=350, type=int)
main_parser.add_argument("--sd_fragment_size", metavar="sd_fragment_size", help="Standard deviation of fragment size", default=50, type=int)
main_parser.add_argument("--vcfs", metavar="vcfs", help="VCF list", nargs="+", default=[])
main_parser.add_argument("--liftover_jar", metavar="liftover_jar", help="LiftOver jar", type=file, default=default_liftover, required=require_liftover)
main_parser.add_argument("--force_five_base_encoding", action="store_true", help="Force bases to be ACTGN")
main_parser.add_argument("--filter", action="store_true", help="Only use PASS variants")
main_parser.add_argument("--keep_temp", action="store_true", help="Keep temporary files")
main_parser.add_argument("--vcfstats_jar", metavar="JAR", help="VCFStats jar", type=file, default=default_vcfstats, required=require_vcfstats)

pipeline_control_group = main_parser.add_argument_group("Pipeline control options. Disable parts of the pipeline.")
pipeline_control_group.add_argument("--disable_rand_vcf", action="store_true", help="Disable RandVCF2VCF")
pipeline_control_group.add_argument("--disable_rand_dgv", action="store_true", help="Disable RandDGV2VCF")
pipeline_control_group.add_argument("--disable_vcf2diploid", action="store_true", help="Disable vcf2diploid")
pipeline_control_group.add_argument("--disable_sim", action="store_true", help="Disable read simulation")

#RandVCF2VCF seed num_SNP num_INS num_DEL num_MNP num_COMPLEX percent_novel min_length_lim max_length_lim reference_file file.vcf
rand_vcf_group = main_parser.add_argument_group("RandVCF2VCF options")
rand_vcf_group.add_argument("--vc_num_snp", metavar="num_snp", help="Number of SNPs", default=0, type=int)
rand_vcf_group.add_argument("--vc_num_ins", metavar="num_ins", help="Number of insertions", default=0, type=int);
rand_vcf_group.add_argument("--vc_num_del", metavar="num_del", help="Number of deletions", default=0, type=int);
rand_vcf_group.add_argument("--vc_num_mnp", metavar="num_mnp", help="Number of MNPs", default=0, type=int)
rand_vcf_group.add_argument("--vc_num_complex", metavar="num_complex", help="Number of complex variants", default=0, type=int)
rand_vcf_group.add_argument("--vc_percent_novel", metavar="percent_novel", help="Percent novel", default=0, type=float)
rand_vcf_group.add_argument("--vc_min_length_lim", metavar="min_length_lim", help="Min length lim", default=0, type=int)
rand_vcf_group.add_argument("--vc_max_length_lim", metavar="max_length_lim", help="Max length lim", default=50, type=int)
rand_vcf_group.add_argument("--vc_in_vcf", metavar="in_vcf", help="Input VCF", type=file, required=True)
rand_vcf_group.add_argument("--vc_prop_het", metavar="vc_prop_het", help="Proportion of heterozygous vars", default=0.6, type=float)
rand_vcf_group.add_argument("--rand_vcf_jar", metavar="rand_vcf_jar", help="RandVCF2VCF jar", type=file, default=default_randvcf2vcf, required=require_randvcf2vcf)

#RandDGV2VCF seed num_INS num_DEL num_DUP num_INV percent_novel min_length_lim max_length_lim reference_file insert_seq.txt dgv_file.txt
rand_dgv_group = main_parser.add_argument_group("RandDGV2VCF options")
rand_dgv_group.add_argument("--sv_num_ins", metavar="num_ins", help="Number of insertions", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_del", metavar="num_del", help="Number of deletions", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_dup", metavar="num_dup", help="Number of duplications", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_inv", metavar="num_inv", help="Number of inversions", default=20, type=int);
rand_dgv_group.add_argument("--sv_percent_novel", metavar="percent_novel", help="Percent novel", default=0, type=float)
rand_dgv_group.add_argument("--sv_min_length_lim", metavar="min_length_lim", help="Min length lim", default=50, type=int)
rand_dgv_group.add_argument("--sv_max_length_lim", metavar="max_length_lim", help="Max length lim", default=1000000, type=int)
rand_dgv_group.add_argument("--sv_insert_seq", metavar="insert_seq", help="Insert seq", type=file, required=True)
rand_dgv_group.add_argument("--sv_dgv", metavar="dgv", help="DGV file", type=file, required=True)
rand_dgv_group.add_argument("--rand_dgv_jar", metavar="rand_dgv_jar", help="RandDGV2VCF jar", type=file, default=default_randdgv2vcf, required=require_randdgv2vcf)

dwgsim_group = main_parser.add_argument_group("DWGSIM options")
dwgsim_group.add_argument("--dwgsim_start_e", metavar="first_base_error_rate", help="Error rate on the first base", default=0.0001, type=float)
dwgsim_group.add_argument("--dwgsim_end_e", metavar="last_base_error_rate", help="Error rate on the last base", default=0.0015, type=float)
dwgsim_group.add_argument("--dwgsim_options", help="DWGSIM command-line options", default="", required=False)
dwgsim_group.add_argument("--dwgsim", metavar="dwgsim_executable", help="DWGSIM executable", type=file, required=False, default=None)

art_group = main_parser.add_argument_group("ART options")
art_group.add_argument("--profile_1", metavar="profile_file1", help="Profile for first end", default=None, type=file)
art_group.add_argument("--profile_2", metavar="profile_file2", help="Profile for second end", default=None, type=file)
art_group.add_argument("--art_options", help="ART command-line options", default="", required=False)
art_group.add_argument("--art", metavar="art_executable", help="ART executable", type=file, required=False, default=None)

args = main_parser.parse_args()

for d in [args.log_dir, args.out_dir, args.work_dir]:
  if not os.path.exists(d):
    os.makedirs(d)

FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
logging.basicConfig(filename=os.path.join(args.log_dir, "varsim.log"), filemode="w", level=logging.DEBUG, format=FORMAT)
logger = logging.getLogger(__name__)

def run_shell_command(cmd, cmd_stdout, cmd_stderr, cmd_dir="."):
  subproc = subprocess.Popen(cmd, stdout=cmd_stdout, stderr=cmd_stderr, cwd=cmd_dir, shell=True, preexec_fn=os.setsid)
  retcode = subproc.wait()
  sys.exit(retcode)

def monitor_multiprocesses(processes, logger):
  for p in processes:
    p.join()
    if p.exitcode != 0:
      logger.error("Process with pid %d failed with exit code %d" % (p.pid, pid.exitcode))
    else:
      logger.info("Process with pid %d finished successfully" % (p.pid))

def monitor_processes(processes, logger):
  while processes:
    time.sleep(1)
    kill_all = False
    processes_running = []
    for p in processes:
      status = p.poll()
      if status is not None:
        logger.info("Process %s exited with code %d" % (p.pid, status))
        if status != 0:
          kill_all = True
          logger.error("Process %s failed. Will kill the remaining processes." % (p.pid))
      else:
        processes_running.append(p)
    if kill_all:
      for p in processes:
        status = p.poll()
        if status is None:
          try:
            os.killpg(p.pid, signal.SIGTERM)
          except OSError, e:
            try:
              p.terminate()
            except OSError, ex:
              logger.write ("Could not kill the process\n")
      sys.exit(1)
    processes = processes_running
  return []

def check_executable(fpath):
  if not os.path.isfile(fpath):
    sys.stderr.write("ERROR: File %s does not exist\n" % (fpath))
    sys.exit(1)
  if not os.access(fpath, os.X_OK):
    sys.stderr.write("ERROR: File %s is not executable\n" % (fpath))
    sys.exit(1)

if args.simulator == "dwgsim":
  if args.dwgsim is None:
    sys.stderr.write("ERROR: Please specify the DWGSIM binary with --dwgsim option\n")
    sys.exit(1)
  check_executable(args.dwgsim.name)
if args.simulator == "art":
  if args.art is None:
    sys.stderr.write("ERROR: Please specify the ART binary with --art option\n")
    sys.exit(1)
  check_executable(args.art.name)

processes = []

t_s = time.time()

args.vcfs = [os.path.realpath(vcf) for vcf in args.vcfs]

if not args.disable_rand_vcf:
  rand_vcf_stdout = open(os.path.join(args.out_dir, "random.vc.vcf"), "w")
  rand_vcf_stderr = open(os.path.join(args.log_dir, "RandVCF2VCF.err"), "w")
  args.vcfs.append(os.path.realpath(rand_vcf_stdout.name))

  rand_vcf_command = ["java", "-jar", os.path.realpath(args.rand_vcf_jar.name), "-seed", str(args.seed),
                 "-num_snp", str(args.vc_num_snp), "-num_ins", str(args.vc_num_ins), "-num_del", str(args.vc_num_del),
                 "-num_mnp", str(args.vc_num_mnp), "-num_complex", str(args.vc_num_complex), "-novel", str(args.vc_percent_novel),
                 "-min_len", str(args.vc_min_length_lim), "-max_len", str(args.vc_max_length_lim), "-ref", os.path.realpath(args.reference.name),
                 "-prop_het", str(args.vc_prop_het), "-vcf", os.path.realpath(args.vc_in_vcf.name)]<commit_msg>Only check files when simulation isn't disabled<commit_after>#!/usr/bin/python

import argparse
import os
import sys
import subprocess
import logging
import shutil
import time
import signal
import multiprocessing as mp
from multiprocessing import Process

def get_contigs_list(reference):
  with open("%s.fai" % (reference)) as fai_file:
    contigs = [line.strip().split()[0] for line in fai_file.readlines()]
  return contigs

my_dir = os.path.dirname(os.path.realpath(__file__))

default_vcf2diploid = os.path.join(my_dir, "target/build/vcf2diploid.jar")
default_randvcf2vcf = os.path.join(my_dir, "target/build/randvcf2vcf.jar")
default_randdgv2vcf = os.path.join(my_dir, "target/build/randdgv2vcf.jar")
default_liftover    = os.path.join(my_dir, "target/build/fastq_liftover.jar")
default_vcfstats    = os.path.join(my_dir, "target/build/vcfstats.jar")

require_randvcf2vcf = not os.path.isfile(default_randvcf2vcf)
require_randdgv2vcf = not os.path.isfile(default_randdgv2vcf)
require_vcf2diploid = not os.path.isfile(default_vcf2diploid)
require_liftover    = not os.path.isfile(default_liftover)
require_vcfstats    = not os.path.isfile(default_vcfstats)

if not os.path.isfile(default_randvcf2vcf): default_randvcf2vcf = None
if not os.path.isfile(default_randdgv2vcf): default_randdgv2vcf = None
if not os.path.isfile(default_vcf2diploid): default_vcf2diploid = None
if not os.path.isfile(default_liftover):    default_liftover    = None
if not os.path.isfile(default_vcfstats):    default_vcfstats    = None

main_parser = argparse.ArgumentParser(description="VarSim: An accurate reads simulator", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
main_parser.add_argument("--out_dir", metavar="Out directory", help="Output directory", required=False, default="out")
main_parser.add_argument("--work_dir", metavar="Work directory", help="Work directory", required=False, default="work")
main_parser.add_argument("--log_dir", metavar="Log directory", help="Directory to log to", required=False, default="log")
main_parser.add_argument("--reference", metavar="Reference", help="Reference file", required=True, type=file)
main_parser.add_argument("--seed", metavar="seed", help="Random number seed", type=int, default=0)
main_parser.add_argument("--sex", metavar="Sex", help="Sex of the person (male/female)", required=False, type=str, choices=["male", "female"], default="male")
main_parser.add_argument("--id", metavar="id", help="Sample ID", required=True)
main_parser.add_argument("--simulator", metavar="simulator", help="Read simulator", required=False, type=str, choices=["art", "dwgsim"], default="art")
main_parser.add_argument("--vcf2diploid_jar", metavar="vcf2diploid_jar", help="vcf2diploid jar", type=file, default=default_vcf2diploid, required=require_vcf2diploid)
main_parser.add_argument("--read_length",    metavar="read_length",    help="Length of reads", default=100, type=int)
main_parser.add_argument("--nlanes",         metavar="nlanes",         help="Number of lanes to generate", default=1, type=int)
main_parser.add_argument("--total_coverage", metavar="total_coverage", help="Total coverage", default=1.0, type=float)
main_parser.add_argument("--mean_fragment_size", metavar="mean_fragment_size", help="Mean fragment size", default=350, type=int)
main_parser.add_argument("--sd_fragment_size", metavar="sd_fragment_size", help="Standard deviation of fragment size", default=50, type=int)
main_parser.add_argument("--vcfs", metavar="vcfs", help="VCF list", nargs="+", default=[])
main_parser.add_argument("--liftover_jar", metavar="liftover_jar", help="LiftOver jar", type=file, default=default_liftover, required=require_liftover)
main_parser.add_argument("--force_five_base_encoding", action="store_true", help="Force bases to be ACTGN")
main_parser.add_argument("--filter", action="store_true", help="Only use PASS variants")
main_parser.add_argument("--keep_temp", action="store_true", help="Keep temporary files")
main_parser.add_argument("--vcfstats_jar", metavar="JAR", help="VCFStats jar", type=file, default=default_vcfstats, required=require_vcfstats)

pipeline_control_group = main_parser.add_argument_group("Pipeline control options. Disable parts of the pipeline.")
pipeline_control_group.add_argument("--disable_rand_vcf", action="store_true", help="Disable RandVCF2VCF")
pipeline_control_group.add_argument("--disable_rand_dgv", action="store_true", help="Disable RandDGV2VCF")
pipeline_control_group.add_argument("--disable_vcf2diploid", action="store_true", help="Disable vcf2diploid")
pipeline_control_group.add_argument("--disable_sim", action="store_true", help="Disable read simulation")

#RandVCF2VCF seed num_SNP num_INS num_DEL num_MNP num_COMPLEX percent_novel min_length_lim max_length_lim reference_file file.vcf
rand_vcf_group = main_parser.add_argument_group("RandVCF2VCF options")
rand_vcf_group.add_argument("--vc_num_snp", metavar="num_snp", help="Number of SNPs", default=0, type=int)
rand_vcf_group.add_argument("--vc_num_ins", metavar="num_ins", help="Number of insertions", default=0, type=int);
rand_vcf_group.add_argument("--vc_num_del", metavar="num_del", help="Number of deletions", default=0, type=int);
rand_vcf_group.add_argument("--vc_num_mnp", metavar="num_mnp", help="Number of MNPs", default=0, type=int)
rand_vcf_group.add_argument("--vc_num_complex", metavar="num_complex", help="Number of complex variants", default=0, type=int)
rand_vcf_group.add_argument("--vc_percent_novel", metavar="percent_novel", help="Percent novel", default=0, type=float)
rand_vcf_group.add_argument("--vc_min_length_lim", metavar="min_length_lim", help="Min length lim", default=0, type=int)
rand_vcf_group.add_argument("--vc_max_length_lim", metavar="max_length_lim", help="Max length lim", default=50, type=int)
rand_vcf_group.add_argument("--vc_in_vcf", metavar="in_vcf", help="Input VCF", type=file, required=True)
rand_vcf_group.add_argument("--vc_prop_het", metavar="vc_prop_het", help="Proportion of heterozygous vars", default=0.6, type=float)
rand_vcf_group.add_argument("--rand_vcf_jar", metavar="rand_vcf_jar", help="RandVCF2VCF jar", type=file, default=default_randvcf2vcf, required=require_randvcf2vcf)

#RandDGV2VCF seed num_INS num_DEL num_DUP num_INV percent_novel min_length_lim max_length_lim reference_file insert_seq.txt dgv_file.txt
rand_dgv_group = main_parser.add_argument_group("RandDGV2VCF options")
rand_dgv_group.add_argument("--sv_num_ins", metavar="num_ins", help="Number of insertions", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_del", metavar="num_del", help="Number of deletions", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_dup", metavar="num_dup", help="Number of duplications", default=20, type=int);
rand_dgv_group.add_argument("--sv_num_inv", metavar="num_inv", help="Number of inversions", default=20, type=int);
rand_dgv_group.add_argument("--sv_percent_novel", metavar="percent_novel", help="Percent novel", default=0, type=float)
rand_dgv_group.add_argument("--sv_min_length_lim", metavar="min_length_lim", help="Min length lim", default=50, type=int)
rand_dgv_group.add_argument("--sv_max_length_lim", metavar="max_length_lim", help="Max length lim", default=1000000, type=int)
rand_dgv_group.add_argument("--sv_insert_seq", metavar="insert_seq", help="Insert seq", type=file, required=True)
rand_dgv_group.add_argument("--sv_dgv", metavar="dgv", help="DGV file", type=file, required=True)
rand_dgv_group.add_argument("--rand_dgv_jar", metavar="rand_dgv_jar", help="RandDGV2VCF jar", type=file, default=default_randdgv2vcf, required=require_randdgv2vcf)

dwgsim_group = main_parser.add_argument_group("DWGSIM options")
dwgsim_group.add_argument("--dwgsim_start_e", metavar="first_base_error_rate", help="Error rate on the first base", default=0.0001, type=float)
dwgsim_group.add_argument("--dwgsim_end_e", metavar="last_base_error_rate", help="Error rate on the last base", default=0.0015, type=float)
dwgsim_group.add_argument("--dwgsim_options", help="DWGSIM command-line options", default="", required=False)
dwgsim_group.add_argument("--dwgsim", metavar="dwgsim_executable", help="DWGSIM executable", type=file, required=False, default=None)

art_group = main_parser.add_argument_group("ART options")
art_group.add_argument("--profile_1", metavar="profile_file1", help="Profile for first end", default=None, type=file)
art_group.add_argument("--profile_2", metavar="profile_file2", help="Profile for second end", default=None, type=file)
art_group.add_argument("--art_options", help="ART command-line options", default="", required=False)
art_group.add_argument("--art", metavar="art_executable", help="ART executable", type=file, required=False, default=None)

args = main_parser.parse_args()

for d in [args.log_dir, args.out_dir, args.work_dir]:
  if not os.path.exists(d):
    os.makedirs(d)

FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
logging.basicConfig(filename=os.path.join(args.log_dir, "varsim.log"), filemode="w", level=logging.DEBUG, format=FORMAT)
logger = logging.getLogger(__name__)

def run_shell_command(cmd, cmd_stdout, cmd_stderr, cmd_dir="."):
  subproc = subprocess.Popen(cmd, stdout=cmd_stdout, stderr=cmd_stderr, cwd=cmd_dir, shell=True, preexec_fn=os.setsid)
  retcode = subproc.wait()
  sys.exit(retcode)

def monitor_multiprocesses(processes, logger):
  for p in processes:
    p.join()
    if p.exitcode != 0:
      logger.error("Process with pid %d failed with exit code %d" % (p.pid, pid.exitcode))
    else:
      logger.info("Process with pid %d finished successfully" % (p.pid))

def monitor_processes(processes, logger):
  while processes:
    time.sleep(1)
    kill_all = False
    processes_running = []
    for p in processes:
      status = p.poll()
      if status is not None:
        logger.info("Process %s exited with code %d" % (p.pid, status))
        if status != 0:
          kill_all = True
          logger.error("Process %s failed. Will kill the remaining processes." % (p.pid))
      else:
        processes_running.append(p)
    if kill_all:
      for p in processes:
        status = p.poll()
        if status is None:
          try:
            os.killpg(p.pid, signal.SIGTERM)
          except OSError, e:
            try:
              p.terminate()
            except OSError, ex:
              logger.write ("Could not kill the process\n")
      sys.exit(1)
    processes = processes_running
  return []

def check_executable(fpath):
  if not os.path.isfile(fpath):
    sys.stderr.write("ERROR: File %s does not exist\n" % (fpath))
    sys.exit(1)
  if not os.access(fpath, os.X_OK):
    sys.stderr.write("ERROR: File %s is not executable\n" % (fpath))
    sys.exit(1)

if not args.disable_sim:
  if args.simulator == "dwgsim":
    if args.dwgsim is None:
      sys.stderr.write("ERROR: Please specify the DWGSIM binary with --dwgsim option\n")
      sys.exit(1)
    check_executable(args.dwgsim.name)
  if args.simulator == "art":
    if args.art is None:
      sys.stderr.write("ERROR: Please specify the ART binary with --art option\n")
      sys.exit(1)
    check_executable(args.art.name)

processes = []

t_s = time.time()

args.vcfs = [os.path.realpath(vcf) for vcf in args.vcfs]

if not args.disable_rand_vcf:
  rand_vcf_stdout = open(os.path.join(args.out_dir, "random.vc.vcf"), "w")
  rand_vcf_stderr = open(os.path.join(args.log_dir, "RandVCF2VCF.err"), "w")
  args.vcfs.append(os.path.realpath(rand_vcf_stdout.name))

  rand_vcf_command = ["java", "-jar", os.path.realpath(args.rand_vcf_jar.name), "-seed", str(args.seed),
                 "-num_snp", str(args.vc_num_snp), "-num_ins", str(args.vc_num_ins), "-num_del", str(args.vc_num_del),
                 "-num_mnp", str(args.vc_num_mnp), "-num_complex", str(args.vc_num_complex), "-novel", str(args.vc_percent_novel),
                 "-min_len", str(args.vc_min_length_lim), "-max_len", str(args.vc_max_length_lim), "-ref", os.path.realpath(args.reference.name),
                 "-prop_het", str(args.vc_prop_het), "-vcf", os.path.realpath(args.vc_in_vcf.name)]
{'fraction_non_alphanumeric': 0.1435472452421605, 'fraction_numerical': 0.008661974763669679, 'mean_word_length': 6.30130738826391, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1391238', 'n_tokens_mistral': 8644, 'n_tokens_neox': 8414, 'n_words': 1552}