<commit_before>#!/usr/bin/python
import argparse, sys, multiprocessing, random, os, re, subprocess
from shutil import rmtree

def main():
  parser = argparse.ArgumentParser(description="Launch GMAP and run it in a temp directory until output is finished.")
  parser.add_argument('input_fasta')
  parser.add_argument('output_psl')
  parser.add_argument('--best',action='store_true',help="Only output the best path")
  parser.add_argument('--gmap_index',required=True,help="Path to gmap index (directory)")
  parser.add_argument('--threads',type=int,default=multiprocessing.cpu_count())
  parser.add_argument('--max_paths',type=int,help="Maximum number of paths to show.")
  parser.add_argument('--max_intron_length',type=int,help="Maximum length of intron.")
  parser.add_argument('--tempdir',default='/tmp')
  args = parser.parse_args()

  args.gmap_index = args.gmap_index.rstrip('/')
  m = re.match('^(.*)$',args.gmap_index)
  if m:
    gmapindexpath = m.group(1)
  m = re.search('([^\/]+)$',args.gmap_index)
  if m:
    gmapindexname = m.group(1)
  else:
    sys.stderr.write("problem reading gmap index "+args.gmap_index+"\n")
    sys.exit()

  maxpathpart = ''
  if args.max_paths:
    maxpathpart = ' -n '+str(args.max_paths)+' '

  maxintronpart = ''
  if args.max_paths:
    maxintronpart = ' -K '+str(args.max_intron_length)+' '

  gmap_cmd = 'gmap -D '+gmapindexpath+' -f 1 -d '+gmapindexname+' -t '+str(args.threads)+' '+maxpathpart+maxintronpart+' '+args.input_fasta
  sys.stderr.write("executing:\n"+gmap_cmd+"\n")
  rnum = random.randint(1,10000000)

  args.tempdir = args.tempdir.rstrip('/')+'/weirathe.'+str(rnum)
  if not os.path.exists(args.tempdir):
    os.makedirs(args.tempdir)

  allpsl = args.tempdir+'/all.psl'
  #subprocess.call(gmap_cmd+' > '+allpsl,shell=True,stderr=subprocess.PIPE)
  ofe = open('/dev/null','w')
  subprocess.call(gmap_cmd+' > '+allpsl,shell=True,stderr=ofe)

  if args.best:
    vals = {}
    with open(allpsl) as inf:
      for line in inf:
        if re.match('^#',line): 
          continue
        f = line.rstrip().split("\t")
        if f[9] not in vals:
          vals[f[9]]=0
        if int(f[0]) > vals[f[9]]: vals[f[9]] = int(f[0])
          
  of = sys.stdout
  if args.output_psl != '-':
    of = open(args.output_psl,'w')
  with open(allpsl) as inf:
    for line in inf:
      f = line.rstrip().split("\t")
      if args.best:
        if f[9] in vals:<commit_msg>make output ordered by default<commit_after>#!/usr/bin/python
import argparse, sys, multiprocessing, random, os, re, subprocess
from shutil import rmtree

def main():
  parser = argparse.ArgumentParser(description="Launch GMAP and run it in a temp directory until output is finished.")
  parser.add_argument('input_fasta')
  parser.add_argument('output_psl')
  parser.add_argument('--best',action='store_true',help="Only output the best path")
  parser.add_argument('--gmap_index',required=True,help="Path to gmap index (directory)")
  parser.add_argument('--threads',type=int,default=multiprocessing.cpu_count())
  parser.add_argument('--max_paths',type=int,help="Maximum number of paths to show.")
  parser.add_argument('--max_intron_length',type=int,help="Maximum length of intron.")
  parser.add_argument('--tempdir',default='/tmp')
  args = parser.parse_args()

  args.gmap_index = args.gmap_index.rstrip('/')
  m = re.match('^(.*)$',args.gmap_index)
  if m:
    gmapindexpath = m.group(1)
  m = re.search('([^\/]+)$',args.gmap_index)
  if m:
    gmapindexname = m.group(1)
  else:
    sys.stderr.write("problem reading gmap index "+args.gmap_index+"\n")
    sys.exit()

  maxpathpart = ''
  if args.max_paths:
    maxpathpart = ' -n '+str(args.max_paths)+' '

  maxintronpart = ''
  if args.max_paths:
    maxintronpart = ' -K '+str(args.max_intron_length)+' '

  gmap_cmd = 'gmap --ordered -D '+gmapindexpath+' -f 1 -d '+gmapindexname+' -t '+str(args.threads)+' '+maxpathpart+maxintronpart+' '+args.input_fasta
  sys.stderr.write("executing:\n"+gmap_cmd+"\n")
  rnum = random.randint(1,10000000)

  args.tempdir = args.tempdir.rstrip('/')+'/weirathe.'+str(rnum)
  if not os.path.exists(args.tempdir):
    os.makedirs(args.tempdir)

  allpsl = args.tempdir+'/all.psl'
  #subprocess.call(gmap_cmd+' > '+allpsl,shell=True,stderr=subprocess.PIPE)
  ofe = open('/dev/null','w')
  subprocess.call(gmap_cmd+' > '+allpsl,shell=True,stderr=ofe)

  if args.best:
    vals = {}
    with open(allpsl) as inf:
      for line in inf:
        if re.match('^#',line): 
          continue
        f = line.rstrip().split("\t")
        if f[9] not in vals:
          vals[f[9]]=0
        if int(f[0]) > vals[f[9]]: vals[f[9]] = int(f[0])
          
  of = sys.stdout
  if args.output_psl != '-':
    of = open(args.output_psl,'w')
  with open(allpsl) as inf:
    for line in inf:
      f = line.rstrip().split("\t")
      if args.best:
        if f[9] in vals:
{'fraction_non_alphanumeric': 0.17614452884417983, 'fraction_numerical': 0.008211866146581811, 'mean_word_length': 3.795275590551181, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5306535', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1822, 'n_words': 336}