<commit_before>#! /usr/bin/env python

import sys
import optparse
import subprocess

import vdj

parser = optparse.OptionParser()
parser.add_option('-c','--cutoff',default=4.5,type='float')
parser.add_option('-l','--linkage',type='choice',choices=['single','complete'],default='single')
parser.add_option('-q','--queue')
parser.add_option('-o','--LSFoutput')
(options, args) = parser.parse_args()

if len(args) == 2:
    inhandle = open(args[0],'r')
    outname = args[0]
    outhandle = open(args[1],'w')
elif len(args) == 1:
    inhandle = open(args[0],'r')
    outname = args[0]
    outhandle = sys.stdout
elif len(args) == 0:
    inhandle = sys.stdin
    outname = 'VJ_parts.vdjxml'
    outhandle = sys.stdout

print >>sys.stderr, "NOTE: chains must be filtered for valid VJ aln and junctions BEFORE clustering."

(VJ_parts,VJ_IDs) = vdj.split_vdjxml_into_VJ_parts(inhandle,outname)

VJ_parts_clustered = []
jobs = []
for (vj_file,vj_id) in zip(VJ_parts,VJ_IDs):
    vj_file_clustered = vj_file + '.clustered'
    VJ_parts_clustered.append(vj_file_clustered)
    params = {'cutoff':options.cutoff,
             'linkage':options.linkage,
                 'tag':vj_id,
              'infile':vj_file,
             'outfile':vj_file_clustered}
    cluster_cmd = r'cluster_cdr3.py --cutoff %(cutoff)f --tag %(tag)s --linkage %(linkage)s %(infile)s %(outfile)s' % params
    jobID = vdj.LSF.submit_to_LSF(options.queue,options.LSFoutput,cluster_cmd)
    jobs.append(jobID)

vdj.LSF.wait_for_LSF_jobs(jobs)

for chain in vdj.parse_VDJXML_parts(VJ_parts_clustered):
    print >>outhandle, chain<commit_msg>Modified cluster_LSF script for intermediate files and file sizes.<commit_after>#! /usr/bin/env python

import sys
import optparse
import subprocess
import os
import tempfile

import vdj

parser = optparse.OptionParser()
parser.add_option('-c','--cutoff',default=4.5,type='float')
parser.add_option('-l','--linkage',type='choice',choices=['single','complete'],default='single')
parser.add_option('-q','--queue')
parser.add_option('-o','--LSFoutput')
(options, args) = parser.parse_args()

if len(args) == 2:
    inhandle = open(args[0],'r')
    outname = args[0]
    outhandle = open(args[1],'w')
elif len(args) == 1:
    inhandle = open(args[0],'r')
    outname = args[0]
    outhandle = sys.stdout
elif len(args) == 0:
    inhandle = sys.stdin
    outname = 'VJ_parts.vdjxml'
    outhandle = sys.stdout

print >>sys.stderr, "NOTE: chains must be filtered for valid VJ aln and junctions BEFORE clustering."

# temporary directory to dump intermediate files
tempdir = tempfile.mkdtemp(prefix=outname+'.intermediate.',dir='.')

(VJ_parts,VJ_IDs) = vdj.split_vdjxml_into_VJ_parts(inhandle,os.path.join(tempdir,outname))

VJ_parts_clustered = []
jobs = []
for (vj_file,vj_id) in zip(VJ_parts,VJ_IDs):
    vj_file_clustered = os.path.join(tempdir,vj_file+'.clustered')
    VJ_parts_clustered.append(vj_file_clustered)
    params = {'cutoff':options.cutoff,
             'linkage':options.linkage,
                 'tag':vj_id,
              'infile':vj_file,
             'outfile':vj_file_clustered}
    cluster_cmd = r'cluster_cdr3.py --cutoff %(cutoff)f --tag %(tag)s --linkage %(linkage)s %(infile)s %(outfile)s' % params
    if os.stat(inpart).st_size < 8e6:
        jobID = vdj.LSF.submit_to_LSF(options.queue,options.LSFoutput,cluster_cmd)
    else:
        jobID = vdj.LSF.submit_to_LSF(options.queue,options.LSFoutput,cluster_cmd,mem_usage=6000)
    jobs.append(jobID)

vdj.LSF.wait_for_LSF_jobs(jobs)

for chain in vdj.parse_VDJXML_parts(VJ_parts_clustered):
    print >>outhandle, chain
{'fraction_non_alphanumeric': 0.15262718932443703, 'fraction_numerical': 0.00778426466499861, 'mean_word_length': 4.346210995542347, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6171454', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1383, 'n_words': 228}