<commit_before>#!/usr/bin/env python
# pseudoreplicator 0.0.1
# Generated by dx-app-wizard.
#
# Basic execution pattern: Your app will run on a single machine from
# beginning to end.
#
# See https://wiki.dnanexus.com/Developer-Portal for documentation and
# tutorials on how to modify this file.
#
# DNAnexus Python Bindings (dxpy) documentation:
#   http://autodoc.dnanexus.com/bindings/python/current/

import os, subprocess, shlex, time, re, gzip
from multiprocessing import Pool, cpu_count
from subprocess import Popen, PIPE #debug only this should only need to be imported into run_pipe
import dxpy

def run_pipe(steps, outfile=None):
    #break this out into a recursive function
    #TODO:  capture stderr
    from subprocess import Popen, PIPE
    p = None
    p_next = None
    first_step_n = 1
    last_step_n = len(steps)
    for n,step in enumerate(steps, start=first_step_n):
        print "step %d: %s" %(n,step)
        if n == first_step_n:
            if n == last_step_n and outfile: #one-step pipeline with outfile
                with open(outfile, 'w') as fh:
                    print "one step shlex: %s to file: %s" %(shlex.split(step), outfile)
                    p = Popen(shlex.split(step), stdout=fh)
                break
            print "first step shlex to stdout: %s" %(shlex.split(step))
            p = Popen(shlex.split(step), stdout=PIPE)
            #need to close p.stdout here?
        elif n == last_step_n and outfile: #only treat the last step specially if you're sending stdout to a file
            with open(outfile, 'w') as fh:
                print "last step shlex: %s to file: %s" %(shlex.split(step), outfile)
                p_last = Popen(shlex.split(step), stdin=p.stdout, stdout=fh)
                p.stdout.close()
                p = p_last
        else: #handles intermediate steps and, in the case of a pipe to stdout, the last step
            print "intermediate step %d shlex to stdout: %s" %(n,shlex.split(step))
            p_next = Popen(shlex.split(step), stdin=p.stdout, stdout=PIPE)
            p.stdout.close()
            p = p_next
    out,err = p.communicate()
    return out,err

@dxpy.entry_point('main')
def main(input_tags):

    # The following line(s) initialize your data object inputs on the platform
    # into dxpy.DXDataObject instances that you can start using immediately.

    input_tags_file = dxpy.DXFile(input_tags)

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    input_tags_filename = input_tags_file.name
    dxpy.download_dxfile(input_tags_file.get_id(), input_tags_filename)

    # introspect the file to determine tagAlign (thus SE) or BEDPE (thus PE)
    # strip extension as appropriate

    print subprocess.check_output('ls', shell=True)
    #out = subprocess.check_output('gzip -dc %s | head -n 1' %(input_tags_filename), shell=True)
    #out,err = run_pipe(['gzip -dc %s' %(input_tags_filename), 'sed -n 1p'])
    with gzip.open(input_tags_filename) as f:
        firstline = f.readline()
    print firstline

    se_cols = 6
    pe_cols = 10
    if re.match('^(\S+[\t\n]){%d}$' %(se_cols), firstline):
        paired_end = False
        input_tags_basename = input_tags_filename.rstrip('.bedpe.gz')
        filename_infix = 'SE'
        print "Single-end data"
    #elif re.match('^([a-zA-Z0-9_:-+]+[\t\n]){%d}$' %(pe_cols), firstline):
    elif re.match('^(\S+[\t\n]){%d}$' %(pe_cols), firstline):
        paired_end = True
        input_tags_basename = input_tags_filename.rstrip('.tagAlign.gz')
        filename_infix = 'PE2SE'
        print "Paired-end data"
    else:
        raise IOError("%s is neither a BEDPE or tagAlign file" %(input_tags_filename))

    pr_ta_filenames = [input_tags_basename + ".%s.pr1.tagAlign.gz" %(filename_infix),
                       input_tags_filename + ".%s.pr2.tagAlign.gz" %(filename_infix)]

    #count lines in the file
    out,err = run_pipe([
        'gzip -dc %s' %(input_tags_filename),
        'wc -l'])
    #number of lines in each split   
    nlines = (int(out)+1)/2
    # Shuffle and split BEDPE file into 2 equal parts
    splits_prefix = 'temp_split'
    out,err = run_pipe([
        'gzip -dc %s' %(input_tags_filename),<commit_msg>Bug in PE/SE<commit_after>#!/usr/bin/env python
# pseudoreplicator 0.0.1
# Generated by dx-app-wizard.
#
# Basic execution pattern: Your app will run on a single machine from
# beginning to end.
#
# See https://wiki.dnanexus.com/Developer-Portal for documentation and
# tutorials on how to modify this file.
#
# DNAnexus Python Bindings (dxpy) documentation:
#   http://autodoc.dnanexus.com/bindings/python/current/

import os, subprocess, shlex, time, re, gzip
from multiprocessing import Pool, cpu_count
from subprocess import Popen, PIPE #debug only this should only need to be imported into run_pipe
import dxpy

def run_pipe(steps, outfile=None):
    #break this out into a recursive function
    #TODO:  capture stderr
    from subprocess import Popen, PIPE
    p = None
    p_next = None
    first_step_n = 1
    last_step_n = len(steps)
    for n,step in enumerate(steps, start=first_step_n):
        print "step %d: %s" %(n,step)
        if n == first_step_n:
            if n == last_step_n and outfile: #one-step pipeline with outfile
                with open(outfile, 'w') as fh:
                    print "one step shlex: %s to file: %s" %(shlex.split(step), outfile)
                    p = Popen(shlex.split(step), stdout=fh)
                break
            print "first step shlex to stdout: %s" %(shlex.split(step))
            p = Popen(shlex.split(step), stdout=PIPE)
            #need to close p.stdout here?
        elif n == last_step_n and outfile: #only treat the last step specially if you're sending stdout to a file
            with open(outfile, 'w') as fh:
                print "last step shlex: %s to file: %s" %(shlex.split(step), outfile)
                p_last = Popen(shlex.split(step), stdin=p.stdout, stdout=fh)
                p.stdout.close()
                p = p_last
        else: #handles intermediate steps and, in the case of a pipe to stdout, the last step
            print "intermediate step %d shlex to stdout: %s" %(n,shlex.split(step))
            p_next = Popen(shlex.split(step), stdin=p.stdout, stdout=PIPE)
            p.stdout.close()
            p = p_next
    out,err = p.communicate()
    return out,err

@dxpy.entry_point('main')
def main(input_tags):

    # The following line(s) initialize your data object inputs on the platform
    # into dxpy.DXDataObject instances that you can start using immediately.

    input_tags_file = dxpy.DXFile(input_tags)

    # The following line(s) download your file inputs to the local file system
    # using variable names for the filenames.

    input_tags_filename = input_tags_file.name
    dxpy.download_dxfile(input_tags_file.get_id(), input_tags_filename)

    # introspect the file to determine tagAlign (thus SE) or BEDPE (thus PE)
    # strip extension as appropriate

    print subprocess.check_output('ls', shell=True)
    #out = subprocess.check_output('gzip -dc %s | head -n 1' %(input_tags_filename), shell=True)
    #out,err = run_pipe(['gzip -dc %s' %(input_tags_filename), 'sed -n 1p'])
    with gzip.open(input_tags_filename) as f:
        firstline = f.readline()
    print firstline

    se_cols = 6
    pe_cols = 10
    if re.match('^(\S+[\t\n]){%d}$' %(se_cols), firstline):
        paired_end = False
        input_tags_basename = input_tags_filename.rstrip('.tagAlign.gz')
        filename_infix = 'SE'
        print "Single-end data"
    #elif re.match('^([a-zA-Z0-9_:-+]+[\t\n]){%d}$' %(pe_cols), firstline):
    elif re.match('^(\S+[\t\n]){%d}$' %(pe_cols), firstline):
        paired_end = True
        input_tags_basename = input_tags_filename.rstrip('.bedpe.gz')
        filename_infix = 'PE2SE'
        print "Paired-end data"
    else:
        raise IOError("%s is neither a BEDPE or tagAlign file" %(input_tags_filename))

    pr_ta_filenames = [input_tags_basename + ".%s.pr1.tagAlign.gz" %(filename_infix),
                       input_tags_filename + ".%s.pr2.tagAlign.gz" %(filename_infix)]

    #count lines in the file
    out,err = run_pipe([
        'gzip -dc %s' %(input_tags_filename),
        'wc -l'])
    #number of lines in each split   
    nlines = (int(out)+1)/2
    # Shuffle and split BEDPE file into 2 equal parts
    splits_prefix = 'temp_split'
    out,err = run_pipe([
        'gzip -dc %s' %(input_tags_filename),
{'fraction_non_alphanumeric': 0.11524511524511524, 'fraction_numerical': 0.003978003978003978, 'mean_word_length': 2.7856510186005314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3864234', 'n_tokens_mistral': 2920, 'n_tokens_neox': 2830, 'n_words': 821}