<commit_before>#!/usr/bin/env python2.7
# launch.py 1.0.1

import argparse,os, sys, json
#import urlparse, subprocess, itertools, logging
from datetime import datetime
from collections import deque

import dxpy
import dxencode

### TODO:
# 1) NEED TO MAKE a --template version not relying on ENCODEd at all!

# NOTES: This command-line utility will run the a pipeline for a single replicate or combined reps.
#      - All results will be written to a folder /<results_folder>/<exp_id>/rep<#>_<#>.
#      - If any results are already in that directory, then the steps that created those results
#        will not be rerun.
#      - If any jobs for the experiment and replicate are already running, nothing new will be
#        launched.
#      - Almost all the code is generic and derived classes will only need to entend or override a 
#        a few things.  It relies upon JSON pipeline descriptions which in simple cases can be 
#        directly read from dx (e.g. srnaLaunch.py).  More compex examples (e.g. lrnaLaunch.py)
#        must hard-code the pipeline json to handle repeated apps and dx name conflicts.
#        Tokens are used to abstract dx.app input/outout file names to avoid collisions.
#        - PIPELINE_BRANCHES[branch_id]["ORDER"] is the list of steps in the pipeline branch
#        - PIPELINE_BRANCHES[branch_id]["STEPS"] contain step definitions and enforces dependencies by:
#          'input' file tokens matching to 'result' file tokens of earlier steps.
#        - FILE_GLOBS is needed for locating result files from prior runs.
#      - Combined replicate processing is optional and depends upon PIPELINE_BRANCHES["COMBINED_REPS"], etc.
#        - The standard combination expected PIPELINE_BRANCH_ORDER[ "REP", "COMBINED_REPS" ].
#        - More complex combinations can be acheived by overriding add_combining_reps() in the derived class.
#        - Tying results from one branch to inputs into another relies on input file tokens ending with '_A', '_B' or '_ABC'.
#      - Control files are supported but most likely will require function overrides in derived
#        classes (e.g. rampageLaunch.py)
#      - In PIPELINE_BRANCHES[branch_id]["STEPS"], both 'inputs' and 'results' are key:value lists where the key is used
#        to attach results of one step to the inputs of another, while values must match dxapp.json names.
#      - The FILE_GLOBS are matched by input/result keys, not values.
#        (e.g. "input": { "pooled_bam": "bam" } matches file_glob "pooled_bam" and dxapp.json input named "bam".
#        This example would allow the same applet to be used on replicate bams and poold bams in the same pipeline.)
#      - RARE cases using 'output_values' and 'param_links' can support non-file outputs as inputs to later steps,
#        even across rep branch boundaries.
#
#   A design note about replicates in the launchers:
#      - "rep" objects should contain (almost) all information necesary to launch processing against them.
#        A few minor things common to the entire experiment are stored directly in the psv (pipeline specific variables) object
#      - Each "rep" object will get processed by one PIPELINE_BRANCH which determines which STEPS are run.
#      - All ENCODEd replicates ready for processing should get loaded into psv["reps"], keyed with letters: "a","b","c"...
#      - These "simple reps" have a "rep_tech" like "rep2_1" (for bio_rep=2, tech_rep=1).
#      - Combined replicates have a "rep_tech" like "reps1_1-2_1" (for combining simple reps rep1_1 and rep2_1).
#      - Combined replicates should be keyed (in sort order) after their "tributary" reps (e.g. "a","b","b-combines-a-b").
#      - The final combined replicate (aka "SEA" into which all rivers flow) is keyed with self.SEA_ID ("zzz")
#      - In the 'standard combination model' (which can be overridden in descendent classes): 
#        - Simple reps are processed by the "REP" pipeline branch.
#        - The one combined rep (aka "SEA") is processed by the "COMBINED_REPS" pipeline branch

#
# Example derived classes: long-rna-seq-pipeline: lrnaLaunch.py, srnaLaunch.py rampageLaunch.py

class Launch(object):
    '''
    Launch module is the common code for pipeline specific launcher classes.  
    Launchers dynamically create and launch a workflow covering any needed steps of a pipeline.
    Steps needed are thos that have available inputs but not already generatedresults.
    '''

    PIPELINE_NAME = "MUST REPLACE IN DREIVED CLASS"
    ''' This must match the assay type returned by dxencode.get_assay_type({exp_id}).'''

    PIPELINE_HELP = "Launches '"+PIPELINE_NAME+"' pipeline " + \
                    "analysis for one replicate or combined replicates. "
    ''' This help title should name pipline and whether combined replicates are supported.'''
                    
    SERVER_DEFAULT = 'www'
    '''At this time there is no need to use the any but the one true server for launching.'''
    
    FOLDER_DEFAULT = '/runs/'
    ''' This the default location to place results folders for each experiment.'''
    
    PIPELINE_BRANCH_ORDER = None # MUST replace in derived class. Example: [ "REP", "COMBINED_REPS" ]
    '''A pipeline is frequently made of branches that flow into each other, such as replicate level to combined replicate.'''
    
    PIPELINE_BRANCHES = None # MUST replace in derived class. 'ORDER' and "STEPS" are required for each branch,
    # Example:               # though simple "STEPS" may be discovered.
    #{    "REP": {
    #            "ORDER": [ "rep_app_one", ... ],
    #     # NOTE: In simple cases "STEPS" can be discovered from dnanexus apps (example: srnaLaunch.py)
    #     # HINT: Try to discover the "STEPS" usubg build_simple_steps(verbose) to print the json, then modify from there  
    #            "STEPS": {
    #                        "rep_app_one": {
    #                            "inputs":  { "reads":    "reads" },
    #                            "app":       "rep_app_1", 
    #                            "params":  { "nthreads": "nthreads"}, 
    #                            "results": { "rep_bam":  "bam_out" }
    #                        }, ...
    #    },
    #    "COMBINED_REPS": ...
    #}
    '''Each branch must define the 'steps' and their (artificially) linear order.'''

    # "STEPS" objects within PIPELINE_BRANCHES contain the following:
    #    "step-name":   { # Must match "ORDER" and may match dx applet
    #        "app":     "dx-applet-name", # Must match applet
    #        "params":  { "param_token": "dx_input_name", ...}, # Non-files: Token must match psv key.
    #        "inputs":  { "input_token": "dx_input_name", ...},  # Files: Token *may* match name
    #        "results": { "result_token": "dx_output_name", ...}, # Files: Token *may* match name
    #        "output_values": { "output_token": "dx_output_name" }, # RARE: non-file output used as input to another step
    #        "param_links": { "param_token": {"name":"output_token", "rep":"sister"} }} # RARE: link input to non-file output 
    # Note, tokens *may* be the same as dx names, but MUST MATCH other tokens to define dependencies as: 
    #     stepA:result_token == stepB:input_token
    # Also result_tokens must be in FILE_GLOB.keys()
    #
    # NOTE: Simple pipelines have no reused steps or dx name to param complications and the 
    #       "STEPS" objects can be generated directly from dx apps.  If unsure, print the json from
    #       self.build_simple_steps(verbose=True) to determine if it will work for your pipeline.
    
    PRUNE_STEPS = None
    '''Rare setting that allows dynamically pruning steps from a pipeline befpre running it.  Used by lrnaLaunch.py.'''
    
    FILE_GLOBS = {"MAY": {}, "NEED": {}, "TO": {}, "REPLACE": {} }
    '''Globs for discovering existing results. Can be discoverd in simple piplines.'''

    CONTROL_FILE_GLOB = None
    '''Glob for discovering control files if experiment has controls. Othewise: None'''

    REFERENCE_FILES = {
        # For looking up reference file names.
        }

    GENOMES_SUPPORTED = ['hg19', 'mm10']
    GENOME_DEFAULT = 'hg19'
    ''' This is the default Genome that experiments are mapped to.'''
    
    STANDARD_LABELS = {
        'long-rna-seq':  { 'long': 'Long-RNA-seq',  'short': 'lrna',  'se_and_pe': True  }, 
        'small-rna-seq': { 'long': 'Small-RNA-seq', 'short': 'srna',  'se_and_pe': False }, 
        'rampage':       { 'long': 'Rampage',       'short': 'ramp',  'se_and_pe': False }, 
        'dnase-seq':     { 'long': 'DNase-seq',     'short': 'dnase', 'se_and_pe': True  },
    }
    '''Standard labelling requires exp_type specific labels.  This can be overridden in descendent classes.'''
    
    SEA_ID = 'zzz'
    '''SEA is the final branch into which all tributaries flow.'''

    def __init__(self):
        '''<commit_msg>Minor change to support launching dme pipeline.<commit_after>#!/usr/bin/env python2.7
# launch.py 1.0.1

import argparse,os, sys, json
#import urlparse, subprocess, itertools, logging
from datetime import datetime
from collections import deque

import dxpy
import dxencode

### TODO:
# 1) NEED TO MAKE a --template version not relying on ENCODEd at all!

# NOTES: This command-line utility will run the a pipeline for a single replicate or combined reps.
#      - All results will be written to a folder /<results_folder>/<exp_id>/rep<#>_<#>.
#      - If any results are already in that directory, then the steps that created those results
#        will not be rerun.
#      - If any jobs for the experiment and replicate are already running, nothing new will be
#        launched.
#      - Almost all the code is generic and derived classes will only need to entend or override a 
#        a few things.  It relies upon JSON pipeline descriptions which in simple cases can be 
#        directly read from dx (e.g. srnaLaunch.py).  More compex examples (e.g. lrnaLaunch.py)
#        must hard-code the pipeline json to handle repeated apps and dx name conflicts.
#        Tokens are used to abstract dx.app input/outout file names to avoid collisions.
#        - PIPELINE_BRANCHES[branch_id]["ORDER"] is the list of steps in the pipeline branch
#        - PIPELINE_BRANCHES[branch_id]["STEPS"] contain step definitions and enforces dependencies by:
#          'input' file tokens matching to 'result' file tokens of earlier steps.
#        - FILE_GLOBS is needed for locating result files from prior runs.
#      - Combined replicate processing is optional and depends upon PIPELINE_BRANCHES["COMBINED_REPS"], etc.
#        - The standard combination expected PIPELINE_BRANCH_ORDER[ "REP", "COMBINED_REPS" ].
#        - More complex combinations can be acheived by overriding add_combining_reps() in the derived class.
#        - Tying results from one branch to inputs into another relies on input file tokens ending with '_A', '_B' or '_ABC'.
#      - Control files are supported but most likely will require function overrides in derived
#        classes (e.g. rampageLaunch.py)
#      - In PIPELINE_BRANCHES[branch_id]["STEPS"], both 'inputs' and 'results' are key:value lists where the key is used
#        to attach results of one step to the inputs of another, while values must match dxapp.json names.
#      - The FILE_GLOBS are matched by input/result keys, not values.
#        (e.g. "input": { "pooled_bam": "bam" } matches file_glob "pooled_bam" and dxapp.json input named "bam".
#        This example would allow the same applet to be used on replicate bams and poold bams in the same pipeline.)
#      - RARE cases using 'output_values' and 'param_links' can support non-file outputs as inputs to later steps,
#        even across rep branch boundaries.
#
#   A design note about replicates in the launchers:
#      - "rep" objects should contain (almost) all information necesary to launch processing against them.
#        A few minor things common to the entire experiment are stored directly in the psv (pipeline specific variables) object
#      - Each "rep" object will get processed by one PIPELINE_BRANCH which determines which STEPS are run.
#      - All ENCODEd replicates ready for processing should get loaded into psv["reps"], keyed with letters: "a","b","c"...
#      - These "simple reps" have a "rep_tech" like "rep2_1" (for bio_rep=2, tech_rep=1).
#      - Combined replicates have a "rep_tech" like "reps1_1-2_1" (for combining simple reps rep1_1 and rep2_1).
#      - Combined replicates should be keyed (in sort order) after their "tributary" reps (e.g. "a","b","b-combines-a-b").
#      - The final combined replicate (aka "SEA" into which all rivers flow) is keyed with self.SEA_ID ("zzz")
#      - In the 'standard combination model' (which can be overridden in descendent classes): 
#        - Simple reps are processed by the "REP" pipeline branch.
#        - The one combined rep (aka "SEA") is processed by the "COMBINED_REPS" pipeline branch

#
# Example derived classes: long-rna-seq-pipeline: lrnaLaunch.py, srnaLaunch.py rampageLaunch.py

class Launch(object):
    '''
    Launch module is the common code for pipeline specific launcher classes.  
    Launchers dynamically create and launch a workflow covering any needed steps of a pipeline.
    Steps needed are thos that have available inputs but not already generatedresults.
    '''

    PIPELINE_NAME = "MUST REPLACE IN DREIVED CLASS"
    ''' This must match the assay type returned by dxencode.get_assay_type({exp_id}).'''

    PIPELINE_HELP = "Launches '"+PIPELINE_NAME+"' pipeline " + \
                    "analysis for one replicate or combined replicates. "
    ''' This help title should name pipline and whether combined replicates are supported.'''
                    
    SERVER_DEFAULT = 'www'
    '''At this time there is no need to use the any but the one true server for launching.'''
    
    FOLDER_DEFAULT = '/runs/'
    ''' This the default location to place results folders for each experiment.'''
    
    PIPELINE_BRANCH_ORDER = None # MUST replace in derived class. Example: [ "REP", "COMBINED_REPS" ]
    '''A pipeline is frequently made of branches that flow into each other, such as replicate level to combined replicate.'''
    
    PIPELINE_BRANCHES = None # MUST replace in derived class. 'ORDER' and "STEPS" are required for each branch,
    # Example:               # though simple "STEPS" may be discovered.
    #{    "REP": {
    #            "ORDER": [ "rep_app_one", ... ],
    #     # NOTE: In simple cases "STEPS" can be discovered from dnanexus apps (example: srnaLaunch.py)
    #     # HINT: Try to discover the "STEPS" usubg build_simple_steps(verbose) to print the json, then modify from there  
    #            "STEPS": {
    #                        "rep_app_one": {
    #                            "inputs":  { "reads":    "reads" },
    #                            "app":       "rep_app_1", 
    #                            "params":  { "nthreads": "nthreads"}, 
    #                            "results": { "rep_bam":  "bam_out" }
    #                        }, ...
    #    },
    #    "COMBINED_REPS": ...
    #}
    '''Each branch must define the 'steps' and their (artificially) linear order.'''

    # "STEPS" objects within PIPELINE_BRANCHES contain the following:
    #    "step-name":   { # Must match "ORDER" and may match dx applet
    #        "app":     "dx-applet-name", # Must match applet
    #        "params":  { "param_token": "dx_input_name", ...}, # Non-files: Token must match psv key.
    #        "inputs":  { "input_token": "dx_input_name", ...},  # Files: Token *may* match name
    #        "results": { "result_token": "dx_output_name", ...}, # Files: Token *may* match name
    #        "output_values": { "output_token": "dx_output_name" }, # RARE: non-file output used as input to another step
    #        "param_links": { "param_token": {"name":"output_token", "rep":"sister"} }} # RARE: link input to non-file output 
    # Note, tokens *may* be the same as dx names, but MUST MATCH other tokens to define dependencies as: 
    #     stepA:result_token == stepB:input_token
    # Also result_tokens must be in FILE_GLOB.keys()
    #
    # NOTE: Simple pipelines have no reused steps or dx name to param complications and the 
    #       "STEPS" objects can be generated directly from dx apps.  If unsure, print the json from
    #       self.build_simple_steps(verbose=True) to determine if it will work for your pipeline.
    
    PRUNE_STEPS = None
    '''Rare setting that allows dynamically pruning steps from a pipeline befpre running it.  Used by lrnaLaunch.py.'''
    
    FILE_GLOBS = {"MAY": {}, "NEED": {}, "TO": {}, "REPLACE": {} }
    '''Globs for discovering existing results. Can be discoverd in simple piplines.'''

    CONTROL_FILE_GLOB = None
    '''Glob for discovering control files if experiment has controls. Othewise: None'''

    REFERENCE_FILES = {
        # For looking up reference file names.
        }

    GENOMES_SUPPORTED = ['hg19', 'mm10']
    GENOME_DEFAULT = 'hg19'
    ''' This is the default Genome that experiments are mapped to.'''
    
    STANDARD_LABELS = {
        'long-rna-seq':  { 'long': 'Long-RNA-seq',  'short': 'lrna',  'se_and_pe': True  }, 
        'small-rna-seq': { 'long': 'Small-RNA-seq', 'short': 'srna',  'se_and_pe': False }, 
        'rampage':       { 'long': 'Rampage',       'short': 'ramp',  'se_and_pe': False }, 
        'dnase-seq':     { 'long': 'DNase-seq',     'short': 'dnase', 'se_and_pe': True  },
        'dna-me':        { 'long': 'DNA-methlyation','short': 'dme',   'se_and_pe': True  },
    }
    '''Standard labelling requires exp_type specific labels.  This can be overridden in descendent classes.'''
    
    SEA_ID = 'zzz'
    '''SEA is the final branch into which all tributaries flow.'''

    def __init__(self):
        '''
{'fraction_non_alphanumeric': 0.10209907640638119, 'fraction_numerical': 0.00279876854184159, 'mean_word_length': 2.807757885763001, 'pattern_counts': {'":': 64, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7403537', 'n_tokens_mistral': 5391, 'n_tokens_neox': 5146, 'n_words': 2026}