<commit_before># -*- coding: utf-8 -*-
import os
import config
import time

if config.mode == "LSF":
    import sys_LSF as manager
elif config.mode == "LOCAL":
    import sys_single as manager
else:
    import sys_OTHER as manager

def change_environment(env):
    if len(env) > 0:
        for var, val in env.iteritems():
            if val[1] == "overwrite":
                os.environ[var] = val[0]
            else:
                if os.environ.has_key(var):
                    os.environ[var] = val[0] + ":" + os.environ[var]
                else:
                    os.environ[var] = val[0]

def project_process(path_base, folder):
    samples = get_samples(path_base, folder, path_base + "/" + folder + "/samples.list")
    # Check main process
    print "## MAIN PROCESS ###########################"
    try:
        f = open(path_base + "/" + folder + "/pid.txt", 'r')
        i = f.readline().strip("\n").split("\t")[1]
        f.close()
        k = manager.job_status(i)
        if k == 1:
            st = "DONE"
        elif k == 0:
            st = "RUN"
        else:
            st = "ERROR"
        print "- Main process (" + i + ") status: " + st
    except:
        print "- Main process not found or already finished"
    # Check subprocesses
    print "## SUBPROCESSES ###########################"
    pids = dict()
    try:
        f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
        for i in f:
            i = i.strip("\n").split("\t")
            pids[i[0]] = [i[1].split("|"),i[2].split("|")]
        f.close()
    except:
        print "- No subprocesses file found"
    f = open(path_base + "/" + folder + "/config.txt", 'r')
    config = dict()
    for i in f:
        if not i.startswith("%"):
            i = i.strip("\n").split("\t")
            if i[0] in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", 'sam2sortbam', "picard_IS", "varscan", "gatk"]:
                i[1] = i[1].split("/")[0]
                if i[1] != "0":
                    config[i[0]] = i[1]
    if (config.has_key("varscan") or config.has_key("gatk") or config.has_key("picard_IS")) and (not config.has_key("sam2sortbam")):
        config["sam2sortbam"] = 1
    if len(config) > 0:
        for pg in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", "sam2sortbam", "picard_IS", "varscan", "gatk"]:
            if config.has_key(pg):
                print "Process:  " + pg
                if not pids.has_key(pg):
                    print "- Already done or waiting for previous module output"
                else:
                    pid = pids[pg]
                    print "- ID:       " + "|".join(pid[0])
                    n = list()
                    for i in pid[1]:
                        k = manager.job_status(i)
                        if k == 1:
                            n.append("DONE")
                        elif k == 0:
                            n.append("RUN")
                        else:
                            n.append("ERROR")
                    print "- Status:   " +  "|".join(n)
                    samples_v, stats = check_samples(samples, path_base, folder, pg, "update")
                    sok = str(round(100 * float(stats[1])/float(stats[0]),2))
                    sko = str(round(100 * float(stats[2])/float(stats[0]),2))
                    pending = str(round(100 * float(stats[0]-stats[1]-stats[2])/float(stats[0]),2))
                    print "- Progress: " + sok + "% succeeded / " + sko + "% exited / " + pending + "% pending"


#############################################################################
# MAIN FUNCTION FOR KILLING ALL THE PROCESSES RELATED TO A PROJECT RUN
#############################################################################
def project_kill(path_base, folder):
    print "Main process:"
    uds = ""
    try:
        f = open(path_base + "/" + folder + "/pid.txt",'r')
        uds = f.readline().strip("\n").split("\t")[1]
        f.close()
        print "- Killing main process ("+uds+")"
        manager.job_kill(uds)
    except:
        print "- No main process to kill. Already finished (" + uds + ")"
    print "Submodule processes:"
    try:
        f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
        for i in f:
            i = i.strip("\n").split("\t")
            if len(i) > 1:
                j = i[1].split("|")
                for jj in j:
                    print "- Killing process " + i[0] + " ("+jj+")"
                    manager.job_kill(jj)
        f.close()
    except:
        print "No submodule processes to kill"
    return 1


#############################################################################
## PARSES DE CONFIGURATION FILE
#############################################################################
def config_file(config, path_base, folder, paths):
    # config: path to configuration file
    # path_base: Absolute path to the location where the project folder has been created
    # folder: Name of the project folder located in 'path_base'
    mandatory_fields = ['genome_build', 'strandedness', 'trimgalore', 'fastqc',
                        'star', 'star-fusion', 'picard', 'htseq-gene', 'htseq-exon',
                        'kallisto', 'sam2sortbam', 'picard_IS', 'gatk', 'varscan',
                        'q', 'wt', 'star_args', 'star2pass', 'starfusion', 'kalboot',
                        'varscan_args', 'gatk_args', 'htseq-gene-mode', 'htseq-exon-mode']
    f = open(config, 'r')
    var = dict()
    for i in f:
       if not i.startswith("%"):
           i = i.strip('\n').split("\t")
           if len(i) > 1:
               var[i[0]] = i[1]
    f.close()
    for i in mandatory_fields:
        if i not in var:
            exit('Field "' + i + '" is missing in the configuration file.')
    return [config, var]


def job_wait(path, secs):
## WAITS UNTIL A JOB ENDS SUCCESSFULLY (IF ERROR -> ABORTS) BASED ON THE LOG FILES CREATED BY THE RESOURCE MANAGER
## Input:
##   path: String with the path to the log files corresponding to the jobs that will be checked (i.e. single job: '/some_path/job1.log', job array: '/some_path/job1.log|/some_path/job2.log')
##   secs: Number of seconds between subsequent checks
## Returns:
##   1: Once all the jobs have finished
## If an error is detected -> exit()
    if "|" in path:
        jobarray = path.split("|")
    else:
        jobarray = [path]
    for path in jobarray:<commit_msg>jsplice programs jsplice arnapipe jsplice vcrparser<commit_after># -*- coding: utf-8 -*-
import os
import config
import time

if config.mode == "LSF":
    import sys_LSF as manager
elif config.mode == "LOCAL":
    import sys_single as manager
else:
    import sys_OTHER as manager

def change_environment(env):
    if len(env) > 0:
        for var, val in env.iteritems():
            if val[1] == "overwrite":
                os.environ[var] = val[0]
            else:
                if os.environ.has_key(var):
                    os.environ[var] = val[0] + ":" + os.environ[var]
                else:
                    os.environ[var] = val[0]

def project_process(path_base, folder):
    samples = get_samples(path_base, folder, path_base + "/" + folder + "/samples.list")
    # Check main process
    print "## MAIN PROCESS ###########################"
    try:
        f = open(path_base + "/" + folder + "/pid.txt", 'r')
        i = f.readline().strip("\n").split("\t")[1]
        f.close()
        k = manager.job_status(i)
        if k == 1:
            st = "DONE"
        elif k == 0:
            st = "RUN"
        else:
            st = "ERROR"
        print "- Main process (" + i + ") status: " + st
    except:
        print "- Main process not found or already finished"
    # Check subprocesses
    print "## SUBPROCESSES ###########################"
    pids = dict()
    try:
        f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
        for i in f:
            i = i.strip("\n").split("\t")
            pids[i[0]] = [i[1].split("|"),i[2].split("|")]
        f.close()
    except:
        print "- No subprocesses file found"
    f = open(path_base + "/" + folder + "/config.txt", 'r')
    config = dict()
    for i in f:
        if not i.startswith("%"):
            i = i.strip("\n").split("\t")
            if i[0] in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", 'sam2sortbam', "picard_IS", "varscan", "gatk", "jsplice"]:
                i[1] = i[1].split("/")[0]
                if i[1] != "0":
                    config[i[0]] = i[1]
    if (config.has_key("varscan") or config.has_key("gatk") or config.has_key("picard_IS")) and (not config.has_key("sam2sortbam")):
        config["sam2sortbam"] = 1
    if len(config) > 0:
        for pg in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", "sam2sortbam", "picard_IS", "varscan", "gatk", "jsplice"]:
            if config.has_key(pg):
                print "Process:  " + pg
                if not pids.has_key(pg):
                    print "- Already done or waiting for previous module output"
                else:
                    pid = pids[pg]
                    print "- ID:       " + "|".join(pid[0])
                    n = list()
                    for i in pid[1]:
                        k = manager.job_status(i)
                        if k == 1:
                            n.append("DONE")
                        elif k == 0:
                            n.append("RUN")
                        else:
                            n.append("ERROR")
                    print "- Status:   " +  "|".join(n)
                    samples_v, stats = check_samples(samples, path_base, folder, pg, "update")
                    sok = str(round(100 * float(stats[1])/float(stats[0]),2))
                    sko = str(round(100 * float(stats[2])/float(stats[0]),2))
                    pending = str(round(100 * float(stats[0]-stats[1]-stats[2])/float(stats[0]),2))
                    print "- Progress: " + sok + "% succeeded / " + sko + "% exited / " + pending + "% pending"


#############################################################################
# MAIN FUNCTION FOR KILLING ALL THE PROCESSES RELATED TO A PROJECT RUN
#############################################################################
def project_kill(path_base, folder):
    print "Main process:"
    uds = ""
    try:
        f = open(path_base + "/" + folder + "/pid.txt",'r')
        uds = f.readline().strip("\n").split("\t")[1]
        f.close()
        print "- Killing main process ("+uds+")"
        manager.job_kill(uds)
    except:
        print "- No main process to kill. Already finished (" + uds + ")"
    print "Submodule processes:"
    try:
        f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
        for i in f:
            i = i.strip("\n").split("\t")
            if len(i) > 1:
                j = i[1].split("|")
                for jj in j:
                    print "- Killing process " + i[0] + " ("+jj+")"
                    manager.job_kill(jj)
        f.close()
    except:
        print "No submodule processes to kill"
    return 1


#############################################################################
## PARSES DE CONFIGURATION FILE
#############################################################################
def config_file(config, path_base, folder, paths):
    # config: path to configuration file
    # path_base: Absolute path to the location where the project folder has been created
    # folder: Name of the project folder located in 'path_base'
    mandatory_fields = ['genome_build', 'strandedness', 'trimgalore', 'fastqc',
                        'star', 'star-fusion', 'picard', 'htseq-gene', 'htseq-exon',
                        'kallisto', 'sam2sortbam', 'picard_IS', 'gatk', 'varscan',
                        'q', 'wt', 'star_args', 'star2pass', 'starfusion', 'kalboot',
                        'varscan_args', 'gatk_args', 'htseq-gene-mode', 'htseq-exon-mode', "jsplice"]
    f = open(config, 'r')
    var = dict()
    for i in f:
       if not i.startswith("%"):
           i = i.strip('\n').split("\t")
           if len(i) > 1:
               var[i[0]] = i[1]
    f.close()
    for i in mandatory_fields:
        if i not in var:
            exit('Field "' + i + '" is missing in the configuration file.')
    return [config, var]


def job_wait(path, secs):
## WAITS UNTIL A JOB ENDS SUCCESSFULLY (IF ERROR -> ABORTS) BASED ON THE LOG FILES CREATED BY THE RESOURCE MANAGER
## Input:
##   path: String with the path to the log files corresponding to the jobs that will be checked (i.e. single job: '/some_path/job1.log', job array: '/some_path/job1.log|/some_path/job2.log')
##   secs: Number of seconds between subsequent checks
## Returns:
##   1: Once all the jobs have finished
## If an error is detected -> exit()
    if "|" in path:
        jobarray = path.split("|")
    else:
        jobarray = [path]
    for path in jobarray:
{'fraction_non_alphanumeric': 0.20231831007397238, 'fraction_numerical': 0.009761305574620606, 'mean_word_length': 1.9703284258210645, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1253924', 'n_tokens_mistral': 4168, 'n_tokens_neox': 3980, 'n_words': 1071}