<commit_before>#!/usr/bin/env python
# Description: submit job to queue
import os
import sys
import myfunc
import subprocess
import time
import math
import json
suq_exec = "/usr/bin/suq"
progname =  os.path.basename(__file__)
wspace = ''.join([" "]*len(progname))


rundir = os.path.dirname(os.path.realpath(__file__))
basedir = os.path.realpath("%s/../"%(rundir))
python_exec = os.path.realpath("%s/../../env/bin/python"%(basedir))
suq_basedir = "/tmp"
if os.path.exists("/scratch"):
    suq_basedir = "/scratch"
elif os.path.exists("/tmp"):
    suq_basedir = "/tmp"
gen_errfile = "%s/static/log/%s.log"%(basedir, progname)

vip_email_file = "%s/config/vip_email.txt"%(basedir)

usage_short="""
Usage: %s -nseq INT -jobid STR -outpath DIR -datapath DIR
       %s -email EMAIL -host IP -baseurl BASE_WWW_URL
       %s -nseq-this-user INT
       %s -only-get-cache [-force]

Description: 
    BASE_WWW_URL e.g. topcons.net
"""%(progname, wspace, wspace, wspace)

usage_ext="""
Description:
    Submit job to queue
    datapath should include query.fa

OPTIONS:
  -only-get-cache   Only get the cached results, this will be run on the front-end
  -force            Do not use cahced result
  -nseq-this-user   Number of sequences in the queue submitted by this user
  -h, --help    Print this help message and exit

Created 2015-01-20, updated 2016-12-07, <NAME>
"""
usage_exp="""
Examples:
    %s -jobid rst_mXLDGD -outpath /data3/result/rst_mXLDGD -datapath /data3/tmp/tmp_dkgSD
"""%(progname)

def PrintHelp(fpout=sys.stdout):#{{{
    print >> fpout, usage_short
    print >> fpout, usage_ext
    print >> fpout, usage_exp#}}}

def SubmitJobToQueue(jobid, datapath, outpath, numseq, numseq_this_user, email, #{{{
        host_ip, base_www_url):
    myfunc.WriteFile("Entering SubmitJobToQueue()\n", g_params['debugfile'],
            "a", True)
    fafile = "%s/query.fa"%(datapath)
    query_parafile = "%s/query.para.txt"%(outpath)

    query_para = {}
    content = myfunc.ReadFile(query_parafile)
    para_str = content
    if content != "":
        query_para = json.loads(content)

    try:
        name_software = query_para['name_software']
    except KeyError:
        name_software = ""


    if numseq == -1:
        numseq = myfunc.CountFastaSeq(fafile)
    if numseq_this_user == -1:
        numseq_this_user = numseq

    runjob = "%s %s/run_job.py"%(python_exec, rundir)
    scriptfile = "%s/runjob,%s,%s,%s,%s,%d.sh"%(outpath, name_software, jobid, host_ip, email, numseq)
    code_str_list = []
    code_str_list.append("#!/bin/bash")
    cmdline = "%s %s -outpath %s -tmpdir %s -jobid %s "%(runjob, fafile, outpath, datapath, jobid)
    if email != "":
        cmdline += "-email \"%s\" "%(email)
    if base_www_url != "":
        cmdline += "-baseurl \"%s\" "%(base_www_url)
    if g_params['isOnlyGetCache']:
        cmdline += "-only-get-cache "
    code_str_list.append(cmdline)

    code = "\n".join(code_str_list)

    msg = "Write scriptfile %s"%(scriptfile)
    myfunc.WriteFile(msg+"\n", g_params['debugfile'], "a", True)

    myfunc.WriteFile(code, scriptfile, mode="w", isFlush=True)
    os.chmod(scriptfile, 0755)

    myfunc.WriteFile("Getting priority"+"\n", g_params['debugfile'], "a", True)
    priority = myfunc.GetSuqPriority(numseq_this_user)

    if email in g_params['vip_user_list']:
        priority = 999999999.0

    myfunc.WriteFile("priority=%d\n"%(priority), g_params['debugfile'], "a",
            True)

    st1 = SubmitSuqJob(suq_basedir, datapath, priority, scriptfile)

    return st1
#}}}
def SubmitSuqJob(suq_basedir, datapath, priority, scriptfile):#{{{
    myfunc.WriteFile("Entering SubmitSuqJob()\n", g_params['debugfile'], "a",
            True)
    rmsg = ""
    cmd = [suq_exec,"-b", suq_basedir, "run", "-d", datapath, "-p", "%d"%(priority), scriptfile]
    cmdline = " ".join(cmd)
    myfunc.WriteFile("cmdline: %s\n\n"%(cmdline), g_params['debugfile'], "a",
            True)
    MAX_TRY = 5
    cnttry = 0
    isSubmitSuccess = False
    while cnttry < MAX_TRY:
        try:
            myfunc.WriteFile("run cmd: cnttry = %d, MAX_TRY=%d\n"%(cnttry,
                MAX_TRY), g_params['debugfile'], "a", True)
            rmsg = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            isSubmitSuccess = True
            break
        except subprocess.CalledProcessError, e:
            print  e
            print rmsg
            myfunc.WriteFile(str(e)+"\n"+rmsg+"\n", g_params['debugfile'],
                    "a", True)
            pass
        cnttry += 1<commit_msg>write scriptfile to result folder<commit_after>#!/usr/bin/env python
# Description: submit job to queue
import os
import sys
import myfunc
import subprocess
import time
import math
import json
suq_exec = "/usr/bin/suq"
progname =  os.path.basename(__file__)
wspace = ''.join([" "]*len(progname))


rundir = os.path.dirname(os.path.realpath(__file__))
basedir = os.path.realpath("%s/../"%(rundir))
python_exec = os.path.realpath("%s/../../env/bin/python"%(basedir))
suq_basedir = "/tmp"
if os.path.exists("/scratch"):
    suq_basedir = "/scratch"
elif os.path.exists("/tmp"):
    suq_basedir = "/tmp"
gen_errfile = "%s/static/log/%s.log"%(basedir, progname)

vip_email_file = "%s/config/vip_email.txt"%(basedir)

usage_short="""
Usage: %s -nseq INT -jobid STR -outpath DIR -datapath DIR
       %s -email EMAIL -host IP -baseurl BASE_WWW_URL
       %s -nseq-this-user INT
       %s -only-get-cache [-force]

Description: 
    BASE_WWW_URL e.g. topcons.net
"""%(progname, wspace, wspace, wspace)

usage_ext="""
Description:
    Submit job to queue
    datapath should include query.fa

OPTIONS:
  -only-get-cache   Only get the cached results, this will be run on the front-end
  -force            Do not use cahced result
  -nseq-this-user   Number of sequences in the queue submitted by this user
  -h, --help    Print this help message and exit

Created 2015-01-20, updated 2016-12-07, <NAME>
"""
usage_exp="""
Examples:
    %s -jobid rst_mXLDGD -outpath /data3/result/rst_mXLDGD -datapath /data3/tmp/tmp_dkgSD
"""%(progname)

def PrintHelp(fpout=sys.stdout):#{{{
    print >> fpout, usage_short
    print >> fpout, usage_ext
    print >> fpout, usage_exp#}}}

def SubmitJobToQueue(jobid, datapath, outpath, numseq, numseq_this_user, email, #{{{
        host_ip, base_www_url):
    myfunc.WriteFile("Entering SubmitJobToQueue()\n", g_params['debugfile'],
            "a", True)
    fafile = "%s/query.fa"%(datapath)
    query_parafile = "%s/query.para.txt"%(outpath)

    query_para = {}
    content = myfunc.ReadFile(query_parafile)
    para_str = content
    if content != "":
        query_para = json.loads(content)

    try:
        name_software = query_para['name_software']
    except KeyError:
        name_software = ""


    if numseq == -1:
        numseq = myfunc.CountFastaSeq(fafile)
    if numseq_this_user == -1:
        numseq_this_user = numseq

    runjob = "%s %s/run_job.py"%(python_exec, rundir)
    scriptfile = "%s/runjob,%s,%s,%s,%s,%d.sh"%(outpath, name_software, jobid, host_ip, email, numseq)
    code_str_list = []
    code_str_list.append("#!/bin/bash")
    cmdline = "%s %s -outpath %s -tmpdir %s -jobid %s "%(runjob, fafile, outpath, datapath, jobid)
    if email != "":
        cmdline += "-email \"%s\" "%(email)
    if base_www_url != "":
        cmdline += "-baseurl \"%s\" "%(base_www_url)
    if g_params['isOnlyGetCache']:
        cmdline += "-only-get-cache "
    code_str_list.append(cmdline)

    code = "\n".join(code_str_list)

    msg = "Write scriptfile %s"%(scriptfile)
    myfunc.WriteFile(msg+"\n", g_params['debugfile'], "a", True)

    myfunc.WriteFile(code, scriptfile, mode="w", isFlush=True)
    os.chmod(scriptfile, 0755)

    myfunc.WriteFile("Getting priority"+"\n", g_params['debugfile'], "a", True)
    priority = myfunc.GetSuqPriority(numseq_this_user)

    if email in g_params['vip_user_list']:
        priority = 999999999.0

    myfunc.WriteFile("priority=%d\n"%(priority), g_params['debugfile'], "a",
            True)

    st1 = SubmitSuqJob(suq_basedir, datapath, priority, scriptfile)

    return st1
#}}}
def SubmitSuqJob(suq_basedir, datapath, priority, scriptfile):#{{{
    myfunc.WriteFile("Entering SubmitSuqJob()\n", g_params['debugfile'], "a",
            True)
    rmsg = ""
    cmd = [suq_exec,"-b", suq_basedir, "run", "-d", outpath, "-p", "%d"%(priority), scriptfile]
    cmdline = " ".join(cmd)
    myfunc.WriteFile("cmdline: %s\n\n"%(cmdline), g_params['debugfile'], "a",
            True)
    MAX_TRY = 5
    cnttry = 0
    isSubmitSuccess = False
    while cnttry < MAX_TRY:
        try:
            myfunc.WriteFile("run cmd: cnttry = %d, MAX_TRY=%d\n"%(cnttry,
                MAX_TRY), g_params['debugfile'], "a", True)
            rmsg = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            isSubmitSuccess = True
            break
        except subprocess.CalledProcessError, e:
            print  e
            print rmsg
            myfunc.WriteFile(str(e)+"\n"+rmsg+"\n", g_params['debugfile'],
                    "a", True)
            pass
        cnttry += 1
{'fraction_non_alphanumeric': 0.15396425046605988, 'fraction_numerical': 0.008553569470336659, 'mean_word_length': 3.0768886902101027, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '507642', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3298, 'n_words': 701}