<commit_before>#!/usr/bin/env python

'''
Oct 10, 2017: <NAME>, The University of Melbourne

Simplifies running orthoMCL with a wrapper and pre-checks the known
formatting issues with FASTA headers to avoid failure in later stages of the run.


'''

import os, sys, optparse, getpass
from multiprocessing import Process, Pipe
from Utils import Base, Fasta

#################################################
def options():
    parser = optparse.OptionParser('usage: %prog -i "proteins1.fa proteins2.fa ... proteinsN.fa" -l "lab1 lab2 ... labN" -p "1 3 ... 1" -e 1e-5 -s 0.6')
    parser.add_option('-d', '--dir', dest='wd', help='Working directory', metavar='DIR', default='TmpOrthoMcl')
    parser.add_option('-i', '--filenames', dest='filenames', help='Names of the files of species containing the proteins (separated by commas)', metavar='FILES', default='')
    parser.add_option('-l', '--labels', dest='labs', help="Labels for each species (separated by commas)", metavar='LABELS', default='')
    parser.add_option('-p', '--positions', dest='positions', help="Default positions of unique identifier in FASTA header separated by |. Default position is 1 for all (separate by commas).", metavar='POSITIONS', default='')
    parser.add_option('-T', '--threads', dest='pCnt', help='Number of parallel threads (default 24)', metavar='THREADS', default='24')
    parser.add_option('-e', '--evalue', dest='evalue', help="E-value used at blast. Default is 1e-5. Use 1e-X format only!", metavar='EVALUE', default='1e-5')
    parser.add_option('-s', '--similarity', dest='sim', help="Required similarity (0 .. 1). Default if 0.5", metavar='SIM', default='0.5')
    parser.add_option('-m', '--minlen', dest='minlen', help="Allowed minimum lenght of a protein. Default is 20.", metavar='MINLEN', default='20')
    parser.add_option('-b', '--noblast', dest='skipBlast', action='store_true', help="Skip BLAST", default=False)
    options, args = parser.parse_args()
    if options.filenames == '' or options.labs == '':
        parser.print_help()
        print '\nE.g.: orthoMcl -i "proteome1.fa proteome2.fa" -l "Tax Tvi" -p "4 4" -e 1e-5 -s 0.5'
        print "Results will be found in %s/Results directory in groups.txt file." %options.wd
        print "Note! The labels must be exactly 3 characters long and preferrably start with an upper case character."
        sys.exit(-1)
    return options


#################################################
def checkResidue(fastaFile):
    '''
    '''
    retVal = "nucleotides"
    try:
        limit = 100
        fasta = Fasta(fastaFile)
        for i in xrange(fasta.cnt()):
            if i > limit: break
            seq = fasta.seq(i).upper()
            for item in seq:
                if item not in ['A', 'T', 'C', 'G', 'N']:
                    retVal = "amino acids"
                    break
    except IOError:
        print >> sys.stderr, "### Fatal error: file %s not found. Exiting..." %fastaFile
        sys.exit(-1)
    return retVal


#################################################
def checkUniqueIds(fastaFile):
    '''
    '''
    fasta = Fasta(fastaFile)
    if fasta.cnt() != len(set(fasta.headers)):
        print >> sys.stderr, "### Fatal error: FASTA sequence identifiers are not unique in %s. Exiting..." %fastaFile
        print >> sys.stderr, "### Probably position for this file is given wrong..."
        sys.exit(-1)


#################################################
def createOrthoMclConfigFile(wd, userName, eValue, similarity):
    '''
    '''
    eValue = eValue.split('e')[1]
    similarity = int(float(similarity) * 100.0)
    handle = open("%s/orthomcl.config" %wd, 'w')
    handle.write("# this config assumes a mysql database named 'orthomcl'.  adjust according\n")
    handle.write("# to your situation.\n")
    handle.write("dbVendor=mysql\n")

    handle.write("dbConnectString=dbi:mysql:database=ortho%s;host=%s;port=3306\n" %(userName, os.environ["MYSQLHOST"]))
    handle.write("dbLogin=ortho%s\n" %userName)
    handle.write("dbPassword=<PASSWORD>")
    handle.write("similarSequencesTable=SimilarSequences\n")
    handle.write("orthologTable=Ortholog\n")
    handle.write("inParalogTable=InParalog\n")
    handle.write("coOrthologTable=CoOrtholog\n")
    handle.write("interTaxonMatchView=InterTaxonMatch\n")
    handle.write("percentMatchCutoff=%d\n" %similarity)
    handle.write("evalueExponentCutoff=%s\n" %eValue)
    handle.write("oracleIndexTblSpc=NONE\n")
    handle.close()


#################################################
def createMySqlScripts(wd, userName):
    '''
    '''
    handle = open("%s/createDb.sql" %wd, 'w')
    handle.write("CREATE USER IF NOT EXISTS 'ortho%s'@'%%' IDENTIFIED BY 'password';\n" %userName)
    handle.write("CREATE DATABASE ortho%s;\n" %userName)
    handle.write("GRANT SELECT,INSERT,UPDATE,DELETE,CREATE VIEW,CREATE,INDEX,DROP on ortho%s.* TO 'ortho%s'@'%%';\n" %(userName, userName))
    handle.close()
    handle = open("%s/dropDb.sql" %wd, 'w')
    handle.write("drop database if exists ortho%s;\n" %userName)
    handle.close()


#################################################
def callShell(base, cmdStr, dummy = None):
    '''
    '''
    base.shell(cmdStr)


#################################################
def main():
    '''
    '''
    opts = options() # files contains exactly two PE files

    pCnt = int(opts.pCnt)
    eValue = opts.evalue
    similarity = opts.sim
    minlen = opts.minlen
    files = opts.filenames.strip().split(',')
    labels = opts.labs.strip().split(',')
    if len(labels) != len(set(labels)):
        print >> sys.stderr, "### Fatal error: duplicate labels found. Exiting..."
        sys.exit(-1)
    if len(files) != len(set(files)):
        print >> sys.stderr, "### Fatal error: duplicate fasta file names found. Exiting..."
        sys.exit(-1)
    positions = None
    if opts.positions != "":
        positions = opts.positions.strip().split(',')
    if positions == None:
        positions = []
        for i in xrange(len(files)):
            positions.append("1")
    if len(files) != len(labels):
        print >> sys.stderr, "### Fatal error: number of files does not match with the number of labels. Exiting..."
        sys.exit(-1)
    if len(positions) != len(labels):
        print >> sys.stderr, "### Fatal error: number of labels does not match with the number of positions of the ids. Exiting..."
        sys.exit(-1)
    for lab in labels:
        if len(lab) != 3:
            print >> sys.stderr, "### Fatal error: labels have to be exactly three characters long. Exiting..."
            sys.exit(-1)

    base = Base()
    wd = "%s/Results" %opts.wd
    wdFasta = "%s/Fasta" %wd
    wdAdds = "%s/Adds" %wd
    base.createDir(wd)
    #logHandle = open("%s/log.txt" %wd, 'w')
    logHandle = open("log.txt" %wd, 'w')
    base.setLogHandle(logHandle)
    base.createDir(wdFasta)
    #userName = getpass.getuser()
    #createOrthoMclConfigFile(wd, userName, eValue, similarity)
    #createMySqlScripts(wd, userName)
    createOrthoMclConfigFile(wd, "root", eValue, similarity)
    createMySqlScripts(wd, "root")

    requiredMolType = "amino acids"
    for myFile in files:
        molType = checkResidue(myFile)
        if requiredMolType != molType:
            print >> sys.stderr, "### Fatal error: files have to all be amino acids. Exiting..."
            print >> sys.stderr, "### File %s failed and was %s." %(myFile, molType)
            sys.exit(-1)<commit_msg>Fixing a syntax error<commit_after>#!/usr/bin/env python

'''
Oct 10, 2017: <NAME>, The University of Melbourne

Simplifies running orthoMCL with a wrapper and pre-checks the known
formatting issues with FASTA headers to avoid failure in later stages of the run.


'''

import os, sys, optparse, getpass
from multiprocessing import Process, Pipe
from Utils import Base, Fasta

#################################################
def options():
    parser = optparse.OptionParser('usage: %prog -i "proteins1.fa proteins2.fa ... proteinsN.fa" -l "lab1 lab2 ... labN" -p "1 3 ... 1" -e 1e-5 -s 0.6')
    parser.add_option('-d', '--dir', dest='wd', help='Working directory', metavar='DIR', default='TmpOrthoMcl')
    parser.add_option('-i', '--filenames', dest='filenames', help='Names of the files of species containing the proteins (separated by commas)', metavar='FILES', default='')
    parser.add_option('-l', '--labels', dest='labs', help="Labels for each species (separated by commas)", metavar='LABELS', default='')
    parser.add_option('-p', '--positions', dest='positions', help="Default positions of unique identifier in FASTA header separated by |. Default position is 1 for all (separate by commas).", metavar='POSITIONS', default='')
    parser.add_option('-T', '--threads', dest='pCnt', help='Number of parallel threads (default 24)', metavar='THREADS', default='24')
    parser.add_option('-e', '--evalue', dest='evalue', help="E-value used at blast. Default is 1e-5. Use 1e-X format only!", metavar='EVALUE', default='1e-5')
    parser.add_option('-s', '--similarity', dest='sim', help="Required similarity (0 .. 1). Default if 0.5", metavar='SIM', default='0.5')
    parser.add_option('-m', '--minlen', dest='minlen', help="Allowed minimum lenght of a protein. Default is 20.", metavar='MINLEN', default='20')
    parser.add_option('-b', '--noblast', dest='skipBlast', action='store_true', help="Skip BLAST", default=False)
    options, args = parser.parse_args()
    if options.filenames == '' or options.labs == '':
        parser.print_help()
        print '\nE.g.: orthoMcl -i "proteome1.fa proteome2.fa" -l "Tax Tvi" -p "4 4" -e 1e-5 -s 0.5'
        print "Results will be found in %s/Results directory in groups.txt file." %options.wd
        print "Note! The labels must be exactly 3 characters long and preferrably start with an upper case character."
        sys.exit(-1)
    return options


#################################################
def checkResidue(fastaFile):
    '''
    '''
    retVal = "nucleotides"
    try:
        limit = 100
        fasta = Fasta(fastaFile)
        for i in xrange(fasta.cnt()):
            if i > limit: break
            seq = fasta.seq(i).upper()
            for item in seq:
                if item not in ['A', 'T', 'C', 'G', 'N']:
                    retVal = "amino acids"
                    break
    except IOError:
        print >> sys.stderr, "### Fatal error: file %s not found. Exiting..." %fastaFile
        sys.exit(-1)
    return retVal


#################################################
def checkUniqueIds(fastaFile):
    '''
    '''
    fasta = Fasta(fastaFile)
    if fasta.cnt() != len(set(fasta.headers)):
        print >> sys.stderr, "### Fatal error: FASTA sequence identifiers are not unique in %s. Exiting..." %fastaFile
        print >> sys.stderr, "### Probably position for this file is given wrong..."
        sys.exit(-1)


#################################################
def createOrthoMclConfigFile(wd, userName, eValue, similarity):
    '''
    '''
    eValue = eValue.split('e')[1]
    similarity = int(float(similarity) * 100.0)
    handle = open("%s/orthomcl.config" %wd, 'w')
    handle.write("# this config assumes a mysql database named 'orthomcl'.  adjust according\n")
    handle.write("# to your situation.\n")
    handle.write("dbVendor=mysql\n")

    handle.write("dbConnectString=dbi:mysql:database=ortho%s;host=%s;port=3306\n" %(userName, os.environ["MYSQLHOST"]))
    handle.write("dbLogin=ortho%s\n" %userName)
    handle.write("dbPassword=<PASSWORD>")
    handle.write("similarSequencesTable=SimilarSequences\n")
    handle.write("orthologTable=Ortholog\n")
    handle.write("inParalogTable=InParalog\n")
    handle.write("coOrthologTable=CoOrtholog\n")
    handle.write("interTaxonMatchView=InterTaxonMatch\n")
    handle.write("percentMatchCutoff=%d\n" %similarity)
    handle.write("evalueExponentCutoff=%s\n" %eValue)
    handle.write("oracleIndexTblSpc=NONE\n")
    handle.close()


#################################################
def createMySqlScripts(wd, userName):
    '''
    '''
    handle = open("%s/createDb.sql" %wd, 'w')
    handle.write("CREATE USER IF NOT EXISTS 'ortho%s'@'%%' IDENTIFIED BY 'password';\n" %userName)
    handle.write("CREATE DATABASE ortho%s;\n" %userName)
    handle.write("GRANT SELECT,INSERT,UPDATE,DELETE,CREATE VIEW,CREATE,INDEX,DROP on ortho%s.* TO 'ortho%s'@'%%';\n" %(userName, userName))
    handle.close()
    handle = open("%s/dropDb.sql" %wd, 'w')
    handle.write("drop database if exists ortho%s;\n" %userName)
    handle.close()


#################################################
def callShell(base, cmdStr, dummy = None):
    '''
    '''
    base.shell(cmdStr)


#################################################
def main():
    '''
    '''
    opts = options() # files contains exactly two PE files

    pCnt = int(opts.pCnt)
    eValue = opts.evalue
    similarity = opts.sim
    minlen = opts.minlen
    files = opts.filenames.strip().split(',')
    labels = opts.labs.strip().split(',')
    if len(labels) != len(set(labels)):
        print >> sys.stderr, "### Fatal error: duplicate labels found. Exiting..."
        sys.exit(-1)
    if len(files) != len(set(files)):
        print >> sys.stderr, "### Fatal error: duplicate fasta file names found. Exiting..."
        sys.exit(-1)
    positions = None
    if opts.positions != "":
        positions = opts.positions.strip().split(',')
    if positions == None:
        positions = []
        for i in xrange(len(files)):
            positions.append("1")
    if len(files) != len(labels):
        print >> sys.stderr, "### Fatal error: number of files does not match with the number of labels. Exiting..."
        sys.exit(-1)
    if len(positions) != len(labels):
        print >> sys.stderr, "### Fatal error: number of labels does not match with the number of positions of the ids. Exiting..."
        sys.exit(-1)
    for lab in labels:
        if len(lab) != 3:
            print >> sys.stderr, "### Fatal error: labels have to be exactly three characters long. Exiting..."
            sys.exit(-1)

    base = Base()
    wd = "%s/Results" %opts.wd
    wdFasta = "%s/Fasta" %wd
    wdAdds = "%s/Adds" %wd
    base.createDir(wd)
    #logHandle = open("%s/log.txt" %wd, 'w')
    logHandle = open("log.txt", 'w')
    base.setLogHandle(logHandle)
    base.createDir(wdFasta)
    #userName = getpass.getuser()
    #createOrthoMclConfigFile(wd, userName, eValue, similarity)
    #createMySqlScripts(wd, userName)
    createOrthoMclConfigFile(wd, "root", eValue, similarity)
    createMySqlScripts(wd, "root")

    requiredMolType = "amino acids"
    for myFile in files:
        molType = checkResidue(myFile)
        if requiredMolType != molType:
            print >> sys.stderr, "### Fatal error: files have to all be amino acids. Exiting..."
            print >> sys.stderr, "### File %s failed and was %s." %(myFile, molType)
            sys.exit(-1)
{'fraction_non_alphanumeric': 0.19784557741201658, 'fraction_numerical': 0.00923323966278603, 'mean_word_length': 3.3387518142235124, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1112588', 'n_tokens_mistral': 4892, 'n_tokens_neox': 4696, 'n_words': 1247}