<commit_before>import os


def fasta2dict(fasta_file, samtools, picard):
    try:
        print "  - genome.dict"
        os.system("java -jar " + picard + "/CreateSequenceDictionary.jar R=" + fasta_file + " O=" + fasta_file.replace(".fa", ".dict") + " &>/dev/null")
        print "  - genome.fa.fai"
        os.system(samtools + " faidx " + fasta_file)
        return 1
    except:
        return 0

def refflat(path_gtf2gp, output_dir):
    try:
        print "  - Global file: 'genesets.refFlat'"
        outa = open(output_dir + "/script.sh", 'w')
        temp = output_dir + "/temp.rf"
        command1 = path_gtf2gp + " -genePredExt -geneNameAsName2 " + output_dir + "/genesets.gtf" + " " + temp
        command2 = "paste <(cut -f 12 " + temp +") <(cut -f 1-10 " + temp +") > " + temp.replace("temp.rf", "genesets.refFlat")
        print >> outa, command1
        print >> outa, command2
        print >> outa, "rm " + temp
        ofold = output_dir + "/refFlats"
        types = dict()
        if not os.path.exists(ofold):
            os.mkdir(ofold)
        f = open(output_dir + "/genesets.gtf", 'r')
        i = ""
        while 1:
            i = f.readline()
            if not i.startswith("#"):
                break
        if len(i) > 0:
            i = i.strip("\n").split("\t")
            t = i[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
            if not types.has_key(t):
                types[t] = 0
            types[t] += 1
            for i in f:
                i = i.strip("\n").split("\t")
                t = i[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
                if not types.has_key(t):
                    types[t]=0
                types[t]+=1
        f.close()
        N = 0
        for i,j in types.iteritems():
            N += j
        out = dict()
        for i,j in sorted(types.iteritems()):
            print "  - Gene\t"+i+"\t"+str(j)+"\t"+str(round(100*float(j)/N,2))+"%"
            if j > 0:
                out[i] = open(ofold+"/"+i+".gtf",'w')

        f = open(output_dir + "/genesets.gtf",'r')
        for i in f:
            if i.startswith("#"):
                for j in out.keys():
                    print >> out[j], i.strip("\n")
            else:
                i = i.strip("\n")
                t = i.split("\t")[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
                if out.has_key(t):
                    print >> out[t], i
        f.close()
        for i in out.keys():
            out[i].close()
        n    = os.listdir(ofold)
        temp = output_dir + "/temp.rf"
        for i in n:
            if i.endswith(".gtf"):
                command1 = path_gtf2gp + " -genePredExt -geneNameAsName2 " + ofold + "/" + i + " " + temp
                command2 = "paste <(cut -f 12 " + temp +") <(cut -f 1-10 " + temp +") > " + ofold + "/" + i.replace(".gtf",".refFlat")
                print >> outa, command1
                print >> outa, command2
                print >> outa, "rm " + temp
                print "  - " + i.replace(".gtf",".refFlat")
        outa.close()
        os.system("bash " + output_dir + "/script.sh")
        os.system("rm " + output_dir + "/script.sh")
        return 1
    except:
        return 0

def annotate_gtf(filename):
    try:
        features    = {"gene", "transcript", "exon"}
        transc2gene = dict() # transcript id to gene id for annotation
        exon2gene   = dict() # exon id to gene id for annotation
        loci        = dict() # chrom, start, end
        data = dict()
        for i in features:
            data[i] = dict()
        f = open(filename, 'r')
        for i in f:
            if not i.startswith("#"):
                i = i.strip("\n").split("\t")
                if len(i) == 9:
                    if i[2] in features: # gene/transcript/exon
                        j = i[8]
                        gid = j.split('gene_id "')[1].split('"')[0]  # associated gene id
                        if i[2]+'_id "' in j:
                            feat_id = j.split(i[2]+'_id "')[1].split('"')[0] # feature id
                        else:
                            if i[2]=='exon':
                                print gid
                                print j.split('transcript_id "')[1].split('"')[0]
                                print j.split('exon_number "')[1].split('"')[0]
                                feat_id = gid + '_' + j.split('transcript_id "')[1].split('"')[0] + '_' + j.split('exon_number "')[1].split('"')[0]
                        loci[feat_id] = [i[0], i[3], i[4]] # chrom, start, end
                        L = [float(i[3]), float(i[4])]
                        data[i[2]][feat_id] = list()
                        if i[2] == "exon":
                            tid = j.split('transcript_id "')[1].split('"')[0]
                            print tid
                            data["gene"][gid].append(L)
                            print L
                            data["transcript"][tid].append(L)
                            data["exon"][feat_id].append(L)
                            print 1
                            exon2gene[feat_id] = gid<commit_msg>genome_build: sove missing 'exon_id' bug<commit_after>import os


def fasta2dict(fasta_file, samtools, picard):
    try:
        print "  - genome.dict"
        os.system("java -jar " + picard + "/CreateSequenceDictionary.jar R=" + fasta_file + " O=" + fasta_file.replace(".fa", ".dict") + " &>/dev/null")
        print "  - genome.fa.fai"
        os.system(samtools + " faidx " + fasta_file)
        return 1
    except:
        return 0

def refflat(path_gtf2gp, output_dir):
    try:
        print "  - Global file: 'genesets.refFlat'"
        outa = open(output_dir + "/script.sh", 'w')
        temp = output_dir + "/temp.rf"
        command1 = path_gtf2gp + " -genePredExt -geneNameAsName2 " + output_dir + "/genesets.gtf" + " " + temp
        command2 = "paste <(cut -f 12 " + temp +") <(cut -f 1-10 " + temp +") > " + temp.replace("temp.rf", "genesets.refFlat")
        print >> outa, command1
        print >> outa, command2
        print >> outa, "rm " + temp
        ofold = output_dir + "/refFlats"
        types = dict()
        if not os.path.exists(ofold):
            os.mkdir(ofold)
        f = open(output_dir + "/genesets.gtf", 'r')
        i = ""
        while 1:
            i = f.readline()
            if not i.startswith("#"):
                break
        if len(i) > 0:
            i = i.strip("\n").split("\t")
            t = i[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
            if not types.has_key(t):
                types[t] = 0
            types[t] += 1
            for i in f:
                i = i.strip("\n").split("\t")
                t = i[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
                if not types.has_key(t):
                    types[t]=0
                types[t]+=1
        f.close()
        N = 0
        for i,j in types.iteritems():
            N += j
        out = dict()
        for i,j in sorted(types.iteritems()):
            print "  - Gene\t"+i+"\t"+str(j)+"\t"+str(round(100*float(j)/N,2))+"%"
            if j > 0:
                out[i] = open(ofold+"/"+i+".gtf",'w')

        f = open(output_dir + "/genesets.gtf",'r')
        for i in f:
            if i.startswith("#"):
                for j in out.keys():
                    print >> out[j], i.strip("\n")
            else:
                i = i.strip("\n")
                t = i.split("\t")[8].split("gene_biotype ")[1].split(";")[0].replace('"','')
                if out.has_key(t):
                    print >> out[t], i
        f.close()
        for i in out.keys():
            out[i].close()
        n    = os.listdir(ofold)
        temp = output_dir + "/temp.rf"
        for i in n:
            if i.endswith(".gtf"):
                command1 = path_gtf2gp + " -genePredExt -geneNameAsName2 " + ofold + "/" + i + " " + temp
                command2 = "paste <(cut -f 12 " + temp +") <(cut -f 1-10 " + temp +") > " + ofold + "/" + i.replace(".gtf",".refFlat")
                print >> outa, command1
                print >> outa, command2
                print >> outa, "rm " + temp
                print "  - " + i.replace(".gtf",".refFlat")
        outa.close()
        os.system("bash " + output_dir + "/script.sh")
        os.system("rm " + output_dir + "/script.sh")
        return 1
    except:
        return 0

def annotate_gtf(filename):
    try:
        features    = {"gene", "transcript", "exon"}
        transc2gene = dict() # transcript id to gene id for annotation
        exon2gene   = dict() # exon id to gene id for annotation
        loci        = dict() # chrom, start, end
        data = dict()
        for i in features:
            data[i] = dict()
        f = open(filename, 'r')
        for i in f:
            if not i.startswith("#"):
                i = i.strip("\n").split("\t")
                if len(i) == 9:
                    if i[2] in features: # gene/transcript/exon
                        j = i[8]
                        gid = j.split('gene_id "')[1].split('"')[0]  # associated gene id
                        if i[2]+'_id "' in j:
                            feat_id = j.split(i[2]+'_id "')[1].split('"')[0] # feature id
                        else:
                            if i[2]=='exon':
                                feat_id = gid + '_' + j.split('transcript_id "')[1].split('"')[0] + '_' + j.split('exon_number "')[1].split('"')[0]
                        print 1, feat_id
                        loci[feat_id] = [i[0], i[3], i[4]] # chrom, start, end
                        print 2
                        L = [float(i[3]), float(i[4])]
                        print 3
                        data[i[2]][feat_id] = list()
                        if i[2] == "exon":
                            tid = j.split('transcript_id "')[1].split('"')[0]
                            print tid
                            data["gene"][gid].append(L)
                            print L
                            data["transcript"][tid].append(L)
                            data["exon"][feat_id].append(L)
                            print 1
                            exon2gene[feat_id] = gid
{'fraction_non_alphanumeric': 0.16955084580983862, 'fraction_numerical': 0.01545790394711258, 'mean_word_length': 1.3096093399191737, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '755502', 'n_tokens_mistral': 3466, 'n_tokens_neox': 3272, 'n_words': 710}