<commit_before>from __future__ import absolute_import
import sys, os, yaml, glob
import subprocess
import pandas as pd
import argparse
import re
import string
import shutil
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from nougat import common
from nougat import pdf
from nougat.pdf.theme import colors, DefaultTheme


def main(args):
    workingDir = os.getcwd()
    validation_dirs = args.validation_dirs
    assemblies_dirs = args.assemblies_dirs
    min_contig_length = args.min_contig_length
    # store all samples folders
    assemblies_samples_dirs = [sample for sample in \
            os.listdir(assemblies_dirs) \
            if os.path.isdir(os.path.join(assemblies_dirs,sample))]
    # store all samples folders
    validation_samples_dirs = [sample for sample in \
            os.listdir(validation_dirs) \
            if os.path.isdir(os.path.join(validation_dirs,sample))]
    if args.no_uppmax:
        collect_results_and_report(validation_dirs, assemblies_dirs, "",
                args.sample_name, min_contig_length, args.no_uppmax)
    else:
        for sample in assemblies_samples_dirs:
            if sample not in validation_samples_dirs:
                system.exit("ATTENTION: sample {} is present in dir {} but \
                        absent in dir {}".format(sample, assemblies_dirs,
                        validation_dirs))
            else: #otherwise I have one or more sample folders
                 #It means that I have assembled with a set of assemblers and
                 #all of them have gone through validation... I can proceed
                os.chdir(workingDir)
                validation_sample_dir = os.path.join(validation_dirs, sample)
                assemblies_sample_dir = os.path.join(assemblies_dirs, sample)
                sample_folder = os.path.join(workingDir, sample)
                if not os.path.exists(sample_folder):
                    os.makedirs(sample_folder)
                os.chdir(sample_folder)
                collect_results_and_report(validation_sample_dir,
                        assemblies_sample_dir, sample_folder, sample,
                        min_contig_length, args.no_uppmax)


def collect_results_and_report(validation_sample_dir, assemblies_sample_dir,
        sample_folder, sample, min_contig_length, no_uppmax):
    assemblers_validation = [assembler for assembler in \
            os.listdir(validation_sample_dir) \
            if os.path.isdir(os.path.join(validation_sample_dir,assembler))]
    assemblers_assemblies = [assembler  for assembler in \
            os.listdir(assemblies_sample_dir) \
            if os.path.isdir(os.path.join(assemblies_sample_dir,assembler))]
    if set(assemblers_validation) != set(assemblers_assemblies):
        sys.exit("Error: different assemblies in assemblies and validation "
                "folder: {} {}".format(
                assemblies_sample_dir,validation_sample_dir))
    assemblers_assemblies = sorted(assemblers_assemblies)
    assemblers_validation = sorted(assemblers_validation)
    #now I am in the folder that will contain all the results
    if not os.path.exists("assemblies"):
        os.makedirs("assemblies")
        for assembler in assemblers_assemblies:
            shutil.copytree(os.path.join(assemblies_sample_dir, assembler),
                    os.path.join("assemblies", assembler))
    ##copied original assemblies
    if not os.path.exists("evaluation"):
        os.makedirs("evaluation")

    # let us start with QAcompute results
    QA_pictures = os.path.join(sample_folder, "evaluation", "QA_pictures")
    if not os.path.exists(QA_pictures):
        os.makedirs(QA_pictures)
    picturesQA = {}
    for assembler in assemblers_assemblies:
        original_QA_dir = os.path.join(validation_sample_dir,
                    assembler, "QAstats")
        if not os.path.exists(original_QA_dir):
            continue
        cur_ass_dir = os.path.join(QA_pictures, assembler)
        if not os.path.exists(cur_ass_dir):
            os.makedirs(cur_ass_dir)
        ##QA pictures
        Original_CoverageDistribution200 = os.path.join(validation_sample_dir,
                assembler, "QAstats", "Coverage_distribution_noOutliers.png")
        Original_GC_vs_Coverage = os.path.join(validation_sample_dir,
                assembler, "QAstats", "GC_vs_Coverage_noOutliers.png")
        Original_GC_vs_CtgLength = os.path.join(validation_sample_dir,
                assembler, "QAstats", "GC_vs_CtgLength.png")
        Original_MedianCov_vs_CtgLength = os.path.join(validation_sample_dir,
                assembler, "QAstats", "MedianCov_vs_CtgLength_noOutliers.png")
        Original_QAstats_gc_result = ""
        QAstats_gc_result_file_name = ""
        if no_uppmax:
            QAstats_gc_result_file_name = [name for name in \
                    os.listdir(os.path.join(validation_sample_dir,
                    assembler, "QAstats")) \
                    if name.endswith(".bam.cov.gc")][0]
            Original_QAstats_gc_result = os.path.join(validation_sample_dir,
                    assembler, "QAstats",
                    "{}".format(QAstats_gc_result_file_name))
        else:
            Original_QAstats_gc_result = os.path.join(validation_sample_dir,
                    assembler, "QAstats", "{}.bam.cov.gc".format(sample))

        Copied_CoverageDistribution200 = os.path.join(cur_ass_dir,
                "Coverage_distribution_noOutliers.png")
        Copied_GC_vs_Coverage = os.path.join(cur_ass_dir,
                "GC_vs_Coverage_noOutliers.png")
        Copied_GC_vs_CtgLength = os.path.join(cur_ass_dir,
                "GC_vs_CtgLength.png")
        Copied_MedianCov_vs_CtgLength = os.path.join(cur_ass_dir,
                "MedianCov_vs_CtgLength_noOutliers.png")
        if no_uppmax:
            Copied_QAstats_gc_result = os.path.join(cur_ass_dir,
                    QAstats_gc_result_file_name)
        else:
            Copied_QAstats_gc_result = os.path.join(cur_ass_dir,
                    "{}.bam.cov.gc".format(sample))

        shutil.copy(Original_CoverageDistribution200,
                Copied_CoverageDistribution200)
        shutil.copy(Original_GC_vs_Coverage,
                Copied_GC_vs_Coverage)
        shutil.copy(Original_GC_vs_CtgLength,
                Copied_GC_vs_CtgLength)
        shutil.copy(Original_MedianCov_vs_CtgLength,
                Copied_MedianCov_vs_CtgLength)
        shutil.copy(Original_QAstats_gc_result,
                Copied_QAstats_gc_result)

        picturesQA[assembler] = [[Copied_CoverageDistribution200,
            "Contig coverage distribtion" ],
            [Copied_GC_vs_Coverage, "GC-content versus contig-coverage"],
            [Copied_GC_vs_CtgLength, "GC-content versus contig-Length"],
            [Copied_MedianCov_vs_CtgLength,
            "Median-coverage vs Contig-Length"]]

    # now FRCurve results
    FRC_folder = os.path.join(sample_folder, "evaluation", "FRCurves")
    if not os.path.exists(FRC_folder):
        os.makedirs(FRC_folder)
    Features = ["_FRC", "COMPR_MP_FRC", "COMPR_PE_FRC", "HIGH_COV_PE_FRC",
            "HIGH_NORM_COV_PE_FRC", "HIGH_OUTIE_MP_FRC", "HIGH_OUTIE_PE_FRC",
            "HIGH_SINGLE_MP_FRC", "HIGH_SINGLE_PE_FRC", "HIGH_SPAN_MP_FRC",
            "HIGH_SPAN_PE_FRC", "LOW_COV_PE_FRC", "LOW_NORM_COV_PE_FRC",
            "STRECH_MP_FRC", "STRECH_PE_FRC"]

    FRC_to_print = ""
    for feature in Features:
        FRCurves = []
        for assembler in assemblers_assemblies:
            FRCurve_Orig_name = os.path.join(validation_sample_dir, assembler,
                    "FRCurve", "{}{}.txt".format(sample, feature))
            FRCurves.append([assembler, FRCurve_Orig_name])
        FRCname = _plotFRCurve(os.path.join(FRC_folder, "{}_{}".format(sample,
            feature)), FRCurves)
        if feature == "_FRC":
            FRC_to_print = FRCname
        FRCurves = []

    #Contiguity stats
    contig_stats = []
    source_stats = []
    for assembler in assemblers_assemblies:
        asm_stats = [assembler]
        stat_file = os.path.join(validation_sample_dir, assembler,
                "contig_stats", "contiguity.out")
        source_stats.append((stat_file, assembler))
        with open(stat_file, "r") as sf:
            for line in sf:
                if line.startswith("scaffolds"):
                    sl = line.strip().split("\t")
                    asm_stats.extend([sl[1], sl[4], sl[8], sl[9], sl[10], sl[2], sl[6]])
        contig_stats.append(asm_stats)
    contig_stats = sorted(contig_stats, key=lambda x: x[0])<commit_msg>Tiny bugfix<commit_after>from __future__ import absolute_import
import sys, os, yaml, glob
import subprocess
import pandas as pd
import argparse
import re
import string
import shutil
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from nougat import common
from nougat import pdf
from nougat.pdf.theme import colors, DefaultTheme


def main(args):
    workingDir = os.getcwd()
    validation_dirs = args.validation_dirs
    assemblies_dirs = args.assemblies_dirs
    min_contig_length = args.min_contig_length
    # store all samples folders
    assemblies_samples_dirs = [sample for sample in \
            os.listdir(assemblies_dirs) \
            if os.path.isdir(os.path.join(assemblies_dirs,sample))]
    # store all samples folders
    validation_samples_dirs = [sample for sample in \
            os.listdir(validation_dirs) \
            if os.path.isdir(os.path.join(validation_dirs,sample))]
    if args.no_uppmax:
        collect_results_and_report(validation_dirs, assemblies_dirs, "",
                args.sample_name, min_contig_length, args.no_uppmax)
    else:
        for sample in assemblies_samples_dirs:
            if sample not in validation_samples_dirs:
                system.exit("ATTENTION: sample {} is present in dir {} but \
                        absent in dir {}".format(sample, assemblies_dirs,
                        validation_dirs))
            else: #otherwise I have one or more sample folders
                 #It means that I have assembled with a set of assemblers and
                 #all of them have gone through validation... I can proceed
                os.chdir(workingDir)
                validation_sample_dir = os.path.join(validation_dirs, sample)
                assemblies_sample_dir = os.path.join(assemblies_dirs, sample)
                sample_folder = os.path.join(workingDir, sample)
                if not os.path.exists(sample_folder):
                    os.makedirs(sample_folder)
                os.chdir(sample_folder)
                collect_results_and_report(validation_sample_dir,
                        assemblies_sample_dir, sample_folder, sample,
                        min_contig_length, args.no_uppmax)


def collect_results_and_report(validation_sample_dir, assemblies_sample_dir,
        sample_folder, sample, min_contig_length, no_uppmax):
    assemblers_validation = [assembler for assembler in \
            os.listdir(validation_sample_dir) \
            if os.path.isdir(os.path.join(validation_sample_dir,assembler))]
    assemblers_assemblies = [assembler  for assembler in \
            os.listdir(assemblies_sample_dir) \
            if os.path.isdir(os.path.join(assemblies_sample_dir,assembler))]
    if set(assemblers_validation) != set(assemblers_assemblies):
        sys.exit("Error: different assemblies in assemblies and validation "
                "folder: {} {}".format(
                assemblies_sample_dir,validation_sample_dir))
    assemblers_assemblies = sorted(assemblers_assemblies)
    assemblers_validation = sorted(assemblers_validation)
    #now I am in the folder that will contain all the results
    if not os.path.exists("assemblies"):
        os.makedirs("assemblies")
        for assembler in assemblers_assemblies:
            shutil.copytree(os.path.join(assemblies_sample_dir, assembler),
                    os.path.join("assemblies", assembler))
    ##copied original assemblies
    if not os.path.exists("evaluation"):
        os.makedirs("evaluation")

    # let us start with QAcompute results
    QA_pictures = os.path.join(sample_folder, "evaluation", "QA_pictures")
    if not os.path.exists(QA_pictures):
        os.makedirs(QA_pictures)
    picturesQA = {}
    for assembler in assemblers_assemblies:
        original_QA_dir = os.path.join(validation_sample_dir,
                    assembler, "QAstats")
        if not os.path.exists(original_QA_dir):
            continue
        cur_ass_dir = os.path.join(QA_pictures, assembler)
        if not os.path.exists(cur_ass_dir):
            os.makedirs(cur_ass_dir)
        ##QA pictures
        Original_CoverageDistribution200 = os.path.join(validation_sample_dir,
                assembler, "QAstats", "Coverage_distribution_noOutliers.png")
        Original_GC_vs_Coverage = os.path.join(validation_sample_dir,
                assembler, "QAstats", "GC_vs_Coverage_noOutliers.png")
        Original_GC_vs_CtgLength = os.path.join(validation_sample_dir,
                assembler, "QAstats", "GC_vs_CtgLength.png")
        Original_MedianCov_vs_CtgLength = os.path.join(validation_sample_dir,
                assembler, "QAstats", "MedianCov_vs_CtgLength_noOutliers.png")
        Original_QAstats_gc_result = ""
        QAstats_gc_result_file_name = ""
        if no_uppmax:
            QAstats_gc_result_file_name = [name for name in \
                    os.listdir(os.path.join(validation_sample_dir,
                    assembler, "QAstats")) \
                    if name.endswith(".bam.cov.gc")][0]
            Original_QAstats_gc_result = os.path.join(validation_sample_dir,
                    assembler, "QAstats",
                    "{}".format(QAstats_gc_result_file_name))
        else:
            Original_QAstats_gc_result = os.path.join(validation_sample_dir,
                    assembler, "QAstats", "{}.bam.cov.gc".format(sample))

        Copied_CoverageDistribution200 = os.path.join(cur_ass_dir,
                "Coverage_distribution_noOutliers.png")
        Copied_GC_vs_Coverage = os.path.join(cur_ass_dir,
                "GC_vs_Coverage_noOutliers.png")
        Copied_GC_vs_CtgLength = os.path.join(cur_ass_dir,
                "GC_vs_CtgLength.png")
        Copied_MedianCov_vs_CtgLength = os.path.join(cur_ass_dir,
                "MedianCov_vs_CtgLength_noOutliers.png")
        if no_uppmax:
            Copied_QAstats_gc_result = os.path.join(cur_ass_dir,
                    QAstats_gc_result_file_name)
        else:
            Copied_QAstats_gc_result = os.path.join(cur_ass_dir,
                    "{}.bam.cov.gc".format(sample))

        shutil.copy(Original_CoverageDistribution200,
                Copied_CoverageDistribution200)
        shutil.copy(Original_GC_vs_Coverage,
                Copied_GC_vs_Coverage)
        shutil.copy(Original_GC_vs_CtgLength,
                Copied_GC_vs_CtgLength)
        shutil.copy(Original_MedianCov_vs_CtgLength,
                Copied_MedianCov_vs_CtgLength)
        shutil.copy(Original_QAstats_gc_result,
                Copied_QAstats_gc_result)

        picturesQA[assembler] = [[Copied_CoverageDistribution200,
            "Contig coverage distribtion" ],
            [Copied_GC_vs_Coverage, "GC-content versus contig-coverage"],
            [Copied_GC_vs_CtgLength, "GC-content versus contig-Length"],
            [Copied_MedianCov_vs_CtgLength,
            "Median-coverage vs Contig-Length"]]

    # now FRCurve results
    FRC_folder = os.path.join(sample_folder, "evaluation", "FRCurves")
    if not os.path.exists(FRC_folder):
        os.makedirs(FRC_folder)
    Features = ["_FRC", "COMPR_MP_FRC", "COMPR_PE_FRC", "HIGH_COV_PE_FRC",
            "HIGH_NORM_COV_PE_FRC", "HIGH_OUTIE_MP_FRC", "HIGH_OUTIE_PE_FRC",
            "HIGH_SINGLE_MP_FRC", "HIGH_SINGLE_PE_FRC", "HIGH_SPAN_MP_FRC",
            "HIGH_SPAN_PE_FRC", "LOW_COV_PE_FRC", "LOW_NORM_COV_PE_FRC",
            "STRECH_MP_FRC", "STRECH_PE_FRC"]

    FRC_to_print = ""
    for feature in Features:
        FRCurves = []
        for assembler in assemblers_assemblies:
            FRCurve_Orig_name = os.path.join(validation_sample_dir, assembler,
                    "FRCurve", "{}{}.txt".format(sample, feature))
            FRCurves.append([assembler, FRCurve_Orig_name])
        FRCname = _plotFRCurve(os.path.join(FRC_folder, "{}_{}".format(sample,
            feature)), FRCurves)
        if feature == "_FRC":
            FRC_to_print = FRCname
        FRCurves = []

    #Contiguity stats
    contig_stats = []
    source_stats = []
    for assembler in assemblers_assemblies:
        asm_stats = [assembler]
        stat_file = os.path.join(validation_sample_dir, assembler,
                "contig_stats", "contiguity.out")
        source_stats.append((stat_file, assembler))
        with open(stat_file, "r") as sf:
            for line in sf:
                if line.startswith("scaffolds"):
                    sl = line.strip().split("\t")
                    asm_stats.extend([sl[1], sl[4], sl[8], sl[9], sl[10], sl[2], sl[5]])
        contig_stats.append(asm_stats)
    contig_stats = sorted(contig_stats, key=lambda x: x[0])
{'fraction_non_alphanumeric': 0.08944128237890579, 'fraction_numerical': 0.0029039377395748637, 'mean_word_length': 2.559115336916081, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3939656', 'n_tokens_mistral': 5732, 'n_tokens_neox': 5453, 'n_words': 920}