<commit_before>#!/usr/bin/env python

"""
Gathers results from simulation experiments.  Run this from the
simulated_reads subdirectory of the qsim-experiments repo.  It descends into
the various experimental subdirectories and parses the Makefiles it finds.

Outputs:
 - "overall.csv" with one big table of summary measures
 - "summary" subdirectory with lots of raw results compiled into a directory
   structure
   + Subdirectories correspond to simulation experiments and contain:
     - for each sampling rate:
       + for each trial:
         - featimport_*.csv -- feature importances for each alignment type
         - params.csv -- feature importances for each model
         - Subdirectories for training/test, each with:
           + roc.csv -- ROC table
           + summary.csv -- summarizes data, model fit
"""

from __future__ import print_function
import sys
import os
import re
import glob
import logging
from os.path import join

# We identify the experiments by fishing through the Makefiles for output
# targets.  When this regex matches a line of a Makefile, we know the
# following lines are giving us the names of targets.
target_re = re.compile('^outs_[_a-zA-Z01-9]*:.*')


def parse_aligner_local(target):
    """ Based on Makefile target name, parse which aligner is involved """
    toks = target.split('_')
    aligner, local = 'bt2', False
    if 'bwamem' in toks[1]:
        aligner = 'bwamem'
        local = True
    if 'snap' in toks[1]:
        aligner = 'snap'
        local = True
    if aligner == 'bt2' and toks[1][-1] == 'l':
        toks[1] = toks[1][:-1]
        local = True
    return aligner, local


def parse_species(target):
    """ Based on Makefile target name, parse which species is involved """
    genome = 'hg'
    toks = target.split('_')
    # Parsing reference species info
    if toks[2] == 'mm':
        genome = 'mm'
    if toks[2] == 'zm':
        genome = 'zm'
    return genome


def parse_sim(target):
    """ Based on Makefile target name, parse which read simulator is involved """
    sim = 'mason'
    toks = target.split('_')
    # Parsing simulator info:
    if toks[2] == 'wgsim':
        sim = 'wgsim'
    if toks[2] == 'art':
        sim = 'art'
    return sim


def parse_sensitivity(target, aligner):
    """ Based on Makefile target name, parse which sensitivity level is involved """
    toks = target.split('_')
    sensitivity = 's'
    if aligner == 'bt2':
        sensitivity = toks[1][3:]
    return sensitivity


def parse_paired(target):
    """ Based on Makefile target name, parse whether the data is paired-end """
    return target.startswith('r12')


def parse_readlen(target):
    """ Based on Makefile target name, parse the read length """
    readlen = target.split('_')[-2]
    return 500 if readlen == '50to500' else int(readlen)


def parse_name_and_target(combined):
    """ Based on Makefile target name, parse the read length """
    toks = combined.split('_')
    roff = 3
    if toks[2] == 'r0' or toks[2] == 'r12':
        roff = 2
    assert toks[roff] == 'r0' or toks[roff] == 'r12'
    return '_'.join(toks[:roff]), '_'.join(toks[roff:])


def mkdir_quiet(dr):
    """ Create directories if needed, quietly """
    import errno
    if not os.path.isdir(dr):
        try:
            os.makedirs(dr)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise


def has_done(dr):
    """ Return true if directory contains DONE file, indicating qsim finished
        running. """
    done_fn = join(dr, 'DONE')
    if not os.path.exists(done_fn):
        raise RuntimeError('Directory "%s" does not contain DONE file' % dr)


def copyfiles(fglob, dest, prefix=''):
    """ Copy files (indicated by the file glob fglob) to the destination
        directory (indicated by dest). """
    assert os.path.isdir(dest) and os.path.exists(dest)
    for fn in glob.glob(fglob):
        os.system('cp -f %s %s' % (fn, join(dest, prefix + os.path.basename(fn))))


def roc_file_to_string(roc_fn, inner_sep=':', outer_sep=';'):
    """ Convert a file with a ROC table into a string with one line per ROC row """
    fields = []
    with open(roc_fn) as fh:
        for ln in fh:
            cor, _, _, incor, mapq, _, _, _, _, _ = ln.rstrip().split(',')
            fields.append(inner_sep.join([mapq, cor, incor]))
    return outer_sep.join(fields)


def compile_line(ofh, combined_target_name, tt, trial, params_fn, summ_fn, roc_fn, roc_round_fn, roc_orig_fn, first):
    """ Put together one line of output and write to ofh (overall.csv)
        """
    name, target = parse_name_and_target(combined_target_name)
    aligner, local = parse_aligner_local(target)
    paired = parse_paired(target)
    sim = parse_sim(target)
    readlen = parse_readlen(target)
    sensitivity = parse_sensitivity(target, aligner)
    species = parse_species(target)
    headers = ['name', 'training', 'trial_no', 'aligner', 'local', 'paired',
               'sim', 'readlen', 'sensitivity', 'species']
    values = [name, 'T' if tt == 'training' else 'F', trial, aligner,
              'T' if local else 'F', 'T' if paired else 'F', sim,
              str(readlen), sensitivity, species]
    for fn in [params_fn, summ_fn]:<commit_msg>skip header when parsing ROC<commit_after>#!/usr/bin/env python

"""
Gathers results from simulation experiments.  Run this from the
simulated_reads subdirectory of the qsim-experiments repo.  It descends into
the various experimental subdirectories and parses the Makefiles it finds.

Outputs:
 - "overall.csv" with one big table of summary measures
 - "summary" subdirectory with lots of raw results compiled into a directory
   structure
   + Subdirectories correspond to simulation experiments and contain:
     - for each sampling rate:
       + for each trial:
         - featimport_*.csv -- feature importances for each alignment type
         - params.csv -- feature importances for each model
         - Subdirectories for training/test, each with:
           + roc.csv -- ROC table
           + summary.csv -- summarizes data, model fit
"""

from __future__ import print_function
import sys
import os
import re
import glob
import logging
from os.path import join

# We identify the experiments by fishing through the Makefiles for output
# targets.  When this regex matches a line of a Makefile, we know the
# following lines are giving us the names of targets.
target_re = re.compile('^outs_[_a-zA-Z01-9]*:.*')


def parse_aligner_local(target):
    """ Based on Makefile target name, parse which aligner is involved """
    toks = target.split('_')
    aligner, local = 'bt2', False
    if 'bwamem' in toks[1]:
        aligner = 'bwamem'
        local = True
    if 'snap' in toks[1]:
        aligner = 'snap'
        local = True
    if aligner == 'bt2' and toks[1][-1] == 'l':
        toks[1] = toks[1][:-1]
        local = True
    return aligner, local


def parse_species(target):
    """ Based on Makefile target name, parse which species is involved """
    genome = 'hg'
    toks = target.split('_')
    # Parsing reference species info
    if toks[2] == 'mm':
        genome = 'mm'
    if toks[2] == 'zm':
        genome = 'zm'
    return genome


def parse_sim(target):
    """ Based on Makefile target name, parse which read simulator is involved """
    sim = 'mason'
    toks = target.split('_')
    # Parsing simulator info:
    if toks[2] == 'wgsim':
        sim = 'wgsim'
    if toks[2] == 'art':
        sim = 'art'
    return sim


def parse_sensitivity(target, aligner):
    """ Based on Makefile target name, parse which sensitivity level is involved """
    toks = target.split('_')
    sensitivity = 's'
    if aligner == 'bt2':
        sensitivity = toks[1][3:]
    return sensitivity


def parse_paired(target):
    """ Based on Makefile target name, parse whether the data is paired-end """
    return target.startswith('r12')


def parse_readlen(target):
    """ Based on Makefile target name, parse the read length """
    readlen = target.split('_')[-2]
    return 500 if readlen == '50to500' else int(readlen)


def parse_name_and_target(combined):
    """ Based on Makefile target name, parse the read length """
    toks = combined.split('_')
    roff = 3
    if toks[2] == 'r0' or toks[2] == 'r12':
        roff = 2
    assert toks[roff] == 'r0' or toks[roff] == 'r12'
    return '_'.join(toks[:roff]), '_'.join(toks[roff:])


def mkdir_quiet(dr):
    """ Create directories if needed, quietly """
    import errno
    if not os.path.isdir(dr):
        try:
            os.makedirs(dr)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise


def has_done(dr):
    """ Return true if directory contains DONE file, indicating qsim finished
        running. """
    done_fn = join(dr, 'DONE')
    if not os.path.exists(done_fn):
        raise RuntimeError('Directory "%s" does not contain DONE file' % dr)


def copyfiles(fglob, dest, prefix=''):
    """ Copy files (indicated by the file glob fglob) to the destination
        directory (indicated by dest). """
    assert os.path.isdir(dest) and os.path.exists(dest)
    for fn in glob.glob(fglob):
        os.system('cp -f %s %s' % (fn, join(dest, prefix + os.path.basename(fn))))


def roc_file_to_string(roc_fn, inner_sep=':', outer_sep=';'):
    """ Convert a file with a ROC table into a string with one line per ROC row """
    fields = []
    with open(roc_fn) as fh:
        for ln in fh:
            cor, _, _, incor, mapq, _, _, _, _, _ = ln.rstrip().split(',')
            if mapq == "mapq":
                continue
            fields.append(inner_sep.join([mapq, cor, incor]))
    return outer_sep.join(fields)


def compile_line(ofh, combined_target_name, tt, trial, params_fn, summ_fn, roc_fn, roc_round_fn, roc_orig_fn, first):
    """ Put together one line of output and write to ofh (overall.csv)
        """
    name, target = parse_name_and_target(combined_target_name)
    aligner, local = parse_aligner_local(target)
    paired = parse_paired(target)
    sim = parse_sim(target)
    readlen = parse_readlen(target)
    sensitivity = parse_sensitivity(target, aligner)
    species = parse_species(target)
    headers = ['name', 'training', 'trial_no', 'aligner', 'local', 'paired',
               'sim', 'readlen', 'sensitivity', 'species']
    values = [name, 'T' if tt == 'training' else 'F', trial, aligner,
              'T' if local else 'F', 'T' if paired else 'F', sim,
              str(readlen), sensitivity, species]
    for fn in [params_fn, summ_fn]:
{'fraction_non_alphanumeric': 0.11929725134599037, 'fraction_numerical': 0.007556437139888543, 'mean_word_length': 2.9041297935103243, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '508870', 'n_tokens_mistral': 3389, 'n_tokens_neox': 3261, 'n_words': 1105}