<commit_before>#!/usr/bin/env python3
"""Sample Sheet generation for BCL2FASTQ pipeline
"""
# --- standard library imports
#
import sys
import os
import logging
import argparse
from collections import namedtuple

#--- third-party imports
#
import yaml
import requests
import xml.etree.ElementTree as ET

#--- project specific imports
#
from pipelines import get_machine_run_flowcell_id
#from bcl2fastq import MuxUnit
# WARNING changes here, must be reflected in bcl2fastq.py as well
MuxUnit = namedtuple('MuxUnit', ['run_id', 'flowcell_id', 'mux_id', 'lane_ids',
                                 'mux_dir', 'barcode_mismatches'])


__author__ = "<NAME>"
__email__ = "<EMAIL>"
__copyright__ = "2016 Genome Institute of Singapore"
__license__ = "The MIT License (MIT)"


# global logger
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    '[{asctime}] {levelname:8s} {filename} {message}', style='{'))
logger.addHandler(handler)


SAMPLESHEET_CSV = "samplesheet.csv"
USEBASES_CFG = "usebases.yaml"
MUXINFO_CFG = "muxinfo.yaml"
DEFAULT_BARCODE_MISMATCHES = None
SAMPLESHEET_HEADER = '[Data]'+'\n'+ 'Lane,Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'


def getdirs(args):
    """gets directories from args and checks existance
    """
    rundir = args.rundir
    if not os.path.exists(rundir):
        logger.fatal("rundir '%s' does not exist under Run directory.\n" % (rundir))
        sys.exit(1)

    runinfo = os.path.join(rundir + '/RunInfo.xml')
    if not os.path.exists(runinfo):
        logger.fatal("RunInfo '%s' does not exist under Run directory.\n" % (runinfo))
        sys.exit(1)

    outdir = args.outdir
    if not os.path.exists(outdir):
        logger.fatal("output directory '%s' does not exist.\n" % (outdir))
        sys.exit(1)

    return(rundir, outdir, runinfo)


def generate_usebases(barcode_lens, runinfo):
    """FIXME:add-doc
    """
    tree = ET.parse(runinfo)
    root = tree.getroot()
    ub_list = []

    # for each lane and its barcode lengths
    for k, v in sorted(barcode_lens.items()):
        # v is list of barcode_len tuples
        assert len(set(v)) == 1, ("Different barcode length in lane {}".format(k))
        bc1, bc2 = v[0]# since all v's are the same

        ub = ""
        #if test:
        for read in root.iter('Read'):
            numcyc = int(read.attrib['NumCycles'])-1
            if read.attrib['IsIndexedRead'] == 'N':
                ub += 'Y' + str(numcyc) + 'n*,'
            elif read.attrib['IsIndexedRead'] == 'Y':
                if read.attrib['Number'] == '2':    ### BC1
                    if bc1 > 0:
                        ub += 'I'+str(bc1)+'n*,'
                    else:
                        ub += 'n*'+','
                if read.attrib['Number'] == '3':    ### BC2
                    if bc2 > 0:
                        ub += 'I'+str(bc2)+'n*,'
                    else:
                        ub += 'n*'+','
        ub = ub[:-1]
        ub_list.append(str(k + ':' + ub))
    return ub_list


def main():
    """
    The main function
    """

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--force-overwrite",
                        action="store_true",
                        help="Force overwriting of output files")
    parser.add_argument("-r", "--rundir",
                        dest="rundir",
                        required=True,
                        help="rundir, e.g. /mnt/seq/userrig/HS004/HS004-PE-R00139_BC6A7HANXX")
    parser.add_argument("-o", "--outdir",
                        required=True,
                        dest="outdir",
                        help="Output directory")
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help="Increase verbosity")
    parser.add_argument('-q', '--quiet', action='count', default=0,
                        help="Decrease verbosity")
    args = parser.parse_args()

    # Repeateable -v and -q for setting logging level.
    # See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
    # and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
    # script -vv -> DEBUG
    # script -v -> INFO
    # script -> WARNING
    # script -q -> ERROR
    # script -qq -> CRITICAL
    # script -qqq -> no logging at all
    logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)

    (rundir, outdir, runinfo) = getdirs(args)
    samplesheet_csv = os.path.join(outdir, SAMPLESHEET_CSV)
    usebases_cfg = os.path.join(outdir, USEBASES_CFG)
    muxinfo_cfg = os.path.join(outdir, MUXINFO_CFG)
    for f in [samplesheet_csv, usebases_cfg, muxinfo_cfg]:
        if not args.force_overwrite and os.path.exists(f):
            logger.fatal("Refusing to overwrite existing file {}".format(f))
            sys.exit(1)

    _, run_num, flowcellid = get_machine_run_flowcell_id(rundir)

    logger.info("Querying ELM for {}".format(run_num))
    #DEVELOPMENT URL
    #rest_url = 'http://dlap51v:8080/elm/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    #PRODUCTION url
    rest_url = 'http://qldb01.gis.a-star.edu.sg:8080/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    #NEW DEVELOPMENT url
    #rest_url = 'http://qldb01:8180/rest/seqrun/illumina/'+ run_num + '/detailanalysis/json'
    response = requests.get(rest_url)
    if response.status_code != requests.codes.ok:
        response.raise_for_status()
    rest_data = response.json()

    run_id = rest_data['runId']
    #counter = 0
    if rest_data['runPass'] != 'Pass':
        logger.warning("Skipping non-passed run")
        # NOTE: exit 0 and missing output files is the upstream signal for a failed run
        sys.exit(0)

    # this is the master samplesheet
    logger.info("Writing to {}".format(samplesheet_csv))
    # keys: lanes, values are barcode lens in lane (always two tuples, -1 if not present)
    barcode_lens = {}
    mux_units = dict()

    with open(samplesheet_csv, 'w') as fh_out:
        fh_out.write(SAMPLESHEET_HEADER + '\n')
        for rows in rest_data['lanes']:
            if rows['lanePass'] != 'Pass':
                continue
            BCL_Mismatch = []
            if "MUX" in rows['libraryId']:
                # multiplexed
                #counter = 0
                for child in rows['Children']:
                    #print(child)
                    #counter += 1
                    #id = 'S' + str(counter)
                    if 'BCL_Mismatch' in child:
                        BCL_Mismatch.append(child['BCL_Mismatch'])
                        # older samples have no values and that's okay

                    if "-" in child['barcode']:
                        # dual index
                        index = child['barcode'].split('-')
                        sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,,'+ index[0] +',,'+ index[1] + ',' +'Project_'+rows['libraryId']+','+child['libtech']
                        index_lens = (len((index[0])), len((index[1])))
                    else:
                        sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,,'+child['barcode']+',,,'+'Project_'+rows['libraryId']+','+child['libtech']
                        index_lens = (len(child['barcode']), -1)
                        print(sample)
                    barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                    fh_out.write(sample+ '\n')

            else:# non-multiplexed
                sample = rows['laneId']+',Sample_'+rows['libraryId']+','+rows['libraryId']+'-NoIndex'+',,,,,,,'+'Project_'+rows['libraryId']+','+rows['libtech']
                index_lens = (-1, -1)
                barcode_lens.setdefault(rows['laneId'], []).append(index_lens)<commit_msg>Removed some print details about sample<commit_after>#!/usr/bin/env python3
"""Sample Sheet generation for BCL2FASTQ pipeline
"""
# --- standard library imports
#
import sys
import os
import logging
import argparse
from collections import namedtuple

#--- third-party imports
#
import yaml
import requests
import xml.etree.ElementTree as ET

#--- project specific imports
#
from pipelines import get_machine_run_flowcell_id
#from bcl2fastq import MuxUnit
# WARNING changes here, must be reflected in bcl2fastq.py as well
MuxUnit = namedtuple('MuxUnit', ['run_id', 'flowcell_id', 'mux_id', 'lane_ids',
                                 'mux_dir', 'barcode_mismatches'])


__author__ = "<NAME>"
__email__ = "<EMAIL>"
__copyright__ = "2016 Genome Institute of Singapore"
__license__ = "The MIT License (MIT)"


# global logger
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    '[{asctime}] {levelname:8s} {filename} {message}', style='{'))
logger.addHandler(handler)


SAMPLESHEET_CSV = "samplesheet.csv"
USEBASES_CFG = "usebases.yaml"
MUXINFO_CFG = "muxinfo.yaml"
DEFAULT_BARCODE_MISMATCHES = None
SAMPLESHEET_HEADER = '[Data]'+'\n'+ 'Lane,Sample_ID,Sample_Name,Sample_Plate,Sample_Well,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project,Description'


def getdirs(args):
    """gets directories from args and checks existance
    """
    rundir = args.rundir
    if not os.path.exists(rundir):
        logger.fatal("rundir '%s' does not exist under Run directory.\n" % (rundir))
        sys.exit(1)

    runinfo = os.path.join(rundir + '/RunInfo.xml')
    if not os.path.exists(runinfo):
        logger.fatal("RunInfo '%s' does not exist under Run directory.\n" % (runinfo))
        sys.exit(1)

    outdir = args.outdir
    if not os.path.exists(outdir):
        logger.fatal("output directory '%s' does not exist.\n" % (outdir))
        sys.exit(1)

    return(rundir, outdir, runinfo)


def generate_usebases(barcode_lens, runinfo):
    """FIXME:add-doc
    """
    tree = ET.parse(runinfo)
    root = tree.getroot()
    ub_list = []

    # for each lane and its barcode lengths
    for k, v in sorted(barcode_lens.items()):
        # v is list of barcode_len tuples
        assert len(set(v)) == 1, ("Different barcode length in lane {}".format(k))
        bc1, bc2 = v[0]# since all v's are the same

        ub = ""
        #if test:
        for read in root.iter('Read'):
            numcyc = int(read.attrib['NumCycles'])-1
            if read.attrib['IsIndexedRead'] == 'N':
                ub += 'Y' + str(numcyc) + 'n*,'
            elif read.attrib['IsIndexedRead'] == 'Y':
                if read.attrib['Number'] == '2':    ### BC1
                    if bc1 > 0:
                        ub += 'I'+str(bc1)+'n*,'
                    else:
                        ub += 'n*'+','
                if read.attrib['Number'] == '3':    ### BC2
                    if bc2 > 0:
                        ub += 'I'+str(bc2)+'n*,'
                    else:
                        ub += 'n*'+','
        ub = ub[:-1]
        ub_list.append(str(k + ':' + ub))
    return ub_list


def main():
    """
    The main function
    """

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--force-overwrite",
                        action="store_true",
                        help="Force overwriting of output files")
    parser.add_argument("-r", "--rundir",
                        dest="rundir",
                        required=True,
                        help="rundir, e.g. /mnt/seq/userrig/HS004/HS004-PE-R00139_BC6A7HANXX")
    parser.add_argument("-o", "--outdir",
                        required=True,
                        dest="outdir",
                        help="Output directory")
    parser.add_argument('-v', '--verbose', action='count', default=0,
                        help="Increase verbosity")
    parser.add_argument('-q', '--quiet', action='count', default=0,
                        help="Decrease verbosity")
    args = parser.parse_args()

    # Repeateable -v and -q for setting logging level.
    # See https://www.reddit.com/r/Python/comments/3nctlm/what_python_tools_should_i_be_using_on_every/
    # and https://gist.github.com/andreas-wilm/b6031a84a33e652680d4
    # script -vv -> DEBUG
    # script -v -> INFO
    # script -> WARNING
    # script -q -> ERROR
    # script -qq -> CRITICAL
    # script -qqq -> no logging at all
    logger.setLevel(logging.WARN + 10*args.quiet - 10*args.verbose)

    (rundir, outdir, runinfo) = getdirs(args)
    samplesheet_csv = os.path.join(outdir, SAMPLESHEET_CSV)
    usebases_cfg = os.path.join(outdir, USEBASES_CFG)
    muxinfo_cfg = os.path.join(outdir, MUXINFO_CFG)
    for f in [samplesheet_csv, usebases_cfg, muxinfo_cfg]:
        if not args.force_overwrite and os.path.exists(f):
            logger.fatal("Refusing to overwrite existing file {}".format(f))
            sys.exit(1)

    _, run_num, flowcellid = get_machine_run_flowcell_id(rundir)

    logger.info("Querying ELM for {}".format(run_num))
    #DEVELOPMENT URL
    #rest_url = 'http://dlap51v:8080/elm/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    #PRODUCTION url
    rest_url = 'http://qldb01.gis.a-star.edu.sg:8080/rest/seqrun/illumina/' + run_num + '/detailanalysis/json'
    #NEW DEVELOPMENT url
    #rest_url = 'http://qldb01:8180/rest/seqrun/illumina/'+ run_num + '/detailanalysis/json'
    response = requests.get(rest_url)
    if response.status_code != requests.codes.ok:
        response.raise_for_status()
    rest_data = response.json()

    run_id = rest_data['runId']
    #counter = 0
    if rest_data['runPass'] != 'Pass':
        logger.warning("Skipping non-passed run")
        # NOTE: exit 0 and missing output files is the upstream signal for a failed run
        sys.exit(0)

    # this is the master samplesheet
    logger.info("Writing to {}".format(samplesheet_csv))
    # keys: lanes, values are barcode lens in lane (always two tuples, -1 if not present)
    barcode_lens = {}
    mux_units = dict()

    with open(samplesheet_csv, 'w') as fh_out:
        fh_out.write(SAMPLESHEET_HEADER + '\n')
        for rows in rest_data['lanes']:
            if rows['lanePass'] != 'Pass':
                continue
            BCL_Mismatch = []
            if "MUX" in rows['libraryId']:
                # multiplexed
                #counter = 0
                for child in rows['Children']:
                    #counter += 1
                    #id = 'S' + str(counter)
                    if 'BCL_Mismatch' in child:
                        BCL_Mismatch.append(child['BCL_Mismatch'])
                        # older samples have no values and that's okay

                    if "-" in child['barcode']:
                        # dual index
                        index = child['barcode'].split('-')
                        sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,,'+ index[0] +',,'+ index[1] + ',' +'Project_'+rows['libraryId']+','+child['libtech']
                        index_lens = (len((index[0])), len((index[1])))
                    else:
                        sample = rows['laneId']+',Sample_'+child['libraryId']+','+child['libraryId']+'-'+child['barcode']+',,,,'+child['barcode']+',,,'+'Project_'+rows['libraryId']+','+child['libtech']
                        index_lens = (len(child['barcode']), -1)
                    barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
                    fh_out.write(sample+ '\n')

            else:# non-multiplexed
                sample = rows['laneId']+',Sample_'+rows['libraryId']+','+rows['libraryId']+'-NoIndex'+',,,,,,,'+'Project_'+rows['libraryId']+','+rows['libtech']
                index_lens = (-1, -1)
                barcode_lens.setdefault(rows['laneId'], []).append(index_lens)
{'fraction_non_alphanumeric': 0.14724816965412774, 'fraction_numerical': 0.01237061348144408, 'mean_word_length': 2.319019689987432, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2252228', 'n_tokens_mistral': 5252, 'n_tokens_neox': 4988, 'n_words': 1052}