<commit_before>#!/usr/bin/env python
__author__ = '<NAME> <<EMAIL>>'

import sys
from urlparse import urlparse
import csv
from datetime import datetime

import os
from os.path import join, splitext
from dateutil import parser as dateparser
import subprocess

import xmltodict

from mytardis_uploader import MyTardisUploader
from mytardis_uploader import setup_logging, get_config, validate_config
# from mytardis_uploader import get_exclude_patterns_as_regex_list


def get_run_metadata(run_path):
    """

    :param run_path: str
    :rtype: dict
    """
    metadata = {}
    end_time, rta_version = rta_complete_parser(run_path)
    metadata['end_time'] = end_time.isoformat(' ')
    metadata['rta_version'] = rta_version
    metadata['run_path'] = run_path
    metadata['run_id'] = os.path.relpath(
        options.path,
        options.storage_base_path
    )

    runinfo_parameters = runinfo_parser(run_path)
    runinfo_parameters['rta_version'] = rta_version
    instrument_config = illumina_config_parser(run_path)

    metadata['instrument_model'] = instrument_config['instrument_model']
    metadata['title'] = "%s sequencing run %s" % (
        metadata['instrument_model'],
        metadata['run_id']
    )

    samplesheet, chemistry = get_samplesheet(join(run_path, 'SampleSheet.csv'))
    samplesheet_common_fields = {'chemistry': chemistry,
                                 'operator_name':
                                     samplesheet[0].get('Operator', '')
                                 }

    parameters = merge_dicts(runinfo_parameters,
                             instrument_config,
                             samplesheet_common_fields)
    parameter_list = dict_to_parameter_list(parameters)

    # parameter_list.append({u'name': param_name, u'value': param_value})
    schema = 'http://www.tardis.edu.au/schemas/ngs/illumina/run'
    metadata['parameter_sets'] = [{u'schema': schema,
                                   u'parameters': parameter_list}]

    return metadata


def get_project_metadata(proj_id,
                         run_metadata,
                         fastqc_summary_json='',
                         schema='http://www.tardis.edu.au/schemas/ngs/project'):
    end_date = run_metadata['end_time'].split()[0]
    project_title = 'Sequencing Project, %s, %s' % (proj_id, end_date)
    if proj_id == 'Undetermined_indices':
        project_title = '%s, %s, %s' % (proj_id,
                                        run_metadata['run_id'],
                                        end_date)
    proj_metadata = {'title': project_title,
                     'description': project_title,
                     'institute': run_metadata['institute'],
                     'end_time': run_metadata['end_time'],
                     }

    # project_parameters = run_metadata
    # Project Dataset parameters are suffixed with '__project'
    # (to prevent name clashes with identical parameters at the
    #  Run Experiment level)
    # project_parameters = add_suffix_to_parameter_set(
    #    run_metadata['parameter_sets'][0]['parameters'], param_suffix)
    # project_parameter_list = dict_to_parameter_list(project_parameters)

    project_parameter_list = list(
        run_metadata['parameter_sets'][0]['parameters']
    )
    project_parameter_list.extend(
        dict_to_parameter_list({'fastqc_summary_json': fastqc_summary_json})
    )

    proj_metadata['parameter_sets'] = [{u'schema': schema,
                                        u'parameters': project_parameter_list}]

    return proj_metadata


def add_suffix_to_parameter_set(parameters, suffix, divider='__'):
    """
    Adds a suffix ('__suffix') to the keys of a dictionary of MyTardis
    parameters. Returns a copy of the dict with suffixes on keys.
    (eg to prevent name clashes with identical parameters at the
        Run Experiment level)
    """
    suffixed_parameters = {}
    for p in parameters:
        suffixed_parameters[u'%s%s%s' % (p['name'],
                                         divider,
                                         suffix)] = p['value']
    return suffixed_parameters


def get_samplesheet(file_path):
    # FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,
    # Operator,SampleProject
    #<commit_msg>Put FastQC summary JSON blob in it's own (hidden) parameter set / schema.<commit_after>#!/usr/bin/env python
__author__ = '<NAME> <<EMAIL>>'

import sys
from urlparse import urlparse
import csv
from datetime import datetime

import os
from os.path import join, splitext
from dateutil import parser as dateparser
import subprocess

import xmltodict

from mytardis_uploader import MyTardisUploader
from mytardis_uploader import setup_logging, get_config, validate_config
# from mytardis_uploader import get_exclude_patterns_as_regex_list


def get_run_metadata(run_path):
    """

    :param run_path: str
    :rtype: dict
    """
    metadata = {}
    end_time, rta_version = rta_complete_parser(run_path)
    metadata['end_time'] = end_time.isoformat(' ')
    metadata['rta_version'] = rta_version
    metadata['run_path'] = run_path
    metadata['run_id'] = os.path.relpath(
        options.path,
        options.storage_base_path
    )

    runinfo_parameters = runinfo_parser(run_path)
    runinfo_parameters['rta_version'] = rta_version
    instrument_config = illumina_config_parser(run_path)

    metadata['instrument_model'] = instrument_config['instrument_model']
    metadata['title'] = "%s sequencing run %s" % (
        metadata['instrument_model'],
        metadata['run_id']
    )

    samplesheet, chemistry = get_samplesheet(join(run_path, 'SampleSheet.csv'))
    samplesheet_common_fields = {'chemistry': chemistry,
                                 'operator_name':
                                     samplesheet[0].get('Operator', '')
                                 }

    parameters = merge_dicts(runinfo_parameters,
                             instrument_config,
                             samplesheet_common_fields)
    parameter_list = dict_to_parameter_list(parameters)

    # parameter_list.append({u'name': param_name, u'value': param_value})
    schema = 'http://www.tardis.edu.au/schemas/ngs/illumina/run'
    metadata['parameter_sets'] = [{u'schema': schema,
                                   u'parameters': parameter_list}]

    return metadata


def get_project_metadata(proj_id,
                         run_metadata,
                         fastqc_summary_json='',
                         schema='http://www.tardis.edu.au/schemas/ngs/project'):
    end_date = run_metadata['end_time'].split()[0]
    project_title = 'Sequencing Project, %s, %s' % (proj_id, end_date)
    if proj_id == 'Undetermined_indices':
        project_title = '%s, %s, %s' % (proj_id,
                                        run_metadata['run_id'],
                                        end_date)
    proj_metadata = {'title': project_title,
                     'description': project_title,
                     'institute': run_metadata['institute'],
                     'end_time': run_metadata['end_time'],
                     }

    # project_parameters = run_metadata
    # Project Dataset parameters are suffixed with '__project'
    # (to prevent name clashes with identical parameters at the
    #  Run Experiment level)
    # project_parameters = add_suffix_to_parameter_set(
    #    run_metadata['parameter_sets'][0]['parameters'], param_suffix)
    # project_parameter_list = dict_to_parameter_list(project_parameters)

    project_parameter_list = list(
        run_metadata['parameter_sets'][0]['parameters']
    )

    # This second (hidden) parameter_set, provides a summary of
    # FastQC results for every sample in the project, used for
    # rendering and overview table
    fastqc_summary_parameters = \
        dict_to_parameter_list({'fastqc_summary_json': fastqc_summary_json})

    fasqc_summary_parameterset = {
        u'schema':
        'http://www.tardis.edu.au/schemas/ngs/project/fastqc_summary',
        u'parameters': fastqc_summary_parameters
    }

    proj_metadata['parameter_sets'] = [{u'schema': schema,
                                        u'parameters': project_parameter_list},
                                       fasqc_summary_parameterset
                                       ]

    return proj_metadata


def add_suffix_to_parameter_set(parameters, suffix, divider='__'):
    """
    Adds a suffix ('__suffix') to the keys of a dictionary of MyTardis
    parameters. Returns a copy of the dict with suffixes on keys.
    (eg to prevent name clashes with identical parameters at the
        Run Experiment level)
    """
    suffixed_parameters = {}
    for p in parameters:
        suffixed_parameters[u'%s%s%s' % (p['name'],
                                         divider,
                                         suffix)] = p['value']
    return suffixed_parameters


def get_samplesheet(file_path):
    # FCID,Lane,SampleID,SampleRef,Index,Description,Control,Recipe,
    # Operator,SampleProject
    #
{'fraction_non_alphanumeric': 0.09560609406049901, 'fraction_numerical': 0.0008831971737690439, 'mean_word_length': 2.5058049535603715, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6051581', 'n_tokens_mistral': 2586, 'n_tokens_neox': 2489, 'n_words': 521}