<commit_before>#!/usr/bin/env python
import unittest
import logging
import filecmp
import mock
import os

from taca.analysis.analysis_nanopore import *
from taca.utils import config as conf


CONFIG = conf.load_yaml_config('data/taca_test_nanopore_cfg.yaml')

class TestNanoporeAnalysis(unittest.TestCase):

    def test_find_runs_to_process(self):
        """Find all expected nanopore runs to process."""
        expected_dirs = ["data/nanopore_data/run1/still_sequencing/20200101_1412_MN19414_AAU641_68125dc2",
                         "data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2",
                         "data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2",
                         "data/nanopore_data/run3/demultiplexing/20200103_1412_MN19414_AAU643_68125dc2",
                         "data/nanopore_data/run7/done_no_sample_sheet/20200107_1412_MN19417_AAU645_68125dc2",
                         "data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2"]
        found_dirs = find_runs_to_process()
        self.assertEqual(sorted(found_dirs), sorted(expected_dirs))

    @mock.patch('taca.analysis.analysis_nanopore.open')
    def test_parse_lims_sample_sheet(self, mock_write):
        """Find and parse lims sample sheet."""
        run_dir = 'data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2'
        parsed_sample_sheet_location = parse_lims_sample_sheet(run_dir)
        self.assertEqual(parsed_sample_sheet_location, 'data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2/SQK-LSK109_sample_sheet.csv')

    def test_get_original_samplesheet(self):
        """Get location of lims sample sheet."""
        run_id = '20200102_1412_MN19414_AAU642_68125dc2'
        found_sample_sheet = get_original_samplesheet(run_id)
        expected_sample_sheet = 'data/nanopore_samplesheets/2020/SQK-LSK109_AAU642_Samplesheet_22-594126.csv'
        self.assertEqual(found_sample_sheet, expected_sample_sheet)

    def test_parse_samplesheet(self):
        """Make nanoseq sample sheet from lims sample sheet."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        lims_samplesheet = 'data/nanopore_samplesheets/2020/SQK-LSK109_AAU644_Samplesheet_24-594126.csv'
        nanoseq_samplesheet = parse_samplesheet(run_dir, lims_samplesheet)
        self.assertTrue(filecmp.cmp(nanoseq_samplesheet, 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'))

    @mock.patch('taca.analysis.analysis_nanopore.get_flowcell_id')
    @mock.patch('taca.analysis.analysis_nanopore.is_multiplexed')
    @mock.patch('taca.analysis.analysis_nanopore.subprocess.Popen')
    def test_start_analysis_pipeline_multiplexed(self, mock_popen, mock_is_multiplexed, mock_get_id):
        """Submit detached nanoseq job for multiplexed data."""
        mock_get_id.return_value = 'FLO-FLG001'
        mock_is_multiplexed.return_value = True
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        start_analysis_pipeline(run_dir, sample_sheet)
        expected_parameters = 'nextflow run nf-core/nanoseq --input ' + sample_sheet + \
            ' --input_path ' + run_dir + '/fast5/ \
            --outdir ' + run_dir + '/nanoseq_output \
            --flowcell FLO-FLG001' + \
            ' --guppy_gpu \
            --skip_alignment \
            --kit SQK-LSK109' + \
            ' --max_cpus 6 \
            --max_memory 20.GB \
            --barcode_kit EXP-NBD104' + \
            ' -profile singularity; echo $? > .exitcode_for_nanoseq'
        mock_popen.assert_called_once_with(expected_parameters, stdout=subprocess.PIPE, shell=True, cwd=run_dir)

    @mock.patch('taca.analysis.analysis_nanopore.get_flowcell_id')
    @mock.patch('taca.analysis.analysis_nanopore.is_multiplexed')
    @mock.patch('taca.analysis.analysis_nanopore.subprocess.Popen')
    def test_start_analysis_pipeline_not_multiplexed(self, mock_popen, mock_is_multiplexed, mock_get_id):
        """Submit detached nanoseq job for non multiplexed data."""
        mock_get_id.return_value = 'FLO-FLG001'
        mock_is_multiplexed.return_value = False
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        start_analysis_pipeline(run_dir, sample_sheet)
        expected_parameters = 'nextflow run nf-core/nanoseq --input ' + sample_sheet + \
        ' --input_path ' + run_dir + '/fast5/ \
        --outdir ' + run_dir + '/nanoseq_output \
        --flowcell FLO-FLG001' + \
        ' --guppy_gpu \
        --skip_alignment \
        --kit SQK-LSK109' + \
        ' --max_cpus 6 \
        --max_memory 20.GB \
        -profile singularity; echo $? > .exitcode_for_nanoseq'
        mock_popen.assert_called_once_with(expected_parameters, stdout=subprocess.PIPE, shell=True, cwd=run_dir)

    def test_get_flowcell_id(self):
        """Get flowcell ID from report.md."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        got_id = get_flowcell_id(run_dir)
        expected_id = 'FLO-FLG001'
        self.assertEqual(got_id, expected_id)

    def test_is_multiplexed(self):
        """Return True if run is multiplexed, else False."""
        multiplexed_sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        non_multiplexed_sample_sheet = 'data/nanopore_data/run3/demultiplexing/20200103_1412_MN19414_AAU643_68125dc2/SQK-LSK109_AAU643_sample_sheet.csv'
        self.assertTrue(is_multiplexed(multiplexed_sample_sheet))
        self.assertFalse(is_multiplexed(non_multiplexed_sample_sheet))

    def test_get_barcode_kit(self):
        """Return EXP-NBD104 or EXP-NBD114 barcode kit based on sample sheet."""
        sample_sheet_104 = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        got_kit_104 = get_barcode_kit(sample_sheet_104)
        sample_sheet_114 = 'data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2/SQK-LSK109_sample_sheet.csv'
        got_kit_114 = get_barcode_kit(sample_sheet_114)
        self.assertEqual(got_kit_104, 'EXP-NBD104')
        self.assertEqual(got_kit_114, 'EXP-NBD114')

    def test_check_exit_status(self):
        """Check nanoseq exit status from file."""
        self.assertTrue(check_exit_status('data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/.exitcode_for_nanoseq'))
        self.assertFalse(check_exit_status('data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2/.exitcode_for_nanoseq'))

    def test_is_not_transferred(self):
        """Check if nanopore run has been transferred."""
        self.assertTrue(is_not_transferred('20200104_1412_MN19414_AAU644_68125dc2', 'data/nanopore_data/transfer.tsv'))
        self.assertFalse(is_not_transferred('20200105_1412_MN19414_AAU645_68125dc2', 'data/nanopore_data/transfer.tsv'))

    @mock.patch('taca.analysis.analysis_nanopore.RsyncAgent')
    def test_transfer_run(self, mock_rsync):
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        transfer_run(run_dir)
        rsync_opts = {'-Lav': None,
                      '--no-o' : None,
                      '--no-g' : None,
                      '--chmod' : 'g+rw',
                      '-r' : None,
                      '--exclude' : 'work'}
        mock_rsync.assert_called_with(run_dir,
                                      dest_path='some_dir',
                                      remote_host='some_host',
                                      remote_user='some_user',
                                      validate=False,
                                      opts=rsync_opts)

    @mock.patch('taca.analysis.analysis_nanopore.shutil.move')
    def test_archive_run(self, mock_move):
        """Move directory to archive."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        archive_run(run_dir)
        mock_move.assert_called_once_with('data/nanopore_data/run4', 'data/nanopore_data/nosync')

    @mock.patch('taca.analysis.analysis_nanopore.parse_lims_sample_sheet')<commit_msg>make rsync test pass<commit_after>#!/usr/bin/env python
import unittest
import logging
import filecmp
import mock
import os

from taca.analysis.analysis_nanopore import *
from taca.utils import config as conf


CONFIG = conf.load_yaml_config('data/taca_test_nanopore_cfg.yaml')

class TestNanoporeAnalysis(unittest.TestCase):

    def test_find_runs_to_process(self):
        """Find all expected nanopore runs to process."""
        expected_dirs = ["data/nanopore_data/run1/still_sequencing/20200101_1412_MN19414_AAU641_68125dc2",
                         "data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2",
                         "data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2",
                         "data/nanopore_data/run3/demultiplexing/20200103_1412_MN19414_AAU643_68125dc2",
                         "data/nanopore_data/run7/done_no_sample_sheet/20200107_1412_MN19417_AAU645_68125dc2",
                         "data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2"]
        found_dirs = find_runs_to_process()
        self.assertEqual(sorted(found_dirs), sorted(expected_dirs))

    @mock.patch('taca.analysis.analysis_nanopore.open')
    def test_parse_lims_sample_sheet(self, mock_write):
        """Find and parse lims sample sheet."""
        run_dir = 'data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2'
        parsed_sample_sheet_location = parse_lims_sample_sheet(run_dir)
        self.assertEqual(parsed_sample_sheet_location, 'data/nanopore_data/run2/done_sequencing/20200102_1412_MN19414_AAU642_68125dc2/SQK-LSK109_sample_sheet.csv')

    def test_get_original_samplesheet(self):
        """Get location of lims sample sheet."""
        run_id = '20200102_1412_MN19414_AAU642_68125dc2'
        found_sample_sheet = get_original_samplesheet(run_id)
        expected_sample_sheet = 'data/nanopore_samplesheets/2020/SQK-LSK109_AAU642_Samplesheet_22-594126.csv'
        self.assertEqual(found_sample_sheet, expected_sample_sheet)

    def test_parse_samplesheet(self):
        """Make nanoseq sample sheet from lims sample sheet."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        lims_samplesheet = 'data/nanopore_samplesheets/2020/SQK-LSK109_AAU644_Samplesheet_24-594126.csv'
        nanoseq_samplesheet = parse_samplesheet(run_dir, lims_samplesheet)
        self.assertTrue(filecmp.cmp(nanoseq_samplesheet, 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'))

    @mock.patch('taca.analysis.analysis_nanopore.get_flowcell_id')
    @mock.patch('taca.analysis.analysis_nanopore.is_multiplexed')
    @mock.patch('taca.analysis.analysis_nanopore.subprocess.Popen')
    def test_start_analysis_pipeline_multiplexed(self, mock_popen, mock_is_multiplexed, mock_get_id):
        """Submit detached nanoseq job for multiplexed data."""
        mock_get_id.return_value = 'FLO-FLG001'
        mock_is_multiplexed.return_value = True
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        start_analysis_pipeline(run_dir, sample_sheet)
        expected_parameters = 'nextflow run nf-core/nanoseq --input ' + sample_sheet + \
            ' --input_path ' + run_dir + '/fast5/ \
            --outdir ' + run_dir + '/nanoseq_output \
            --flowcell FLO-FLG001' + \
            ' --guppy_gpu \
            --skip_alignment \
            --kit SQK-LSK109' + \
            ' --max_cpus 6 \
            --max_memory 20.GB \
            --barcode_kit EXP-NBD104' + \
            ' -profile singularity; echo $? > .exitcode_for_nanoseq'
        mock_popen.assert_called_once_with(expected_parameters, stdout=subprocess.PIPE, shell=True, cwd=run_dir)

    @mock.patch('taca.analysis.analysis_nanopore.get_flowcell_id')
    @mock.patch('taca.analysis.analysis_nanopore.is_multiplexed')
    @mock.patch('taca.analysis.analysis_nanopore.subprocess.Popen')
    def test_start_analysis_pipeline_not_multiplexed(self, mock_popen, mock_is_multiplexed, mock_get_id):
        """Submit detached nanoseq job for non multiplexed data."""
        mock_get_id.return_value = 'FLO-FLG001'
        mock_is_multiplexed.return_value = False
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        start_analysis_pipeline(run_dir, sample_sheet)
        expected_parameters = 'nextflow run nf-core/nanoseq --input ' + sample_sheet + \
        ' --input_path ' + run_dir + '/fast5/ \
        --outdir ' + run_dir + '/nanoseq_output \
        --flowcell FLO-FLG001' + \
        ' --guppy_gpu \
        --skip_alignment \
        --kit SQK-LSK109' + \
        ' --max_cpus 6 \
        --max_memory 20.GB \
        -profile singularity; echo $? > .exitcode_for_nanoseq'
        mock_popen.assert_called_once_with(expected_parameters, stdout=subprocess.PIPE, shell=True, cwd=run_dir)

    def test_get_flowcell_id(self):
        """Get flowcell ID from report.md."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        got_id = get_flowcell_id(run_dir)
        expected_id = 'FLO-FLG001'
        self.assertEqual(got_id, expected_id)

    def test_is_multiplexed(self):
        """Return True if run is multiplexed, else False."""
        multiplexed_sample_sheet = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        non_multiplexed_sample_sheet = 'data/nanopore_data/run3/demultiplexing/20200103_1412_MN19414_AAU643_68125dc2/SQK-LSK109_AAU643_sample_sheet.csv'
        self.assertTrue(is_multiplexed(multiplexed_sample_sheet))
        self.assertFalse(is_multiplexed(non_multiplexed_sample_sheet))

    def test_get_barcode_kit(self):
        """Return EXP-NBD104 or EXP-NBD114 barcode kit based on sample sheet."""
        sample_sheet_104 = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/SQK-LSK109_sample_sheet.csv'
        got_kit_104 = get_barcode_kit(sample_sheet_104)
        sample_sheet_114 = 'data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2/SQK-LSK109_sample_sheet.csv'
        got_kit_114 = get_barcode_kit(sample_sheet_114)
        self.assertEqual(got_kit_104, 'EXP-NBD104')
        self.assertEqual(got_kit_114, 'EXP-NBD114')

    def test_check_exit_status(self):
        """Check nanoseq exit status from file."""
        self.assertTrue(check_exit_status('data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2/.exitcode_for_nanoseq'))
        self.assertFalse(check_exit_status('data/nanopore_data/run8/demux_failed/20200108_1412_MN19414_AAU648_68125dc2/.exitcode_for_nanoseq'))

    def test_is_not_transferred(self):
        """Check if nanopore run has been transferred."""
        self.assertTrue(is_not_transferred('20200104_1412_MN19414_AAU644_68125dc2', 'data/nanopore_data/transfer.tsv'))
        self.assertFalse(is_not_transferred('20200105_1412_MN19414_AAU645_68125dc2', 'data/nanopore_data/transfer.tsv'))

    @mock.patch('taca.analysis.analysis_nanopore.RsyncAgent')
    def test_transfer_run(self, mock_rsync):
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        transfer_run(run_dir)
        rsync_opts = {'-Lav': None,
                      '--chown': ':ngi2016003',
                      '--chmod' : 'Dg+s,g+rw',
                      '-r' : None,
                      '--exclude' : 'work'}
        mock_rsync.assert_called_with(run_dir,
                                      dest_path='some_dir',
                                      remote_host='some_host',
                                      remote_user='some_user',
                                      validate=False,
                                      opts=rsync_opts)

    @mock.patch('taca.analysis.analysis_nanopore.shutil.move')
    def test_archive_run(self, mock_move):
        """Move directory to archive."""
        run_dir = 'data/nanopore_data/run4/done_demuxing/20200104_1412_MN19414_AAU644_68125dc2'
        archive_run(run_dir)
        mock_move.assert_called_once_with('data/nanopore_data/run4', 'data/nanopore_data/nosync')

    @mock.patch('taca.analysis.analysis_nanopore.parse_lims_sample_sheet')
{'fraction_non_alphanumeric': 0.09513607411696584, 'fraction_numerical': 0.09664157498552403, 'mean_word_length': 3.3591620393740533, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3807643', 'n_tokens_mistral': 7264, 'n_tokens_neox': 6148, 'n_words': 654}