<commit_before>version: "2.0" # mistral version
name: arteria-packs.ngi_uu_workflow
description: The ngi workflow, from sequencer to remote system...

workflows:
    main:
        type: direct
        input:
            - host
            - runfolder
            - ignore_sisyphus_qc_result
            - bcl2fastq_version
            - barcode_mismatches
            - tiles
            - use_base_mask
            - bcl2fastq_additional_arguments
            - conversionstats_swap_read_nr
            - conversionstats_swap_read_from
            - conversionstats_swap_read_to
        output:
            output_the_whole_workflow_context: <% $ %>
        task-defaults:
            on-error:
                - oh_shit_error
                - mark_as_failed

        tasks:
            ### GENERAL TASKS START ###
            note_workflow_repo_version:
              action: core.local
              input:
                cmd: git rev-parse HEAD
                cwd: /opt/stackstorm/packs/arteria-packs/
              on-success:
                - get_config

            get_config:
              action: arteria-packs.get_pack_config
              publish:
                hermes_base_url: <% task(get_config).result.result.hermes_base_url %>
                hermes_user: <% task(get_config).result.result.hermes_user %>
                hermes_token: <% task(get_config).result.result.hermes_token %>
                remote_host: <% task(get_config).result.result.remote_host %>
                remote_user: <% task(get_config).result.result.remote_user%>
                remote_destination: <% task(get_config).result.result.remote_destination %>
                summary_host: <% task(get_config).result.result.summary_host %>
                summary_user: <% task(get_config).result.result.summary_user %>
                summary_destination: <% task(get_config).result.result.summary_destination %>
                siswrap_service_port: <% task(get_config).result.result.siswrap_service_port %>
                bcl2fastq_service_port: <% task(get_config).result.result.bcl2fastq_service_port %>
                runfolder_service_port: <% task(get_config).result.result.runfolder_service_port %>
                send_mail_to: <% task(get_config).result.result.send_mail_to %>
                remote_sisyphus_location: <% task(get_config).result.result.remote_sisyphus_location %>
                irma_remote_path: <% task(get_config).result.result.irma_remote_path %>
                irma_api_key: <% task(get_config).result.result.irma_api_key %>
                irma_checksum_base_url: <% task(get_config).result.result.irma_checksum_base_url %>
                irma_siswrap_base_url: <% task(get_config).result.result.irma_siswrap_base_url %>
                ngi_pipeline_url: <% task(get_config).result.result.ngi_pipeline_url %>
                irma_replace_expressions: <% task(get_config).result.result.irma_replace_expressions %>
                remote_host_key: <% task(get_config).result.result.remote_host_key %>
              on-success:
                 - mark_as_started

            mark_as_started:
              action: core.http
              input:
                url: http://<% $.host %>:<% $.runfolder_service_port %>/api/1.0/runfolders/path<% $.runfolder %>
                body: '{"state": "started"}'
                method: "POST"
              on-success:
                - get_runfolder_name

            get_runfolder_name:
              action: core.local
              input:
                cmd: python -c 'import sys; sys.stdout.write("<% $.runfolder %>".split("/")[-1])'
              publish:
                runfolder_name: "<% task(get_runfolder_name).result.stdout %>"
              on-success:
                - get_flowcell_name

            get_flowcell_name:
                action: arteria-packs.get_flowcell_from_runfolder_name
                input:
                    runfolder_name: <% $.runfolder_name %>
                publish:
                    flowcell_name: <% task(get_flowcell_name).result.result %>
                on-success:
                    - download_samplesheet

            ### GENERAL TASKS END ###
            ### DEMULTIPLEX START ###
            download_samplesheet:
                action: core.http
                input:
                    url: <% $.hermes_base_url %>/<% $.flowcell_name %>/samplesheetfile
                    headers:
                        USER: <% $.hermes_user %>
                        X-XSRF-TOKEN: <% $.hermes_token %>
                publish:
                    samplesheet_string: <% task(download_samplesheet).result.body.samplesheet %>
                on-success:
                    - construct_bcl2fastq_body

            # Since we don't want to pass empty values into
            # bcl2fastq-ws, we remove any empty keys from
            # this body.
            construct_bcl2fastq_body:
                action: arteria-packs.parse_bcl2fastq_args
                input:
                    samplesheet: "<% $.samplesheet_string %>"
                    bcl2fastq_version: "<% $.bcl2fastq_version %>"
                    barcode_mismatches: "<% $.barcode_mismatches %>"
                    tiles: "<% $.tiles %>"
                    use_base_mask: "<% $.use_base_mask %>"
                    additional_arguments: "<% $.bcl2fastq_additional_arguments %>"
                publish:
                    bcl2fastq_body: <% task(construct_bcl2fastq_body).result.stdout %>
                on-success:
                    - run_bcl2fastq

            run_bcl2fastq:
                action: core.http
                input:
                    url: "http://<% $.host %>:<% $.bcl2fastq_service_port %>/api/1.0/start/<% $.runfolder_name %>"
                    method: "POST"
                    headers:
                        Content-Type: application/json
                    body: <% $.bcl2fastq_body %>
                publish:
                    bcl2fastq_status_url: <% task(run_bcl2fastq).result.body.link %>
                on-success:
                    - poll_demultiplex_status

            poll_demultiplex_status:
                action: arteria-packs.poll_status
                input:
                    url: <% $.bcl2fastq_status_url %>
                    timeout: 86400 # Wait for 24 h before timing out.
                on-success:
                    - conversionstats_workaround: <% $.conversionstats_swap_read_nr = true %>
                    - download_sisyphus_config: <% $.conversionstats_swap_read_nr = false %>

            conversionstats_workaround:
                action: core.local
                input:
                    cmd: ssh <% $.host %> "bash -s" < /opt/stackstorm/packs/arteria-packs/actions/lib/swap_read_nr_in_conversionstats.sh <% $.runfolder %> <% $.conversionstats_swap_read_from %> <% $.conversionstats_swap_read_to %>
                on-success:
                    - download_sisyphus_config

            ## DEMULTIPLEX END ###
            ### QUICK REPORT START ###
            download_sisyphus_config:
                action: core.http
                input:
                    url: <% $.hermes_base_url %>/<% $.flowcell_name %>/configfile
                    headers:<commit_msg>Add a comment to the action that explains its purpose<commit_after>version: "2.0" # mistral version
name: arteria-packs.ngi_uu_workflow
description: The ngi workflow, from sequencer to remote system...

workflows:
    main:
        type: direct
        input:
            - host
            - runfolder
            - ignore_sisyphus_qc_result
            - bcl2fastq_version
            - barcode_mismatches
            - tiles
            - use_base_mask
            - bcl2fastq_additional_arguments
            - conversionstats_swap_read_nr
            - conversionstats_swap_read_from
            - conversionstats_swap_read_to
        output:
            output_the_whole_workflow_context: <% $ %>
        task-defaults:
            on-error:
                - oh_shit_error
                - mark_as_failed

        tasks:
            ### GENERAL TASKS START ###
            note_workflow_repo_version:
              action: core.local
              input:
                cmd: git rev-parse HEAD
                cwd: /opt/stackstorm/packs/arteria-packs/
              on-success:
                - get_config

            get_config:
              action: arteria-packs.get_pack_config
              publish:
                hermes_base_url: <% task(get_config).result.result.hermes_base_url %>
                hermes_user: <% task(get_config).result.result.hermes_user %>
                hermes_token: <% task(get_config).result.result.hermes_token %>
                remote_host: <% task(get_config).result.result.remote_host %>
                remote_user: <% task(get_config).result.result.remote_user%>
                remote_destination: <% task(get_config).result.result.remote_destination %>
                summary_host: <% task(get_config).result.result.summary_host %>
                summary_user: <% task(get_config).result.result.summary_user %>
                summary_destination: <% task(get_config).result.result.summary_destination %>
                siswrap_service_port: <% task(get_config).result.result.siswrap_service_port %>
                bcl2fastq_service_port: <% task(get_config).result.result.bcl2fastq_service_port %>
                runfolder_service_port: <% task(get_config).result.result.runfolder_service_port %>
                send_mail_to: <% task(get_config).result.result.send_mail_to %>
                remote_sisyphus_location: <% task(get_config).result.result.remote_sisyphus_location %>
                irma_remote_path: <% task(get_config).result.result.irma_remote_path %>
                irma_api_key: <% task(get_config).result.result.irma_api_key %>
                irma_checksum_base_url: <% task(get_config).result.result.irma_checksum_base_url %>
                irma_siswrap_base_url: <% task(get_config).result.result.irma_siswrap_base_url %>
                ngi_pipeline_url: <% task(get_config).result.result.ngi_pipeline_url %>
                irma_replace_expressions: <% task(get_config).result.result.irma_replace_expressions %>
                remote_host_key: <% task(get_config).result.result.remote_host_key %>
              on-success:
                 - mark_as_started

            mark_as_started:
              action: core.http
              input:
                url: http://<% $.host %>:<% $.runfolder_service_port %>/api/1.0/runfolders/path<% $.runfolder %>
                body: '{"state": "started"}'
                method: "POST"
              on-success:
                - get_runfolder_name

            get_runfolder_name:
              action: core.local
              input:
                cmd: python -c 'import sys; sys.stdout.write("<% $.runfolder %>".split("/")[-1])'
              publish:
                runfolder_name: "<% task(get_runfolder_name).result.stdout %>"
              on-success:
                - get_flowcell_name

            get_flowcell_name:
                action: arteria-packs.get_flowcell_from_runfolder_name
                input:
                    runfolder_name: <% $.runfolder_name %>
                publish:
                    flowcell_name: <% task(get_flowcell_name).result.result %>
                on-success:
                    - download_samplesheet

            ### GENERAL TASKS END ###
            ### DEMULTIPLEX START ###
            download_samplesheet:
                action: core.http
                input:
                    url: <% $.hermes_base_url %>/<% $.flowcell_name %>/samplesheetfile
                    headers:
                        USER: <% $.hermes_user %>
                        X-XSRF-TOKEN: <% $.hermes_token %>
                publish:
                    samplesheet_string: <% task(download_samplesheet).result.body.samplesheet %>
                on-success:
                    - construct_bcl2fastq_body

            # Since we don't want to pass empty values into
            # bcl2fastq-ws, we remove any empty keys from
            # this body.
            construct_bcl2fastq_body:
                action: arteria-packs.parse_bcl2fastq_args
                input:
                    samplesheet: "<% $.samplesheet_string %>"
                    bcl2fastq_version: "<% $.bcl2fastq_version %>"
                    barcode_mismatches: "<% $.barcode_mismatches %>"
                    tiles: "<% $.tiles %>"
                    use_base_mask: "<% $.use_base_mask %>"
                    additional_arguments: "<% $.bcl2fastq_additional_arguments %>"
                publish:
                    bcl2fastq_body: <% task(construct_bcl2fastq_body).result.stdout %>
                on-success:
                    - run_bcl2fastq

            run_bcl2fastq:
                action: core.http
                input:
                    url: "http://<% $.host %>:<% $.bcl2fastq_service_port %>/api/1.0/start/<% $.runfolder_name %>"
                    method: "POST"
                    headers:
                        Content-Type: application/json
                    body: <% $.bcl2fastq_body %>
                publish:
                    bcl2fastq_status_url: <% task(run_bcl2fastq).result.body.link %>
                on-success:
                    - poll_demultiplex_status

            poll_demultiplex_status:
                action: arteria-packs.poll_status
                input:
                    url: <% $.bcl2fastq_status_url %>
                    timeout: 86400 # Wait for 24 h before timing out.
                on-success:
                    - conversionstats_workaround: <% $.conversionstats_swap_read_nr = true %>
                    - download_sisyphus_config: <% $.conversionstats_swap_read_nr = false %>

            # Due to a bug in bcl2fastq some read numbers in Unaligned/Stats/ConversionStats.xml will sometimes have the wrong number. 
            # If the Arteria operator manually sets "conversionstats_swap_read"_nr to "true" then this workaround action will swap 
            # the incorrect value "conversionstats_swap_read_from" to the requested correct value "conversionstats_swap_read_to". 
            conversionstats_workaround:
                action: core.local
                input:
                    cmd: ssh <% $.host %> "bash -s" < /opt/stackstorm/packs/arteria-packs/actions/lib/swap_read_nr_in_conversionstats.sh <% $.runfolder %> <% $.conversionstats_swap_read_from %> <% $.conversionstats_swap_read_to %>
                on-success:
                    - download_sisyphus_config

            ## DEMULTIPLEX END ###
            ### QUICK REPORT START ###
            download_sisyphus_config:
                action: core.http
                input:
                    url: <% $.hermes_base_url %>/<% $.flowcell_name %>/configfile
                    headers:
{'fraction_non_alphanumeric': 0.09706138295735993, 'fraction_numerical': 0.004618783051074369, 'mean_word_length': 1.5951016154247004, 'pattern_counts': {'":': 2, '<': 115, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4653868', 'n_tokens_mistral': 4445, 'n_tokens_neox': 4076, 'n_words': 612}