<commit_before>version: "2.0" # mistral version
name: arteria-packs.fill_projman
description: Downloads summary statistics data from irma and fills the projman database.

workflows:
    main:
        type: direct
        input:
          - year

        tasks:
            note_workflow_version:
              action: core.local
              input:
                cmd: git rev-parse HEAD
                cwd: /opt/stackstorm/packs/arteria-packs/
              on-success:
                  - get_config

            get_config:
              action: arteria-packs.get_pack_config
              publish:
                summary_host: <% task(get_config).result.result.summary_host %>
                summary_user: <% task(get_config).result.result.summary_user %>
                summary_host_key: <% task(get_config).result.result.summary_host_key %>
                summary_destination: <% task(get_config).result.result.summary_destination %>
                irma_remote_path: <% task(get_config).result.result.irma_remote_path %>
              on-success:
                 - get_year

            get_year:
              action: core.local
              input:
                # -1 is the default value of year - indicating that the current year should be used
                cmd: if [ -1 -eq <% $.year %> ]; then date +%Y; else echo <% $.year %>; fi
              publish:
                current_year: <% task(get_year).result.stdout %>
              on-success:
                - check_for_runfolders

            check_for_runfolders:
              action: arteria-packs.get_remote_folder_list
              input:
                directory: <% $.summary_destination %>/<% $.current_year %>/
                username: <% $.summary_user %>
                hosts: <% $.summary_host %>
                private_key: <% $.summary_host_key %>
              publish:
                runfolders: <% task(check_for_runfolders).result.get($.summary_host).stdout %>
              on-success:
                - rsync_from_irma

            rsync_from_irma:
              action: core.remote
              with-items: runfolder in <% $.runfolders %>
              input:
                # Note:
                # - that modtimes are preserved here so that we can check downstream
                #   that required files are old enough.
                # - that this will always return 0 exit status and that this is a potential source or problems.
                #   Since we cannot know before hand if a report directory exists or not, we will attempt to transfer
                #   regardless of this. If this fails (and rsync returns code 23) we will ignore this, and this might
                #   potentially leading to us missing real errors.
                # - Filtering for downstream insertion into the database is carried out by the check_reports_old_enough
                #   which will only return the runfolders with valid summary reports for processing.
                cmd: rsync -e "ssh -i /home/seqsum/.ssh/mm-xlas002" -r --times <EMAIL>:<% $.irma_remote_path %>/<% $.runfolder %>/Summary <% $.summary_destination %>/<% $.current_year %>/<% $.runfolder %>/; if (( $? == 0 || $? == 23 )) ; then true; else false; fi
                hosts: <% $.summary_host %>
                username: <% $.summary_user %>
                private_key: <% $.summary_host_key %>
              on-complete:
                - check_reports_old_enough

            check_reports_old_enough:
              action: arteria-packs.check_summary_report_modtime
              input:
                directory: <% $.summary_destination %>/<% $.current_year %>/
                username: <% $.summary_user %>
                hosts: <% $.summary_host %>
                private_key: <% $.summary_host_key %>
                # Require the summary report file to be at least 20 min old.
                # This avoids trying to read a report into the db
                # for which the report job has not yet been finished.
                modtime: 1200
              publish:
                filtered_runfolders: <% task(check_reports_old_enough).result.get($.summary_host).stdout %>
              on-success:
                - load_data_into_projman
<commit_msg>Limit no of concurrent logins to mm-xlas002 so that it doesn't get spammed<commit_after>version: "2.0" # mistral version
name: arteria-packs.fill_projman
description: Downloads summary statistics data from irma and fills the projman database.

workflows:
    main:
        type: direct
        input:
          - year

        tasks:
            note_workflow_version:
              action: core.local
              input:
                cmd: git rev-parse HEAD
                cwd: /opt/stackstorm/packs/arteria-packs/
              on-success:
                  - get_config

            get_config:
              action: arteria-packs.get_pack_config
              publish:
                summary_host: <% task(get_config).result.result.summary_host %>
                summary_user: <% task(get_config).result.result.summary_user %>
                summary_host_key: <% task(get_config).result.result.summary_host_key %>
                summary_destination: <% task(get_config).result.result.summary_destination %>
                irma_remote_path: <% task(get_config).result.result.irma_remote_path %>
              on-success:
                 - get_year

            get_year:
              action: core.local
              input:
                # -1 is the default value of year - indicating that the current year should be used
                cmd: if [ -1 -eq <% $.year %> ]; then date +%Y; else echo <% $.year %>; fi
              publish:
                current_year: <% task(get_year).result.stdout %>
              on-success:
                - check_for_runfolders

            check_for_runfolders:
              action: arteria-packs.get_remote_folder_list
              input:
                directory: <% $.summary_destination %>/<% $.current_year %>/
                username: <% $.summary_user %>
                hosts: <% $.summary_host %>
                private_key: <% $.summary_host_key %>
              publish:
                runfolders: <% task(check_for_runfolders).result.get($.summary_host).stdout %>
              on-success:
                - rsync_from_irma

            rsync_from_irma:
              action: core.remote
              with-items: runfolder in <% $.runfolders %>
              input:
                # Note:
                # - that modtimes are preserved here so that we can check downstream
                #   that required files are old enough.
                # - that this will always return 0 exit status and that this is a potential source or problems.
                #   Since we cannot know before hand if a report directory exists or not, we will attempt to transfer
                #   regardless of this. If this fails (and rsync returns code 23) we will ignore this, and this might
                #   potentially leading to us missing real errors.
                # - Filtering for downstream insertion into the database is carried out by the check_reports_old_enough
                #   which will only return the runfolders with valid summary reports for processing.
                cmd: rsync -e "ssh -i /home/seqsum/.ssh/mm-xlas002" -r --times <EMAIL>:<% $.irma_remote_path %>/<% $.runfolder %>/Summary <% $.summary_destination %>/<% $.current_year %>/<% $.runfolder %>/; if (( $? == 0 || $? == 23 )) ; then true; else false; fi
                hosts: <% $.summary_host %>
                username: <% $.summary_user %>
                private_key: <% $.summary_host_key %>
              concurrency: 5
              on-complete:
                - check_reports_old_enough

            check_reports_old_enough:
              action: arteria-packs.check_summary_report_modtime
              input:
                directory: <% $.summary_destination %>/<% $.current_year %>/
                username: <% $.summary_user %>
                hosts: <% $.summary_host %>
                private_key: <% $.summary_host_key %>
                # Require the summary report file to be at least 20 min old.
                # This avoids trying to read a report into the db
                # for which the report job has not yet been finished.
                modtime: 1200
              publish:
                filtered_runfolders: <% task(check_reports_old_enough).result.get($.summary_host).stdout %>
              on-success:
                - load_data_into_projman

{'fraction_non_alphanumeric': 0.09049615653389238, 'fraction_numerical': 0.004891684136967156, 'mean_word_length': 1.6147990255785627, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022319', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2169, 'n_words': 618}