<commit_before>---
- scm:
    name: govuk-taxonomy-supervised-learning
    scm:
        - git:
            url: <EMAIL>:alphagov/govuk-taxonomy-supervised-learning.git
            branches:
              - master

- job:
    name: govuk_taxonomy_supervised_learning
    display-name: Taxonomy Supervised Learning Pipeline
    project-type: freestyle
    description: |
      This job runs the taxonomy supervised learning pipeline
    logrotate:
      numToKeep: 20
    scm:
      - govuk-taxonomy-supervised-learning
    triggers:
        - timed: 'H 2 * * 1-5'
    builders:
      - shell: |
          #!/bin/bash
          git checkout -B ${GIT_BRANCH} origin/${GIT_BRANCH}

          rm -rf ./venv
          virtualenv --python=python3 --no-site-packages ./venv
          export VIRTUAL_ENV="$PWD/venv"
          export PATH="$PWD/venv/bin:$PATH"
          pip3 install --upgrade pip

          make pip_install

          export PLEK_SERVICE_RUMMAGER_URI=https://<%= @rummager_api %>
          export PLEK_SERVICE_DRAFT_CONTENT_STORE_URI=https://<%= @draft_content_store_api %>

          make clean
          make data/taxons.json.gz
          make data/content.json.gz
    parameters:
        - string:
            name: GIT_BRANCH
            description: The Git branch of the govuk-taxonomy-supervised-learning repository to use.
            default: master<commit_msg>Add measuring average taxons to taxonomy pipeline jenkins job<commit_after>---
- scm:
    name: govuk-taxonomy-supervised-learning
    scm:
        - git:
            url: <EMAIL>:alphagov/govuk-taxonomy-supervised-learning.git
            branches:
              - master

- job:
    name: govuk_taxonomy_supervised_learning
    display-name: Taxonomy Supervised Learning Pipeline
    project-type: freestyle
    description: |
      This job runs the taxonomy supervised learning pipeline
    logrotate:
      numToKeep: 20
    scm:
      - govuk-taxonomy-supervised-learning
    triggers:
        - timed: 'H 2 * * 1-5'
    builders:
      - shell: |
          #!/bin/bash
          git checkout -B ${GIT_BRANCH} origin/${GIT_BRANCH}

          rm -rf ./venv
          virtualenv --python=python3 --no-site-packages ./venv
          export VIRTUAL_ENV="$PWD/venv"
          export PATH="$PWD/venv/bin:$PATH"
          pip3 install --upgrade pip

          make pip_install

          export PLEK_SERVICE_RUMMAGER_URI=https://<%= @rummager_api %>
          export PLEK_SERVICE_DRAFT_CONTENT_STORE_URI=https://<%= @draft_content_store_api %>

          make clean
          make data/taxons.json.gz
          make data/content.json.gz
          make measure_average_taxons
    parameters:
        - string:
            name: GIT_BRANCH
            description: The Git branch of the govuk-taxonomy-supervised-learning repository to use.
            default: master
{'fraction_non_alphanumeric': 0.09071729957805907, 'fraction_numerical': 0.004922644163150493, 'mean_word_length': 1.9915878023133544, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1222728', 'n_tokens_mistral': 907, 'n_tokens_neox': 865, 'n_words': 193}