<commit_before>#!/usr/bin/env python3

import argparse
import copy
import json
import os
import re
import shutil
import subprocess
import sys
import traceback
import logging

try:
    import yaml
except ImportError:
    print("PyYAML missing, try running 'sudo pip3 install pyyaml'.")
    sys.exit(2)

# Disable pager during menu navigation.
os.environ['GIT_PAGER'] = "cat"

# This is basically a YAML file which contains the state of the release tool.
# The easiest way to understand its format is by just looking at it after the
# key fields have been filled in. This is updated continuously while the script
# is operating.
# The repositories are indexed by their Git repository names.
RELEASE_TOOL_STATE = None

JENKINS_SERVER = "https://mender-jenkins.mender.io"
JENKINS_JOB = "job/mender-builder"
JENKINS_USER = os.getenv("JENKINS_USER")
JENKINS_PASSWORD = os.getenv("JENKINS_PASSWORD")

# What we use in commits messages when bumping versions.
VERSION_BUMP_STRING = "Bump versions for Mender"

# Whether or not pushes should really happen.
PUSH = True
# Whether this is a dry-run.
DRY_RUN = False

class RepoName:
    """An object that contains a pair of links for the docker and git names of a
    repository."""

    # Name of container in docker-compose file.
    container = None
    # Name of image in docker hub
    docker = None
    # Name of repository in Git. (what we index by)
    git = None
    # Whether or not this repository has a Docker container.
    has_container = None

    def __init__(self, container, docker, git, has_container):
        self.container = container
        self.docker = docker
        self.git = git
        self.has_container = has_container

# All our repos, and also a map from docker-compose image name to all
# names. The key is container name, and thereafter the order is the order of the
# RepoName constructor, just above.
#
# This is the main list of repos that will be used throughout the script. If you
# add anything here, make sure to also update REPO_ALIASES (if there are
# alternate names) and GIT_TO_BUILDPARAM_MAP (which tells the tool how to
# trigger Jenkins jobs.
REPOS = {
    "mender-api-gateway": RepoName("mender-api-gateway", "api-gateway", "mender-api-gateway-docker", True),
    "mender-client": RepoName("mender-client", "mender-client-qemu", "mender", True),
    "mender-deployments": RepoName("mender-deployments", "deployments", "deployments", True),
    "mender-device-adm": RepoName("mender-device-adm", "deviceadm", "deviceadm", True),
    "mender-device-auth": RepoName("mender-device-auth", "deviceauth", "deviceauth", True),
    "mender-gui": RepoName("mender-gui", "gui", "gui", True),
    "mender-inventory": RepoName("mender-inventory", "inventory", "inventory", True),
    "mender-useradm": RepoName("mender-useradm", "useradm", "useradm", True),

    # These ones doesn't have a Docker name, but just use same as Git for
    # indexing purposes.
    "mender-artifact": RepoName("mender-artifact", "mender-artifact", "mender-artifact", False),
    "mender-integration": RepoName("mender-integration", "integration", "integration", False),
}

# These are optional repositories that aren't included when iterating over
# repositories, but that are available for querying.
OPTIONAL_REPOS = {
    "mender-tenantadm": RepoName("mender-tenantadm", "tenantadm", "tenantadm", True),
}

# Some convenient aliases, mainly because Git phrasing differs slightly from
# Docker.
REPO_ALIASES = {
    "api-gateway-docker": "mender-api-gateway",
    "deviceadm": "mender-device-adm",
    "deviceauth": "mender-device-auth",
    "mender": "mender-client",
    "mender-client-qemu": "mender-client",
    "mender-api-gateway-docker": "mender-api-gateway",
}

# A map from git repo name to build parameter name in Jenkins.
GIT_TO_BUILDPARAM_MAP = {
    "mender-api-gateway-docker": "MENDER_API_GATEWAY_DOCKER_REV",
    "deployments": "DEPLOYMENTS_REV",
    "deviceadm": "DEVICEADM_REV",
    "deviceauth": "DEVICEAUTH_REV",
    "gui": "GUI_REV",
    "inventory": "INVENTORY_REV",
    "useradm": "USERADM_REV",

    "mender": "MENDER_REV",
    "mender-artifact": "MENDER_ARTIFACT_REV",

    "integration": "INTEGRATION_REV",
}

# These will be saved along with the state if they are changed.
EXTRA_BUILDPARAMS = {
    "BUILD_BBB": "on",
    "BUILD_QEMU": "on",
    "BUILD_QEMU_RAW_FLASH": "on",
    "BUILD_RPI3": "on",
    "CLEAN_BUILD_CACHE": "",
    "MENDER_QA_REV": "master",
    "MENDER_STRESS_TEST_CLIENT_REV": "master",
    "META_MENDER_REV": "pyro",
    "META_OPENEMBEDDED_REV": "pyro",
    "META_RASPBERRYPI_REV": "pyro",
    "POKY_REV": "pyro",
    "PUSH_CONTAINERS": "on",
    "RUN_INTEGRATION_TESTS": "on",
    "TENANTADM_REV": "master",
    "TEST_BBB": "",
    "TEST_QEMU": "on",
    "TEST_RPI3": "on",
    "TESTS_IN_PARALLEL": "8",
    "UPLOAD_OUTPUT": "",
}

def integration_dir():
    """Return the location of the integration repository."""

    if os.path.isabs(sys.argv[0]):
        return os.path.normpath(os.path.dirname(os.path.dirname(sys.argv[0])))
    else:
        return os.path.normpath(os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), ".."))

def ask(text):
    """Ask a question and return the reply."""<commit_msg>release_tool: Adapt Jenkins parameters to new pipeline.<commit_after>#!/usr/bin/env python3

import argparse
import copy
import json
import os
import re
import shutil
import subprocess
import sys
import traceback
import logging

try:
    import yaml
except ImportError:
    print("PyYAML missing, try running 'sudo pip3 install pyyaml'.")
    sys.exit(2)

# Disable pager during menu navigation.
os.environ['GIT_PAGER'] = "cat"

# This is basically a YAML file which contains the state of the release tool.
# The easiest way to understand its format is by just looking at it after the
# key fields have been filled in. This is updated continuously while the script
# is operating.
# The repositories are indexed by their Git repository names.
RELEASE_TOOL_STATE = None

JENKINS_SERVER = "https://mender-jenkins.mender.io"
JENKINS_JOB = "job/mender-builder"
JENKINS_USER = os.getenv("JENKINS_USER")
JENKINS_PASSWORD = os.getenv("JENKINS_PASSWORD")

# What we use in commits messages when bumping versions.
VERSION_BUMP_STRING = "Bump versions for Mender"

# Whether or not pushes should really happen.
PUSH = True
# Whether this is a dry-run.
DRY_RUN = False

class RepoName:
    """An object that contains a pair of links for the docker and git names of a
    repository."""

    # Name of container in docker-compose file.
    container = None
    # Name of image in docker hub
    docker = None
    # Name of repository in Git. (what we index by)
    git = None
    # Whether or not this repository has a Docker container.
    has_container = None

    def __init__(self, container, docker, git, has_container):
        self.container = container
        self.docker = docker
        self.git = git
        self.has_container = has_container

# All our repos, and also a map from docker-compose image name to all
# names. The key is container name, and thereafter the order is the order of the
# RepoName constructor, just above.
#
# This is the main list of repos that will be used throughout the script. If you
# add anything here, make sure to also update REPO_ALIASES (if there are
# alternate names) and GIT_TO_BUILDPARAM_MAP (which tells the tool how to
# trigger Jenkins jobs.
REPOS = {
    "mender-api-gateway": RepoName("mender-api-gateway", "api-gateway", "mender-api-gateway-docker", True),
    "mender-client": RepoName("mender-client", "mender-client-qemu", "mender", True),
    "mender-deployments": RepoName("mender-deployments", "deployments", "deployments", True),
    "mender-device-adm": RepoName("mender-device-adm", "deviceadm", "deviceadm", True),
    "mender-device-auth": RepoName("mender-device-auth", "deviceauth", "deviceauth", True),
    "mender-gui": RepoName("mender-gui", "gui", "gui", True),
    "mender-inventory": RepoName("mender-inventory", "inventory", "inventory", True),
    "mender-useradm": RepoName("mender-useradm", "useradm", "useradm", True),

    # These ones doesn't have a Docker name, but just use same as Git for
    # indexing purposes.
    "mender-artifact": RepoName("mender-artifact", "mender-artifact", "mender-artifact", False),
    "mender-integration": RepoName("mender-integration", "integration", "integration", False),
}

# These are optional repositories that aren't included when iterating over
# repositories, but that are available for querying.
OPTIONAL_REPOS = {
    "mender-tenantadm": RepoName("mender-tenantadm", "tenantadm", "tenantadm", True),
}

# Some convenient aliases, mainly because Git phrasing differs slightly from
# Docker.
REPO_ALIASES = {
    "api-gateway-docker": "mender-api-gateway",
    "deviceadm": "mender-device-adm",
    "deviceauth": "mender-device-auth",
    "mender": "mender-client",
    "mender-client-qemu": "mender-client",
    "mender-api-gateway-docker": "mender-api-gateway",
}

# A map from git repo name to build parameter name in Jenkins.
GIT_TO_BUILDPARAM_MAP = {
    "mender-api-gateway-docker": "MENDER_API_GATEWAY_DOCKER_REV",
    "deployments": "DEPLOYMENTS_REV",
    "deviceadm": "DEVICEADM_REV",
    "deviceauth": "DEVICEAUTH_REV",
    "gui": "GUI_REV",
    "inventory": "INVENTORY_REV",
    "useradm": "USERADM_REV",

    "mender": "MENDER_REV",
    "mender-artifact": "MENDER_ARTIFACT_REV",

    "integration": "INTEGRATION_REV",
}

# These will be saved along with the state if they are changed.
EXTRA_BUILDPARAMS = {
    "BUILD_BEAGLEBONEBLACK": "on",
    "BUILD_QEMU_SDIMG": "on",
    "BUILD_QEMU_RAW_FLASH": "on",
    "BUILD_RASPBERRYPI3": "on",
    "CLEAN_BUILD_CACHE": "",
    "MENDER_QA_REV": "master",
    "MENDER_STRESS_TEST_CLIENT_REV": "master",
    "META_MENDER_REV": "rocko",
    "META_OPENEMBEDDED_REV": "rocko",
    "META_RASPBERRYPI_REV": "rocko",
    "POKY_REV": "rocko",
    "PUBLISH_ARTIFACTS": "on",
    "RUN_INTEGRATION_TESTS": "on",
    "TENANTADM_REV": "master",
    "TEST_BEAGLEBONEBLACK": "on",
    "TEST_QEMU_SDIMG": "on",
    "TEST_QEMU_RAW_FLASH": "on",
    "TEST_RASPBERRYPI3": "on",
    "TESTS_IN_PARALLEL": "7",
}

def integration_dir():
    """Return the location of the integration repository."""

    if os.path.isabs(sys.argv[0]):
        return os.path.normpath(os.path.dirname(os.path.dirname(sys.argv[0])))
    else:
        return os.path.normpath(os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), ".."))

def ask(text):
    """Ask a question and return the reply."""
{'fraction_non_alphanumeric': 0.1270623933244832, 'fraction_numerical': 0.0017068082685378343, 'mean_word_length': 4.14990234375, 'pattern_counts': {'":': 92, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514372', 'n_tokens_mistral': 3730, 'n_tokens_neox': 3590, 'n_words': 1009}