<commit_before>#!/bin/bash

THIS_SCRIPT=$0
EXECUTION_MODE=$1

SHARED_SCRIPTS_HOME="$(cd `dirname $THIS_SCRIPT` && pwd)"
CONFIG_DIR="$(cd $SHARED_SCRIPTS_HOME/../config && pwd)"

source "$SHARED_SCRIPTS_HOME/verifiers/exit_if_execution_mode_missing.sh" "`basename $THIS_SCRIPT`" $EXECUTION_MODE
source "$SHARED_SCRIPTS_HOME/verifiers/exit_if_not_using_sudo.sh"

source "$CONFIG_DIR/load_config.sh" "python_system.config" $EXECUTION_MODE

PIP_REQUIREMENTS_DIR="$(cd "$SHARED_SCRIPTS_HOME/../../pip/requirements" && pwd)"

function exit_if_pip_not_installed
{
    PIP_PATH=`which pip`

    # check if pip is installed
    if [ -z "$PIP_PATH" ]; then
        printf "\n>> pip not installed or not on the PATH\n\n"
        exit -1
    fi
}

function install_packages_with_pip
{
    # function parameters:
    REQUIREMENTS_FILE_NAME=$1
    PACKAGE_SET_NAME=$2

    cd "$PACKAGE_DOWNLOAD_DIR"
    printf "\n>> Installing/upgrading $PACKAGE_SET_NAME packages:\n"
    pip install -M -r "$PIP_REQUIREMENTS_DIR/$REQUIREMENTS_FILE_NAME"
}

function install_system_packages
{
    printf "\n>> Current system packages:\n"
    pip freeze

    install_packages_with_pip "0_system.txt" "system"
    install_packages_with_pip "1_deployment.txt" "deployment"

    printf "\n>> Installed system packages:\n"
    pip freeze
}


exit_if_pip_not_installed
install_system_packages
<commit_msg>[#23633289] Removed the deployment Python packages from the system package installation since these will now be in a separate CI virtualenv instead.<commit_after>#!/bin/bash

THIS_SCRIPT=$0
EXECUTION_MODE=$1

SHARED_SCRIPTS_HOME="$(cd `dirname $THIS_SCRIPT` && pwd)"
CONFIG_DIR="$(cd $SHARED_SCRIPTS_HOME/../config && pwd)"

source "$SHARED_SCRIPTS_HOME/verifiers/exit_if_execution_mode_missing.sh" "`basename $THIS_SCRIPT`" $EXECUTION_MODE
source "$SHARED_SCRIPTS_HOME/verifiers/exit_if_not_using_sudo.sh"

source "$CONFIG_DIR/load_config.sh" "python_system.config" $EXECUTION_MODE

PIP_REQUIREMENTS_DIR="$(cd "$SHARED_SCRIPTS_HOME/../../pip/requirements" && pwd)"

function exit_if_pip_not_installed
{
    PIP_PATH=`which pip`

    # check if pip is installed
    if [ -z "$PIP_PATH" ]; then
        printf "\n>> pip not installed or not on the PATH\n\n"
        exit -1
    fi
}

function install_packages_with_pip
{
    # function parameters:
    REQUIREMENTS_FILE_NAME=$1
    PACKAGE_SET_NAME=$2

    cd "$PACKAGE_DOWNLOAD_DIR"
    printf "\n>> Installing/upgrading $PACKAGE_SET_NAME packages:\n"
    pip install -M -r "$PIP_REQUIREMENTS_DIR/$REQUIREMENTS_FILE_NAME"
}

function install_system_packages
{
    printf "\n>> Current system packages:\n"
    pip freeze

    install_packages_with_pip "0_system.txt" "system"

    printf "\n>> Installed system packages:\n"
    pip freeze
}


exit_if_pip_not_installed
install_system_packages

{'fraction_non_alphanumeric': 0.10992529348986126, 'fraction_numerical': 0.007470651013874066, 'mean_word_length': 4.703853955375253, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3434460', 'n_tokens_mistral': 1065, 'n_tokens_neox': 999, 'n_words': 202}