<commit_before>import argparse
import json
import logging
import os
import re
import shlex
import socket
import sys
import uuid
from typing import Any
from typing import Dict
from typing import List
from typing import Mapping
from typing import Optional
from typing import Tuple
from typing import Union

import yaml
from boto3.exceptions import Boto3Error
from service_configuration_lib.spark_config import get_aws_credentials
from service_configuration_lib.spark_config import get_dra_configs
from service_configuration_lib.spark_config import get_history_url
from service_configuration_lib.spark_config import get_resources_requested
from service_configuration_lib.spark_config import get_signalfx_url
from service_configuration_lib.spark_config import get_spark_conf
from service_configuration_lib.spark_config import get_spark_hourly_cost
from service_configuration_lib.spark_config import send_and_calculate_resources_cost
from service_configuration_lib.spark_config import UnsupportedClusterManagerException

from paasta_tools.cli.cmds.check import makefile_responds_to
from paasta_tools.cli.cmds.cook_image import paasta_cook_image
from paasta_tools.cli.utils import get_instance_config
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_instances
from paasta_tools.clusterman import get_clusterman_metrics
from paasta_tools.kubernetes_tools import limit_size_with_hash
from paasta_tools.spark_tools import DEFAULT_SPARK_SERVICE
from paasta_tools.spark_tools import get_volumes_from_spark_k8s_configs
from paasta_tools.spark_tools import get_volumes_from_spark_mesos_configs
from paasta_tools.spark_tools import get_webui_url
from paasta_tools.spark_tools import inject_spark_conf_str
from paasta_tools.utils import _run
from paasta_tools.utils import DEFAULT_SOA_DIR
from paasta_tools.utils import get_possible_launched_by_user_variable_from_env
from paasta_tools.utils import get_username
from paasta_tools.utils import InstanceConfig
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoConfigurationForServiceError
from paasta_tools.utils import NoDeploymentsAvailable
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PaastaNotConfiguredError
from paasta_tools.utils import SystemPaastaConfig


DEFAULT_AWS_REGION = "us-west-2"
DEFAULT_SPARK_WORK_DIR = "/spark_driver"
DEFAULT_SPARK_DOCKER_IMAGE_PREFIX = "paasta-spark-run"
DEFAULT_SPARK_DOCKER_REGISTRY = "docker-dev.yelpcorp.com"
SENSITIVE_ENV = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
clusterman_metrics, CLUSTERMAN_YAML_FILE_PATH = get_clusterman_metrics()
CLUSTER_MANAGER_MESOS = "mesos"
CLUSTER_MANAGER_K8S = "kubernetes"
CLUSTER_MANAGER_LOCAL = "local"
CLUSTER_MANAGERS = {CLUSTER_MANAGER_MESOS, CLUSTER_MANAGER_K8S, CLUSTER_MANAGER_LOCAL}
# Reference: https://spark.apache.org/docs/latest/configuration.html#application-properties
DEFAULT_DRIVER_CORES_BY_SPARK = 1
DEFAULT_DRIVER_MEMORY_BY_SPARK = "1g"
# Extra room for memory overhead and for any other running inside container
DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2

# Mass enable DRA configs
EXECUTOR_RANGES = {(0, 8)}
<commit_msg>MLCOMPUTE-216 | increase dra mass rolout executor range to 16<commit_after>import argparse
import json
import logging
import os
import re
import shlex
import socket
import sys
import uuid
from typing import Any
from typing import Dict
from typing import List
from typing import Mapping
from typing import Optional
from typing import Tuple
from typing import Union

import yaml
from boto3.exceptions import Boto3Error
from service_configuration_lib.spark_config import get_aws_credentials
from service_configuration_lib.spark_config import get_dra_configs
from service_configuration_lib.spark_config import get_history_url
from service_configuration_lib.spark_config import get_resources_requested
from service_configuration_lib.spark_config import get_signalfx_url
from service_configuration_lib.spark_config import get_spark_conf
from service_configuration_lib.spark_config import get_spark_hourly_cost
from service_configuration_lib.spark_config import send_and_calculate_resources_cost
from service_configuration_lib.spark_config import UnsupportedClusterManagerException

from paasta_tools.cli.cmds.check import makefile_responds_to
from paasta_tools.cli.cmds.cook_image import paasta_cook_image
from paasta_tools.cli.utils import get_instance_config
from paasta_tools.cli.utils import lazy_choices_completer
from paasta_tools.cli.utils import list_instances
from paasta_tools.clusterman import get_clusterman_metrics
from paasta_tools.kubernetes_tools import limit_size_with_hash
from paasta_tools.spark_tools import DEFAULT_SPARK_SERVICE
from paasta_tools.spark_tools import get_volumes_from_spark_k8s_configs
from paasta_tools.spark_tools import get_volumes_from_spark_mesos_configs
from paasta_tools.spark_tools import get_webui_url
from paasta_tools.spark_tools import inject_spark_conf_str
from paasta_tools.utils import _run
from paasta_tools.utils import DEFAULT_SOA_DIR
from paasta_tools.utils import get_possible_launched_by_user_variable_from_env
from paasta_tools.utils import get_username
from paasta_tools.utils import InstanceConfig
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoConfigurationForServiceError
from paasta_tools.utils import NoDeploymentsAvailable
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PaastaNotConfiguredError
from paasta_tools.utils import SystemPaastaConfig


DEFAULT_AWS_REGION = "us-west-2"
DEFAULT_SPARK_WORK_DIR = "/spark_driver"
DEFAULT_SPARK_DOCKER_IMAGE_PREFIX = "paasta-spark-run"
DEFAULT_SPARK_DOCKER_REGISTRY = "docker-dev.yelpcorp.com"
SENSITIVE_ENV = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
clusterman_metrics, CLUSTERMAN_YAML_FILE_PATH = get_clusterman_metrics()
CLUSTER_MANAGER_MESOS = "mesos"
CLUSTER_MANAGER_K8S = "kubernetes"
CLUSTER_MANAGER_LOCAL = "local"
CLUSTER_MANAGERS = {CLUSTER_MANAGER_MESOS, CLUSTER_MANAGER_K8S, CLUSTER_MANAGER_LOCAL}
# Reference: https://spark.apache.org/docs/latest/configuration.html#application-properties
DEFAULT_DRIVER_CORES_BY_SPARK = 1
DEFAULT_DRIVER_MEMORY_BY_SPARK = "1g"
# Extra room for memory overhead and for any other running inside container
DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2

# Mass enable DRA configs
EXECUTOR_RANGES = {(0, 16)}

{'fraction_non_alphanumeric': 0.036550370034737954, 'fraction_numerical': 0.004228968433771334, 'mean_word_length': 9.140888208269525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3499133', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2067, 'n_words': 488}