<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import sys
import os
import re
import subprocess
import shutil
from optparse import OptionParser


class Host:
  """
  Abstraction of the elements unique to each Ambari Agent running on this VM.
  """
  def __init__(self, host_name, ping_port, home_dir):
    self.host_name = host_name
    self.ping_port = ping_port
    self.home_dir = home_dir


class Multiplier:
  """
  In order to perform scale testing, this class bootstraps multiple Ambari Agents to run on the same host.
  Each Ambari Agent has its own home directory with subdirectories for configs, logs, etc.
  Further, each agent is given a unique port number.

  Usage: python agent-multiplier.py [command]
  [command] = start | stop | restart | status

  Optional flags:
  -v --verbose : Increase logging

  """
  CONFIG_FILE = "/etc/ambari-agent/conf/agent-multiplier.conf"

  def __init__(self, args):
    parser = OptionParser()
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                      help="Verbose logging")
    (options, args) = parser.parse_args(args)
    self.verbose = options.verbose

    self.home_dir = "/home/"
    # Subdirectories to create inside the home dir for the given agent.
    self.log_dir = "/var/log/ambari-agent"
    self.config_dir = "/etc/ambari-agent/conf"
    self.pid_file = "/var/run/ambari-agent/ambari-agent.pid"

    # Ambari Agent config file to use as a template
    # Will change hostname and port after copying
    self.source_config_file = "/etc/ambari-agent/conf/ambari-agent.ini"
    self.base_ping_port = 5000

    self.start = 0
    self.num = 0
    self.prefix = None

    # Parse above params from config file, which must exist
    self.parse_configs()

    if len(args) != 2:
      print "Sample Usage: python agent_multiplier.py [action]\n" \
            "actions: start|stop|restart|status"
    self.command = args[1]

    # Validate configs
    self.validate()

    print "*** Params ***"
    print "Start: %d" % self.start
    print "Num: %d" % self.num
    print "Prefix: %s" % self.prefix
    print "Command: %s" % self.command

    # All hostnames that will be managed by Ambari Agents on this host
    self.hosts = []
    for i in range(self.start, self.start + self.num):
      host_name = "%s-%04d" % (self.prefix, i)
      host_home_dir = os.path.join(self.home_dir, host_name)
      host = Host(host_name, self.base_ping_port + i, host_home_dir)
      self.hosts.append(host)

    self.bootstrap()

  def parse_configs(self):
    """
    Parse the configuration file to set the config params.
    """
    if not os.path.exists(self.CONFIG_FILE):
      print "Did not find Agent Multiplier config file: %s" % str(self.CONFIG_FILE)
      sys.exit(-1)

    params = {}
    with open(self.CONFIG_FILE, "r") as f:
      for line in f.readlines():
        index = line.index("=") if "=" in line else None
        if index is not None:
          config = line[0:index].strip()
          value = line[index+1:].strip()
          params[config] = value

    # Convert some configs to ints
    if "start" in params:
      self.start = int(params["start"])

    if "num" in params:
      self.num = int(params["num"])

    if "prefix" in params:
      self.prefix = params["prefix"].strip().lower()

  def validate(self):
    """
    Validate the configs are non-empty and contain correct ranges.
    On error, will exit with code -1
    """
    errors = []
    if self.start <= 0:
      errors.append("Start must be a positive integer")
    if self.num <= 0:
      errors.append("Number of agents on this host must be a positive integer")
    if self.prefix is None or self.prefix.strip() == "":
      errors.append("Prefix is a required field")
    
    if not os.path.isfile(self.source_config_file):
      errors.append("Ambari Agent config file does not exist at %s" % self.source_config_file)

    valid_commands = set(["start", "stop", "restart", "status"])
    if self.command is None or self.command not in valid_commands:
      errors.append("Command must be one of %s" % ", ".join(valid_commands))

    if len(errors) > 0:
      print "Error:"
      print "\n".join(errors)
      sys.exit(-1)

  def bootstrap(self):
    """
    Bootstrap each Ambari Agent that will run on this host with the directories it needs and configuration file.
    """
    for host in self.hosts:
      host_name = host.host_name
      host_home_dir = host.home_dir
      host_log_dir = host_home_dir + self.log_dir
      host_config_dir = host_home_dir + self.config_dir
      host_pid_file = host_home_dir + self.pid_file
      host_pid_dir = os.path.dirname(host_pid_file)

      if self.verbose:
        print "Analyzing host %s with port %d" % (host_name, host.ping_port)

      for dir in [host_home_dir, host_log_dir, host_config_dir, host_pid_dir]:
        if not os.path.isdir(dir):
          print "Creating dir %s" % (dir)
          os.makedirs(dir)

      # Copy config file
      host_config_file = os.path.join(host_config_dir, "ambari-agent.ini")
      if not os.path.isfile(host_config_file):
        print "Copying config file %s" % str(host_config_file)
        shutil.copyfile(self.source_config_file, host_config_file)

      # Create hostname.sh script to use custom FQDN for each agent.
      host_name_script = os.path.join(host_config_dir, "hostname.sh")
      self.create_host_name_script(host_name, host_name_script)

      # Overwrite the port and hostname
      config_dict = {"ping_port": host.ping_port,
                     "hostname_script": host_name_script,
                     "public_hostname_script": host_name_script,
                     "logdir": host_log_dir,
                     "piddir": host_pid_dir}
      self.change_config(host_config_file, config_dict)<commit_msg>AMBARI-18993. Perf: Status commands return INSTALLED even if component in STARTED state.(vbrodetskyi)<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import sys
import os
import re
import subprocess
import shutil
from optparse import OptionParser


class Host:
  """
  Abstraction of the elements unique to each Ambari Agent running on this VM.
  """
  def __init__(self, host_name, ping_port, home_dir):
    self.host_name = host_name
    self.ping_port = ping_port
    self.home_dir = home_dir


class Multiplier:
  """
  In order to perform scale testing, this class bootstraps multiple Ambari Agents to run on the same host.
  Each Ambari Agent has its own home directory with subdirectories for configs, logs, etc.
  Further, each agent is given a unique port number.

  Usage: python agent-multiplier.py [command]
  [command] = start | stop | restart | status

  Optional flags:
  -v --verbose : Increase logging

  """
  CONFIG_FILE = "/etc/ambari-agent/conf/agent-multiplier.conf"

  def __init__(self, args):
    parser = OptionParser()
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                      help="Verbose logging")
    (options, args) = parser.parse_args(args)
    self.verbose = options.verbose

    self.home_dir = "/home/"
    # Subdirectories to create inside the home dir for the given agent.
    self.log_dir = "/var/log/ambari-agent"
    self.config_dir = "/etc/ambari-agent/conf"
    self.pid_file = "/var/run/ambari-agent/ambari-agent.pid"
    self.prefix_dir = "/var/lib/ambari-agent/data"

    # Ambari Agent config file to use as a template
    # Will change hostname and port after copying
    self.source_config_file = "/etc/ambari-agent/conf/ambari-agent.ini"
    self.source_version_file = "/var/lib/ambari-agent/data/version"
    self.base_ping_port = 5000

    self.start = 0
    self.num = 0
    self.prefix = None

    # Parse above params from config file, which must exist
    self.parse_configs()

    if len(args) != 2:
      print "Sample Usage: python agent_multiplier.py [action]\n" \
            "actions: start|stop|restart|status"
    self.command = args[1]

    # Validate configs
    self.validate()

    print "*** Params ***"
    print "Start: %d" % self.start
    print "Num: %d" % self.num
    print "Prefix: %s" % self.prefix
    print "Command: %s" % self.command

    # All hostnames that will be managed by Ambari Agents on this host
    self.hosts = []
    for i in range(self.start, self.start + self.num):
      host_name = "%s-%04d" % (self.prefix, i)
      host_home_dir = os.path.join(self.home_dir, host_name)
      host = Host(host_name, self.base_ping_port + i, host_home_dir)
      self.hosts.append(host)

    self.bootstrap()

  def parse_configs(self):
    """
    Parse the configuration file to set the config params.
    """
    if not os.path.exists(self.CONFIG_FILE):
      print "Did not find Agent Multiplier config file: %s" % str(self.CONFIG_FILE)
      sys.exit(-1)

    params = {}
    with open(self.CONFIG_FILE, "r") as f:
      for line in f.readlines():
        index = line.index("=") if "=" in line else None
        if index is not None:
          config = line[0:index].strip()
          value = line[index+1:].strip()
          params[config] = value

    # Convert some configs to ints
    if "start" in params:
      self.start = int(params["start"])

    if "num" in params:
      self.num = int(params["num"])

    if "prefix" in params:
      self.prefix = params["prefix"].strip().lower()

  def validate(self):
    """
    Validate the configs are non-empty and contain correct ranges.
    On error, will exit with code -1
    """
    errors = []
    if self.start <= 0:
      errors.append("Start must be a positive integer")
    if self.num <= 0:
      errors.append("Number of agents on this host must be a positive integer")
    if self.prefix is None or self.prefix.strip() == "":
      errors.append("Prefix is a required field")
    
    if not os.path.isfile(self.source_config_file):
      errors.append("Ambari Agent config file does not exist at %s" % self.source_config_file)

    valid_commands = set(["start", "stop", "restart", "status"])
    if self.command is None or self.command not in valid_commands:
      errors.append("Command must be one of %s" % ", ".join(valid_commands))

    if len(errors) > 0:
      print "Error:"
      print "\n".join(errors)
      sys.exit(-1)

  def bootstrap(self):
    """
    Bootstrap each Ambari Agent that will run on this host with the directories it needs and configuration file.
    """
    for host in self.hosts:
      host_name = host.host_name
      host_home_dir = host.home_dir
      host_log_dir = host_home_dir + self.log_dir
      host_config_dir = host_home_dir + self.config_dir
      host_pid_file = host_home_dir + self.pid_file
      host_pid_dir = os.path.dirname(host_pid_file)
      host_prefix = host_home_dir + self.prefix_dir

      if self.verbose:
        print "Analyzing host %s with port %d" % (host_name, host.ping_port)

      for dir in [host_home_dir, host_log_dir, host_config_dir, host_pid_dir, host_prefix]:
        if not os.path.isdir(dir):
          print "Creating dir %s" % (dir)
          os.makedirs(dir)

      # Copy config file
      host_config_file = os.path.join(host_config_dir, "ambari-agent.ini")
      if not os.path.isfile(host_config_file):
        print "Copying config file %s" % str(host_config_file)
        shutil.copyfile(self.source_config_file, host_config_file)

      # Copy version file
      version_file = os.path.join(host_prefix, "version")
      if not os.path.isfile(version_file):
        print "Copying version file %s" % str(version_file)
        shutil.copyfile(self.source_version_file, version_file)

      # Create hostname.sh script to use custom FQDN for each agent.
      host_name_script = os.path.join(host_config_dir, "hostname.sh")
      self.create_host_name_script(host_name, host_name_script)

      # Overwrite the port and hostname
      config_dict = {"ping_port": host.ping_port,
                     "hostname_script": host_name_script,
                     "public_hostname_script": host_name_script,
                     "logdir": host_log_dir,
                     "piddir": host_pid_dir,
                     "prefix": host_prefix}
      self.change_config(host_config_file, config_dict)
{'fraction_non_alphanumeric': 0.10125474175663846, 'fraction_numerical': 0.0035745550043770062, 'mean_word_length': 2.9495822529530393, 'pattern_counts': {'":': 13, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1875017', 'n_tokens_mistral': 4250, 'n_tokens_neox': 4139, 'n_words': 1315}