<commit_before>#!/usr/bin/perl
# -*- cperl -*-

# This is a transformation of the "mysql-test-run" Bourne shell script
# to Perl. This is just an intermediate step, the goal is to rewrite
# the Perl script to C. The complexity of the mysql-test-run script
# makes it a bit hard to write and debug it as a C program directly,
# so this is considered a prototype.
#
# Because of this the Perl coding style may in some cases look a bit
# funny. The rules used are
#
#   - The coding style is as close as possible to the C/C++ MySQL
#     coding standard.
#
#   - Where NULL is to be returned, the undefined value is used.
#
#   - Regexp comparisons are simple and can be translated to strcmp
#     and other string functions. To ease this transformation matching
#     is done in the lib "lib/mtr_match.pl", i.e. regular expressions
#     should be avoided in the main program.
#
#   - The "unless" construct is not to be used. It is the same as "if !".
#
#   - opendir/readdir/closedir is used instead of glob()/<*>.
#
#   - All lists of arguments to send to commands are Perl lists/arrays,
#     not strings we append args to. Within reason, most string
#     concatenation for arguments should be avoided.
#
#   - sprintf() is to be used, within reason, for all string creation.
#     This mtr_add_arg() function is also based on sprintf(), i.e. you
#     use a format string and put the variable argument in the argument
#     list.
#
#   - Functions defined in the main program are not to be prefixed,
#     functions in "library files" are to be prefixed with "mtr_" (for
#     Mysql-Test-Run). There are some exceptions, code that fits best in
#     the main program, but are put into separate files to avoid
#     clutter, may be without prefix.
#
#   - All stat/opendir/-f/ is to be kept in collect_test_cases(). It
#     will create a struct that the rest of the program can use to get
#     the information. This separates the "find information" from the
#     "do the work" and makes the program more easy to maintain.
#
#   - At the moment, there are tons of "global" variables that control
#     this script, even accessed from the files in "lib/*.pl". This
#     will change over time, for now global variables are used instead
#     of using %opt, %path and %exe hashes, because I want more
#     compile time checking, that hashes would not give me. Once this
#     script is debugged, hashes will be used and passed as parameters
#     to functions, to more closely mimic how it would be coded in C
#     using structs.
#
#   - The rule when it comes to the logic of this program is
#
#       command_line_setup() - is to handle the logic between flags
#       collect_test_cases() - is to do its best to select what tests
#                              to run, dig out options, if needs restart etc.
#       run_testcase()       - is to run a single testcase, and follow the
#                              logic set in both above. No, or rare file
#                              system operations. If a test seems complex,
#                              it should probably not be here.
#
# A nice way to trace the execution of this script while debugging
# is to use the Devel::Trace package found at
# "http://www.plover.com/~mjd/perl/Trace/" and run this script like
# "perl -d:Trace mysql-test-run.pl"
#
# FIXME Save a PID file from this code as well, to record the process
#       id we think it has. In Cygwin, a fork creates one Cygwin process,
#       and then the real Win32 process. Cygwin Perl can only kill Cygwin
#       processes. And "mysqld --bootstrap ..." doesn't save a PID file.

$Devel::Trace::TRACE= 0;       # Don't trace boring init stuff

#require 5.6.1;
use File::Path;
use File::Basename;
use Cwd;
use Getopt::Long;
use Sys::Hostname;
#use Carp;
use IO::Socket;
use IO::Socket::INET;
use Data::Dumper;
use strict;
#use diagnostics;

require "lib/mtr_cases.pl";
require "lib/mtr_process.pl";
require "lib/mtr_io.pl";
require "lib/mtr_gcov.pl";
require "lib/mtr_gprof.pl";
require "lib/mtr_report.pl";
require "lib/mtr_diff.pl";
require "lib/mtr_match.pl";
require "lib/mtr_misc.pl";

$Devel::Trace::TRACE= 1;

# Used by gcov
our @mysqld_src_dirs=
  (
   "strings",
   "mysys",
   "include",
   "extra",
   "regex",
   "isam",
   "merge",
   "myisam",
   "myisammrg",
   "heap",
   "sql",
  );

##############################################################################
#
#  Default settings
#
##############################################################################

# We are to use handle_options() in "mysys/my_getopt.c" for the C version
#
# In the C version we want to use structs and, in some cases, arrays of
# structs. We let each struct be a separate hash.

# Misc global variables

our $glob_win32=                  0; # OS and native Win32 executables
our $glob_win32_perl=             0; # ActiveState Win32 Perl
our $glob_cygwin_perl=            0; # Cygwin Perl
our $glob_cygwin_shell=           undef;
our $glob_mysql_test_dir=         undef;
our $glob_mysql_bench_dir=        undef;
our $glob_hostname=               undef;
our $glob_scriptname=             undef;
our $glob_use_running_server=     0;
our $glob_use_running_ndbcluster= 0;
our $glob_use_embedded_server=    0;

our $glob_basedir;

# The total result

our $path_charsetsdir;
our $path_client_bindir;
our $path_language;
our $path_timefile;
our $path_manager_log;           # Used by mysqldadmin
our $path_slave_load_tmpdir;     # What is this?!
our $path_my_basedir;
our $opt_vardir;                 # A path but set directly on cmd line
our $opt_tmpdir;                 # A path but set directly on cmd line

our $opt_usage;
our $opt_suite;

our $opt_netware;

our $opt_script_debug= 0;  # Script debugging, enable with --script-debug

# Options FIXME not all....

our $exe_master_mysqld;
our $exe_mysql;
our $exe_mysqladmin;
our $exe_mysqlbinlog;
our $exe_mysql_client_test;
our $exe_mysqld;
our $exe_mysqldump;              # Called from test case
our $exe_mysqlshow;              # Called from test case
our $exe_mysql_fix_system_tables;
our $exe_mysqltest;
our $exe_slave_mysqld;

our $opt_bench= 0;
our $opt_small_bench= 0;
our $opt_big_test= 0;            # Send --big-test to mysqltest

our @opt_extra_mysqld_opt;

our $opt_compress;
our $opt_current_test;
our $opt_ddd;
our $opt_debug;
our $opt_do_test;
our @opt_cases;                  # The test cases names in argv
our $opt_embedded_server;
our $opt_extern;
our $opt_fast;
our $opt_force;
our $opt_reorder;

our $opt_gcov;
our $opt_gcov_err;
our $opt_gcov_msg;

our $opt_gdb;
our $opt_client_gdb;
our $opt_manual_gdb;

our $opt_gprof;
our $opt_gprof_dir;
our $opt_gprof_master;
our $opt_gprof_slave;

our $opt_local;
our $opt_local_master;

our $master;                    # Will be struct in C
our $slave;

our $opt_ndbcluster_port;
our $opt_ndbconnectstring;

our $opt_no_manager;            # Does nothing now, we never use manager
our $opt_manager_port;          # Does nothing now, we never use manager

our $opt_old_master;

our $opt_record;

our $opt_result_ext;

our $opt_skip;
our $opt_skip_rpl;
our $opt_skip_test;

our $opt_sleep;
our $opt_ps_protocol;

our $opt_sleep_time_after_restart= 1;
our $opt_sleep_time_for_delete=    10;

our $opt_socket;

our $opt_source_dist;

our $opt_start_and_exit;
our $opt_start_dirty;
our $opt_start_from;

our $opt_strace_client;

our $opt_timer;

our $opt_user;
our $opt_user_test;

our $opt_valgrind;
our $opt_valgrind_all;
our $opt_valgrind_options;

our $opt_verbose;

our $opt_wait_for_master;
our $opt_wait_for_slave;
our $opt_wait_timeout=  10;

our $opt_warnings;

our $opt_udiff;

our $opt_skip_ndbcluster;
our $opt_with_ndbcluster;
our $opt_with_openssl;

our $exe_ndb_mgm;
our $path_ndb_tools_dir;
our $path_ndb_backup_dir;
our $file_ndb_testrun_log;
our $flag_ndb_status_ok= 1;

######################################################################
#
#  Function declarations
#
######################################################################

sub main ();
sub initial_setup ();
sub command_line_setup ();
sub executable_setup ();
sub environment_setup ();
sub kill_running_server ();
sub kill_and_cleanup ();
sub ndbcluster_install ();
sub ndbcluster_start ();
sub ndbcluster_stop ();
sub run_benchmarks ($);
sub run_tests ();
sub mysql_install_db ();
sub install_db ($$);
sub run_testcase ($);
sub report_failure_and_restart ($);
sub do_before_start_master ($$);
sub do_before_start_slave ($$);
sub mysqld_start ($$$$);
sub mysqld_arguments ($$$$$);
sub stop_masters_slaves ();
sub stop_masters ();
sub stop_slaves ();
sub run_mysqltest ($);
sub usage ($);

######################################################################
#
#  Main program
#
######################################################################

main();

sub main () {

  initial_setup();
  command_line_setup();
  executable_setup();
  environment_setup();
  signal_setup();

  if ( $opt_gcov )
  {
    gcov_prepare();
  }

  if ( $opt_gprof )
  {
    gprof_prepare();
  }

  if ( ! $glob_use_running_server )
  {

    if ( $opt_start_dirty )
    {
      kill_running_server();
    }
    else
    {
      kill_and_cleanup();
      mysql_install_db();

#    mysql_loadstd();  FIXME copying from "std_data" .frm and
#                      .MGR but there are none?!
    }
  }

  if ( $opt_start_and_exit or $opt_start_dirty )
  {
    if ( ndbcluster_start() )
    {
      mtr_error("Can't start ndbcluster");
    }
    if ( mysqld_start('master',0,[],[]) )
    {
      mtr_report("Servers started, exiting");
    }
    else
    {
      mtr_error("Can't start the mysqld server");
    }
  }
  else
  {
    if ( $opt_bench )
    {
      run_benchmarks(shift);      # Shift what? Extra arguments?!
    }
    else
    {
      run_tests();
    }
  }

  mtr_exit(0);
}

##############################################################################
#
#  Initial setup independent on command line arguments
#
##############################################################################

sub initial_setup () {

  select(STDOUT);
  $| = 1;                       # Make unbuffered

  $glob_scriptname=  basename($0);

  $glob_win32_perl=  ($^O eq "MSWin32");
  $glob_cygwin_perl= ($^O eq "cygwin");
  $glob_win32=       ($glob_win32_perl or $glob_cygwin_perl);

  # We require that we are in the "mysql-test" directory
  # to run mysql-test-run

  if (! -f $glob_scriptname)
  {
    mtr_error("Can't find the location for the mysql-test-run script\n" .
              "Go to to the mysql-test directory and execute the script " .
              "as follows:\n./$glob_scriptname");
  }

  if ( -d "../sql" )
  {
    $opt_source_dist=  1;
  }

  $glob_hostname=  mtr_full_hostname();

  # 'basedir' is always parent of "mysql-test" directory
  $glob_mysql_test_dir=  cwd();
  if ( $glob_cygwin_perl )
  {
    # Windows programs like 'mysqld' needs Windows paths
    $glob_mysql_test_dir= `cygpath -m $glob_mysql_test_dir`;
    my $shell= $ENV{'SHELL'} || "/bin/bash";
    $glob_cygwin_shell=   `cygpath -w $shell`; # The Windows path c:\...
    chomp($glob_mysql_test_dir);
    chomp($glob_cygwin_shell);
  }
  $glob_basedir=         dirname($glob_mysql_test_dir);
  $glob_mysql_bench_dir= "$glob_basedir/mysql-bench"; # FIXME make configurable

  # needs to be same length to test logging (FIXME what???)
  $path_slave_load_tmpdir=  "../../var/tmp";

  $path_my_basedir=
    $opt_source_dist ? $glob_mysql_test_dir : $glob_basedir;
}



##############################################################################
#
#  Default settings
#
##############################################################################

sub command_line_setup () {

  # These are defaults for things that are set on the command line

  $opt_suite=        "main";    # Special default suite
  my $opt_master_myport= 9306;
  my $opt_slave_myport=  9308;
  $opt_ndbcluster_port=  9350;

  # Read the command line
  # Note: Keep list, and the order, in sync with usage at end of this file

  GetOptions(
             # Control what engine/variation to run
             'embedded-server'          => \$opt_embedded_server,
             'ps-protocol'              => \$opt_ps_protocol,
             'bench'                    => \$opt_bench,
             'small-bench'              => \$opt_small_bench,
             'no-manager'               => \$opt_no_manager, # Currently not used

             # Control what test suites or cases to run
             'force'                    => \$opt_force,
             'with-ndbcluster'          => \$opt_with_ndbcluster,
             'skip-ndbcluster|skip-ndb' => \$opt_skip_ndbcluster,
             'do-test=s'                => \$opt_do_test,
             'suite=s'                  => \$opt_suite,
             'skip-rpl'                 => \$opt_skip_rpl,
             'skip-test=s'              => \$opt_skip_test,

             # Specify ports
             'master_port=i'            => \$opt_master_myport,
             'slave_port=i'             => \$opt_slave_myport,
             'ndbcluster_port=i'        => \$opt_ndbcluster_port,
             'manager-port=i'           => \$opt_manager_port, # Currently not used

             # Test case authoring
             'record'                   => \$opt_record,

             # ???
             'mysqld=s'                 => \@opt_extra_mysqld_opt,

             # Run test on running server
             'extern'                   => \$opt_extern,
             'ndbconnectstring=s'       => \$opt_ndbconnectstring,

             # Debugging
             'gdb'                      => \$opt_gdb,
             'manual-gdb'               => \$opt_manual_gdb,
             'client-gdb'               => \$opt_client_gdb,
             'ddd'                      => \$opt_ddd,
             'strace-client'            => \$opt_strace_client,
             'master-binary=s'          => \$exe_master_mysqld,
             'slave-binary=s'           => \$exe_slave_mysqld,

             # Coverage, profiling etc
             'gcov'                     => \$opt_gcov,
             'gprof'                    => \$opt_gprof,
             'valgrind'                 => \$opt_valgrind,
             'valgrind-all'             => \$opt_valgrind_all,
             'valgrind-options=s'       => \$opt_valgrind_options,

             # Misc
             'big-test'                 => \$opt_big_test,
             'compress'                 => \$opt_compress,
             'debug'                    => \$opt_debug,
             'fast'                     => \$opt_fast,
             'local'                    => \$opt_local,
             'local-master'             => \$opt_local_master,
             'netware'                  => \$opt_netware,
             'old-master'               => \$opt_old_master,
             'reorder'                  => \$opt_reorder,
             'script-debug'             => \$opt_script_debug,
             'sleep=i'                  => \$opt_sleep,
             'socket=s'                 => \$opt_socket,
             'start-dirty'              => \$opt_start_dirty,
             'start-and-exit'           => \$opt_start_and_exit,
             'start-from=s'             => \$opt_start_from,
             'timer'                    => \$opt_timer,
             'tmpdir=s'                 => \$opt_tmpdir,
             'unified-diff|udiff'       => \$opt_udiff,
             'user-test=s'              => \$opt_user_test,
             'user=s'                   => \$opt_user,
             'vardir=s'                 => \$opt_vardir,
             'verbose'                  => \$opt_verbose,
             'wait-timeout=i'           => \$opt_wait_timeout,
             'warnings|log-warnings'    => \$opt_warnings,
             'with-openssl'             => \$opt_with_openssl,

             'help|h'                   => \$opt_usage,
            ) or usage("Can't read options");

  if ( $opt_usage )
  {
    usage("");
  }

  @opt_cases= @ARGV;

  # --------------------------------------------------------------------------
  # Set the "var/" directory, as it is the base for everything else
  # --------------------------------------------------------------------------

  if ( ! $opt_vardir )
  {
    $opt_vardir= "$glob_mysql_test_dir/var";
  }

  # We make the path absolute, as the server will do a chdir() before usage
  unless ( $opt_vardir =~ m,^/, or
           ($glob_win32 and $opt_vardir =~ m,^[a-z]:/,i) )
  {
    # Make absolute path, relative test dir
    $opt_vardir= "$glob_mysql_test_dir/$opt_vardir";
  }

  # --------------------------------------------------------------------------
  # If not set, set these to defaults
  # --------------------------------------------------------------------------

  $opt_tmpdir=       "$opt_vardir/tmp" unless $opt_tmpdir;
  # FIXME maybe not needed?
  $path_manager_log= "$opt_vardir/log/manager.log"
    unless $path_manager_log;
  $opt_current_test= "$opt_vardir/log/current_test"
    unless $opt_current_test;

  # --------------------------------------------------------------------------
  # Do sanity checks of command line arguments
  # --------------------------------------------------------------------------

  if ( $opt_extern and $opt_local )
  {
    mtr_error("Can't use --extern and --local at the same time");
  }

  if ( ! $opt_socket )
  {     # FIXME set default before reading options?
#    $opt_socket=  '@MYSQL_UNIX_ADDR@';
    $opt_socket=  "/tmp/mysql.sock"; # FIXME
  }

  # --------------------------------------------------------------------------
  # Look at the command line options and set script flags
  # --------------------------------------------------------------------------

  if ( $opt_record and ! @opt_cases )
  {
    mtr_error("Will not run in record mode without a specific test case");
  }

  if ( $opt_embedded_server )
  {
    $glob_use_embedded_server= 1;
    $opt_skip_rpl= 1;              # We never run replication with embedded

    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --embedded-server");
    }
  }

  # FIXME don't understand what this is
#  if ( $opt_local_master )
#  {
#    $opt_master_myport=  3306;
#  }

  if ( $opt_small_bench )
  {
    $opt_bench=  1;
  }

  if ( $opt_sleep )
  {
    $opt_sleep_time_after_restart= $opt_sleep;
  }

  if ( $opt_gcov and ! $opt_source_dist )
  {
    mtr_error("Coverage test needs the source - please use source dist");
  }

  if ( $glob_use_embedded_server and ! $opt_source_dist )
  {
    mtr_error("Embedded server needs source tree - please use source dist");
  }

  if ( $opt_gdb )
  {
    $opt_wait_timeout=  300;
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --gdb");
    }
  }

  if ( $opt_manual_gdb )
  {
    $opt_gdb=  1;
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --manual-gdb");
    }
  }

  if ( $opt_ddd )
  {
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --ddd");
    }
  }

  if ( $opt_ndbconnectstring )
  {
    $glob_use_running_ndbcluster= 1;
    $opt_with_ndbcluster= 1;
  }
  else
  {
    $opt_ndbconnectstring= "host=localhost:$opt_ndbcluster_port";
  }

  if ( $opt_skip_ndbcluster )
  {
    $opt_with_ndbcluster= 0;
  }

  # FIXME

  #if ( $opt_valgrind or $opt_valgrind_all )
  #{
    # VALGRIND=`which valgrind` # this will print an error if not found FIXME
    # Give good warning to the user and stop
  #  if ( ! $VALGRIND )
  #  {
  #    print "You need to have the 'valgrind' program in your PATH to run mysql-test-run with option --valgrind. Valgrind's home page is http://valgrind.kde.org.\n"
  #    exit 1
  #  }
    # >=2.1.2 requires the --tool option, some versions write to stdout, some to stderr
  #  valgrind --help 2>&1 | grep "\-\-tool" > /dev/null && VALGRIND="$VALGRIND --tool=memcheck"
  #  VALGRIND="$VALGRIND --alignment=8 --leak-check=yes --num-callers=16"
  #  $opt_extra_mysqld_opt.= " --skip-safemalloc --skip-bdb";
  #  SLEEP_TIME_AFTER_RESTART=10
  #  $opt_sleep_time_for_delete=  60
  #  $glob_use_running_server= ""
  #  if ( "$1"=  "--valgrind-all" )
  #  {
  #    VALGRIND="$VALGRIND -v --show-reachable=yes"
  #  }
  #}

  if ( ! $opt_user )
  {
    if ( $glob_use_running_server )
    {
      $opt_user= "test";
    }
    else
    {
      $opt_user= "root"; # We want to do FLUSH xxx commands
    }
  }

  # Put this into a hash, will be a C struct

  $master->[0]->{'path_myddir'}=  "$opt_vardir/master-data";
  $master->[0]->{'path_myerr'}=   "$opt_vardir/log/master.err";
  $master->[0]->{'path_mylog'}=   "$opt_vardir/log/master.log";
  $master->[0]->{'path_mypid'}=   "$opt_vardir/run/master.pid";
  $master->[0]->{'path_mysock'}=  "$opt_tmpdir/master.sock";
  $master->[0]->{'path_myport'}=   $opt_master_myport;
  $master->[0]->{'start_timeout'}= 400; # enough time create innodb tables

  $master->[0]->{'ndbcluster'}= 1; # ndbcluster not started

  $master->[1]->{'path_myddir'}=  "$opt_vardir/master1-data";
  $master->[1]->{'path_myerr'}=   "$opt_vardir/log/master1.err";
  $master->[1]->{'path_mylog'}=   "$opt_vardir/log/master1.log";
  $master->[1]->{'path_mypid'}=   "$opt_vardir/run/master1.pid";
  $master->[1]->{'path_mysock'}=  "$opt_tmpdir/master1.sock";
  $master->[1]->{'path_myport'}=   $opt_master_myport + 1;
  $master->[1]->{'start_timeout'}= 400; # enough time create innodb tables

  $slave->[0]->{'path_myddir'}=   "$opt_vardir/slave-data";
  $slave->[0]->{'path_myerr'}=    "$opt_vardir/log/slave.err";
  $slave->[0]->{'path_mylog'}=    "$opt_vardir/log/slave.log";
  $slave->[0]->{'path_mypid'}=    "$opt_vardir/run/slave.pid";
  $slave->[0]->{'path_mysock'}=   "$opt_tmpdir/slave.sock";
  $slave->[0]->{'path_myport'}=    $opt_slave_myport;
  $slave->[0]->{'start_timeout'}=  400;

  $slave->[1]->{'path_myddir'}=   "$opt_vardir/slave1-data";
  $slave->[1]->{'path_myerr'}=    "$opt_vardir/log/slave1.err";
  $slave->[1]->{'path_mylog'}=    "$opt_vardir/log/slave1.log";
  $slave->[1]->{'path_mypid'}=    "$opt_vardir/run/slave1.pid";
  $slave->[1]->{'path_mysock'}=   "$opt_tmpdir/slave1.sock";
  $slave->[1]->{'path_myport'}=    $opt_slave_myport + 1;
  $slave->[1]->{'start_timeout'}=  300;

  $slave->[2]->{'path_myddir'}=   "$opt_vardir/slave2-data";
  $slave->[2]->{'path_myerr'}=    "$opt_vardir/log/slave2.err";
  $slave->[2]->{'path_mylog'}=    "$opt_vardir/log/slave2.log";
  $slave->[2]->{'path_mypid'}=    "$opt_vardir/run/slave2.pid";
  $slave->[2]->{'path_mysock'}=   "$opt_tmpdir/slave2.sock";
  $slave->[2]->{'path_myport'}=    $opt_slave_myport + 2;
  $slave->[2]->{'start_timeout'}=  300;

  if ( $opt_extern )
  {
    $glob_use_running_server=  1;
    $opt_skip_rpl= 1;                   # We don't run rpl test cases
    $master->[0]->{'path_mysock'}=  $opt_socket;
  }

  $path_timefile=  "$opt_vardir/log/mysqltest-time";
}


##############################################################################
#
#  Set paths to various executable programs
#
##############################################################################

sub executable_setup () {

  if ( $opt_source_dist )
  {
    if ( $glob_win32 )
    {
      $path_client_bindir= mtr_path_exists("$glob_basedir/client_release",
                                           "$glob_basedir/bin");
      $exe_mysqld=         mtr_exe_exists ("$path_client_bindir/mysqld-nt");
      $path_language=      mtr_path_exists("$glob_basedir/share/english/");
      $path_charsetsdir=   mtr_path_exists("$glob_basedir/share/charsets");
    }
    else
    {
      $path_client_bindir= mtr_path_exists("$glob_basedir/client");
      $exe_mysqld=         mtr_exe_exists ("$glob_basedir/sql/mysqld");
      $path_language=      mtr_path_exists("$glob_basedir/sql/share/english/");
      $path_charsetsdir=   mtr_path_exists("$glob_basedir/sql/share/charsets");
    }

    if ( $glob_use_embedded_server )
    {
      my $path_examples= "$glob_basedir/libmysqld/examples";
      $exe_mysqltest=    mtr_exe_exists("$path_examples/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$path_examples/mysql_client_test_embedded",
		       "/usr/bin/false");
    }
    else
    {
      $exe_mysqltest=  mtr_exe_exists("$path_client_bindir/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$glob_basedir/tests/mysql_client_test",
		       "/usr/bin/false");
    }
    $exe_mysqldump=      mtr_exe_exists("$path_client_bindir/mysqldump");
    $exe_mysqlshow=      mtr_exe_exists("$path_client_bindir/mysqlshow");
    $exe_mysqlbinlog=    mtr_exe_exists("$path_client_bindir/mysqlbinlog");
    $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
    $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
    $exe_mysql_fix_system_tables=
      mtr_script_exists("$glob_basedir/scripts/mysql_fix_privilege_tables");
    $path_ndb_tools_dir= mtr_path_exists("$glob_basedir/ndb/tools");
    $exe_ndb_mgm=        "$glob_basedir/ndb/src/mgmclient/ndb_mgm";
  }
  else
  {
    $path_client_bindir= mtr_path_exists("$glob_basedir/bin");
    $exe_mysqltest=      mtr_exe_exists("$path_client_bindir/mysqltest");
    $exe_mysqldump=      mtr_exe_exists("$path_client_bindir/mysqldump");
    $exe_mysqlshow=      mtr_exe_exists("$path_client_bindir/mysqlshow");
    $exe_mysqlbinlog=    mtr_exe_exists("$path_client_bindir/mysqlbinlog");
    $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
    $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
    $exe_mysql_fix_system_tables=
      mtr_script_exists("$path_client_bindir/mysql_fix_privilege_tables",
			"$glob_basedir/scripts/mysql_fix_privilege_tables");

    $path_language=      mtr_path_exists("$glob_basedir/share/mysql/english/",
                                         "$glob_basedir/share/english/");
    $path_charsetsdir=   mtr_path_exists("$glob_basedir/share/mysql/charsets",
                                         "$glob_basedir/share/charsets");
    $exe_mysqld=         mtr_exe_exists ("$glob_basedir/libexec/mysqld",
                                         "$glob_basedir/bin/mysqld");

    if ( $glob_use_embedded_server )
    {
      $exe_mysqltest= mtr_exe_exists("$path_client_bindir/mysqltest_embedded");
      $exe_mysql_client_test=
        mtr_exe_exists("$glob_basedir/tests/mysql_client_test_embedded",
                       "$path_client_bindir/mysql_client_test_embedded",
		       "/usr/bin/false");
    }
    else
    {
      $exe_mysqltest= mtr_exe_exists("$path_client_bindir/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$path_client_bindir/mysql_client_test",
		       "/usr/bin/false"); # FIXME temporary
    }

    $path_ndb_tools_dir=  "$glob_basedir/bin";
    $exe_ndb_mgm=         "$glob_basedir/bin/ndb_mgm";
  }

  $exe_master_mysqld= $exe_master_mysqld || $exe_mysqld;
  $exe_slave_mysqld=  $exe_slave_mysqld  || $exe_mysqld;

  $path_ndb_backup_dir=
    "$opt_vardir/ndbcluster-$opt_ndbcluster_port";
  $file_ndb_testrun_log= "$opt_vardir/log/ndb_testrun.log";
}


##############################################################################
#
#  Set environment to be used by childs of this process
#
##############################################################################

# Note that some env is setup in spawn/run, in "mtr_process.pl"

sub environment_setup () {

  # --------------------------------------------------------------------------
  # We might not use a standard installation directory, like /usr/lib.
  # Set LD_LIBRARY_PATH to make sure we find our installed libraries.
  # --------------------------------------------------------------------------

  unless ( $opt_source_dist )
  {
    $ENV{'LD_LIBRARY_PATH'}=
      "$glob_basedir/lib" .
        ($ENV{'LD_LIBRARY_PATH'} ? ":$ENV{'LD_LIBRARY_PATH'}" : "");
    $ENV{'DYLD_LIBRARY_PATH'}=
      "$glob_basedir/lib" .
        ($ENV{'DYLD_LIBRARY_PATH'} ? ":$ENV{'DYLD_LIBRARY_PATH'}" : "");
  }

  # --------------------------------------------------------------------------
  # Also command lines in .opt files may contain env vars
  # --------------------------------------------------------------------------

  $ENV{'UMASK'}=              "0660"; # The octal *string*
  $ENV{'UMASK_DIR'}=          "0770"; # The octal *string*
  $ENV{'LC_COLLATE'}=         "C";
  $ENV{'USE_RUNNING_SERVER'}= $glob_use_running_server;
  $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
  $ENV{'MYSQL_TEST_WINDIR'}=  $glob_mysql_test_dir;
  $ENV{'MASTER_MYSOCK'}=      $master->[0]->{'path_mysock'};
  $ENV{'MASTER_WINMYSOCK'}=   $master->[0]->{'path_mysock'};
  $ENV{'MASTER_MYSOCK1'}=     $master->[1]->{'path_mysock'};
  $ENV{'MASTER_MYPORT'}=      $master->[0]->{'path_myport'};
  $ENV{'MASTER_MYPORT1'}=     $master->[1]->{'path_myport'};
  $ENV{'SLAVE_MYPORT'}=       $slave->[0]->{'path_myport'};
# $ENV{'MYSQL_TCP_PORT'}=     '@MYSQL_TCP_PORT@'; # FIXME
  $ENV{'MYSQL_TCP_PORT'}=     3306;

  if ( $glob_cygwin_perl )
  {
    foreach my $key ('MYSQL_TEST_WINDIR','MASTER_MYSOCK')
    {
      $ENV{$key}= `cygpath -w $ENV{$key}`;
      $ENV{$key} =~ s,\\,\\\\,g;
      chomp($ENV{$key});
    }
  }
}


##############################################################################
#
#  If we get a ^C, we try to clean up before termination
#
##############################################################################
# FIXME check restrictions what to do in a signal handler

sub signal_setup () {
  $SIG{INT}= \&handle_int_signal;
}

sub handle_int_signal () {
  $SIG{INT}= 'DEFAULT';         # If we get a ^C again, we die...
  mtr_warning("got INT signal, cleaning up.....");
  stop_masters_slaves();
  mtr_error("We die from ^C signal from user");
}


##############################################################################
#
#  Handle left overs from previous runs
#
##############################################################################

sub kill_running_server () {

  if ( $opt_fast or $glob_use_embedded_server )
  {
    # FIXME is embedded server really using PID files?!
    unlink($master->[0]->{'path_mypid'});
    unlink($master->[1]->{'path_mypid'});
    unlink($slave->[0]->{'path_mypid'});
    unlink($slave->[1]->{'path_mypid'});
    unlink($slave->[2]->{'path_mypid'});
  }
  else
  {
    # Ensure that no old mysqld test servers are running
    # This is different from terminating processes we have
    # started from ths run of the script, this is terminating
    # leftovers from previous runs.

    mtr_report("Killing Possible Leftover Processes");
    mkpath("$opt_vardir/log"); # Needed for mysqladmin log
    mtr_kill_leftovers();

    ndbcluster_stop();
    $master->[0]->{'ndbcluster'}= 1;
  }
}

sub kill_and_cleanup () {

  kill_running_server ();

  mtr_report("Removing Stale Files");

  rmtree("$opt_vardir/log");
  rmtree("$opt_vardir/ndbcluster-$opt_ndbcluster_port");
  rmtree("$opt_vardir/run");
  rmtree("$opt_vardir/tmp");

  mkpath("$opt_vardir/log");
  mkpath("$opt_vardir/run");
  mkpath("$opt_vardir/tmp");
  mkpath($opt_tmpdir) if $opt_tmpdir ne "$opt_vardir/tmp";

  # FIXME do we really need to create these all, or are they
  # created for us when tables are created?

  rmtree("$master->[0]->{'path_myddir'}");
  mkpath("$master->[0]->{'path_myddir'}/mysql");
  mkpath("$master->[0]->{'path_myddir'}/test");

  rmtree("$master->[1]->{'path_myddir'}");
  mkpath("$master->[1]->{'path_myddir'}/mysql");
  mkpath("$master->[1]->{'path_myddir'}/test");

  rmtree("$slave->[0]->{'path_myddir'}");
  mkpath("$slave->[0]->{'path_myddir'}/mysql");
  mkpath("$slave->[0]->{'path_myddir'}/test");

  rmtree("$slave->[1]->{'path_myddir'}");
  mkpath("$slave->[1]->{'path_myddir'}/mysql");
  mkpath("$slave->[1]->{'path_myddir'}/test");

  rmtree("$slave->[2]->{'path_myddir'}");
  mkpath("$slave->[2]->{'path_myddir'}/mysql");
  mkpath("$slave->[2]->{'path_myddir'}/test");

  # To make some old test cases work, we create a soft
  # link from the old "var" location to the new one

  if ( ! $glob_win32 and $opt_vardir ne "$glob_mysql_test_dir/var" )
  {
    # FIXME why bother with the above, why not always remove all of var?!
    rmtree("$glob_mysql_test_dir/var"); # Clean old var, FIXME or rename it?!
    symlink($opt_vardir, "$glob_mysql_test_dir/var");
  }
}


##############################################################################
#
#  Start the ndb cluster
#
##############################################################################

# FIXME why is there a different start below?!

sub ndbcluster_install () {

  if ( ! $opt_with_ndbcluster or $glob_use_running_ndbcluster )
  {
    return 0;
  }
  mtr_report("Install ndbcluster");
  my $ndbcluster_opts=  $opt_bench ? "" : "--small";
  if (  mtr_run("$glob_mysql_test_dir/ndb/ndbcluster",
		["--port=$opt_ndbcluster_port",
		 "--data-dir=$opt_vardir",
		 $ndbcluster_opts,
		 "--initial"],
		"", "", "", "") )
  {
    mtr_error("Error ndbcluster_install");
    return 1;<commit_msg>Bug #11903, make perl test script run ndbcluster tests by default<commit_after>#!/usr/bin/perl
# -*- cperl -*-

# This is a transformation of the "mysql-test-run" Bourne shell script
# to Perl. This is just an intermediate step, the goal is to rewrite
# the Perl script to C. The complexity of the mysql-test-run script
# makes it a bit hard to write and debug it as a C program directly,
# so this is considered a prototype.
#
# Because of this the Perl coding style may in some cases look a bit
# funny. The rules used are
#
#   - The coding style is as close as possible to the C/C++ MySQL
#     coding standard.
#
#   - Where NULL is to be returned, the undefined value is used.
#
#   - Regexp comparisons are simple and can be translated to strcmp
#     and other string functions. To ease this transformation matching
#     is done in the lib "lib/mtr_match.pl", i.e. regular expressions
#     should be avoided in the main program.
#
#   - The "unless" construct is not to be used. It is the same as "if !".
#
#   - opendir/readdir/closedir is used instead of glob()/<*>.
#
#   - All lists of arguments to send to commands are Perl lists/arrays,
#     not strings we append args to. Within reason, most string
#     concatenation for arguments should be avoided.
#
#   - sprintf() is to be used, within reason, for all string creation.
#     This mtr_add_arg() function is also based on sprintf(), i.e. you
#     use a format string and put the variable argument in the argument
#     list.
#
#   - Functions defined in the main program are not to be prefixed,
#     functions in "library files" are to be prefixed with "mtr_" (for
#     Mysql-Test-Run). There are some exceptions, code that fits best in
#     the main program, but are put into separate files to avoid
#     clutter, may be without prefix.
#
#   - All stat/opendir/-f/ is to be kept in collect_test_cases(). It
#     will create a struct that the rest of the program can use to get
#     the information. This separates the "find information" from the
#     "do the work" and makes the program more easy to maintain.
#
#   - At the moment, there are tons of "global" variables that control
#     this script, even accessed from the files in "lib/*.pl". This
#     will change over time, for now global variables are used instead
#     of using %opt, %path and %exe hashes, because I want more
#     compile time checking, that hashes would not give me. Once this
#     script is debugged, hashes will be used and passed as parameters
#     to functions, to more closely mimic how it would be coded in C
#     using structs.
#
#   - The rule when it comes to the logic of this program is
#
#       command_line_setup() - is to handle the logic between flags
#       collect_test_cases() - is to do its best to select what tests
#                              to run, dig out options, if needs restart etc.
#       run_testcase()       - is to run a single testcase, and follow the
#                              logic set in both above. No, or rare file
#                              system operations. If a test seems complex,
#                              it should probably not be here.
#
# A nice way to trace the execution of this script while debugging
# is to use the Devel::Trace package found at
# "http://www.plover.com/~mjd/perl/Trace/" and run this script like
# "perl -d:Trace mysql-test-run.pl"
#
# FIXME Save a PID file from this code as well, to record the process
#       id we think it has. In Cygwin, a fork creates one Cygwin process,
#       and then the real Win32 process. Cygwin Perl can only kill Cygwin
#       processes. And "mysqld --bootstrap ..." doesn't save a PID file.

$Devel::Trace::TRACE= 0;       # Don't trace boring init stuff

#require 5.6.1;
use File::Path;
use File::Basename;
use Cwd;
use Getopt::Long;
use Sys::Hostname;
#use Carp;
use IO::Socket;
use IO::Socket::INET;
use Data::Dumper;
use strict;
#use diagnostics;

require "lib/mtr_cases.pl";
require "lib/mtr_process.pl";
require "lib/mtr_io.pl";
require "lib/mtr_gcov.pl";
require "lib/mtr_gprof.pl";
require "lib/mtr_report.pl";
require "lib/mtr_diff.pl";
require "lib/mtr_match.pl";
require "lib/mtr_misc.pl";

$Devel::Trace::TRACE= 1;

# Used by gcov
our @mysqld_src_dirs=
  (
   "strings",
   "mysys",
   "include",
   "extra",
   "regex",
   "isam",
   "merge",
   "myisam",
   "myisammrg",
   "heap",
   "sql",
  );

##############################################################################
#
#  Default settings
#
##############################################################################

# We are to use handle_options() in "mysys/my_getopt.c" for the C version
#
# In the C version we want to use structs and, in some cases, arrays of
# structs. We let each struct be a separate hash.

# Misc global variables

our $glob_win32=                  0; # OS and native Win32 executables
our $glob_win32_perl=             0; # ActiveState Win32 Perl
our $glob_cygwin_perl=            0; # Cygwin Perl
our $glob_cygwin_shell=           undef;
our $glob_mysql_test_dir=         undef;
our $glob_mysql_bench_dir=        undef;
our $glob_hostname=               undef;
our $glob_scriptname=             undef;
our $glob_use_running_server=     0;
our $glob_use_running_ndbcluster= 0;
our $glob_use_embedded_server=    0;

our $glob_basedir;

# The total result

our $path_charsetsdir;
our $path_client_bindir;
our $path_language;
our $path_timefile;
our $path_manager_log;           # Used by mysqldadmin
our $path_slave_load_tmpdir;     # What is this?!
our $path_my_basedir;
our $opt_vardir;                 # A path but set directly on cmd line
our $opt_tmpdir;                 # A path but set directly on cmd line

our $opt_usage;
our $opt_suite;

our $opt_netware;

our $opt_script_debug= 0;  # Script debugging, enable with --script-debug

# Options FIXME not all....

our $exe_master_mysqld;
our $exe_mysql;
our $exe_mysqladmin;
our $exe_mysqlbinlog;
our $exe_mysql_client_test;
our $exe_mysqld;
our $exe_mysqldump;              # Called from test case
our $exe_mysqlshow;              # Called from test case
our $exe_mysql_fix_system_tables;
our $exe_mysqltest;
our $exe_slave_mysqld;

our $opt_bench= 0;
our $opt_small_bench= 0;
our $opt_big_test= 0;            # Send --big-test to mysqltest

our @opt_extra_mysqld_opt;

our $opt_compress;
our $opt_current_test;
our $opt_ddd;
our $opt_debug;
our $opt_do_test;
our @opt_cases;                  # The test cases names in argv
our $opt_embedded_server;
our $opt_extern;
our $opt_fast;
our $opt_force;
our $opt_reorder;

our $opt_gcov;
our $opt_gcov_err;
our $opt_gcov_msg;

our $opt_gdb;
our $opt_client_gdb;
our $opt_manual_gdb;

our $opt_gprof;
our $opt_gprof_dir;
our $opt_gprof_master;
our $opt_gprof_slave;

our $opt_local;
our $opt_local_master;

our $master;                    # Will be struct in C
our $slave;

our $opt_ndbcluster_port;
our $opt_ndbconnectstring;

our $opt_no_manager;            # Does nothing now, we never use manager
our $opt_manager_port;          # Does nothing now, we never use manager

our $opt_old_master;

our $opt_record;

our $opt_result_ext;

our $opt_skip;
our $opt_skip_rpl;
our $opt_skip_test;

our $opt_sleep;
our $opt_ps_protocol;

our $opt_sleep_time_after_restart= 1;
our $opt_sleep_time_for_delete=    10;

our $opt_socket;

our $opt_source_dist;

our $opt_start_and_exit;
our $opt_start_dirty;
our $opt_start_from;

our $opt_strace_client;

our $opt_timer;

our $opt_user;
our $opt_user_test;

our $opt_valgrind;
our $opt_valgrind_all;
our $opt_valgrind_options;

our $opt_verbose;

our $opt_wait_for_master;
our $opt_wait_for_slave;
our $opt_wait_timeout=  10;

our $opt_warnings;

our $opt_udiff;

our $opt_skip_ndbcluster;
our $opt_with_ndbcluster;
our $opt_with_openssl;

our $exe_ndb_mgm;
our $path_ndb_tools_dir;
our $path_ndb_backup_dir;
our $file_ndb_testrun_log;
our $flag_ndb_status_ok= 1;

######################################################################
#
#  Function declarations
#
######################################################################

sub main ();
sub initial_setup ();
sub command_line_setup ();
sub executable_setup ();
sub environment_setup ();
sub kill_running_server ();
sub kill_and_cleanup ();
sub ndbcluster_support ();
sub ndbcluster_install ();
sub ndbcluster_start ();
sub ndbcluster_stop ();
sub run_benchmarks ($);
sub run_tests ();
sub mysql_install_db ();
sub install_db ($$);
sub run_testcase ($);
sub report_failure_and_restart ($);
sub do_before_start_master ($$);
sub do_before_start_slave ($$);
sub mysqld_start ($$$$);
sub mysqld_arguments ($$$$$);
sub stop_masters_slaves ();
sub stop_masters ();
sub stop_slaves ();
sub run_mysqltest ($);
sub usage ($);

######################################################################
#
#  Main program
#
######################################################################

main();

sub main () {

  initial_setup();
  command_line_setup();
  executable_setup();
  
  if (! $opt_skip_ndbcluster and ! $opt_with_ndbcluster)
  {
    $opt_with_ndbcluster= ndbcluster_support();
  }

  environment_setup();
  signal_setup();

  if ( $opt_gcov )
  {
    gcov_prepare();
  }

  if ( $opt_gprof )
  {
    gprof_prepare();
  }

  if ( ! $glob_use_running_server )
  {

    if ( $opt_start_dirty )
    {
      kill_running_server();
    }
    else
    {
      kill_and_cleanup();
      mysql_install_db();

#    mysql_loadstd();  FIXME copying from "std_data" .frm and
#                      .MGR but there are none?!
    }
  }

  if ( $opt_start_and_exit or $opt_start_dirty )
  {
    if ( ndbcluster_start() )
    {
      mtr_error("Can't start ndbcluster");
    }
    if ( mysqld_start('master',0,[],[]) )
    {
      mtr_report("Servers started, exiting");
    }
    else
    {
      mtr_error("Can't start the mysqld server");
    }
  }
  else
  {
    if ( $opt_bench )
    {
      run_benchmarks(shift);      # Shift what? Extra arguments?!
    }
    else
    {
      run_tests();
    }
  }

  mtr_exit(0);
}

##############################################################################
#
#  Initial setup independent on command line arguments
#
##############################################################################

sub initial_setup () {

  select(STDOUT);
  $| = 1;                       # Make unbuffered

  $glob_scriptname=  basename($0);

  $glob_win32_perl=  ($^O eq "MSWin32");
  $glob_cygwin_perl= ($^O eq "cygwin");
  $glob_win32=       ($glob_win32_perl or $glob_cygwin_perl);

  # We require that we are in the "mysql-test" directory
  # to run mysql-test-run

  if (! -f $glob_scriptname)
  {
    mtr_error("Can't find the location for the mysql-test-run script\n" .
              "Go to to the mysql-test directory and execute the script " .
              "as follows:\n./$glob_scriptname");
  }

  if ( -d "../sql" )
  {
    $opt_source_dist=  1;
  }

  $glob_hostname=  mtr_full_hostname();

  # 'basedir' is always parent of "mysql-test" directory
  $glob_mysql_test_dir=  cwd();
  if ( $glob_cygwin_perl )
  {
    # Windows programs like 'mysqld' needs Windows paths
    $glob_mysql_test_dir= `cygpath -m $glob_mysql_test_dir`;
    my $shell= $ENV{'SHELL'} || "/bin/bash";
    $glob_cygwin_shell=   `cygpath -w $shell`; # The Windows path c:\...
    chomp($glob_mysql_test_dir);
    chomp($glob_cygwin_shell);
  }
  $glob_basedir=         dirname($glob_mysql_test_dir);
  $glob_mysql_bench_dir= "$glob_basedir/mysql-bench"; # FIXME make configurable

  # needs to be same length to test logging (FIXME what???)
  $path_slave_load_tmpdir=  "../../var/tmp";

  $path_my_basedir=
    $opt_source_dist ? $glob_mysql_test_dir : $glob_basedir;
}



##############################################################################
#
#  Default settings
#
##############################################################################

sub command_line_setup () {

  # These are defaults for things that are set on the command line

  $opt_suite=        "main";    # Special default suite
  my $opt_master_myport= 9306;
  my $opt_slave_myport=  9308;
  $opt_ndbcluster_port=  9350;

  # Read the command line
  # Note: Keep list, and the order, in sync with usage at end of this file

  GetOptions(
             # Control what engine/variation to run
             'embedded-server'          => \$opt_embedded_server,
             'ps-protocol'              => \$opt_ps_protocol,
             'bench'                    => \$opt_bench,
             'small-bench'              => \$opt_small_bench,
             'no-manager'               => \$opt_no_manager, # Currently not used

             # Control what test suites or cases to run
             'force'                    => \$opt_force,
             'with-ndbcluster'          => \$opt_with_ndbcluster,
             'skip-ndbcluster|skip-ndb' => \$opt_skip_ndbcluster,
             'do-test=s'                => \$opt_do_test,
             'suite=s'                  => \$opt_suite,
             'skip-rpl'                 => \$opt_skip_rpl,
             'skip-test=s'              => \$opt_skip_test,

             # Specify ports
             'master_port=i'            => \$opt_master_myport,
             'slave_port=i'             => \$opt_slave_myport,
             'ndbcluster_port=i'        => \$opt_ndbcluster_port,
             'manager-port=i'           => \$opt_manager_port, # Currently not used

             # Test case authoring
             'record'                   => \$opt_record,

             # ???
             'mysqld=s'                 => \@opt_extra_mysqld_opt,

             # Run test on running server
             'extern'                   => \$opt_extern,
             'ndbconnectstring=s'       => \$opt_ndbconnectstring,

             # Debugging
             'gdb'                      => \$opt_gdb,
             'manual-gdb'               => \$opt_manual_gdb,
             'client-gdb'               => \$opt_client_gdb,
             'ddd'                      => \$opt_ddd,
             'strace-client'            => \$opt_strace_client,
             'master-binary=s'          => \$exe_master_mysqld,
             'slave-binary=s'           => \$exe_slave_mysqld,

             # Coverage, profiling etc
             'gcov'                     => \$opt_gcov,
             'gprof'                    => \$opt_gprof,
             'valgrind'                 => \$opt_valgrind,
             'valgrind-all'             => \$opt_valgrind_all,
             'valgrind-options=s'       => \$opt_valgrind_options,

             # Misc
             'big-test'                 => \$opt_big_test,
             'compress'                 => \$opt_compress,
             'debug'                    => \$opt_debug,
             'fast'                     => \$opt_fast,
             'local'                    => \$opt_local,
             'local-master'             => \$opt_local_master,
             'netware'                  => \$opt_netware,
             'old-master'               => \$opt_old_master,
             'reorder'                  => \$opt_reorder,
             'script-debug'             => \$opt_script_debug,
             'sleep=i'                  => \$opt_sleep,
             'socket=s'                 => \$opt_socket,
             'start-dirty'              => \$opt_start_dirty,
             'start-and-exit'           => \$opt_start_and_exit,
             'start-from=s'             => \$opt_start_from,
             'timer'                    => \$opt_timer,
             'tmpdir=s'                 => \$opt_tmpdir,
             'unified-diff|udiff'       => \$opt_udiff,
             'user-test=s'              => \$opt_user_test,
             'user=s'                   => \$opt_user,
             'vardir=s'                 => \$opt_vardir,
             'verbose'                  => \$opt_verbose,
             'wait-timeout=i'           => \$opt_wait_timeout,
             'warnings|log-warnings'    => \$opt_warnings,
             'with-openssl'             => \$opt_with_openssl,

             'help|h'                   => \$opt_usage,
            ) or usage("Can't read options");

  if ( $opt_usage )
  {
    usage("");
  }

  @opt_cases= @ARGV;

  # --------------------------------------------------------------------------
  # Set the "var/" directory, as it is the base for everything else
  # --------------------------------------------------------------------------

  if ( ! $opt_vardir )
  {
    $opt_vardir= "$glob_mysql_test_dir/var";
  }

  # We make the path absolute, as the server will do a chdir() before usage
  unless ( $opt_vardir =~ m,^/, or
           ($glob_win32 and $opt_vardir =~ m,^[a-z]:/,i) )
  {
    # Make absolute path, relative test dir
    $opt_vardir= "$glob_mysql_test_dir/$opt_vardir";
  }

  # --------------------------------------------------------------------------
  # If not set, set these to defaults
  # --------------------------------------------------------------------------

  $opt_tmpdir=       "$opt_vardir/tmp" unless $opt_tmpdir;
  # FIXME maybe not needed?
  $path_manager_log= "$opt_vardir/log/manager.log"
    unless $path_manager_log;
  $opt_current_test= "$opt_vardir/log/current_test"
    unless $opt_current_test;

  # --------------------------------------------------------------------------
  # Do sanity checks of command line arguments
  # --------------------------------------------------------------------------

  if ( $opt_extern and $opt_local )
  {
    mtr_error("Can't use --extern and --local at the same time");
  }

  if ( ! $opt_socket )
  {     # FIXME set default before reading options?
#    $opt_socket=  '@MYSQL_UNIX_ADDR@';
    $opt_socket=  "/tmp/mysql.sock"; # FIXME
  }

  # --------------------------------------------------------------------------
  # Look at the command line options and set script flags
  # --------------------------------------------------------------------------

  if ( $opt_record and ! @opt_cases )
  {
    mtr_error("Will not run in record mode without a specific test case");
  }

  if ( $opt_embedded_server )
  {
    $glob_use_embedded_server= 1;
    $opt_skip_rpl= 1;              # We never run replication with embedded

    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --embedded-server");
    }
  }

  # FIXME don't understand what this is
#  if ( $opt_local_master )
#  {
#    $opt_master_myport=  3306;
#  }

  if ( $opt_small_bench )
  {
    $opt_bench=  1;
  }

  if ( $opt_sleep )
  {
    $opt_sleep_time_after_restart= $opt_sleep;
  }

  if ( $opt_gcov and ! $opt_source_dist )
  {
    mtr_error("Coverage test needs the source - please use source dist");
  }

  if ( $glob_use_embedded_server and ! $opt_source_dist )
  {
    mtr_error("Embedded server needs source tree - please use source dist");
  }

  if ( $opt_gdb )
  {
    $opt_wait_timeout=  300;
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --gdb");
    }
  }

  if ( $opt_manual_gdb )
  {
    $opt_gdb=  1;
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --manual-gdb");
    }
  }

  if ( $opt_ddd )
  {
    if ( $opt_extern )
    {
      mtr_error("Can't use --extern with --ddd");
    }
  }

  if ( $opt_ndbconnectstring )
  {
    $glob_use_running_ndbcluster= 1;
    $opt_with_ndbcluster= 1;
  }
  else
  {
    $opt_ndbconnectstring= "host=localhost:$opt_ndbcluster_port";
  }

  if ( $opt_skip_ndbcluster )
  {
    $opt_with_ndbcluster= 0;
  }

  # FIXME

  #if ( $opt_valgrind or $opt_valgrind_all )
  #{
    # VALGRIND=`which valgrind` # this will print an error if not found FIXME
    # Give good warning to the user and stop
  #  if ( ! $VALGRIND )
  #  {
  #    print "You need to have the 'valgrind' program in your PATH to run mysql-test-run with option --valgrind. Valgrind's home page is http://valgrind.kde.org.\n"
  #    exit 1
  #  }
    # >=2.1.2 requires the --tool option, some versions write to stdout, some to stderr
  #  valgrind --help 2>&1 | grep "\-\-tool" > /dev/null && VALGRIND="$VALGRIND --tool=memcheck"
  #  VALGRIND="$VALGRIND --alignment=8 --leak-check=yes --num-callers=16"
  #  $opt_extra_mysqld_opt.= " --skip-safemalloc --skip-bdb";
  #  SLEEP_TIME_AFTER_RESTART=10
  #  $opt_sleep_time_for_delete=  60
  #  $glob_use_running_server= ""
  #  if ( "$1"=  "--valgrind-all" )
  #  {
  #    VALGRIND="$VALGRIND -v --show-reachable=yes"
  #  }
  #}

  if ( ! $opt_user )
  {
    if ( $glob_use_running_server )
    {
      $opt_user= "test";
    }
    else
    {
      $opt_user= "root"; # We want to do FLUSH xxx commands
    }
  }

  # Put this into a hash, will be a C struct

  $master->[0]->{'path_myddir'}=  "$opt_vardir/master-data";
  $master->[0]->{'path_myerr'}=   "$opt_vardir/log/master.err";
  $master->[0]->{'path_mylog'}=   "$opt_vardir/log/master.log";
  $master->[0]->{'path_mypid'}=   "$opt_vardir/run/master.pid";
  $master->[0]->{'path_mysock'}=  "$opt_tmpdir/master.sock";
  $master->[0]->{'path_myport'}=   $opt_master_myport;
  $master->[0]->{'start_timeout'}= 400; # enough time create innodb tables

  $master->[0]->{'ndbcluster'}= 1; # ndbcluster not started

  $master->[1]->{'path_myddir'}=  "$opt_vardir/master1-data";
  $master->[1]->{'path_myerr'}=   "$opt_vardir/log/master1.err";
  $master->[1]->{'path_mylog'}=   "$opt_vardir/log/master1.log";
  $master->[1]->{'path_mypid'}=   "$opt_vardir/run/master1.pid";
  $master->[1]->{'path_mysock'}=  "$opt_tmpdir/master1.sock";
  $master->[1]->{'path_myport'}=   $opt_master_myport + 1;
  $master->[1]->{'start_timeout'}= 400; # enough time create innodb tables

  $slave->[0]->{'path_myddir'}=   "$opt_vardir/slave-data";
  $slave->[0]->{'path_myerr'}=    "$opt_vardir/log/slave.err";
  $slave->[0]->{'path_mylog'}=    "$opt_vardir/log/slave.log";
  $slave->[0]->{'path_mypid'}=    "$opt_vardir/run/slave.pid";
  $slave->[0]->{'path_mysock'}=   "$opt_tmpdir/slave.sock";
  $slave->[0]->{'path_myport'}=    $opt_slave_myport;
  $slave->[0]->{'start_timeout'}=  400;

  $slave->[1]->{'path_myddir'}=   "$opt_vardir/slave1-data";
  $slave->[1]->{'path_myerr'}=    "$opt_vardir/log/slave1.err";
  $slave->[1]->{'path_mylog'}=    "$opt_vardir/log/slave1.log";
  $slave->[1]->{'path_mypid'}=    "$opt_vardir/run/slave1.pid";
  $slave->[1]->{'path_mysock'}=   "$opt_tmpdir/slave1.sock";
  $slave->[1]->{'path_myport'}=    $opt_slave_myport + 1;
  $slave->[1]->{'start_timeout'}=  300;

  $slave->[2]->{'path_myddir'}=   "$opt_vardir/slave2-data";
  $slave->[2]->{'path_myerr'}=    "$opt_vardir/log/slave2.err";
  $slave->[2]->{'path_mylog'}=    "$opt_vardir/log/slave2.log";
  $slave->[2]->{'path_mypid'}=    "$opt_vardir/run/slave2.pid";
  $slave->[2]->{'path_mysock'}=   "$opt_tmpdir/slave2.sock";
  $slave->[2]->{'path_myport'}=    $opt_slave_myport + 2;
  $slave->[2]->{'start_timeout'}=  300;

  if ( $opt_extern )
  {
    $glob_use_running_server=  1;
    $opt_skip_rpl= 1;                   # We don't run rpl test cases
    $master->[0]->{'path_mysock'}=  $opt_socket;
  }

  $path_timefile=  "$opt_vardir/log/mysqltest-time";
}


##############################################################################
#
#  Set paths to various executable programs
#
##############################################################################

sub executable_setup () {

  if ( $opt_source_dist )
  {
    if ( $glob_win32 )
    {
      $path_client_bindir= mtr_path_exists("$glob_basedir/client_release",
                                           "$glob_basedir/bin");
      $exe_mysqld=         mtr_exe_exists ("$path_client_bindir/mysqld-nt");
      $path_language=      mtr_path_exists("$glob_basedir/share/english/");
      $path_charsetsdir=   mtr_path_exists("$glob_basedir/share/charsets");
    }
    else
    {
      $path_client_bindir= mtr_path_exists("$glob_basedir/client");
      $exe_mysqld=         mtr_exe_exists ("$glob_basedir/sql/mysqld");
      $path_language=      mtr_path_exists("$glob_basedir/sql/share/english/");
      $path_charsetsdir=   mtr_path_exists("$glob_basedir/sql/share/charsets");
    }

    if ( $glob_use_embedded_server )
    {
      my $path_examples= "$glob_basedir/libmysqld/examples";
      $exe_mysqltest=    mtr_exe_exists("$path_examples/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$path_examples/mysql_client_test_embedded",
		       "/usr/bin/false");
    }
    else
    {
      $exe_mysqltest=  mtr_exe_exists("$path_client_bindir/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$glob_basedir/tests/mysql_client_test",
		       "/usr/bin/false");
    }
    $exe_mysqldump=      mtr_exe_exists("$path_client_bindir/mysqldump");
    $exe_mysqlshow=      mtr_exe_exists("$path_client_bindir/mysqlshow");
    $exe_mysqlbinlog=    mtr_exe_exists("$path_client_bindir/mysqlbinlog");
    $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
    $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
    $exe_mysql_fix_system_tables=
      mtr_script_exists("$glob_basedir/scripts/mysql_fix_privilege_tables");
    $path_ndb_tools_dir= mtr_path_exists("$glob_basedir/ndb/tools");
    $exe_ndb_mgm=        "$glob_basedir/ndb/src/mgmclient/ndb_mgm";
  }
  else
  {
    $path_client_bindir= mtr_path_exists("$glob_basedir/bin");
    $exe_mysqltest=      mtr_exe_exists("$path_client_bindir/mysqltest");
    $exe_mysqldump=      mtr_exe_exists("$path_client_bindir/mysqldump");
    $exe_mysqlshow=      mtr_exe_exists("$path_client_bindir/mysqlshow");
    $exe_mysqlbinlog=    mtr_exe_exists("$path_client_bindir/mysqlbinlog");
    $exe_mysqladmin=     mtr_exe_exists("$path_client_bindir/mysqladmin");
    $exe_mysql=          mtr_exe_exists("$path_client_bindir/mysql");
    $exe_mysql_fix_system_tables=
      mtr_script_exists("$path_client_bindir/mysql_fix_privilege_tables",
			"$glob_basedir/scripts/mysql_fix_privilege_tables");

    $path_language=      mtr_path_exists("$glob_basedir/share/mysql/english/",
                                         "$glob_basedir/share/english/");
    $path_charsetsdir=   mtr_path_exists("$glob_basedir/share/mysql/charsets",
                                         "$glob_basedir/share/charsets");
    $exe_mysqld=         mtr_exe_exists ("$glob_basedir/libexec/mysqld",
                                         "$glob_basedir/bin/mysqld");

    if ( $glob_use_embedded_server )
    {
      $exe_mysqltest= mtr_exe_exists("$path_client_bindir/mysqltest_embedded");
      $exe_mysql_client_test=
        mtr_exe_exists("$glob_basedir/tests/mysql_client_test_embedded",
                       "$path_client_bindir/mysql_client_test_embedded",
		       "/usr/bin/false");
    }
    else
    {
      $exe_mysqltest= mtr_exe_exists("$path_client_bindir/mysqltest");
      $exe_mysql_client_test=
        mtr_exe_exists("$path_client_bindir/mysql_client_test",
		       "/usr/bin/false"); # FIXME temporary
    }

    $path_ndb_tools_dir=  "$glob_basedir/bin";
    $exe_ndb_mgm=         "$glob_basedir/bin/ndb_mgm";
  }

  $exe_master_mysqld= $exe_master_mysqld || $exe_mysqld;
  $exe_slave_mysqld=  $exe_slave_mysqld  || $exe_mysqld;

  $path_ndb_backup_dir=
    "$opt_vardir/ndbcluster-$opt_ndbcluster_port";
  $file_ndb_testrun_log= "$opt_vardir/log/ndb_testrun.log";
}


##############################################################################
#
#  Set environment to be used by childs of this process
#
##############################################################################

# Note that some env is setup in spawn/run, in "mtr_process.pl"

sub environment_setup () {

  # --------------------------------------------------------------------------
  # We might not use a standard installation directory, like /usr/lib.
  # Set LD_LIBRARY_PATH to make sure we find our installed libraries.
  # --------------------------------------------------------------------------

  unless ( $opt_source_dist )
  {
    $ENV{'LD_LIBRARY_PATH'}=
      "$glob_basedir/lib" .
        ($ENV{'LD_LIBRARY_PATH'} ? ":$ENV{'LD_LIBRARY_PATH'}" : "");
    $ENV{'DYLD_LIBRARY_PATH'}=
      "$glob_basedir/lib" .
        ($ENV{'DYLD_LIBRARY_PATH'} ? ":$ENV{'DYLD_LIBRARY_PATH'}" : "");
  }

  # --------------------------------------------------------------------------
  # Also command lines in .opt files may contain env vars
  # --------------------------------------------------------------------------

  $ENV{'UMASK'}=              "0660"; # The octal *string*
  $ENV{'UMASK_DIR'}=          "0770"; # The octal *string*
  $ENV{'LC_COLLATE'}=         "C";
  $ENV{'USE_RUNNING_SERVER'}= $glob_use_running_server;
  $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
  $ENV{'MYSQL_TEST_WINDIR'}=  $glob_mysql_test_dir;
  $ENV{'MASTER_MYSOCK'}=      $master->[0]->{'path_mysock'};
  $ENV{'MASTER_WINMYSOCK'}=   $master->[0]->{'path_mysock'};
  $ENV{'MASTER_MYSOCK1'}=     $master->[1]->{'path_mysock'};
  $ENV{'MASTER_MYPORT'}=      $master->[0]->{'path_myport'};
  $ENV{'MASTER_MYPORT1'}=     $master->[1]->{'path_myport'};
  $ENV{'SLAVE_MYPORT'}=       $slave->[0]->{'path_myport'};
# $ENV{'MYSQL_TCP_PORT'}=     '@MYSQL_TCP_PORT@'; # FIXME
  $ENV{'MYSQL_TCP_PORT'}=     3306;

  if ( $glob_cygwin_perl )
  {
    foreach my $key ('MYSQL_TEST_WINDIR','MASTER_MYSOCK')
    {
      $ENV{$key}= `cygpath -w $ENV{$key}`;
      $ENV{$key} =~ s,\\,\\\\,g;
      chomp($ENV{$key});
    }
  }
}


##############################################################################
#
#  If we get a ^C, we try to clean up before termination
#
##############################################################################
# FIXME check restrictions what to do in a signal handler

sub signal_setup () {
  $SIG{INT}= \&handle_int_signal;
}

sub handle_int_signal () {
  $SIG{INT}= 'DEFAULT';         # If we get a ^C again, we die...
  mtr_warning("got INT signal, cleaning up.....");
  stop_masters_slaves();
  mtr_error("We die from ^C signal from user");
}


##############################################################################
#
#  Handle left overs from previous runs
#
##############################################################################

sub kill_running_server () {

  if ( $opt_fast or $glob_use_embedded_server )
  {
    # FIXME is embedded server really using PID files?!
    unlink($master->[0]->{'path_mypid'});
    unlink($master->[1]->{'path_mypid'});
    unlink($slave->[0]->{'path_mypid'});
    unlink($slave->[1]->{'path_mypid'});
    unlink($slave->[2]->{'path_mypid'});
  }
  else
  {
    # Ensure that no old mysqld test servers are running
    # This is different from terminating processes we have
    # started from ths run of the script, this is terminating
    # leftovers from previous runs.

    mtr_report("Killing Possible Leftover Processes");
    mkpath("$opt_vardir/log"); # Needed for mysqladmin log
    mtr_kill_leftovers();

    ndbcluster_stop();
    $master->[0]->{'ndbcluster'}= 1;
  }
}

sub kill_and_cleanup () {

  kill_running_server ();

  mtr_report("Removing Stale Files");

  rmtree("$opt_vardir/log");
  rmtree("$opt_vardir/ndbcluster-$opt_ndbcluster_port");
  rmtree("$opt_vardir/run");
  rmtree("$opt_vardir/tmp");

  mkpath("$opt_vardir/log");
  mkpath("$opt_vardir/run");
  mkpath("$opt_vardir/tmp");
  mkpath($opt_tmpdir) if $opt_tmpdir ne "$opt_vardir/tmp";

  # FIXME do we really need to create these all, or are they
  # created for us when tables are created?

  rmtree("$master->[0]->{'path_myddir'}");
  mkpath("$master->[0]->{'path_myddir'}/mysql");
  mkpath("$master->[0]->{'path_myddir'}/test");

  rmtree("$master->[1]->{'path_myddir'}");
  mkpath("$master->[1]->{'path_myddir'}/mysql");
  mkpath("$master->[1]->{'path_myddir'}/test");

  rmtree("$slave->[0]->{'path_myddir'}");
  mkpath("$slave->[0]->{'path_myddir'}/mysql");
  mkpath("$slave->[0]->{'path_myddir'}/test");

  rmtree("$slave->[1]->{'path_myddir'}");
  mkpath("$slave->[1]->{'path_myddir'}/mysql");
  mkpath("$slave->[1]->{'path_myddir'}/test");

  rmtree("$slave->[2]->{'path_myddir'}");
  mkpath("$slave->[2]->{'path_myddir'}/mysql");
  mkpath("$slave->[2]->{'path_myddir'}/test");

  # To make some old test cases work, we create a soft
  # link from the old "var" location to the new one

  if ( ! $glob_win32 and $opt_vardir ne "$glob_mysql_test_dir/var" )
  {
    # FIXME why bother with the above, why not always remove all of var?!
    rmtree("$glob_mysql_test_dir/var"); # Clean old var, FIXME or rename it?!
    symlink($opt_vardir, "$glob_mysql_test_dir/var");
  }
}


##############################################################################
#
#  Start the ndb cluster
#
##############################################################################

sub ndbcluster_support () {

  # check ndbcluster support by testing using a switch
  # that is only available in that case
  if ( mtr_run($exe_mysqld,
	       ["--no-defaults",
	        "--ndb-use-exact-count",
	        "--help"],
	       "", "/dev/null", "/dev/null", "") != 0 )
  {
    mtr_report("No ndbcluster support");
    return 0;
  }
  mtr_report("Has ndbcluster support");
  return 1;
}

# FIXME why is there a different start below?!

sub ndbcluster_install () {

  if ( ! $opt_with_ndbcluster or $glob_use_running_ndbcluster )
  {
    return 0;
  }
  mtr_report("Install ndbcluster");
  my $ndbcluster_opts=  $opt_bench ? "" : "--small";
  if (  mtr_run("$glob_mysql_test_dir/ndb/ndbcluster",
		["--port=$opt_ndbcluster_port",
		 "--data-dir=$opt_vardir",
		 $ndbcluster_opts,
		 "--initial"],
		"", "", "", "") )
  {
    mtr_error("Error ndbcluster_install");
    return 1;
{'fraction_non_alphanumeric': 0.20160986985631535, 'fraction_numerical': 0.006349206349206349, 'mean_word_length': 2.6093402117838718, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 383, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5804043', 'n_tokens_mistral': 23108, 'n_tokens_neox': 22116, 'n_words': 4754}