<commit_before>/* Copyright (C) 2001-2004 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/* 

   TODO: print the catalog (some USE catalog.db ????).

   Standalone program to read a MySQL binary log (or relay log);
   can read files produced by 3.23, 4.x, 5.0 servers. 

   Can read binlogs from 3.23/4.x/5.0 and relay logs from 4.x/5.0.
   Should be able to read any file of these categories, even with
   --start-position.
   An important fact: the Format_desc event of the log is at most the 3rd event
   of the log; if it is the 3rd then there is this combination:
   Format_desc_of_slave, Rotate_of_master, Format_desc_of_master.
*/

#define MYSQL_CLIENT
#undef MYSQL_SERVER
#include "client_priv.h"
#include <my_time.h>
/* That one is necessary for defines of OPTION_NO_FOREIGN_KEY_CHECKS etc */
#include "mysql_priv.h" 
#include "log_event.h"

#define BIN_LOG_HEADER_SIZE	4
#define PROBE_HEADER_LEN	(EVENT_LEN_OFFSET+4)


#define CLIENT_CAPABILITIES	(CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG | CLIENT_LOCAL_FILES)

char server_version[SERVER_VERSION_LENGTH];
ulong server_id = 0;

// needed by net_serv.c
ulong bytes_sent = 0L, bytes_received = 0L;
ulong mysqld_net_retry_count = 10L;
ulong open_files_limit;
uint test_flags = 0; 
static uint opt_protocol= 0;
static FILE *result_file;

#ifndef DBUG_OFF
static const char* default_dbug_option = "d:t:o,/tmp/mysqlbinlog.trace";
#endif
static const char *load_default_groups[]= { "mysqlbinlog","client",0 };

void sql_print_error(const char *format, ...);

static bool one_database=0, to_last_remote_log= 0, disable_log_bin= 0;
static bool opt_hexdump= 0;
static bool opt_base64_output= 0;
static const char* database= 0;
static my_bool force_opt= 0, short_form= 0, remote_opt= 0;
static ulonglong offset = 0;
static const char* host = 0;
static int port= 0;
static const char* sock= 0;
static const char* user = 0;
static char* pass = 0;

static ulonglong start_position, stop_position;
#define start_position_mot ((my_off_t)start_position)
#define stop_position_mot  ((my_off_t)stop_position)

static char *start_datetime_str, *stop_datetime_str;
static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX;
static ulonglong rec_count= 0;
static short binlog_flags = 0; 
static MYSQL* mysql = NULL;
static const char* dirname_for_local_load= 0;
static bool stop_passed= 0;

/*
  check_header() will set the pointer below.
  Why do we need here a pointer on an event instead of an event ?
  This is because the event will be created (alloced) in read_log_event()
  (which returns a pointer) in check_header().
*/
Format_description_log_event* description_event; 

static int dump_local_log_entries(const char* logname);
static int dump_remote_log_entries(const char* logname);
static int dump_log_entries(const char* logname);
static int dump_remote_file(NET* net, const char* fname);
static void die(const char* fmt, ...);
static MYSQL* safe_connect();


class Load_log_processor
{
  char target_dir_name[FN_REFLEN];
  int target_dir_name_len;

  /*
    When we see first event corresponding to some LOAD DATA statement in
    binlog, we create temporary file to store data to be loaded.
    We add name of this file to file_names array using its file_id as index.
    If we have Create_file event (i.e. we have binary log in pre-5.0.3
    format) we also store save event object to be able which is needed to
    emit LOAD DATA statement when we will meet Exec_load_data event.
    If we have Begin_load_query event we simply store 0 in
    File_name_record::event field.
  */
  struct File_name_record
  {
    char *fname;
    Create_file_log_event *event;
  };
  DYNAMIC_ARRAY file_names;

  /*
    Looking for new uniquie filename that doesn't exist yet by 
    adding postfix -%x

    SYNOPSIS 
       create_unique_file()
       
       filename       buffer for filename
       file_name_end  tail of buffer that should be changed
                      should point to a memory enough to printf("-%x",..)

    RETURN VALUES
      values less than 0      - can't find new filename
      values great or equal 0 - created file with found filename
  */
  File create_unique_file(char *filename, char *file_name_end)
    {
      File res;
      /* If we have to try more than 1000 times, something is seriously wrong */
      for (uint version= 0; version<1000; version++)
      {
	sprintf(file_name_end,"-%x",version);
	if ((res= my_create(filename,0,
			    O_CREAT|O_EXCL|O_BINARY|O_WRONLY,MYF(0)))!=-1)
	  return res;
      }
      return -1;
    }

public:
  Load_log_processor() {}
  ~Load_log_processor()
  {
    destroy();
    delete_dynamic(&file_names);
  }

  int init()
  {
    return init_dynamic_array(&file_names, sizeof(File_name_record),
			      100,100 CALLER_INFO);
  }

  void init_by_dir_name(const char *dir)
    {
      target_dir_name_len= (convert_dirname(target_dir_name, dir, NullS) -
			    target_dir_name);
    }
  void init_by_cur_dir()
    {
      if (my_getwd(target_dir_name,sizeof(target_dir_name),MYF(MY_WME)))
	exit(1);
      target_dir_name_len= strlen(target_dir_name);
    }
  void destroy()
    {
      File_name_record *ptr= (File_name_record *)file_names.buffer;
      File_name_record *end= ptr + file_names.elements;
      for (; ptr<end; ptr++)
      {
	if (ptr->fname)
	{
          my_free(ptr->fname, MYF(MY_WME));
          delete ptr->event;
          bzero((char *)ptr, sizeof(File_name_record));
	}
      }
    }

  /*
    Obtain Create_file event for LOAD DATA statement by its file_id.

    SYNOPSIS
      grab_event()
        file_id - file_id identifiying LOAD DATA statement

    DESCRIPTION
      Checks whenever we have already seen Create_file event for this file_id.
      If yes then returns pointer to it and removes it from array describing
      active temporary files. Since this moment caller is responsible for
      freeing memory occupied by this event and associated file name.

    RETURN VALUES
      Pointer to Create_file event or 0 if there was no such event
      with this file_id.
  */
  Create_file_log_event *grab_event(uint file_id)
    {
      File_name_record *ptr;
      Create_file_log_event *res;

      if (file_id >= file_names.elements)
        return 0;
      ptr= dynamic_element(&file_names, file_id, File_name_record*);
      if ((res= ptr->event))
        bzero((char *)ptr, sizeof(File_name_record));
      return res;
    }

  /*
    Obtain file name of temporary file for LOAD DATA statement by its file_id.

    SYNOPSIS
      grab_fname()
        file_id - file_id identifiying LOAD DATA statement

    DESCRIPTION
      Checks whenever we have already seen Begin_load_query event for this
      file_id. If yes then returns file name of corresponding temporary file.
      Removes record about this file from the array of active temporary files.
      Since this moment caller is responsible for freeing memory occupied by
      this name.

    RETURN VALUES
      String with name of temporary file or 0 if we have not seen Begin_load_query
      event with this file_id.
  */
  char *grab_fname(uint file_id)
    {
      File_name_record *ptr;
      char *res= 0;

      if (file_id >= file_names.elements)
        return 0;
      ptr= dynamic_element(&file_names, file_id, File_name_record*);
      if (!ptr->event)
      {
        res= ptr->fname;
        bzero((char *)ptr, sizeof(File_name_record));
      }
      return res;
    }
  int process(Create_file_log_event *ce);
  int process(Begin_load_query_log_event *ce);
  int process(Append_block_log_event *ae);
  File prepare_new_file_for_old_format(Load_log_event *le, char *filename);
  int load_old_format_file(NET* net, const char *server_fname,
			   uint server_fname_len, File file);
  int process_first_event(const char *bname, uint blen, const char *block,
                          uint block_len, uint file_id,
                          Create_file_log_event *ce);
};



File Load_log_processor::prepare_new_file_for_old_format(Load_log_event *le,
							 char *filename)
{
  uint len;
  char *tail;
  File file;
  
  fn_format(filename, le->fname, target_dir_name, "", 1);
  len= strlen(filename);
  tail= filename + len;
  
  if ((file= create_unique_file(filename,tail)) < 0)
  {
    sql_print_error("Could not construct local filename %s",filename);
    return -1;
  }
  
  le->set_fname_outside_temp_buf(filename,len+strlen(tail));
  
  return file;
}


int Load_log_processor::load_old_format_file(NET* net, const char*server_fname,
					     uint server_fname_len, File file)
{
  char buf[FN_REFLEN+1];
  buf[0] = 0;
  memcpy(buf + 1, server_fname, server_fname_len + 1);
  if (my_net_write(net, buf, server_fname_len +2) || net_flush(net))
  {
    sql_print_error("Failed  requesting the remote dump of %s", server_fname);
    return -1;
  }
  
  for (;;)
  {
    ulong packet_len = my_net_read(net);
    if (packet_len == 0)
    {
      if (my_net_write(net, "", 0) || net_flush(net))
      {
	sql_print_error("Failed sending the ack packet");
	return -1;
      }
      /*
	we just need to send something, as the server will read but
	not examine the packet - this is because mysql_load() sends 
	an OK when it is done
      */
      break;
    }
    else if (packet_len == packet_error)
    {
      sql_print_error("Failed reading a packet during the dump of %s ", 
		      server_fname);
      return -1;
    }
    
    if (packet_len > UINT_MAX)
    {
      sql_print_error("Illegal length of packet read from net");
      return -1;
    }
    if (my_write(file, (byte*) net->read_pos, 
		 (uint) packet_len, MYF(MY_WME|MY_NABP)))
      return -1;
  }
  
  return 0;
}


/*
  Process first event in the sequence of events representing LOAD DATA
  statement.

  SYNOPSIS
    process_first_event()
      bname     - base name for temporary file to be created
      blen      - base name length
      block     - first block of data to be loaded
      block_len - first block length
      file_id   - identifies LOAD DATA statement
      ce        - pointer to Create_file event object if we are processing
                  this type of event.

  DESCRIPTION
    Creates temporary file to be used in LOAD DATA and writes first block of
    data to it. Registers its file name (and optional Create_file event)
    in the array of active temporary files.

  RETURN VALUES
    0     - success
    non-0 - error
*/

int Load_log_processor::process_first_event(const char *bname, uint blen,
                                            const char *block, uint block_len,
                                            uint file_id,
                                            Create_file_log_event *ce)
{
  uint full_len= target_dir_name_len + blen + 9 + 9 + 1;
  int error= 0;
  char *fname, *ptr;
  File file;
  File_name_record rec;
  DBUG_ENTER("Load_log_processor::process_first_event");

  if (!(fname= my_malloc(full_len,MYF(MY_WME))))
    DBUG_RETURN(-1);

  memcpy(fname, target_dir_name, target_dir_name_len);
  ptr= fname + target_dir_name_len;
  memcpy(ptr,bname,blen);
  ptr+= blen;
  ptr+= my_sprintf(ptr, (ptr, "-%x", file_id));

  if ((file= create_unique_file(fname,ptr)) < 0)
  {
    sql_print_error("Could not construct local filename %s%s",
		    target_dir_name,bname);
    DBUG_RETURN(-1);
  }

  rec.fname= fname;
  rec.event= ce;

  if (set_dynamic(&file_names, (gptr)&rec, file_id))
  {
    sql_print_error("Could not construct local filename %s%s",
		    target_dir_name, bname);
    DBUG_RETURN(-1);
  }

  if (ce)
    ce->set_fname_outside_temp_buf(fname, strlen(fname));

  if (my_write(file, (byte*)block, block_len, MYF(MY_WME|MY_NABP)))
    error= -1;
  if (my_close(file, MYF(MY_WME)))
    error= -1;
  DBUG_RETURN(error);
}


int Load_log_processor::process(Create_file_log_event *ce)
{
  const char *bname= ce->fname + dirname_length(ce->fname);
  uint blen= ce->fname_len - (bname-ce->fname);

  return process_first_event(bname, blen, ce->block, ce->block_len,
                             ce->file_id, ce);
}


int Load_log_processor::process(Begin_load_query_log_event *blqe)
{
  return process_first_event("SQL_LOAD_MB", 11, blqe->block, blqe->block_len,
                             blqe->file_id, 0);
}


int Load_log_processor::process(Append_block_log_event *ae)
{
  DBUG_ENTER("Load_log_processor::process");
  const char* fname= ((ae->file_id < file_names.elements) ?
                       dynamic_element(&file_names, ae->file_id,
                                       File_name_record*)->fname : 0);

  if (fname)
  {
    File file;
    int error= 0;
    if (((file= my_open(fname,
			O_APPEND|O_BINARY|O_WRONLY,MYF(MY_WME))) < 0))
      DBUG_RETURN(-1);
    if (my_write(file,(byte*)ae->block,ae->block_len,MYF(MY_WME|MY_NABP)))
      error= -1;
    if (my_close(file,MYF(MY_WME)))
      error= -1;
    DBUG_RETURN(error);
  }

  /*
    There is no Create_file event (a bad binlog or a big
    --start-position). Assuming it's a big --start-position, we just do
    nothing and print a warning.
  */
  fprintf(stderr,"Warning: ignoring Append_block as there is no \
Create_file event for file_id: %u\n",ae->file_id);
  DBUG_RETURN(-1);
}


Load_log_processor load_processor;


static bool check_database(const char *log_dbname)
{
  return one_database &&
         (log_dbname != NULL) &&
         strcmp(log_dbname, database);
}


/*
  Process an event

  SYNOPSIS
    process_event()

  RETURN
    0           ok and continue
    1           error and terminate
    -1          ok and terminate
 
  TODO
    This function returns 0 even in some error cases. This should be changed.
*/



int process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
                  my_off_t pos)
{
  char ll_buff[21];
  Log_event_type ev_type= ev->get_type_code();
  DBUG_ENTER("process_event");
  print_event_info->short_form= short_form;

  /*
    Format events are not concerned by --offset and such, we always need to
    read them to be able to process the wanted events.
  */
  if ((rec_count >= offset) &&
      ((my_time_t)(ev->when) >= start_datetime) ||
      (ev_type == FORMAT_DESCRIPTION_EVENT))
  {
    if (ev_type != FORMAT_DESCRIPTION_EVENT)
    {
      /*
        We have found an event after start_datetime, from now on print
        everything (in case the binlog has timestamps increasing and
        decreasing, we do this to avoid cutting the middle).
      */
      start_datetime= 0;
      offset= 0; // print everything and protect against cycling rec_count
    }
    if (server_id && (server_id != ev->server_id)) {
      DBUG_RETURN(0);
    }
    if (((my_time_t)(ev->when) >= stop_datetime)
        || (pos >= stop_position_mot))
    {
      stop_passed= 1; // skip all next binlogs
      DBUG_RETURN(-1);
    }
    if (!short_form)
      fprintf(result_file, "# at %s\n",llstr(pos,ll_buff));

    if (!opt_hexdump)
      print_event_info->hexdump_from= 0; /* Disabled */
    else
      print_event_info->hexdump_from= pos;

    print_event_info->base64_output= opt_base64_output;

    switch (ev_type) {
    case QUERY_EVENT:
      if (check_database(((Query_log_event*)ev)->db))
        goto end;
      if (opt_base64_output)
      {
        ev->print_header(result_file, print_event_info);
        ev->print_base64(result_file, print_event_info);
      }
      else
        ev->print(result_file, print_event_info);
      break;
    case CREATE_FILE_EVENT:
    {
      Create_file_log_event* ce= (Create_file_log_event*)ev;
      /*
        We test if this event has to be ignored. If yes, we don't save
        this event; this will have the good side-effect of ignoring all
        related Append_block and Exec_load.
        Note that Load event from 3.23 is not tested.
      */
      if (check_database(ce->db))
        goto end;                // Next event
      /*
	We print the event, but with a leading '#': this is just to inform 
	the user of the original command; the command we want to execute 
	will be a derivation of this original command (we will change the 
	filename and use LOCAL), prepared in the 'case EXEC_LOAD_EVENT' 
	below.
      */
      if (opt_base64_output)
      {
        ce->print_header(result_file, print_event_info);
        ce->print_base64(result_file, print_event_info);
      }
      else
        ce->print(result_file, print_event_info, TRUE);

      // If this binlog is not 3.23 ; why this test??
      if (description_event->binlog_version >= 3)
      {
	if (load_processor.process(ce))
	  break;				// Error
	ev= 0;
      }
      break;
    }
    case APPEND_BLOCK_EVENT:
      ev->print(result_file, print_event_info);
      if (load_processor.process((Append_block_log_event*) ev))
	break;					// Error
      break;
    case EXEC_LOAD_EVENT:
    {
      ev->print(result_file, print_event_info);
      Execute_load_log_event *exv= (Execute_load_log_event*)ev;
      Create_file_log_event *ce= load_processor.grab_event(exv->file_id);
      /*
	if ce is 0, it probably means that we have not seen the Create_file
	event (a bad binlog, or most probably --start-position is after the
	Create_file event). Print a warning comment.
      */
      if (ce)
      {
	ce->print(result_file, print_event_info, TRUE);
	my_free((char*)ce->fname,MYF(MY_WME));
	delete ce;
      }
      else
	fprintf(stderr,"Warning: ignoring Exec_load as there is no \
Create_file event for file_id: %u\n",exv->file_id);
      break;
    }
    case FORMAT_DESCRIPTION_EVENT:
      delete description_event;
      description_event= (Format_description_log_event*) ev;
      print_event_info->common_header_len= description_event->common_header_len;
      ev->print(result_file, print_event_info);
      /*
        We don't want this event to be deleted now, so let's hide it (I
        (Guilhem) should later see if this triggers a non-serious Valgrind
        error). Not serious error, because we will free description_event
        later.
      */
      ev= 0;
      break;
    case BEGIN_LOAD_QUERY_EVENT:
      ev->print(result_file, print_event_info);
      load_processor.process((Begin_load_query_log_event*) ev);
      break;
    case EXECUTE_LOAD_QUERY_EVENT:
    {
      Execute_load_query_log_event *exlq= (Execute_load_query_log_event*)ev;
      char *fname= load_processor.grab_fname(exlq->file_id);

      if (check_database(exlq->db))
      {
        if (fname)
          my_free(fname, MYF(MY_WME));
        goto end;
      }

      if (fname)
      {
	exlq->print(result_file, print_event_info, fname);
	my_free(fname, MYF(MY_WME));
      }
      else
	fprintf(stderr,"Warning: ignoring Execute_load_query as there is no \
Begin_load_query event for file_id: %u\n", exlq->file_id);
      break;
    }
    default:
      ev->print(result_file, print_event_info);
    }
  }

end:
  rec_count++;
  if (ev)
    delete ev;
  DBUG_RETURN(0);
}


static struct my_option my_long_options[] =
{

#ifdef __NETWARE__
  {"autoclose", OPT_AUTO_CLOSE, "Auto close the screen on exit for Netware.",
   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#endif
  {"base64-output", OPT_BASE64_OUTPUT,
   "Print all binlog entries using base64 encoding. "
   "This is for debugging only. Logs produced using this option "
   "should not be applied on production systems.",
   (gptr*) &opt_base64_output, (gptr*) &opt_base64_output, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  /*
    mysqlbinlog needs charsets knowledge, to be able to convert a charset
    number found in binlog to a charset name (to be able to print things
    like this:
    SET @`a`:=_cp850 0x4DFC6C6C6572 COLLATE `cp850_general_ci`;
  */
  {"character-sets-dir", OPT_CHARSETS_DIR,
   "Directory where character sets are.", (gptr*) &charsets_dir,
   (gptr*) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
  {"debug", '#', "Output debug log.", (gptr*) &default_dbug_option,
   (gptr*) &default_dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
#endif
  {"database", 'd', "List entries for just this database (local log only).",
   (gptr*) &database, (gptr*) &database, 0, GET_STR_ALLOC, REQUIRED_ARG,
   0, 0, 0, 0, 0, 0},
  {"disable-log-bin", 'D', "Disable binary log. This is useful, if you "
    "enabled --to-last-log and are sending the output to the same MySQL server. "
    "This way you could avoid an endless loop. You would also like to use it "
    "when restoring after a crash to avoid duplication of the statements you "
    "already have. NOTE: you will need a SUPER privilege to use this option.",
   (gptr*) &disable_log_bin, (gptr*) &disable_log_bin, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  {"force-read", 'f', "Force reading unknown binlog events.",
   (gptr*) &force_opt, (gptr*) &force_opt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"help", '?', "Display this help and exit.",
   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"hexdump", 'H', "Augment output with hexadecimal and ASCII event dump.",
   (gptr*) &opt_hexdump, (gptr*) &opt_hexdump, 0, GET_BOOL, NO_ARG,
   0, 0, 0, 0, 0, 0},
  {"host", 'h', "Get the binlog from server.", (gptr*) &host, (gptr*) &host,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"offset", 'o', "Skip the first N entries.", (gptr*) &offset, (gptr*) &offset,
   0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"password", 'p', "Password to connect to remote server.",
   0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
  {"port", 'P', "Use port to connect to the remote server.",
   (gptr*) &port, (gptr*) &port, 0, GET_INT, REQUIRED_ARG, 0, 0, 0,
   0, 0, 0},
  {"position", 'j', "Deprecated. Use --start-position instead.",
   (gptr*) &start_position, (gptr*) &start_position, 0, GET_ULL,
   REQUIRED_ARG, BIN_LOG_HEADER_SIZE, BIN_LOG_HEADER_SIZE,
   /* COM_BINLOG_DUMP accepts only 4 bytes for the position */
   (ulonglong)(~(uint32)0), 0, 0, 0},
  {"protocol", OPT_MYSQL_PROTOCOL,
   "The protocol of connection (tcp,socket,pipe,memory).",
   0, 0, 0, GET_STR,  REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"result-file", 'r', "Direct output to a given file.", 0, 0, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"read-from-remote-server", 'R', "Read binary logs from a MySQL server",
   (gptr*) &remote_opt, (gptr*) &remote_opt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"open_files_limit", OPT_OPEN_FILES_LIMIT,
   "Used to reserve file descriptors for usage by this program",
   (gptr*) &open_files_limit, (gptr*) &open_files_limit, 0, GET_ULONG,
   REQUIRED_ARG, MY_NFILE, 8, OS_FILE_LIMIT, 0, 1, 0},
  {"short-form", 's', "Just show the queries, no extra info.",
   (gptr*) &short_form, (gptr*) &short_form, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"socket", 'S', "Socket file to use for connection.",
   (gptr*) &sock, (gptr*) &sock, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 
   0, 0},
  {"start-datetime", OPT_START_DATETIME,
   "Start reading the binlog at first event having a datetime equal or "
   "posterior to the argument; the argument must be a date and time "
   "in the local time zone, in any format accepted by the MySQL server "
   "for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 "
   "(you should probably use quotes for your shell to set it properly).",
   (gptr*) &start_datetime_str, (gptr*) &start_datetime_str,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"stop-datetime", OPT_STOP_DATETIME,
   "Stop reading the binlog at first event having a datetime equal or "
   "posterior to the argument; the argument must be a date and time "
   "in the local time zone, in any format accepted by the MySQL server "
   "for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 "
   "(you should probably use quotes for your shell to set it properly).",
   (gptr*) &stop_datetime_str, (gptr*) &stop_datetime_str,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"start-position", OPT_START_POSITION,
   "Start reading the binlog at position N. Applies to the first binlog "
   "passed on the command line.",
   (gptr*) &start_position, (gptr*) &start_position, 0, GET_ULL,
   REQUIRED_ARG, BIN_LOG_HEADER_SIZE, BIN_LOG_HEADER_SIZE,
   /* COM_BINLOG_DUMP accepts only 4 bytes for the position */
   (ulonglong)(~(uint32)0), 0, 0, 0},
  {"stop-position", OPT_STOP_POSITION,
   "Stop reading the binlog at position N. Applies to the last binlog "
   "passed on the command line.",
   (gptr*) &stop_position, (gptr*) &stop_position, 0, GET_ULL,
   REQUIRED_ARG, (ulonglong)(~(my_off_t)0), BIN_LOG_HEADER_SIZE,
   (ulonglong)(~(my_off_t)0), 0, 0, 0},
  {"server-id", OPT_SERVER_ID,
   "Only extract binlog entries created by a certain server id "
   "passed on the command line.",
   (gptr*) &server_id, (gptr*) &server_id, 0, GET_ULONG,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"to-last-log", 't', "Requires -R. Will not stop at the end of the \
requested binlog but rather continue printing until the end of the last \
binlog of the MySQL server. If you send the output to the same MySQL server, \
that may lead to an endless loop.",
   (gptr*) &to_last_remote_log, (gptr*) &to_last_remote_log, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  {"user", 'u', "Connect to the remote server as username.",
   (gptr*) &user, (gptr*) &user, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0,
   0, 0},
  {"local-load", 'l', "Prepare local temporary files for LOAD DATA INFILE in the specified directory.",
   (gptr*) &dirname_for_local_load, (gptr*) &dirname_for_local_load, 0,
   GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"version", 'V', "Print version and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
   0, 0, 0, 0, 0},
  {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};


void sql_print_error(const char *format,...)
{
  va_list args;
  va_start(args, format);
  fprintf(stderr, "ERROR: ");
  vfprintf(stderr, format, args);
  fprintf(stderr, "\n");
  va_end(args);
}

static void cleanup()
{
  my_free(pass,MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) database, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) host, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) user, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) dirname_for_local_load, MYF(MY_ALLOW_ZERO_PTR));
}<commit_msg>mysqlbinlog.cc:   Put options in standard order: help first, then lexically,   then variables.<commit_after>/* Copyright (C) 2001-2004 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/* 

   TODO: print the catalog (some USE catalog.db ????).

   Standalone program to read a MySQL binary log (or relay log);
   can read files produced by 3.23, 4.x, 5.0 servers. 

   Can read binlogs from 3.23/4.x/5.0 and relay logs from 4.x/5.0.
   Should be able to read any file of these categories, even with
   --start-position.
   An important fact: the Format_desc event of the log is at most the 3rd event
   of the log; if it is the 3rd then there is this combination:
   Format_desc_of_slave, Rotate_of_master, Format_desc_of_master.
*/

#define MYSQL_CLIENT
#undef MYSQL_SERVER
#include "client_priv.h"
#include <my_time.h>
/* That one is necessary for defines of OPTION_NO_FOREIGN_KEY_CHECKS etc */
#include "mysql_priv.h" 
#include "log_event.h"

#define BIN_LOG_HEADER_SIZE	4
#define PROBE_HEADER_LEN	(EVENT_LEN_OFFSET+4)


#define CLIENT_CAPABILITIES	(CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG | CLIENT_LOCAL_FILES)

char server_version[SERVER_VERSION_LENGTH];
ulong server_id = 0;

// needed by net_serv.c
ulong bytes_sent = 0L, bytes_received = 0L;
ulong mysqld_net_retry_count = 10L;
ulong open_files_limit;
uint test_flags = 0; 
static uint opt_protocol= 0;
static FILE *result_file;

#ifndef DBUG_OFF
static const char* default_dbug_option = "d:t:o,/tmp/mysqlbinlog.trace";
#endif
static const char *load_default_groups[]= { "mysqlbinlog","client",0 };

void sql_print_error(const char *format, ...);

static bool one_database=0, to_last_remote_log= 0, disable_log_bin= 0;
static bool opt_hexdump= 0;
static bool opt_base64_output= 0;
static const char* database= 0;
static my_bool force_opt= 0, short_form= 0, remote_opt= 0;
static ulonglong offset = 0;
static const char* host = 0;
static int port= 0;
static const char* sock= 0;
static const char* user = 0;
static char* pass = 0;

static ulonglong start_position, stop_position;
#define start_position_mot ((my_off_t)start_position)
#define stop_position_mot  ((my_off_t)stop_position)

static char *start_datetime_str, *stop_datetime_str;
static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX;
static ulonglong rec_count= 0;
static short binlog_flags = 0; 
static MYSQL* mysql = NULL;
static const char* dirname_for_local_load= 0;
static bool stop_passed= 0;

/*
  check_header() will set the pointer below.
  Why do we need here a pointer on an event instead of an event ?
  This is because the event will be created (alloced) in read_log_event()
  (which returns a pointer) in check_header().
*/
Format_description_log_event* description_event; 

static int dump_local_log_entries(const char* logname);
static int dump_remote_log_entries(const char* logname);
static int dump_log_entries(const char* logname);
static int dump_remote_file(NET* net, const char* fname);
static void die(const char* fmt, ...);
static MYSQL* safe_connect();


class Load_log_processor
{
  char target_dir_name[FN_REFLEN];
  int target_dir_name_len;

  /*
    When we see first event corresponding to some LOAD DATA statement in
    binlog, we create temporary file to store data to be loaded.
    We add name of this file to file_names array using its file_id as index.
    If we have Create_file event (i.e. we have binary log in pre-5.0.3
    format) we also store save event object to be able which is needed to
    emit LOAD DATA statement when we will meet Exec_load_data event.
    If we have Begin_load_query event we simply store 0 in
    File_name_record::event field.
  */
  struct File_name_record
  {
    char *fname;
    Create_file_log_event *event;
  };
  DYNAMIC_ARRAY file_names;

  /*
    Looking for new uniquie filename that doesn't exist yet by 
    adding postfix -%x

    SYNOPSIS 
       create_unique_file()
       
       filename       buffer for filename
       file_name_end  tail of buffer that should be changed
                      should point to a memory enough to printf("-%x",..)

    RETURN VALUES
      values less than 0      - can't find new filename
      values great or equal 0 - created file with found filename
  */
  File create_unique_file(char *filename, char *file_name_end)
    {
      File res;
      /* If we have to try more than 1000 times, something is seriously wrong */
      for (uint version= 0; version<1000; version++)
      {
	sprintf(file_name_end,"-%x",version);
	if ((res= my_create(filename,0,
			    O_CREAT|O_EXCL|O_BINARY|O_WRONLY,MYF(0)))!=-1)
	  return res;
      }
      return -1;
    }

public:
  Load_log_processor() {}
  ~Load_log_processor()
  {
    destroy();
    delete_dynamic(&file_names);
  }

  int init()
  {
    return init_dynamic_array(&file_names, sizeof(File_name_record),
			      100,100 CALLER_INFO);
  }

  void init_by_dir_name(const char *dir)
    {
      target_dir_name_len= (convert_dirname(target_dir_name, dir, NullS) -
			    target_dir_name);
    }
  void init_by_cur_dir()
    {
      if (my_getwd(target_dir_name,sizeof(target_dir_name),MYF(MY_WME)))
	exit(1);
      target_dir_name_len= strlen(target_dir_name);
    }
  void destroy()
    {
      File_name_record *ptr= (File_name_record *)file_names.buffer;
      File_name_record *end= ptr + file_names.elements;
      for (; ptr<end; ptr++)
      {
	if (ptr->fname)
	{
          my_free(ptr->fname, MYF(MY_WME));
          delete ptr->event;
          bzero((char *)ptr, sizeof(File_name_record));
	}
      }
    }

  /*
    Obtain Create_file event for LOAD DATA statement by its file_id.

    SYNOPSIS
      grab_event()
        file_id - file_id identifiying LOAD DATA statement

    DESCRIPTION
      Checks whenever we have already seen Create_file event for this file_id.
      If yes then returns pointer to it and removes it from array describing
      active temporary files. Since this moment caller is responsible for
      freeing memory occupied by this event and associated file name.

    RETURN VALUES
      Pointer to Create_file event or 0 if there was no such event
      with this file_id.
  */
  Create_file_log_event *grab_event(uint file_id)
    {
      File_name_record *ptr;
      Create_file_log_event *res;

      if (file_id >= file_names.elements)
        return 0;
      ptr= dynamic_element(&file_names, file_id, File_name_record*);
      if ((res= ptr->event))
        bzero((char *)ptr, sizeof(File_name_record));
      return res;
    }

  /*
    Obtain file name of temporary file for LOAD DATA statement by its file_id.

    SYNOPSIS
      grab_fname()
        file_id - file_id identifiying LOAD DATA statement

    DESCRIPTION
      Checks whenever we have already seen Begin_load_query event for this
      file_id. If yes then returns file name of corresponding temporary file.
      Removes record about this file from the array of active temporary files.
      Since this moment caller is responsible for freeing memory occupied by
      this name.

    RETURN VALUES
      String with name of temporary file or 0 if we have not seen Begin_load_query
      event with this file_id.
  */
  char *grab_fname(uint file_id)
    {
      File_name_record *ptr;
      char *res= 0;

      if (file_id >= file_names.elements)
        return 0;
      ptr= dynamic_element(&file_names, file_id, File_name_record*);
      if (!ptr->event)
      {
        res= ptr->fname;
        bzero((char *)ptr, sizeof(File_name_record));
      }
      return res;
    }
  int process(Create_file_log_event *ce);
  int process(Begin_load_query_log_event *ce);
  int process(Append_block_log_event *ae);
  File prepare_new_file_for_old_format(Load_log_event *le, char *filename);
  int load_old_format_file(NET* net, const char *server_fname,
			   uint server_fname_len, File file);
  int process_first_event(const char *bname, uint blen, const char *block,
                          uint block_len, uint file_id,
                          Create_file_log_event *ce);
};



File Load_log_processor::prepare_new_file_for_old_format(Load_log_event *le,
							 char *filename)
{
  uint len;
  char *tail;
  File file;
  
  fn_format(filename, le->fname, target_dir_name, "", 1);
  len= strlen(filename);
  tail= filename + len;
  
  if ((file= create_unique_file(filename,tail)) < 0)
  {
    sql_print_error("Could not construct local filename %s",filename);
    return -1;
  }
  
  le->set_fname_outside_temp_buf(filename,len+strlen(tail));
  
  return file;
}


int Load_log_processor::load_old_format_file(NET* net, const char*server_fname,
					     uint server_fname_len, File file)
{
  char buf[FN_REFLEN+1];
  buf[0] = 0;
  memcpy(buf + 1, server_fname, server_fname_len + 1);
  if (my_net_write(net, buf, server_fname_len +2) || net_flush(net))
  {
    sql_print_error("Failed  requesting the remote dump of %s", server_fname);
    return -1;
  }
  
  for (;;)
  {
    ulong packet_len = my_net_read(net);
    if (packet_len == 0)
    {
      if (my_net_write(net, "", 0) || net_flush(net))
      {
	sql_print_error("Failed sending the ack packet");
	return -1;
      }
      /*
	we just need to send something, as the server will read but
	not examine the packet - this is because mysql_load() sends 
	an OK when it is done
      */
      break;
    }
    else if (packet_len == packet_error)
    {
      sql_print_error("Failed reading a packet during the dump of %s ", 
		      server_fname);
      return -1;
    }
    
    if (packet_len > UINT_MAX)
    {
      sql_print_error("Illegal length of packet read from net");
      return -1;
    }
    if (my_write(file, (byte*) net->read_pos, 
		 (uint) packet_len, MYF(MY_WME|MY_NABP)))
      return -1;
  }
  
  return 0;
}


/*
  Process first event in the sequence of events representing LOAD DATA
  statement.

  SYNOPSIS
    process_first_event()
      bname     - base name for temporary file to be created
      blen      - base name length
      block     - first block of data to be loaded
      block_len - first block length
      file_id   - identifies LOAD DATA statement
      ce        - pointer to Create_file event object if we are processing
                  this type of event.

  DESCRIPTION
    Creates temporary file to be used in LOAD DATA and writes first block of
    data to it. Registers its file name (and optional Create_file event)
    in the array of active temporary files.

  RETURN VALUES
    0     - success
    non-0 - error
*/

int Load_log_processor::process_first_event(const char *bname, uint blen,
                                            const char *block, uint block_len,
                                            uint file_id,
                                            Create_file_log_event *ce)
{
  uint full_len= target_dir_name_len + blen + 9 + 9 + 1;
  int error= 0;
  char *fname, *ptr;
  File file;
  File_name_record rec;
  DBUG_ENTER("Load_log_processor::process_first_event");

  if (!(fname= my_malloc(full_len,MYF(MY_WME))))
    DBUG_RETURN(-1);

  memcpy(fname, target_dir_name, target_dir_name_len);
  ptr= fname + target_dir_name_len;
  memcpy(ptr,bname,blen);
  ptr+= blen;
  ptr+= my_sprintf(ptr, (ptr, "-%x", file_id));

  if ((file= create_unique_file(fname,ptr)) < 0)
  {
    sql_print_error("Could not construct local filename %s%s",
		    target_dir_name,bname);
    DBUG_RETURN(-1);
  }

  rec.fname= fname;
  rec.event= ce;

  if (set_dynamic(&file_names, (gptr)&rec, file_id))
  {
    sql_print_error("Could not construct local filename %s%s",
		    target_dir_name, bname);
    DBUG_RETURN(-1);
  }

  if (ce)
    ce->set_fname_outside_temp_buf(fname, strlen(fname));

  if (my_write(file, (byte*)block, block_len, MYF(MY_WME|MY_NABP)))
    error= -1;
  if (my_close(file, MYF(MY_WME)))
    error= -1;
  DBUG_RETURN(error);
}


int Load_log_processor::process(Create_file_log_event *ce)
{
  const char *bname= ce->fname + dirname_length(ce->fname);
  uint blen= ce->fname_len - (bname-ce->fname);

  return process_first_event(bname, blen, ce->block, ce->block_len,
                             ce->file_id, ce);
}


int Load_log_processor::process(Begin_load_query_log_event *blqe)
{
  return process_first_event("SQL_LOAD_MB", 11, blqe->block, blqe->block_len,
                             blqe->file_id, 0);
}


int Load_log_processor::process(Append_block_log_event *ae)
{
  DBUG_ENTER("Load_log_processor::process");
  const char* fname= ((ae->file_id < file_names.elements) ?
                       dynamic_element(&file_names, ae->file_id,
                                       File_name_record*)->fname : 0);

  if (fname)
  {
    File file;
    int error= 0;
    if (((file= my_open(fname,
			O_APPEND|O_BINARY|O_WRONLY,MYF(MY_WME))) < 0))
      DBUG_RETURN(-1);
    if (my_write(file,(byte*)ae->block,ae->block_len,MYF(MY_WME|MY_NABP)))
      error= -1;
    if (my_close(file,MYF(MY_WME)))
      error= -1;
    DBUG_RETURN(error);
  }

  /*
    There is no Create_file event (a bad binlog or a big
    --start-position). Assuming it's a big --start-position, we just do
    nothing and print a warning.
  */
  fprintf(stderr,"Warning: ignoring Append_block as there is no \
Create_file event for file_id: %u\n",ae->file_id);
  DBUG_RETURN(-1);
}


Load_log_processor load_processor;


static bool check_database(const char *log_dbname)
{
  return one_database &&
         (log_dbname != NULL) &&
         strcmp(log_dbname, database);
}


/*
  Process an event

  SYNOPSIS
    process_event()

  RETURN
    0           ok and continue
    1           error and terminate
    -1          ok and terminate
 
  TODO
    This function returns 0 even in some error cases. This should be changed.
*/



int process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
                  my_off_t pos)
{
  char ll_buff[21];
  Log_event_type ev_type= ev->get_type_code();
  DBUG_ENTER("process_event");
  print_event_info->short_form= short_form;

  /*
    Format events are not concerned by --offset and such, we always need to
    read them to be able to process the wanted events.
  */
  if ((rec_count >= offset) &&
      ((my_time_t)(ev->when) >= start_datetime) ||
      (ev_type == FORMAT_DESCRIPTION_EVENT))
  {
    if (ev_type != FORMAT_DESCRIPTION_EVENT)
    {
      /*
        We have found an event after start_datetime, from now on print
        everything (in case the binlog has timestamps increasing and
        decreasing, we do this to avoid cutting the middle).
      */
      start_datetime= 0;
      offset= 0; // print everything and protect against cycling rec_count
    }
    if (server_id && (server_id != ev->server_id)) {
      DBUG_RETURN(0);
    }
    if (((my_time_t)(ev->when) >= stop_datetime)
        || (pos >= stop_position_mot))
    {
      stop_passed= 1; // skip all next binlogs
      DBUG_RETURN(-1);
    }
    if (!short_form)
      fprintf(result_file, "# at %s\n",llstr(pos,ll_buff));

    if (!opt_hexdump)
      print_event_info->hexdump_from= 0; /* Disabled */
    else
      print_event_info->hexdump_from= pos;

    print_event_info->base64_output= opt_base64_output;

    switch (ev_type) {
    case QUERY_EVENT:
      if (check_database(((Query_log_event*)ev)->db))
        goto end;
      if (opt_base64_output)
      {
        ev->print_header(result_file, print_event_info);
        ev->print_base64(result_file, print_event_info);
      }
      else
        ev->print(result_file, print_event_info);
      break;
    case CREATE_FILE_EVENT:
    {
      Create_file_log_event* ce= (Create_file_log_event*)ev;
      /*
        We test if this event has to be ignored. If yes, we don't save
        this event; this will have the good side-effect of ignoring all
        related Append_block and Exec_load.
        Note that Load event from 3.23 is not tested.
      */
      if (check_database(ce->db))
        goto end;                // Next event
      /*
	We print the event, but with a leading '#': this is just to inform 
	the user of the original command; the command we want to execute 
	will be a derivation of this original command (we will change the 
	filename and use LOCAL), prepared in the 'case EXEC_LOAD_EVENT' 
	below.
      */
      if (opt_base64_output)
      {
        ce->print_header(result_file, print_event_info);
        ce->print_base64(result_file, print_event_info);
      }
      else
        ce->print(result_file, print_event_info, TRUE);

      // If this binlog is not 3.23 ; why this test??
      if (description_event->binlog_version >= 3)
      {
	if (load_processor.process(ce))
	  break;				// Error
	ev= 0;
      }
      break;
    }
    case APPEND_BLOCK_EVENT:
      ev->print(result_file, print_event_info);
      if (load_processor.process((Append_block_log_event*) ev))
	break;					// Error
      break;
    case EXEC_LOAD_EVENT:
    {
      ev->print(result_file, print_event_info);
      Execute_load_log_event *exv= (Execute_load_log_event*)ev;
      Create_file_log_event *ce= load_processor.grab_event(exv->file_id);
      /*
	if ce is 0, it probably means that we have not seen the Create_file
	event (a bad binlog, or most probably --start-position is after the
	Create_file event). Print a warning comment.
      */
      if (ce)
      {
	ce->print(result_file, print_event_info, TRUE);
	my_free((char*)ce->fname,MYF(MY_WME));
	delete ce;
      }
      else
	fprintf(stderr,"Warning: ignoring Exec_load as there is no \
Create_file event for file_id: %u\n",exv->file_id);
      break;
    }
    case FORMAT_DESCRIPTION_EVENT:
      delete description_event;
      description_event= (Format_description_log_event*) ev;
      print_event_info->common_header_len= description_event->common_header_len;
      ev->print(result_file, print_event_info);
      /*
        We don't want this event to be deleted now, so let's hide it (I
        (Guilhem) should later see if this triggers a non-serious Valgrind
        error). Not serious error, because we will free description_event
        later.
      */
      ev= 0;
      break;
    case BEGIN_LOAD_QUERY_EVENT:
      ev->print(result_file, print_event_info);
      load_processor.process((Begin_load_query_log_event*) ev);
      break;
    case EXECUTE_LOAD_QUERY_EVENT:
    {
      Execute_load_query_log_event *exlq= (Execute_load_query_log_event*)ev;
      char *fname= load_processor.grab_fname(exlq->file_id);

      if (check_database(exlq->db))
      {
        if (fname)
          my_free(fname, MYF(MY_WME));
        goto end;
      }

      if (fname)
      {
	exlq->print(result_file, print_event_info, fname);
	my_free(fname, MYF(MY_WME));
      }
      else
	fprintf(stderr,"Warning: ignoring Execute_load_query as there is no \
Begin_load_query event for file_id: %u\n", exlq->file_id);
      break;
    }
    default:
      ev->print(result_file, print_event_info);
    }
  }

end:
  rec_count++;
  if (ev)
    delete ev;
  DBUG_RETURN(0);
}


static struct my_option my_long_options[] =
{
  {"help", '?', "Display this help and exit.",
   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifdef __NETWARE__
  {"autoclose", OPT_AUTO_CLOSE, "Auto close the screen on exit for Netware.",
   0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
#endif
  {"base64-output", OPT_BASE64_OUTPUT,
   "Print all binlog entries using base64 encoding. "
   "This is for debugging only. Logs produced using this option "
   "should not be applied on production systems.",
   (gptr*) &opt_base64_output, (gptr*) &opt_base64_output, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  /*
    mysqlbinlog needs charsets knowledge, to be able to convert a charset
    number found in binlog to a charset name (to be able to print things
    like this:
    SET @`a`:=_cp850 0x4DFC6C6C6572 COLLATE `cp850_general_ci`;
  */
  {"character-sets-dir", OPT_CHARSETS_DIR,
   "Directory where character sets are.", (gptr*) &charsets_dir,
   (gptr*) &charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"database", 'd', "List entries for just this database (local log only).",
   (gptr*) &database, (gptr*) &database, 0, GET_STR_ALLOC, REQUIRED_ARG,
   0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
  {"debug", '#', "Output debug log.", (gptr*) &default_dbug_option,
   (gptr*) &default_dbug_option, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
#endif
  {"disable-log-bin", 'D', "Disable binary log. This is useful, if you "
    "enabled --to-last-log and are sending the output to the same MySQL server. "
    "This way you could avoid an endless loop. You would also like to use it "
    "when restoring after a crash to avoid duplication of the statements you "
    "already have. NOTE: you will need a SUPER privilege to use this option.",
   (gptr*) &disable_log_bin, (gptr*) &disable_log_bin, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  {"force-read", 'f', "Force reading unknown binlog events.",
   (gptr*) &force_opt, (gptr*) &force_opt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"hexdump", 'H', "Augment output with hexadecimal and ASCII event dump.",
   (gptr*) &opt_hexdump, (gptr*) &opt_hexdump, 0, GET_BOOL, NO_ARG,
   0, 0, 0, 0, 0, 0},
  {"host", 'h', "Get the binlog from server.", (gptr*) &host, (gptr*) &host,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"local-load", 'l', "Prepare local temporary files for LOAD DATA INFILE in the specified directory.",
   (gptr*) &dirname_for_local_load, (gptr*) &dirname_for_local_load, 0,
   GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"offset", 'o', "Skip the first N entries.", (gptr*) &offset, (gptr*) &offset,
   0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"password", 'p', "Password to connect to remote server.",
   0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
  {"port", 'P', "Use port to connect to the remote server.",
   (gptr*) &port, (gptr*) &port, 0, GET_INT, REQUIRED_ARG, 0, 0, 0,
   0, 0, 0},
  {"position", 'j', "Deprecated. Use --start-position instead.",
   (gptr*) &start_position, (gptr*) &start_position, 0, GET_ULL,
   REQUIRED_ARG, BIN_LOG_HEADER_SIZE, BIN_LOG_HEADER_SIZE,
   /* COM_BINLOG_DUMP accepts only 4 bytes for the position */
   (ulonglong)(~(uint32)0), 0, 0, 0},
  {"protocol", OPT_MYSQL_PROTOCOL,
   "The protocol of connection (tcp,socket,pipe,memory).",
   0, 0, 0, GET_STR,  REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"read-from-remote-server", 'R', "Read binary logs from a MySQL server",
   (gptr*) &remote_opt, (gptr*) &remote_opt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"result-file", 'r', "Direct output to a given file.", 0, 0, 0, GET_STR,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"server-id", OPT_SERVER_ID,
   "Extract only binlog entries created by the server having the given id.",
   (gptr*) &server_id, (gptr*) &server_id, 0, GET_ULONG,
   REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"short-form", 's', "Just show the queries, no extra info.",
   (gptr*) &short_form, (gptr*) &short_form, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
   0, 0},
  {"socket", 'S', "Socket file to use for connection.",
   (gptr*) &sock, (gptr*) &sock, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 
   0, 0},
  {"start-datetime", OPT_START_DATETIME,
   "Start reading the binlog at first event having a datetime equal or "
   "posterior to the argument; the argument must be a date and time "
   "in the local time zone, in any format accepted by the MySQL server "
   "for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 "
   "(you should probably use quotes for your shell to set it properly).",
   (gptr*) &start_datetime_str, (gptr*) &start_datetime_str,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"start-position", OPT_START_POSITION,
   "Start reading the binlog at position N. Applies to the first binlog "
   "passed on the command line.",
   (gptr*) &start_position, (gptr*) &start_position, 0, GET_ULL,
   REQUIRED_ARG, BIN_LOG_HEADER_SIZE, BIN_LOG_HEADER_SIZE,
   /* COM_BINLOG_DUMP accepts only 4 bytes for the position */
   (ulonglong)(~(uint32)0), 0, 0, 0},
  {"stop-datetime", OPT_STOP_DATETIME,
   "Stop reading the binlog at first event having a datetime equal or "
   "posterior to the argument; the argument must be a date and time "
   "in the local time zone, in any format accepted by the MySQL server "
   "for DATETIME and TIMESTAMP types, for example: 2004-12-25 11:25:56 "
   "(you should probably use quotes for your shell to set it properly).",
   (gptr*) &stop_datetime_str, (gptr*) &stop_datetime_str,
   0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  {"stop-position", OPT_STOP_POSITION,
   "Stop reading the binlog at position N. Applies to the last binlog "
   "passed on the command line.",
   (gptr*) &stop_position, (gptr*) &stop_position, 0, GET_ULL,
   REQUIRED_ARG, (ulonglong)(~(my_off_t)0), BIN_LOG_HEADER_SIZE,
   (ulonglong)(~(my_off_t)0), 0, 0, 0},
  {"to-last-log", 't', "Requires -R. Will not stop at the end of the \
requested binlog but rather continue printing until the end of the last \
binlog of the MySQL server. If you send the output to the same MySQL server, \
that may lead to an endless loop.",
   (gptr*) &to_last_remote_log, (gptr*) &to_last_remote_log, 0, GET_BOOL,
   NO_ARG, 0, 0, 0, 0, 0, 0},
  {"user", 'u', "Connect to the remote server as username.",
   (gptr*) &user, (gptr*) &user, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0,
   0, 0},
  {"version", 'V', "Print version and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
   0, 0, 0, 0, 0},
  {"open_files_limit", OPT_OPEN_FILES_LIMIT,
   "Used to reserve file descriptors for usage by this program",
   (gptr*) &open_files_limit, (gptr*) &open_files_limit, 0, GET_ULONG,
   REQUIRED_ARG, MY_NFILE, 8, OS_FILE_LIMIT, 0, 1, 0},
  {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};


void sql_print_error(const char *format,...)
{
  va_list args;
  va_start(args, format);
  fprintf(stderr, "ERROR: ");
  vfprintf(stderr, format, args);
  fprintf(stderr, "\n");
  va_end(args);
}

static void cleanup()
{
  my_free(pass,MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) database, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) host, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) user, MYF(MY_ALLOW_ZERO_PTR));
  my_free((char*) dirname_for_local_load, MYF(MY_ALLOW_ZERO_PTR));
}
{'fraction_non_alphanumeric': 0.10774637356064005, 'fraction_numerical': 0.017010617616270374, 'mean_word_length': 3.221669823232323, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5591467', 'n_tokens_mistral': 19285, 'n_tokens_neox': 18200, 'n_words': 5658}