<commit_before>/** Copyright 2008, 2009, 2010, 2011, 2012 <NAME>
*
* This file is part of Overpass_API.
*
* Overpass_API is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Overpass_API is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Overpass_API.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "../../expat/expat_justparse_interface.h"
#include "../../template_db/dispatcher.h"
#include "../frontend/console_output.h"
#include "../frontend/user_interface.h"
#include "../frontend/web_output.h"
#include "../osm-backend/clone_database.h"
#include "../statements/osm_script.h"
#include "../statements/statement.h"
#include "resource_manager.h"
#include "scripting_core.h"

#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/select.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

using namespace std;

int main(int argc, char *argv[])
{  
  // read command line arguments
  string db_dir = "";
  string clone_db_dir = "";
  uint log_level = Error_Output::ASSISTING;
  Debug_Level debug_level = parser_execute;
  int area_level = 0;
  bool respect_timeout = true;
  
  int argpos = 1;
  while (argpos < argc)
  {
    if (!(strncmp(argv[argpos], "--db-dir=", 9)))
    {
      db_dir = ((string)argv[argpos]).substr(9);
      if ((db_dir.size() > 0) && (db_dir[db_dir.size()-1] != '/'))
	db_dir += '/';
    }
    else if (!(strcmp(argv[argpos], "--quiet")))
      log_level = Error_Output::QUIET;
    else if (!(strcmp(argv[argpos], "--concise")))
      log_level = Error_Output::CONCISE;
    else if (!(strcmp(argv[argpos], "--progress")))
      log_level = Error_Output::PROGRESS;
    else if (!(strcmp(argv[argpos], "--verbose")))
      log_level = Error_Output::VERBOSE;
    else if (!(strcmp(argv[argpos], "--rules")))
    {
      area_level = 2;
      respect_timeout = false;
    }
    else if (!(strcmp(argv[argpos], "--dump-xml")))
      debug_level = parser_dump_xml;
    else if (!(strcmp(argv[argpos], "--dump-pretty-map-ql")))
      debug_level = parser_dump_pretty_map_ql;
    else if (!(strcmp(argv[argpos], "--dump-compact-map-ql")))
      debug_level = parser_dump_compact_map_ql;
    else if (!(strcmp(argv[argpos], "--dump-bbox-map-ql")))
      debug_level = parser_dump_bbox_map_ql;
    else if (!(strncmp(argv[argpos], "--clone=", 8)))
    {
      clone_db_dir = ((string)argv[argpos]).substr(8);
      if ((clone_db_dir.size() > 0) && (clone_db_dir[clone_db_dir.size()-1] != '/'))
	clone_db_dir += '/';
    }
    else
    {
      cout<<"Unknown argument: "<<argv[argpos]<<"\n\n"
      "Accepted arguments are:\n"
      "  --db-dir=$DB_DIR: The directory where the database resides. If you set this parameter\n"
      "        then osm3s_query will read from the database without using the dispatcher management.\n"
      "  --dump-xml: Don't execute the query but only dump the query in XML format.\n"
      "  --dump-pretty-map-ql: Don't execute the query but only dump the query in pretty QL format.\n"
      "  --dump-compact-map-ql: Don't execute the query but only dump the query in compact QL format.\n"
      "  --dump-bbox-map-ql: Don't execute the query but only dump the query in a suitable form\n"
      "        for an OpenLayers slippy map.\n"
      "  --clone=$TARGET_DIR: Write a consistent copy of the entire database to the given $TARGET_DIR.\n"
      "  --rules: Ignore all time limits and allow area creation by this query.\n"
      "  --quiet: Don't print anything on stderr.\n"
      "  --concise: Print concise information on stderr.\n"
      "  --progress: Print also progress information on stderr.\n"
      "  --verbose: Print everything that happens on stderr.\n";
      
      return 0;
    }
    ++argpos;
  }
  
  Error_Output* error_output(new Console_Output(log_level));
  Statement::set_error_output(error_output);
  
  // connect to dispatcher and get database dir
  try
  {
    if (clone_db_dir != "")
    {
      // open read transaction and log this.
      area_level = determine_area_level(error_output, area_level);
      Dispatcher_Stub dispatcher(db_dir, error_output, "-- clone database --",
				 get_uses_meta_data(), area_level, 24*60*60, 1024*1024*1024);
      copy_file(db_dir + "/replicate_id", clone_db_dir + "/replicate_id");
      
      clone_database(*dispatcher.resource_manager().get_transaction(), clone_db_dir);<commit_msg>Corrected source directory.<commit_after>/** Copyright 2008, 2009, 2010, 2011, 2012 <NAME>
*
* This file is part of Overpass_API.
*
* Overpass_API is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Overpass_API is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Overpass_API.  If not, see <http://www.gnu.org/licenses/>.
*/

#include "../../expat/expat_justparse_interface.h"
#include "../../template_db/dispatcher.h"
#include "../frontend/console_output.h"
#include "../frontend/user_interface.h"
#include "../frontend/web_output.h"
#include "../osm-backend/clone_database.h"
#include "../statements/osm_script.h"
#include "../statements/statement.h"
#include "resource_manager.h"
#include "scripting_core.h"

#include <errno.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/select.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

using namespace std;

int main(int argc, char *argv[])
{  
  // read command line arguments
  string db_dir = "";
  string clone_db_dir = "";
  uint log_level = Error_Output::ASSISTING;
  Debug_Level debug_level = parser_execute;
  int area_level = 0;
  bool respect_timeout = true;
  
  int argpos = 1;
  while (argpos < argc)
  {
    if (!(strncmp(argv[argpos], "--db-dir=", 9)))
    {
      db_dir = ((string)argv[argpos]).substr(9);
      if ((db_dir.size() > 0) && (db_dir[db_dir.size()-1] != '/'))
	db_dir += '/';
    }
    else if (!(strcmp(argv[argpos], "--quiet")))
      log_level = Error_Output::QUIET;
    else if (!(strcmp(argv[argpos], "--concise")))
      log_level = Error_Output::CONCISE;
    else if (!(strcmp(argv[argpos], "--progress")))
      log_level = Error_Output::PROGRESS;
    else if (!(strcmp(argv[argpos], "--verbose")))
      log_level = Error_Output::VERBOSE;
    else if (!(strcmp(argv[argpos], "--rules")))
    {
      area_level = 2;
      respect_timeout = false;
    }
    else if (!(strcmp(argv[argpos], "--dump-xml")))
      debug_level = parser_dump_xml;
    else if (!(strcmp(argv[argpos], "--dump-pretty-map-ql")))
      debug_level = parser_dump_pretty_map_ql;
    else if (!(strcmp(argv[argpos], "--dump-compact-map-ql")))
      debug_level = parser_dump_compact_map_ql;
    else if (!(strcmp(argv[argpos], "--dump-bbox-map-ql")))
      debug_level = parser_dump_bbox_map_ql;
    else if (!(strncmp(argv[argpos], "--clone=", 8)))
    {
      clone_db_dir = ((string)argv[argpos]).substr(8);
      if ((clone_db_dir.size() > 0) && (clone_db_dir[clone_db_dir.size()-1] != '/'))
	clone_db_dir += '/';
    }
    else
    {
      cout<<"Unknown argument: "<<argv[argpos]<<"\n\n"
      "Accepted arguments are:\n"
      "  --db-dir=$DB_DIR: The directory where the database resides. If you set this parameter\n"
      "        then osm3s_query will read from the database without using the dispatcher management.\n"
      "  --dump-xml: Don't execute the query but only dump the query in XML format.\n"
      "  --dump-pretty-map-ql: Don't execute the query but only dump the query in pretty QL format.\n"
      "  --dump-compact-map-ql: Don't execute the query but only dump the query in compact QL format.\n"
      "  --dump-bbox-map-ql: Don't execute the query but only dump the query in a suitable form\n"
      "        for an OpenLayers slippy map.\n"
      "  --clone=$TARGET_DIR: Write a consistent copy of the entire database to the given $TARGET_DIR.\n"
      "  --rules: Ignore all time limits and allow area creation by this query.\n"
      "  --quiet: Don't print anything on stderr.\n"
      "  --concise: Print concise information on stderr.\n"
      "  --progress: Print also progress information on stderr.\n"
      "  --verbose: Print everything that happens on stderr.\n";
      
      return 0;
    }
    ++argpos;
  }
  
  Error_Output* error_output(new Console_Output(log_level));
  Statement::set_error_output(error_output);
  
  // connect to dispatcher and get database dir
  try
  {
    if (clone_db_dir != "")
    {
      // open read transaction and log this.
      area_level = determine_area_level(error_output, area_level);
      Dispatcher_Stub dispatcher(db_dir, error_output, "-- clone database --",
				 get_uses_meta_data(), area_level, 24*60*60, 1024*1024*1024);
      copy_file(dispatcher.resource_manager().get_transaction()->get_db_dir() + "/replicate_id",
		clone_db_dir + "/replicate_id");
      
      clone_database(*dispatcher.resource_manager().get_transaction(), clone_db_dir);
{'fraction_non_alphanumeric': 0.14308665528667536, 'fraction_numerical': 0.01044281554372929, 'mean_word_length': 3.3703378674857394, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6527662', 'n_tokens_mistral': 3360, 'n_tokens_neox': 3243, 'n_words': 913}