<commit_before>#include "copy_elements.hpp"
#include "dump_archive.hpp"
#include "output_writer.hpp"
#include "xml_writer.hpp"
#include "pbf_writer.hpp"
#include "history_filter.hpp"
#include "changeset_filter.hpp"
#include "config.h"

#include <boost/shared_ptr.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/make_shared.hpp>

#include <boost/foreach.hpp>
#include <string>
#include <map>
#include <fstream>
#include <stdexcept>

namespace bt = boost::posix_time;
namespace po = boost::program_options;

/**
 * get command line options, handle --help and usage, validate options.
 */
static void get_options(int argc, char **argv, po::variables_map &vm) {
  po::options_description desc(PACKAGE_STRING ": Allowed options");

  desc.add_options()
    ("help,h", "display help text and exit")
    ("compress-command,c", po::value<std::string>()->default_value("bzip2 -c"),
     "program used to compress XML output, must read from stdin and write to stdout")
    ("xml,x", po::value<std::string>(), "planet XML output file (without history)")
    ("history-xml,X", po::value<std::string>(), "history XML output file")
    ("pbf,p", po::value<std::string>(), "planet PBF output file (without history)")
    ("history-pbf,P", po::value<std::string>(), "history PBF output file")
    ("changesets,C", po::value<std::string>(), "changeset XML output file")
    ("changeset-discussions,D", po::value<std::string>(),
     "changeset discussions XML output file")
    ("xml-no-userinfo", po::value<std::string>(), "planet XML output file (without history or user data)")
    ("history-xml-no-userinfo", po::value<std::string>(), "history XML output file (without user data)")
    ("pbf-no-userinfo", po::value<std::string>(), "planet PBF output file (without history or user data)")
    ("history-pbf-no-userinfo", po::value<std::string>(), "history PBF output file (without user data)")
    ("changesets-no-userinfo", po::value<std::string>(), "changeset XML output file (without user data)")
    ("changeset-discussions-no-userinfo", po::value<std::string>(),
     "changeset discussions XML output file (without user data)")
    ("dense-nodes,d", po::value<bool>()->default_value("true"), "use dense nodes for PBF output")
    ("dump-file,f", po::value<std::string>(), "PostgreSQL table dump to read")
    ("generator", po::value<std::string>()->default_value(PACKAGE_STRING),
     "Override the generator string used by the program. Used by the tests to "
     "ensure consistent output, probably shouldn't be used in normal usage.")
    ("resume", "If this argument is present, then planet-dump-ng will attempt "
     "to resume processing from partial data. If not present, then it will "
     "start from scratch.")
    ("max-concurrency", po::value<unsigned int>()->default_value(16),
      "Maximum number of disk writing threads to run for *each* table.")
    ;

  po::store(po::parse_command_line(argc, argv, desc), vm);
  po::notify(vm);

  if (vm.count("help")) {
    std::cout << desc << std::endl;
    exit(0);
  }

  if (vm.count("dump-file") == 0) {
    BOOST_THROW_EXCEPTION(std::runtime_error("A PostgreSQL table dump file (--dump-file) must be provided."));
  }

  if ((vm.count("xml") + vm.count("history-xml") +
       vm.count("pbf") + vm.count("history-pbf") + 
       vm.count("changesets") + vm.count("changeset-discussions") +
       vm.count("xml-no-userinfo") + vm.count("history-xml-no-userinfo") +
       vm.count("pbf-no-userinfo") + vm.count("history-pbf-no-userinfo") + 
       vm.count("changesets-no-userinfo") + vm.count("changeset-discussions-no-userinfo")) == 0) {
    std::cerr <<
      "No output file provided! You must provide one or more of "
      "--xml, --history-xml, --pbf, --history-pbf, --changesets, "
      "--changeset-discussions (or the respective -no-userinfo options) to get output.\n\n";
    std::cerr << desc << std::endl;
    exit(1);
  }
}

/**
 * read the dump file in parallel to get all of the elements into on-disk
 * databases. this is primarily so that the data is sorted, which is not
 * guaranteed in the PostgreSQL dump file. returns the maximum time seen
 * in a timestamp of any element in the dump file.
 */
bt::ptime setup_databases(const std::string &dump_file, bool resume, unsigned int max_concurrency) {
  std::list<boost::shared_ptr<base_thread> > threads;
  
#define THREAD_RUN(type,table) threads.push_back(boost::make_shared<run_thread<type> >(table, dump_file, resume, max_concurrency))

  THREAD_RUN(changeset, "changesets");
  THREAD_RUN(node, "nodes");
  THREAD_RUN(way, "ways");
  THREAD_RUN(relation, "relations");
  
  THREAD_RUN(current_tag, "changeset_tags");
  THREAD_RUN(old_tag, "node_tags");
  THREAD_RUN(old_tag, "way_tags");
  THREAD_RUN(old_tag, "relation_tags");
  THREAD_RUN(way_node, "way_nodes");
  THREAD_RUN(relation_member, "relation_members");
  
  THREAD_RUN(user, "users");
  THREAD_RUN(changeset_comment, "changeset_comments");

#undef THREAD_RUN
  
  bt::ptime max_time(bt::neg_infin);
  BOOST_FOREACH(boost::shared_ptr<base_thread> &thr, threads) {
    max_time = std::max(max_time, thr->join());
    thr.reset();
  }
  threads.clear();

  return max_time;
}

int main(int argc, char *argv[]) {
  try {
    po::variables_map options;
    get_options(argc, argv, options);

    // workaround for https://svn.boost.org/trac/boost/ticket/5638
    boost::gregorian::greg_month::get_month_map_ptr();

    // extract data from the dump file for the "sorted" data tables, like nodes,
    // ways, relations, changesets and their associated tags, etc...
    const bool resume = options.count("resume") > 0;
    unsigned int max_concurrency = options["max-concurrency"].as<unsigned int>();
    const std::string dump_file(options["dump-file"].as<std::string>());<commit_msg>Remove no longer required workaround in boost 1.73 and later<commit_after>#include "copy_elements.hpp"
#include "dump_archive.hpp"
#include "output_writer.hpp"
#include "xml_writer.hpp"
#include "pbf_writer.hpp"
#include "history_filter.hpp"
#include "changeset_filter.hpp"
#include "config.h"

#include <boost/shared_ptr.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/make_shared.hpp>

#include <boost/foreach.hpp>
#include <string>
#include <map>
#include <fstream>
#include <stdexcept>

namespace bt = boost::posix_time;
namespace po = boost::program_options;

/**
 * get command line options, handle --help and usage, validate options.
 */
static void get_options(int argc, char **argv, po::variables_map &vm) {
  po::options_description desc(PACKAGE_STRING ": Allowed options");

  desc.add_options()
    ("help,h", "display help text and exit")
    ("compress-command,c", po::value<std::string>()->default_value("bzip2 -c"),
     "program used to compress XML output, must read from stdin and write to stdout")
    ("xml,x", po::value<std::string>(), "planet XML output file (without history)")
    ("history-xml,X", po::value<std::string>(), "history XML output file")
    ("pbf,p", po::value<std::string>(), "planet PBF output file (without history)")
    ("history-pbf,P", po::value<std::string>(), "history PBF output file")
    ("changesets,C", po::value<std::string>(), "changeset XML output file")
    ("changeset-discussions,D", po::value<std::string>(),
     "changeset discussions XML output file")
    ("xml-no-userinfo", po::value<std::string>(), "planet XML output file (without history or user data)")
    ("history-xml-no-userinfo", po::value<std::string>(), "history XML output file (without user data)")
    ("pbf-no-userinfo", po::value<std::string>(), "planet PBF output file (without history or user data)")
    ("history-pbf-no-userinfo", po::value<std::string>(), "history PBF output file (without user data)")
    ("changesets-no-userinfo", po::value<std::string>(), "changeset XML output file (without user data)")
    ("changeset-discussions-no-userinfo", po::value<std::string>(),
     "changeset discussions XML output file (without user data)")
    ("dense-nodes,d", po::value<bool>()->default_value("true"), "use dense nodes for PBF output")
    ("dump-file,f", po::value<std::string>(), "PostgreSQL table dump to read")
    ("generator", po::value<std::string>()->default_value(PACKAGE_STRING),
     "Override the generator string used by the program. Used by the tests to "
     "ensure consistent output, probably shouldn't be used in normal usage.")
    ("resume", "If this argument is present, then planet-dump-ng will attempt "
     "to resume processing from partial data. If not present, then it will "
     "start from scratch.")
    ("max-concurrency", po::value<unsigned int>()->default_value(16),
      "Maximum number of disk writing threads to run for *each* table.")
    ;

  po::store(po::parse_command_line(argc, argv, desc), vm);
  po::notify(vm);

  if (vm.count("help")) {
    std::cout << desc << std::endl;
    exit(0);
  }

  if (vm.count("dump-file") == 0) {
    BOOST_THROW_EXCEPTION(std::runtime_error("A PostgreSQL table dump file (--dump-file) must be provided."));
  }

  if ((vm.count("xml") + vm.count("history-xml") +
       vm.count("pbf") + vm.count("history-pbf") + 
       vm.count("changesets") + vm.count("changeset-discussions") +
       vm.count("xml-no-userinfo") + vm.count("history-xml-no-userinfo") +
       vm.count("pbf-no-userinfo") + vm.count("history-pbf-no-userinfo") + 
       vm.count("changesets-no-userinfo") + vm.count("changeset-discussions-no-userinfo")) == 0) {
    std::cerr <<
      "No output file provided! You must provide one or more of "
      "--xml, --history-xml, --pbf, --history-pbf, --changesets, "
      "--changeset-discussions (or the respective -no-userinfo options) to get output.\n\n";
    std::cerr << desc << std::endl;
    exit(1);
  }
}

/**
 * read the dump file in parallel to get all of the elements into on-disk
 * databases. this is primarily so that the data is sorted, which is not
 * guaranteed in the PostgreSQL dump file. returns the maximum time seen
 * in a timestamp of any element in the dump file.
 */
bt::ptime setup_databases(const std::string &dump_file, bool resume, unsigned int max_concurrency) {
  std::list<boost::shared_ptr<base_thread> > threads;
  
#define THREAD_RUN(type,table) threads.push_back(boost::make_shared<run_thread<type> >(table, dump_file, resume, max_concurrency))

  THREAD_RUN(changeset, "changesets");
  THREAD_RUN(node, "nodes");
  THREAD_RUN(way, "ways");
  THREAD_RUN(relation, "relations");
  
  THREAD_RUN(current_tag, "changeset_tags");
  THREAD_RUN(old_tag, "node_tags");
  THREAD_RUN(old_tag, "way_tags");
  THREAD_RUN(old_tag, "relation_tags");
  THREAD_RUN(way_node, "way_nodes");
  THREAD_RUN(relation_member, "relation_members");
  
  THREAD_RUN(user, "users");
  THREAD_RUN(changeset_comment, "changeset_comments");

#undef THREAD_RUN
  
  bt::ptime max_time(bt::neg_infin);
  BOOST_FOREACH(boost::shared_ptr<base_thread> &thr, threads) {
    max_time = std::max(max_time, thr->join());
    thr.reset();
  }
  threads.clear();

  return max_time;
}

int main(int argc, char *argv[]) {
  try {
    po::variables_map options;
    get_options(argc, argv, options);

#if BOOST_VERSION < 107300
    // workaround for https://svn.boost.org/trac/boost/ticket/5638
    boost::gregorian::greg_month::get_month_map_ptr();
#endif

    // extract data from the dump file for the "sorted" data tables, like nodes,
    // ways, relations, changesets and their associated tags, etc...
    const bool resume = options.count("resume") > 0;
    unsigned int max_concurrency = options["max-concurrency"].as<unsigned int>();
    const std::string dump_file(options["dump-file"].as<std::string>());
{'fraction_non_alphanumeric': 0.16208344041805878, 'fraction_numerical': 0.002827036751477769, 'mean_word_length': 4.6560077519379846, 'pattern_counts': {'":': 2, '<': 88, '<?xml version=': 0, '>': 79, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3995188', 'n_tokens_mistral': 3851, 'n_tokens_neox': 3763, 'n_words': 978}