<commit_before>

#ifdef HAVE_CONFIG_H
#include <config.h>
#undef VERSION
#endif

#ifdef HAVE_LIBOSMIUM

#include "../../expat/escape_xml.h"
#include "../core/settings.h"
#include "../frontend/basic_formats.h"
#include "output_osmium.h"

#include <future>
#include <sys/types.h>
#include <sys/stat.h>

#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/opl_output.hpp>
#include <osmium/io/pbf_output.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/gzip_compression.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/builder/attr.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/types.hpp>


/*
 * Libosmium/FastCGI repeater Libosmium writes directly to stdout fd rather
 * than using streambuf. This doesn't work out of the box with FastCGI.
 * A dedicated thread is being used to redirect libosmium results to cout
 * via a pipe. Additional overhead due to additional copying, etc.
 * has only insignificant implications in real life.
 *
 * Workaround for https://github.com/osmcode/libosmium/issues/174
 *
 */

void Output_Osmium::setup_pipe()
{
  if (!fastcgi_enabled())
    return;

  saved_stdout = dup(1);

  int rc = pipe(fd);
  if (rc != 0)
    throw File_Error(errno, "", "Output_Osmium::setup_pipe::pipe");

  dup2(fd[1], STDOUT_FILENO);   // redirect stdout to the writing end of the pipe
  close(fd[1]);

  repeater = std::async(std::launch::async, [](int readFd)
  {
    ssize_t len = 0;
    char buffer[PIPE_BUF];

    try {

      while(true)
      {
        len = read(readFd, &buffer, sizeof(buffer));
        if (len < 0) {
          throw File_Error(errno, "", "print_target::osmium::read");
        }

        if (len == 0)
          break;

        std::cout.write(&buffer[0], len);
      }
    } catch (...) {
      close(readFd);
      throw;
    }
    close(readFd);

    std::cout << std::flush;

  }, fd[0]);
}

void Output_Osmium::shutdown_pipe()
{
  if (!fastcgi_enabled())
    return;

  dup2(saved_stdout, 1);
  close(saved_stdout);

  if (repeater.valid())
    repeater.get();
}

bool Output_Osmium::write_http_headers()
{
  if (output_format == "pbf")
     std::cout<<"Content-type: application/vnd.openstreetmap.data.pbf\n" << std::flush;
  else if (output_format == "opl")
     std::cout<<"Content-type: application/vnd.openstreetmap.data.opl\n" << std::flush;
  return true;
}

void Output_Osmium::write_payload_header
    (const std::string& db_dir, const std::string& timestamp, const std::string& area_timestamp)
{<commit_msg>Add missing content-type for xml output<commit_after>

#ifdef HAVE_CONFIG_H
#include <config.h>
#undef VERSION
#endif

#ifdef HAVE_LIBOSMIUM

#include "../../expat/escape_xml.h"
#include "../core/settings.h"
#include "../frontend/basic_formats.h"
#include "output_osmium.h"

#include <future>
#include <sys/types.h>
#include <sys/stat.h>

#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/opl_output.hpp>
#include <osmium/io/pbf_output.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/gzip_compression.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/builder/attr.hpp>
#include <osmium/osm/box.hpp>
#include <osmium/osm/item_type.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/node.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/types.hpp>
#include <osmium/osm/way.hpp>
#include <osmium/osm.hpp>
#include <osmium/osm/types.hpp>


/*
 * Libosmium/FastCGI repeater Libosmium writes directly to stdout fd rather
 * than using streambuf. This doesn't work out of the box with FastCGI.
 * A dedicated thread is being used to redirect libosmium results to cout
 * via a pipe. Additional overhead due to additional copying, etc.
 * has only insignificant implications in real life.
 *
 * Workaround for https://github.com/osmcode/libosmium/issues/174
 *
 */

void Output_Osmium::setup_pipe()
{
  if (!fastcgi_enabled())
    return;

  saved_stdout = dup(1);

  int rc = pipe(fd);
  if (rc != 0)
    throw File_Error(errno, "", "Output_Osmium::setup_pipe::pipe");

  dup2(fd[1], STDOUT_FILENO);   // redirect stdout to the writing end of the pipe
  close(fd[1]);

  repeater = std::async(std::launch::async, [](int readFd)
  {
    ssize_t len = 0;
    char buffer[PIPE_BUF];

    try {

      while(true)
      {
        len = read(readFd, &buffer, sizeof(buffer));
        if (len < 0) {
          throw File_Error(errno, "", "print_target::osmium::read");
        }

        if (len == 0)
          break;

        std::cout.write(&buffer[0], len);
      }
    } catch (...) {
      close(readFd);
      throw;
    }
    close(readFd);

    std::cout << std::flush;

  }, fd[0]);
}

void Output_Osmium::shutdown_pipe()
{
  if (!fastcgi_enabled())
    return;

  dup2(saved_stdout, 1);
  close(saved_stdout);

  if (repeater.valid())
    repeater.get();
}

bool Output_Osmium::write_http_headers()
{
  if (output_format == "pbf")
     std::cout<<"Content-type: application/vnd.openstreetmap.data.pbf\n" << std::flush;
  else if (output_format == "opl")
     std::cout<<"Content-type: application/vnd.openstreetmap.data.opl\n" << std::flush;
  else if (output_format == "xml")
     std::cout<<"Content-type: application/osm3s+xml\n" << std::flush;
  return true;
}

void Output_Osmium::write_payload_header
    (const std::string& db_dir, const std::string& timestamp, const std::string& area_timestamp)
{
{'fraction_non_alphanumeric': 0.1553109713487072, 'fraction_numerical': 0.005415793151642208, 'mean_word_length': 3.8028523489932886, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 47, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1209178', 'n_tokens_mistral': 2179, 'n_tokens_neox': 2076, 'n_words': 440}