<commit_before>/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/*
 * This file is part of the libetonyek project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include "libetonyek_xml.h"

#include <cassert>

#include <boost/lexical_cast.hpp>
#include <boost/none.hpp>
#include <boost/optional.hpp>

#include "libetonyek_utils.h"
#include "IWORKToken.h"
#include "IWORKTokenizer.h"

using boost::bad_lexical_cast;
using boost::lexical_cast;
using boost::none;
using boost::optional;

using std::string;

extern "C" int readFromStream(void *context, char *buffer, int len)
{
  try
  {
    auto *const input = reinterpret_cast<librevenge::RVNGInputStream *>(context);

    unsigned long bytesRead = 0;
    const unsigned char *const bytes = input->read((unsigned long)len, bytesRead);

    std::memcpy(buffer, bytes, static_cast<size_t>(bytesRead));
    return static_cast<int>(bytesRead);
  }
  catch (...)
  {
  }

  return -1;
}

extern "C" int closeStream(void * /* context */)
{
  return 0;
}

namespace libetonyek
{

std::unique_ptr<xmlTextReader, void (*)(xmlTextReaderPtr)> xmlReaderForStream(const RVNGInputStreamPtr_t &input)
{
  return std::unique_ptr<xmlTextReader, void (*)(xmlTextReaderPtr)>(
           xmlReaderForIO(readFromStream, closeStream, input.get(), "", nullptr, XML_PARSE_NONET),
           xmlFreeTextReader
         );
}

bool bool_cast(const char *value)
{
  return get(try_bool_cast(value));
}

boost::optional<bool> try_bool_cast(const char *value)
{
  const IWORKTokenizer &tok(IWORKToken::getTokenizer());
  switch (tok.getId(value))<commit_msg>ignore blank nodes too<commit_after>/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/*
 * This file is part of the libetonyek project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include "libetonyek_xml.h"

#include <cassert>

#include <boost/lexical_cast.hpp>
#include <boost/none.hpp>
#include <boost/optional.hpp>

#include "libetonyek_utils.h"
#include "IWORKToken.h"
#include "IWORKTokenizer.h"

using boost::bad_lexical_cast;
using boost::lexical_cast;
using boost::none;
using boost::optional;

using std::string;

extern "C" int readFromStream(void *context, char *buffer, int len)
{
  try
  {
    auto *const input = reinterpret_cast<librevenge::RVNGInputStream *>(context);

    unsigned long bytesRead = 0;
    const unsigned char *const bytes = input->read((unsigned long)len, bytesRead);

    std::memcpy(buffer, bytes, static_cast<size_t>(bytesRead));
    return static_cast<int>(bytesRead);
  }
  catch (...)
  {
  }

  return -1;
}

extern "C" int closeStream(void * /* context */)
{
  return 0;
}

namespace libetonyek
{

std::unique_ptr<xmlTextReader, void (*)(xmlTextReaderPtr)> xmlReaderForStream(const RVNGInputStreamPtr_t &input)
{
  return std::unique_ptr<xmlTextReader, void (*)(xmlTextReaderPtr)>(
           xmlReaderForIO(readFromStream, closeStream, input.get(), "", nullptr, XML_PARSE_NOBLANKS | XML_PARSE_NONET),
           xmlFreeTextReader
         );
}

bool bool_cast(const char *value)
{
  return get(try_bool_cast(value));
}

boost::optional<bool> try_bool_cast(const char *value)
{
  const IWORKTokenizer &tok(IWORKToken::getTokenizer());
  switch (tok.getId(value))
{'fraction_non_alphanumeric': 0.13682148852825965, 'fraction_numerical': 0.005036373810856184, 'mean_word_length': 4.466360856269113, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '894140', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1207, 'n_words': 307}