<commit_before>/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/*
 * This file is part of the libabw project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <limits>
#include <vector>

#include <boost/algorithm/string.hpp>

#include <librevenge/librevenge.h>
#include "ABWStylesCollector.h"
#include "libabw_internal.h"

#define ABW_EPSILON 1.0E-06

namespace libabw
{

namespace
{

enum ABWListType
{
  NUMBERED_LIST = 0,
  LOWERCASE_LIST = 1,
  UPPERCASE_LIST = 2,
  LOWERROMAN_LIST = 3,
  UPPERROMAN_LIST = 4,

  BULLETED_LIST = 5,
  DASHED_LIST = 6,
  SQUARE_LIST = 7,
  TRIANGLE_LIST = 8,
  DIAMOND_LIST = 9,
  STAR_LIST = 10,
  IMPLIES_LIST = 11,
  TICK_LIST = 12,
  BOX_LIST = 13,
  HAND_LIST = 14,
  HEART_LIST = 15,
  ARROWHEAD_LIST = 16,

  LAST_BULLETED_LIST = 17,
  OTHER_NUMBERED_LISTS = 0x7f,
  ARABICNUMBERED_LIST = 0x80,
  HEBREW_LIST = 0x81,
  NOT_A_LIST = 0xff
};

static int abw_unichar_to_utf8(unsigned c, char *outbuf)
{
  unsigned char len = 1;
  unsigned char first = 0;

  if (c < 0x80)
  {
    first = 0;
    len = 1;
  }
  else if (c < 0x800)
  {
    first = 0xc0;
    len = 2;
  }
  else if (c < 0x10000)
  {
    first = 0xe0;
    len = 3;
  }
  else if (c < 0x200000)
  {
    first = 0xf0;
    len = 4;
  }
  else if (c < 0x4000000)
  {
    first = 0xf8;
    len = 5;
  }
  else
  {
    first = 0xfc;
    len = 6;
  }

  if (outbuf)
  {
    for (auto i = (unsigned char)(len - 1); i > 0; --i)
    {
      outbuf[i] = (char)((c & 0x3f) | 0x80);
      c >>= 6;
    }
    outbuf[0] = (char)(c | first);
  }

  return len;
}

static void appendUCS4(librevenge::RVNGString &str, unsigned ucs4)
{
  int charLength = abw_unichar_to_utf8(ucs4, nullptr);
  std::vector<char> utf8(charLength+1);
  utf8[charLength] = '\0';
  abw_unichar_to_utf8(ucs4, utf8.data());
  str.append(utf8.data());
}

} // anonymous namespace

} // namespace libabw

libabw::ABWStylesTableState::ABWStylesTableState() :
  m_currentCellProperties(),
  m_currentTableWidth(0),
  m_currentTableRow(-1),
  m_currentTableId(-1) {}

libabw::ABWStylesTableState::ABWStylesTableState(const ABWStylesTableState &ts) :
  m_currentCellProperties(ts.m_currentCellProperties),
  m_currentTableWidth(ts.m_currentTableWidth),
  m_currentTableRow(ts.m_currentTableRow),
  m_currentTableId(ts.m_currentTableId) {}

libabw::ABWStylesTableState::~ABWStylesTableState() {}

libabw::ABWStylesParsingState::ABWStylesParsingState() :
  m_tableStates() {}

libabw::ABWStylesParsingState::ABWStylesParsingState(const ABWStylesParsingState &ps) :
  m_tableStates(ps.m_tableStates) {}

libabw::ABWStylesParsingState::~ABWStylesParsingState() {}

libabw::ABWStylesCollector::ABWStylesCollector(std::map<int, int> &tableSizes,
                                               std::map<std::string, ABWData> &data,
                                               std::map<int, std::shared_ptr<ABWListElement>> &listElements) :
  m_ps(new ABWStylesParsingState),
  m_tableSizes(tableSizes),
  m_data(data),
  m_tableCounter(0),
  m_listElements(listElements) {}

libabw::ABWStylesCollector::~ABWStylesCollector()
{
}

void libabw::ABWStylesCollector::openTable(const char *)
{
  m_ps->m_tableStates.push(ABWStylesTableState());
  m_ps->m_tableStates.top().m_currentTableId = m_tableCounter++;
  m_ps->m_tableStates.top().m_currentTableRow = -1;
  m_ps->m_tableStates.top().m_currentTableWidth = 0;
}

void libabw::ABWStylesCollector::closeTable()
{
  if (!m_ps->m_tableStates.empty())
  {
    auto curWidth = m_ps->m_tableStates.top().m_currentTableWidth;
    m_tableSizes[m_ps->m_tableStates.top().m_currentTableId] = curWidth > 0 ? curWidth : 0;
    m_ps->m_tableStates.pop();
  }
}

void libabw::ABWStylesCollector::openCell(const char *props)
{
  if (!m_ps->m_tableStates.empty())
  {
    if (props)
      parsePropString(props, m_ps->m_tableStates.top().m_currentCellProperties);
    int currentRow(0);
    if (!findInt(_findCellProperty("top-attach"), currentRow))
      currentRow = m_ps->m_tableStates.top().m_currentTableRow + 1;<commit_msg>ofz#4420 avoid signed integer overflow<commit_after>/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/*
 * This file is part of the libabw project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

#include <limits>
#include <vector>

#include <boost/algorithm/string.hpp>

#include <librevenge/librevenge.h>
#include "ABWStylesCollector.h"
#include "libabw_internal.h"

#define ABW_EPSILON 1.0E-06

namespace libabw
{

namespace
{

enum ABWListType
{
  NUMBERED_LIST = 0,
  LOWERCASE_LIST = 1,
  UPPERCASE_LIST = 2,
  LOWERROMAN_LIST = 3,
  UPPERROMAN_LIST = 4,

  BULLETED_LIST = 5,
  DASHED_LIST = 6,
  SQUARE_LIST = 7,
  TRIANGLE_LIST = 8,
  DIAMOND_LIST = 9,
  STAR_LIST = 10,
  IMPLIES_LIST = 11,
  TICK_LIST = 12,
  BOX_LIST = 13,
  HAND_LIST = 14,
  HEART_LIST = 15,
  ARROWHEAD_LIST = 16,

  LAST_BULLETED_LIST = 17,
  OTHER_NUMBERED_LISTS = 0x7f,
  ARABICNUMBERED_LIST = 0x80,
  HEBREW_LIST = 0x81,
  NOT_A_LIST = 0xff
};

static int abw_unichar_to_utf8(unsigned c, char *outbuf)
{
  unsigned char len = 1;
  unsigned char first = 0;

  if (c < 0x80)
  {
    first = 0;
    len = 1;
  }
  else if (c < 0x800)
  {
    first = 0xc0;
    len = 2;
  }
  else if (c < 0x10000)
  {
    first = 0xe0;
    len = 3;
  }
  else if (c < 0x200000)
  {
    first = 0xf0;
    len = 4;
  }
  else if (c < 0x4000000)
  {
    first = 0xf8;
    len = 5;
  }
  else
  {
    first = 0xfc;
    len = 6;
  }

  if (outbuf)
  {
    for (auto i = (unsigned char)(len - 1); i > 0; --i)
    {
      outbuf[i] = (char)((c & 0x3f) | 0x80);
      c >>= 6;
    }
    outbuf[0] = (char)(c | first);
  }

  return len;
}

static void appendUCS4(librevenge::RVNGString &str, unsigned ucs4)
{
  int charLength = abw_unichar_to_utf8(ucs4, nullptr);
  std::vector<char> utf8(charLength+1);
  utf8[charLength] = '\0';
  abw_unichar_to_utf8(ucs4, utf8.data());
  str.append(utf8.data());
}

} // anonymous namespace

} // namespace libabw

libabw::ABWStylesTableState::ABWStylesTableState() :
  m_currentCellProperties(),
  m_currentTableWidth(0),
  m_currentTableRow(-1),
  m_currentTableId(-1) {}

libabw::ABWStylesTableState::ABWStylesTableState(const ABWStylesTableState &ts) :
  m_currentCellProperties(ts.m_currentCellProperties),
  m_currentTableWidth(ts.m_currentTableWidth),
  m_currentTableRow(ts.m_currentTableRow),
  m_currentTableId(ts.m_currentTableId) {}

libabw::ABWStylesTableState::~ABWStylesTableState() {}

libabw::ABWStylesParsingState::ABWStylesParsingState() :
  m_tableStates() {}

libabw::ABWStylesParsingState::ABWStylesParsingState(const ABWStylesParsingState &ps) :
  m_tableStates(ps.m_tableStates) {}

libabw::ABWStylesParsingState::~ABWStylesParsingState() {}

libabw::ABWStylesCollector::ABWStylesCollector(std::map<int, int> &tableSizes,
                                               std::map<std::string, ABWData> &data,
                                               std::map<int, std::shared_ptr<ABWListElement>> &listElements) :
  m_ps(new ABWStylesParsingState),
  m_tableSizes(tableSizes),
  m_data(data),
  m_tableCounter(0),
  m_listElements(listElements) {}

libabw::ABWStylesCollector::~ABWStylesCollector()
{
}

void libabw::ABWStylesCollector::openTable(const char *)
{
  m_ps->m_tableStates.push(ABWStylesTableState());
  m_ps->m_tableStates.top().m_currentTableId = m_tableCounter++;
  m_ps->m_tableStates.top().m_currentTableRow = -1;
  m_ps->m_tableStates.top().m_currentTableWidth = 0;
}

void libabw::ABWStylesCollector::closeTable()
{
  if (!m_ps->m_tableStates.empty())
  {
    auto curWidth = m_ps->m_tableStates.top().m_currentTableWidth;
    m_tableSizes[m_ps->m_tableStates.top().m_currentTableId] = curWidth > 0 ? curWidth : 0;
    m_ps->m_tableStates.pop();
  }
}

void libabw::ABWStylesCollector::openCell(const char *props)
{
  if (!m_ps->m_tableStates.empty())
  {
    if (props)
      parsePropString(props, m_ps->m_tableStates.top().m_currentCellProperties);
    int currentRow(0);
    if (!findInt(_findCellProperty("top-attach"), currentRow))
    {
      currentRow = m_ps->m_tableStates.top().m_currentTableRow;
      if (currentRow < std::numeric_limits<int>::max())
        ++currentRow;
    }
{'fraction_non_alphanumeric': 0.13511298442805292, 'fraction_numerical': 0.029153494906919566, 'mean_word_length': 3.7297895902547067, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197729', 'n_tokens_mistral': 3521, 'n_tokens_neox': 3390, 'n_words': 592}