<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "precompiled.h"

#include <cctype>
#include <sstream>

#include <gumbo.h>

#include "flatui.h"
#include "font_manager.h"
#include "font_util.h"
#include "fplbase/fpl_common.h"

namespace flatui {

using mathfu::vec3;
using mathfu::vec3_packed;

// Replace whitespace with a single space.
// This emulates how HTML processes raw text fields.
// Appends trimmed text to `out`.
// Returns reference to `out`.
std::string &TrimHtmlWhitespace(const char *text, bool trim_leading_whitespace,
                                std::string *out) {
  const char *t = text;

  if (trim_leading_whitespace) {
    while (std::isspace(*t)) ++t;
  }

  for (;;) {
    // Output non-whitespace.
    while (!std::isspace(*t) && *t != '\0') out->push_back(*t++);

    // Break out if at end of input. Note that we do *not* trim all
    // trailing whitespace.
    if (*t == '\0') break;

    // Skip whitespace.
    while (std::isspace(*t)) ++t;

    // Insert single space to compensate for trimmed whitespace.
    out->push_back(' ');
  }
  return *out;
}

// Remove trailing whitespace. Then add a `prefix` if there is any preceeding
// text.
static std::string &StartHtmlLine(const char *prefix, std::string *out) {
  // Trim trailing whitespace.
  while (std::isspace(out->back())) out->pop_back();

  // Append the new lines in `prefix`.
  if (!out->empty()) {
    out->append(prefix);
  }
  return *out;
}

// The very first text output we want to trim the leading whitespace.
// We should also trim if the previous text ends in whitespace.
static bool ShouldTrimLeadingWhitespace(const std::vector<HtmlSection> &s) {
  if (s.size() <= 1) return true;

  const std::string &prev_text =
      s.back().text().empty() ? s[s.size() - 2].text() : s.back().text();
  return std::isspace(prev_text.back());
}

static bool ParseColorValue(const char *value, uint32_t *color) {
  // We only support a hex formatted parameter.
  if (*value++ != '#') {
    return false;
  }
  uint32_t ret = std::stoi(value, nullptr, 16);
  if (ret >= 0xffffff) {
    return false;
  }
  // Convert RGB value to RGBA.
  ret = ret << 8 | 0xff;
  *color = ret;
  return true;
}

static void GumboTreeToHtmlSections(const GumboNode *node,
                                    std::vector<HtmlSection> *s,
                                    HtmlSection *current_setting) {
  auto original_face = current_setting->face();
  auto original_color = current_setting->color();
  auto original_size = current_setting->size();

  switch (node->type) {
    // Process non-text elements, possibly recursing into child nodes.
    case GUMBO_NODE_ELEMENT: {
      const GumboElement &element = node->v.element;

      // Tree prefix processing.
      switch (element.tag) {
        case GUMBO_TAG_A: {
          // Record the link address.
          GumboAttribute *href =
              gumbo_get_attribute(&element.attributes, "href");
          if (href != nullptr) {
            // Start a new section for the anchor.
            if (!s->back().text().empty()) {
              s->push_back(HtmlSection());
            }
            s->back().set_link(href->value);
          }
          break;
        }
        case GUMBO_TAG_FONT: {
          // Record the properties.
          GumboAttribute *face =
              gumbo_get_attribute(&element.attributes, "face");
          GumboAttribute *color =
              gumbo_get_attribute(&element.attributes, "color");
          GumboAttribute *size =
              gumbo_get_attribute(&element.attributes, "size");

          // Start a new section for the font tag.
          bool new_params =
              face != nullptr || color != nullptr || size != nullptr;
          if (new_params && !s->back().text().empty()) {
            s->push_back(HtmlSection());
          }
          if (face != nullptr) {
            // Keep current setting and update font face setting.
            original_face = current_setting->face();
            s->back().set_face(face->value);
            current_setting->set_face(face->value);
          } else {
            s->back().set_face(current_setting->face());
          }
          if (color != nullptr) {
            uint32_t color_value = kDefaultColor;
            if (ParseColorValue(color->value, &color_value)) {
              // Keep current setting and update font face setting.
              original_color = current_setting->color();
              s->back().set_color(color_value);
              current_setting->set_color(color_value);
            } else {
              LogInfo("Failed to parse a value: %s", color->value);
            }
          } else {
            s->back().set_color(current_setting->color());
          }
          if (size != nullptr) {
            auto value = std::stoi(size->value, nullptr, 10);
            if (value) {
              // Convert the virtual size to physical size.
              value = VirtualToPhysical(vec2(0, value)).y();
<commit_msg>Changing conversion functions to use strtol.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "precompiled.h"

#include <cctype>
#include <sstream>

#include <gumbo.h>

#include "flatui.h"
#include "font_manager.h"
#include "font_util.h"
#include "fplbase/fpl_common.h"

namespace flatui {

using mathfu::vec3;
using mathfu::vec3_packed;

// Replace whitespace with a single space.
// This emulates how HTML processes raw text fields.
// Appends trimmed text to `out`.
// Returns reference to `out`.
std::string &TrimHtmlWhitespace(const char *text, bool trim_leading_whitespace,
                                std::string *out) {
  const char *t = text;

  if (trim_leading_whitespace) {
    while (std::isspace(*t)) ++t;
  }

  for (;;) {
    // Output non-whitespace.
    while (!std::isspace(*t) && *t != '\0') out->push_back(*t++);

    // Break out if at end of input. Note that we do *not* trim all
    // trailing whitespace.
    if (*t == '\0') break;

    // Skip whitespace.
    while (std::isspace(*t)) ++t;

    // Insert single space to compensate for trimmed whitespace.
    out->push_back(' ');
  }
  return *out;
}

// Remove trailing whitespace. Then add a `prefix` if there is any preceeding
// text.
static std::string &StartHtmlLine(const char *prefix, std::string *out) {
  // Trim trailing whitespace.
  while (std::isspace(out->back())) out->pop_back();

  // Append the new lines in `prefix`.
  if (!out->empty()) {
    out->append(prefix);
  }
  return *out;
}

// The very first text output we want to trim the leading whitespace.
// We should also trim if the previous text ends in whitespace.
static bool ShouldTrimLeadingWhitespace(const std::vector<HtmlSection> &s) {
  if (s.size() <= 1) return true;

  const std::string &prev_text =
      s.back().text().empty() ? s[s.size() - 2].text() : s.back().text();
  return std::isspace(prev_text.back());
}

static bool ParseColorValue(const char *value, uint32_t *color) {
  // We only support a hex formatted parameter.
  if (*value++ != '#') {
    return false;
  }
  uint32_t ret = static_cast<uint32_t>(std::strtol(value, nullptr, 16));
  if (ret >= 0xffffff) {
    return false;
  }
  // Convert RGB value to RGBA.
  ret = ret << 8 | 0xff;
  *color = ret;
  return true;
}

static void GumboTreeToHtmlSections(const GumboNode *node,
                                    std::vector<HtmlSection> *s,
                                    HtmlSection *current_setting) {
  auto original_face = current_setting->face();
  auto original_color = current_setting->color();
  auto original_size = current_setting->size();

  switch (node->type) {
    // Process non-text elements, possibly recursing into child nodes.
    case GUMBO_NODE_ELEMENT: {
      const GumboElement &element = node->v.element;

      // Tree prefix processing.
      switch (element.tag) {
        case GUMBO_TAG_A: {
          // Record the link address.
          GumboAttribute *href =
              gumbo_get_attribute(&element.attributes, "href");
          if (href != nullptr) {
            // Start a new section for the anchor.
            if (!s->back().text().empty()) {
              s->push_back(HtmlSection());
            }
            s->back().set_link(href->value);
          }
          break;
        }
        case GUMBO_TAG_FONT: {
          // Record the properties.
          GumboAttribute *face =
              gumbo_get_attribute(&element.attributes, "face");
          GumboAttribute *color =
              gumbo_get_attribute(&element.attributes, "color");
          GumboAttribute *size =
              gumbo_get_attribute(&element.attributes, "size");

          // Start a new section for the font tag.
          bool new_params =
              face != nullptr || color != nullptr || size != nullptr;
          if (new_params && !s->back().text().empty()) {
            s->push_back(HtmlSection());
          }
          if (face != nullptr) {
            // Keep current setting and update font face setting.
            original_face = current_setting->face();
            s->back().set_face(face->value);
            current_setting->set_face(face->value);
          } else {
            s->back().set_face(current_setting->face());
          }
          if (color != nullptr) {
            uint32_t color_value = kDefaultColor;
            if (ParseColorValue(color->value, &color_value)) {
              // Keep current setting and update font face setting.
              original_color = current_setting->color();
              s->back().set_color(color_value);
              current_setting->set_color(color_value);
            } else {
              LogInfo("Failed to parse a value: %s", color->value);
            }
          } else {
            s->back().set_color(current_setting->color());
          }
          if (size != nullptr) {
            auto value =
                static_cast<int>(std::strtol(size->value, nullptr, 10));
            if (value) {
              // Convert the virtual size to physical size.
              value = VirtualToPhysical(vec2(0, value)).y();

{'fraction_non_alphanumeric': 0.1284084833363037, 'fraction_numerical': 0.005346640527535198, 'mean_word_length': 2.3253333333333335, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1730104', 'n_tokens_mistral': 3301, 'n_tokens_neox': 3186, 'n_words': 967}