<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "MethodProfiles.h"

#include <boost/algorithm/string.hpp>
#include <charconv>
#include <fstream>
#include <iostream>
#include <stdio.h>
#include <stdlib.h>

#include "CppUtil.h"
#include "Show.h"

using namespace method_profiles;

extern int errno;

namespace {

template <class Func>
bool parse_cells(std::string_view line, const Func& parse_cell) {
  char* save_ptr = nullptr;
  uint32_t i = 0;
  // Assuming there are no quoted strings containing commas!
  for (auto cell : split_string(line, ",")) {
    if (cell.back() == '\n') {
      cell.remove_suffix(1);
    }
    bool success = parse_cell(cell, i);
    if (!success) {
      return false;
    }
    ++i;
  }
  return true;
}

bool empty_column(std::string_view sv) { return sv.empty() || sv == "\n"; }

} // namespace

const StatsMap& MethodProfiles::method_stats(
    const std::string& interaction_id) const {
  const auto& search1 = m_method_stats.find(interaction_id);
  if (search1 != m_method_stats.end()) {
    return search1->second;
  }
  if (interaction_id == COLD_START) {
    // Originally, the stats file had no interaction_id column and it only
    // covered coldstart. Search for the default (empty string) for backwards
    // compatibility when we're searching for coldstart but it's not found.
    const auto& search2 = m_method_stats.find("");
    if (search2 != m_method_stats.end()) {
      return search2->second;
    }
  }

  static StatsMap empty_map = {};
  return empty_map;
}

bool MethodProfiles::parse_stats_file(const std::string& csv_filename) {
  TRACE(METH_PROF, 3, "input csv filename: %s", csv_filename.c_str());
  if (csv_filename.empty()) {
    TRACE(METH_PROF, 2, "No csv file given");
    return false;
  }

  // Using C-style file reading and parsing because it's faster than the
  // iostreams equivalent and we expect to read very large csv files.
  std::ifstream ifs(csv_filename);
  if (!ifs.good()) {
    std::cerr << "FAILED to open " << csv_filename << std::endl;
    return false;
  }

  // getline will allocate a buffer and put a pointer to it here
  std::string line;
  while (std::getline(ifs, line)) {
    bool success = false;
    if (m_mode == NONE) {
      success = parse_header(line);
    } else {
      success = parse_line(line);
    }
    if (!success) {
      return false;
    }
  }
  if (ifs.bad()) {
    std::cerr << "FAILED to read a line!" << std::endl;
    return false;
  }

  TRACE(METH_PROF, 1,
        "MethodProfiles successfully parsed %zu rows; %zu unresolved lines",
        size(), unresolved_size());
  return true;
}

// This should have worked for double too as described in p0067r5.
// But libc++ 7 did not implement floating point conversions.
template <typename IntType = int64_t>
IntType parse_int(std::string_view tok) {
  IntType result{};
  auto [ptr, ec]{std::from_chars(tok.data(), tok.data() + tok.size(), result)};
  std::string_view rest(ptr, tok.size() - (ptr - tok.data()));

  always_assert_log(ec == std::errc(), "can't parse %s into an int: %s",
                    SHOW(tok), std::make_error_condition(ec).message().c_str());
  always_assert_log(empty_column(rest), "can't parse %s into an int",
                    SHOW(tok));
  return result;
}

// `strtod` requires c string to be null terminated, std::string_view::data()
// doesn't have this guarantee. Our `string_view`s are taken from
// `std::string`s. This should be safe.
double parse_double(std::string_view tok) {
  char* ptr = nullptr;
  const auto result = strtod(tok.data(), &ptr);
  always_assert_log(
      ptr <= (tok.data() + tok.size()),
      "strtod went over std::string_view boundary for string_view %s",
      SHOW(tok));

  std::string_view rest(ptr, tok.size() - (ptr - tok.data()));
  always_assert_log(ptr != tok.data(), "can't parse %s into a double",
                    SHOW(tok));
  always_assert_log(empty_column(rest), "can't parse %s into a double",
                    SHOW(tok));
  return result;
}

bool MethodProfiles::parse_metadata(std::string_view line) {
  always_assert(m_mode == METADATA);
  uint32_t interaction_count{0};
  auto parse_cell = [&](std::string_view cell, uint32_t col) -> bool {
    switch (col) {
    case 0:
      m_interaction_id = std::string(cell);
      return true;
    case 1: {
      auto parsed = parse_int(cell);
      always_assert(parsed <= std::numeric_limits<uint32_t>::max());
      always_assert(parsed >= 0);
      interaction_count = static_cast<uint32_t>(parsed);
      return true;
    }
    default: {
      bool ok = empty_column(cell);
      if (!ok) {
        std::cerr << "Unexpected extra value in metadata: " << cell
                  << std::endl;
      }
      return ok;
    }
    }
  };
  bool success = parse_cells(line, parse_cell);
  if (!success) {
    return false;
  }<commit_msg>Fix OSS build failure due to charconv parsing in MethodProfiles (#657)<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

#include "MethodProfiles.h"

#include <boost/algorithm/string.hpp>
#include <fstream>
#include <iostream>
#include <stdio.h>
#include <stdlib.h>

#include "CppUtil.h"
#include "Show.h"

using namespace method_profiles;

extern int errno;

namespace {

template <class Func>
bool parse_cells(std::string_view line, const Func& parse_cell) {
  char* save_ptr = nullptr;
  uint32_t i = 0;
  // Assuming there are no quoted strings containing commas!
  for (auto cell : split_string(line, ",")) {
    if (cell.back() == '\n') {
      cell.remove_suffix(1);
    }
    bool success = parse_cell(cell, i);
    if (!success) {
      return false;
    }
    ++i;
  }
  return true;
}

bool empty_column(std::string_view sv) { return sv.empty() || sv == "\n"; }

} // namespace

const StatsMap& MethodProfiles::method_stats(
    const std::string& interaction_id) const {
  const auto& search1 = m_method_stats.find(interaction_id);
  if (search1 != m_method_stats.end()) {
    return search1->second;
  }
  if (interaction_id == COLD_START) {
    // Originally, the stats file had no interaction_id column and it only
    // covered coldstart. Search for the default (empty string) for backwards
    // compatibility when we're searching for coldstart but it's not found.
    const auto& search2 = m_method_stats.find("");
    if (search2 != m_method_stats.end()) {
      return search2->second;
    }
  }

  static StatsMap empty_map = {};
  return empty_map;
}

bool MethodProfiles::parse_stats_file(const std::string& csv_filename) {
  TRACE(METH_PROF, 3, "input csv filename: %s", csv_filename.c_str());
  if (csv_filename.empty()) {
    TRACE(METH_PROF, 2, "No csv file given");
    return false;
  }

  // Using C-style file reading and parsing because it's faster than the
  // iostreams equivalent and we expect to read very large csv files.
  std::ifstream ifs(csv_filename);
  if (!ifs.good()) {
    std::cerr << "FAILED to open " << csv_filename << std::endl;
    return false;
  }

  // getline will allocate a buffer and put a pointer to it here
  std::string line;
  while (std::getline(ifs, line)) {
    bool success = false;
    if (m_mode == NONE) {
      success = parse_header(line);
    } else {
      success = parse_line(line);
    }
    if (!success) {
      return false;
    }
  }
  if (ifs.bad()) {
    std::cerr << "FAILED to read a line!" << std::endl;
    return false;
  }

  TRACE(METH_PROF, 1,
        "MethodProfiles successfully parsed %zu rows; %zu unresolved lines",
        size(), unresolved_size());
  return true;
}

// `strtol` and `strtod` requires c string to be null terminated,
// std::string_view::data() doesn't have this guarantee. Our `string_view`s are
// taken from `std::string`s. This should be safe.
template <typename IntType>
IntType parse_int(std::string_view tok) {
  char* ptr = nullptr;
  const auto parsed = strtol(tok.data(), &ptr, 10);
  always_assert_log(
      ptr <= (tok.data() + tok.size()),
      "strtod went over std::string_view boundary for string_view %s",
      SHOW(tok));

  std::string_view rest(ptr, tok.size() - (ptr - tok.data()));
  always_assert_log(ptr != tok.data(), "can't parse %s into a int", SHOW(tok));
  always_assert_log(empty_column(rest), "can't parse %s into a int", SHOW(tok));
  always_assert(parsed <= std::numeric_limits<IntType>::max());
  always_assert(parsed >= std::numeric_limits<IntType>::min());
  return static_cast<IntType>(parsed);
}

double parse_double(std::string_view tok) {
  char* ptr = nullptr;
  const auto result = strtod(tok.data(), &ptr);
  always_assert_log(
      ptr <= (tok.data() + tok.size()),
      "strtod went over std::string_view boundary for string_view %s",
      SHOW(tok));

  std::string_view rest(ptr, tok.size() - (ptr - tok.data()));
  always_assert_log(ptr != tok.data(), "can't parse %s into a double",
                    SHOW(tok));
  always_assert_log(empty_column(rest), "can't parse %s into a double",
                    SHOW(tok));
  return result;
}

bool MethodProfiles::parse_metadata(std::string_view line) {
  always_assert(m_mode == METADATA);
  uint32_t interaction_count{0};
  auto parse_cell = [&](std::string_view cell, uint32_t col) -> bool {
    switch (col) {
    case 0:
      m_interaction_id = std::string(cell);
      return true;
    case 1: {
      interaction_count = parse_int<uint32_t>(cell);
      return true;
    }
    default: {
      bool ok = empty_column(cell);
      if (!ok) {
        std::cerr << "Unexpected extra value in metadata: " << cell
                  << std::endl;
      }
      return ok;
    }
    }
  };
  bool success = parse_cells(line, parse_cell);
  if (!success) {
    return false;
  }
{'fraction_non_alphanumeric': 0.13606672696211436, 'fraction_numerical': 0.006029544769369913, 'mean_word_length': 3.313827481577807, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1752916', 'n_tokens_mistral': 3333, 'n_tokens_neox': 3225, 'n_words': 942}