<commit_before>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/format/zeek.hpp"

#include "vast/concept/parseable/vast/port.hpp"
#include "vast/concept/printable/numeric.hpp"
#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/data.hpp"
#include "vast/concept/printable/vast/type.hpp"
#include "vast/concept/printable/vast/view.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/escapers.hpp"
#include "vast/detail/fdostream.hpp"
#include "vast/detail/string.hpp"
#include "vast/error.hpp"
#include "vast/event.hpp"
#include "vast/logger.hpp"
#include "vast/table_slice.hpp"
#include "vast/table_slice_builder.hpp"

#include <caf/none.hpp>

#include <fstream>
#include <iomanip>

namespace vast::format::zeek {

namespace {

// The type name prefix to preprend to zeek log names when transleting them
// into VAST types.
constexpr std::string_view type_name_prefix = "zeek.";

// Creates a VAST type from an ASCII Zeek type in a log header.
caf::expected<type> parse_type(std::string_view zeek_type) {
  type t;
  if (zeek_type == "enum" || zeek_type == "string" || zeek_type == "file")
    t = string_type{};
  else if (zeek_type == "bool")
    t = bool_type{};
  else if (zeek_type == "int")
    t = integer_type{};
  else if (zeek_type == "count")
    t = count_type{};
  else if (zeek_type == "double")
    t = real_type{};
  else if (zeek_type == "time")
    t = time_type{};
  else if (zeek_type == "interval")
    t = duration_type{};
  else if (zeek_type == "pattern")
    t = pattern_type{};
  else if (zeek_type == "addr")
    t = address_type{};
  else if (zeek_type == "subnet")
    t = subnet_type{};
  else if (zeek_type == "port")
    t = port_type{};
  if (caf::holds_alternative<none_type>(t)
      && (detail::starts_with(zeek_type, "vector")
          || detail::starts_with(zeek_type, "set")
          || detail::starts_with(zeek_type, "table"))) {
    // Zeek's logging framwork cannot log nested vectors/sets/tables, so we can
    // safely assume that we're dealing with a basic type inside the brackets.
    // If this will ever change, we'll have to enhance this simple parser.
    auto open = zeek_type.find("[");
    auto close = zeek_type.rfind("]");
    if (open == std::string::npos || close == std::string::npos)
      return make_error(ec::format_error, "missing container brackets:",
                        std::string{zeek_type});
    auto elem = parse_type(zeek_type.substr(open + 1, close - open - 1));
    if (!elem)
      return elem.error();
    // Zeek sometimes logs sets as tables, e.g., represents set[string] as
    // table[string]. We iron out this inconsistency by normalizing the type to
    // a set.
    if (detail::starts_with(zeek_type, "vector"))
      t = vector_type{*elem};
    else
      t = set_type{*elem};
  }
  if (caf::holds_alternative<none_type>(t))
    return make_error(ec::format_error, "failed to parse type: ",
                      std::string{zeek_type});
  return t;
}

struct zeek_type_printer {
  template <class T>
  std::string operator()(const T& x) const {
    return to_string(x);
  }

  std::string operator()(const real_type&) {
    return "double";
  }

  std::string operator()(const time_type&) {
    return "time";
  }

  std::string operator()(const duration_type&) {
    return "interval";
  }

  std::string operator()(const vector_type& t) const {
    return "vector[" + caf::visit(*this, t.value_type) + ']';
  }

  std::string operator()(const set_type& t) const {
    return "set[" + caf::visit(*this, t.value_type) + ']';
  }

  std::string operator()(const alias_type& t) const {
    return caf::visit(*this, t.value_type);
  }
};

auto to_zeek_string(const type& t) {
  return caf::visit(zeek_type_printer{}, t);
}

constexpr char separator = '\x09';
constexpr char set_separator = ',';
constexpr auto empty_field = "(empty)";
constexpr auto unset_field = '-';

struct time_factory {<commit_msg>Do not print attributes in Zeek type header<commit_after>/******************************************************************************
 *                    _   _____   __________                                  *
 *                   | | / / _ | / __/_  __/     Visibility                   *
 *                   | |/ / __ |_\ \  / /          Across                     *
 *                   |___/_/ |_/___/ /_/       Space and Time                 *
 *                                                                            *
 * This file is part of VAST. It is subject to the license terms in the       *
 * LICENSE file found in the top-level directory of this distribution and at  *
 * http://vast.io/license. No part of VAST, including this file, may be       *
 * copied, modified, propagated, or distributed except according to the terms *
 * contained in the LICENSE file.                                             *
 ******************************************************************************/

#include "vast/format/zeek.hpp"

#include "vast/concept/parseable/vast/port.hpp"
#include "vast/concept/printable/numeric.hpp"
#include "vast/concept/printable/to_string.hpp"
#include "vast/concept/printable/vast/data.hpp"
#include "vast/concept/printable/vast/type.hpp"
#include "vast/concept/printable/vast/view.hpp"
#include "vast/detail/assert.hpp"
#include "vast/detail/escapers.hpp"
#include "vast/detail/fdostream.hpp"
#include "vast/detail/string.hpp"
#include "vast/error.hpp"
#include "vast/event.hpp"
#include "vast/logger.hpp"
#include "vast/table_slice.hpp"
#include "vast/table_slice_builder.hpp"

#include <caf/none.hpp>

#include <fstream>
#include <iomanip>

namespace vast::format::zeek {

namespace {

// The type name prefix to preprend to zeek log names when transleting them
// into VAST types.
constexpr std::string_view type_name_prefix = "zeek.";

// Creates a VAST type from an ASCII Zeek type in a log header.
caf::expected<type> parse_type(std::string_view zeek_type) {
  type t;
  if (zeek_type == "enum" || zeek_type == "string" || zeek_type == "file")
    t = string_type{};
  else if (zeek_type == "bool")
    t = bool_type{};
  else if (zeek_type == "int")
    t = integer_type{};
  else if (zeek_type == "count")
    t = count_type{};
  else if (zeek_type == "double")
    t = real_type{};
  else if (zeek_type == "time")
    t = time_type{};
  else if (zeek_type == "interval")
    t = duration_type{};
  else if (zeek_type == "pattern")
    t = pattern_type{};
  else if (zeek_type == "addr")
    t = address_type{};
  else if (zeek_type == "subnet")
    t = subnet_type{};
  else if (zeek_type == "port")
    t = port_type{};
  if (caf::holds_alternative<none_type>(t)
      && (detail::starts_with(zeek_type, "vector")
          || detail::starts_with(zeek_type, "set")
          || detail::starts_with(zeek_type, "table"))) {
    // Zeek's logging framwork cannot log nested vectors/sets/tables, so we can
    // safely assume that we're dealing with a basic type inside the brackets.
    // If this will ever change, we'll have to enhance this simple parser.
    auto open = zeek_type.find("[");
    auto close = zeek_type.rfind("]");
    if (open == std::string::npos || close == std::string::npos)
      return make_error(ec::format_error, "missing container brackets:",
                        std::string{zeek_type});
    auto elem = parse_type(zeek_type.substr(open + 1, close - open - 1));
    if (!elem)
      return elem.error();
    // Zeek sometimes logs sets as tables, e.g., represents set[string] as
    // table[string]. We iron out this inconsistency by normalizing the type to
    // a set.
    if (detail::starts_with(zeek_type, "vector"))
      t = vector_type{*elem};
    else
      t = set_type{*elem};
  }
  if (caf::holds_alternative<none_type>(t))
    return make_error(ec::format_error, "failed to parse type: ",
                      std::string{zeek_type});
  return t;
}

struct zeek_type_printer {
  template <class T>
  std::string operator()(const T& x) const {
    return kind(x);
  }

  std::string operator()(const real_type&) {
    return "double";
  }

  std::string operator()(const time_type&) {
    return "time";
  }

  std::string operator()(const duration_type&) {
    return "interval";
  }

  std::string operator()(const address_type&) {
    return "addr";
  }

  std::string operator()(const vector_type& t) const {
    return "vector[" + caf::visit(*this, t.value_type) + ']';
  }

  std::string operator()(const set_type& t) const {
    return "set[" + caf::visit(*this, t.value_type) + ']';
  }

  std::string operator()(const alias_type& t) const {
    return caf::visit(*this, t.value_type);
  }
};

auto to_zeek_string(const type& t) {
  return caf::visit(zeek_type_printer{}, t);
}

constexpr char separator = '\x09';
constexpr char set_separator = ',';
constexpr auto empty_field = "(empty)";
constexpr auto unset_field = '-';

struct time_factory {
{'fraction_non_alphanumeric': 0.16753180661577607, 'fraction_numerical': 0.0008142493638676844, 'mean_word_length': 2.79969064191802, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2615987', 'n_tokens_mistral': 3198, 'n_tokens_neox': 3044, 'n_words': 802}