<commit_before>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 <NAME>
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <fnord-msg/MessageDecoder.h>
#include <fnord-base/inspect.h>

namespace fnord {
namespace msg {

void MessageDecoder::decode(
    const Buffer& buf,
    const MessageSchema& schema,
    MessageObject* msg) {
  fnord::iputs("----------", 1);
  util::BinaryMessageReader reader(buf.data(), buf.size());

  auto num_fields = *reader.readUInt32();
  Vector<Pair<uint32_t, uint64_t>> fields;

  for (int i = 0; i < num_fields; ++i) {
    auto id = *reader.readUInt32();
    auto end = *reader.readUInt32();
    fields.emplace_back(id, end);
  }

  decodeObject(0, 0, reader.remaining(), fields, schema, &reader, msg);
}

void MessageDecoder::decodeObject(
    size_t idx,
    uint64_t begin,
    uint64_t end,
    const Vector<Pair<uint32_t, uint64_t>>& fields,
    const MessageSchema& schema,
    util::BinaryMessageReader* reader,
    MessageObject* msg) {
  auto fbegin = begin;
  auto fid = fields[idx].first;
  auto fend = fields[idx].second;

  switch (schema.type(fid)) {

    case FieldType::OBJECT: {
      fnord::iputs("load obj $2 ($0..$1)", fbegin, fend, schema.name(fid));
      auto obegin = fbegin;
      for (int i = idx + 1; i < fields.size(); ++i) {
        auto oend = fields[i].second;

        if (oend < obegin) {
          continue;
        }

        if (oend > fend) {
          break;
        }

        decodeObject(i, obegin, oend, fields, schema, reader, msg);

        obegin = oend;
      }
      break;
    }

    case FieldType::UINT32: {
      fnord::iputs("load uint32 $2 ($0..$1)", fbegin, fend, schema.name(fid));
      break;
    }

    case FieldType::BOOLEAN: {
      fnord::iputs("load bool $2 ($0..$1)", fbegin, fend, schema.name(fid));
      break;
    }

  }
}


} // namespace msg
} // namespace fnord<commit_msg>MessageDecoder<commit_after>/**
 * This file is part of the "libfnord" project
 *   Copyright (c) 2015 <NAME>
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <fnord-msg/MessageDecoder.h>
#include <fnord-base/inspect.h>

namespace fnord {
namespace msg {

void MessageDecoder::decode(
    const Buffer& buf,
    const MessageSchema& schema,
    MessageObject* msg) {
  util::BinaryMessageReader reader(buf.data(), buf.size());

  auto num_fields = *reader.readUInt32();
  Vector<Pair<uint32_t, uint64_t>> fields;

  for (int i = 0; i < num_fields; ++i) {
    auto id = *reader.readUInt32();
    auto end = *reader.readUInt32();
    fields.emplace_back(id, end);
  }

  decodeObject(0, 0, reader.remaining(), fields, schema, &reader, msg);
}

void MessageDecoder::decodeObject(
    size_t idx,
    uint64_t begin,
    uint64_t end,
    const Vector<Pair<uint32_t, uint64_t>>& fields,
    const MessageSchema& schema,
    util::BinaryMessageReader* reader,
    MessageObject* msg) {
  auto fbegin = begin;
  auto fid = fields[idx].first;
  auto fend = fields[idx].second;

  switch (schema.type(fid)) {

    case FieldType::OBJECT: {
      MessageObject* nxt;
      if (fid == 0) {
        nxt = msg;
      } else {
        nxt = &msg->addChild(fid);
      }

      auto obegin = fbegin;
      for (int i = idx + 1; i < fields.size(); ++i) {
        auto oend = fields[i].second;

        if (oend <= obegin) {
          continue;
        }

        if (oend > fend) {
          break;
        }

        decodeObject(i, obegin, oend, fields, schema, reader, nxt);

        obegin = oend;
      }
      break;
    }

    case FieldType::UINT32: {
      uint32_t val = *reader->readUInt32();
      msg->addChild(fid, val);
      break;
    }

    case FieldType::BOOLEAN: {
      uint8_t val = *reader->readUInt8();
      msg->addChild(fid, val == 1);
      break;
    }

    case FieldType::STRING: {
      auto val = reader->read(fend - fbegin);
      msg->addChild(fid, String((const char*) val, fend - fbegin));
      break;
    }

  }
}


} // namespace msg
} // namespace fnord
{'fraction_non_alphanumeric': 0.13984760197220977, 'fraction_numerical': 0.017929179740026894, 'mean_word_length': 2.6914805624483042, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4981983', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1489, 'n_words': 410}