<commit_before>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 <NAME>, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <fnord-sstable/binaryformat.h>
#include <fnord-sstable/fileheaderreader.h>
#include <fnord-base/exception.h>
#include <fnord-base/fnv.h>

namespace fnord {
namespace sstable {

FileHeaderReader::FileHeaderReader(
    void* buf,
    size_t buf_size) :
    fnord::util::BinaryMessageReader(buf, buf_size) {
  auto magic_bytes = *readUInt32();
  if (magic_bytes != BinaryFormat::kMagicBytes) {
    RAISE(kIllegalStateError, "not a valid sstable");
  }

  auto version = *readUInt16();
  if (version != BinaryFormat::kVersion) {
    RAISE(kIllegalStateError, "unsupported sstable version");
  }

  body_size_ = *readUInt64();
  userdata_checksum_ = *readUInt32();
  userdata_size_ = *readUInt32();
  userdata_offset_ = pos_;
}

bool FileHeaderReader::verify() {
  if (userdata_offset_ + userdata_size_ > size_) {
    return false;
  }

  const void* userdata;
  size_t userdata_size;
  readUserdata(&userdata, &userdata_size);

  hash::FNV<uint32_t> fnv;
  uint32_t userdata_checksum = fnv.hash(userdata, userdata_size);

  return userdata_checksum == userdata_checksum_;
}<commit_msg>fix sstable::FileHeaderReader::verify for empty userdata<commit_after>/**
 * This file is part of the "FnordMetric" project
 *   Copyright (c) 2014 <NAME>smuth, Google Inc.
 *
 * FnordMetric is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License v3.0. You should have received a
 * copy of the GNU General Public License along with this program. If not, see
 * <http://www.gnu.org/licenses/>.
 */
#include <fnord-sstable/binaryformat.h>
#include <fnord-sstable/fileheaderreader.h>
#include <fnord-base/exception.h>
#include <fnord-base/fnv.h>

namespace fnord {
namespace sstable {

FileHeaderReader::FileHeaderReader(
    void* buf,
    size_t buf_size) :
    fnord::util::BinaryMessageReader(buf, buf_size) {
  auto magic_bytes = *readUInt32();
  if (magic_bytes != BinaryFormat::kMagicBytes) {
    RAISE(kIllegalStateError, "not a valid sstable");
  }

  auto version = *readUInt16();
  if (version != BinaryFormat::kVersion) {
    RAISE(kIllegalStateError, "unsupported sstable version");
  }

  body_size_ = *readUInt64();
  userdata_checksum_ = *readUInt32();
  userdata_size_ = *readUInt32();
  userdata_offset_ = pos_;
}

bool FileHeaderReader::verify() {
  if (userdata_offset_ + userdata_size_ > size_) {
    return false;
  }

  if (userdata_size_ == 0) {
    return true;
  }

  const void* userdata;
  size_t userdata_size;
  readUserdata(&userdata, &userdata_size);

  hash::FNV<uint32_t> fnv;
  uint32_t userdata_checksum = fnv.hash(userdata, userdata_size);

  return userdata_checksum == userdata_checksum_;
}
{'fraction_non_alphanumeric': 0.11574828822954027, 'fraction_numerical': 0.013368112161721553, 'mean_word_length': 4.557971014492754, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1862618', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1027, 'n_words': 265}