<commit_before>//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#define __STDC_FORMAT_MACROS

#ifndef GFLAGS
#include <cstdio>
int main() {
  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
  return 1;
}
#else

#include <gflags/gflags.h>

#include <atomic>
#include <iostream>
#include <memory>
#include <thread>
#include <type_traits>
#include <vector>

#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/writebuffer.h"
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/comparator.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/options.h"
#include "rocksdb/slice_transform.h"
#include "util/arena.h"
#include "util/mutexlock.h"
#include "util/stop_watch.h"
#include "util/testutil.h"

using GFLAGS::ParseCommandLineFlags;
using GFLAGS::RegisterFlagValidator;
using GFLAGS::SetUsageMessage;

DEFINE_string(benchmarks, "fillrandom",
              "Comma-separated list of benchmarks to run. Options:\n"
              "\tfillrandom             -- write N random values\n"
              "\tfillseq                -- write N values in sequential order\n"
              "\treadrandom             -- read N values in random order\n"
              "\treadseq                -- scan the DB\n"
              "\treadwrite              -- 1 thread writes while N - 1 threads "
              "do random\n"
              "\t                          reads\n"
              "\tseqreadwrite           -- 1 thread writes while N - 1 threads "
              "do scans\n");

DEFINE_string(memtablerep, "skiplist",
              "Which implementation of memtablerep to use. See "
              "include/memtablerep.h for\n"
              "  more details. Options:\n"
              "\tskiplist            -- backed by a skiplist\n"
              "\tvector              -- backed by an std::vector\n"
              "\thashskiplist        -- backed by a hash skip list\n"
              "\thashlinklist        -- backed by a hash linked list\n"
              "\tcuckoo              -- backed by a cuckoo hash table");

DEFINE_int64(bucket_count, 1000000,
             "bucket_count parameter to pass into NewHashSkiplistRepFactory or "
             "NewHashLinkListRepFactory");

DEFINE_int32(
    hashskiplist_height, 4,
    "skiplist_height parameter to pass into NewHashSkiplistRepFactory");

DEFINE_int32(
    hashskiplist_branching_factor, 4,
    "branching_factor parameter to pass into NewHashSkiplistRepFactory");

DEFINE_int32(
    huge_page_tlb_size, 0,
    "huge_page_tlb_size parameter to pass into NewHashLinkListRepFactory");

DEFINE_int32(bucket_entries_logging_threshold, 4096,
             "bucket_entries_logging_threshold parameter to pass into "
             "NewHashLinkListRepFactory");

DEFINE_bool(if_log_bucket_dist_when_flash, true,
            "if_log_bucket_dist_when_flash parameter to pass into "
            "NewHashLinkListRepFactory");

DEFINE_int32(
    threshold_use_skiplist, 256,
    "threshold_use_skiplist parameter to pass into NewHashLinkListRepFactory");

DEFINE_int64(
    write_buffer_size, 256,
    "write_buffer_size parameter to pass into NewHashCuckooRepFactory");

DEFINE_int64(
    average_data_size, 64,
    "average_data_size parameter to pass into NewHashCuckooRepFactory");

DEFINE_int64(
    hash_function_count, 4,
    "hash_function_count parameter to pass into NewHashCuckooRepFactory");

DEFINE_int32(
    num_threads, 1,
    "Number of concurrent threads to run. If the benchmark includes writes,\n"
    "then at most one thread will be a writer");

DEFINE_int32(num_operations, 1000000,
             "Number of operations to do for write and random read benchmarks");

DEFINE_int32(num_scans, 10,
             "Number of times for each thread to scan the memtablerep for "
             "sequential read "
             "benchmarks");

DEFINE_int32(item_size, 100, "Number of bytes each item should be");

DEFINE_int32(prefix_length, 8,
             "Prefix length to pass into NewFixedPrefixTransform");

/* VectorRep settings */
DEFINE_int64(vectorrep_count, 0,
             "Number of entries to reserve on VectorRep initialization");

DEFINE_int64(seed, 0,
             "Seed base for random number generators. "
             "When 0 it is deterministic.");

static rocksdb::Env* FLAGS_env = rocksdb::Env::Default();

namespace rocksdb {

namespace {
struct CallbackVerifyArgs {
  bool found;
  LookupKey* key;
  MemTableRep* table;
  InternalKeyComparator* comparator;
};
}  // namespace

// Helper for quickly generating random data.
class RandomGenerator {
 private:
  std::string data_;
  unsigned int pos_;

 public:
  RandomGenerator() {
    Random rnd(301);
    auto size = (unsigned)std::max(1048576, FLAGS_item_size);
    test::RandomString(&rnd, size, &data_);<commit_msg>Fix introduced in 2ab7065 was reverted by 18285c1.<commit_after>//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
//  This source code is licensed under the BSD-style license found in the
//  LICENSE file in the root directory of this source tree. An additional grant
//  of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#define __STDC_FORMAT_MACROS

#ifndef GFLAGS
#include <cstdio>
int main() {
  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
  return 1;
}
#else

#include <gflags/gflags.h>

#include <atomic>
#include <iostream>
#include <memory>
#include <thread>
#include <type_traits>
#include <vector>

#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/writebuffer.h"
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/comparator.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/options.h"
#include "rocksdb/slice_transform.h"
#include "util/arena.h"
#include "util/mutexlock.h"
#include "util/stop_watch.h"
#include "util/testutil.h"

using GFLAGS::ParseCommandLineFlags;
using GFLAGS::RegisterFlagValidator;
using GFLAGS::SetUsageMessage;

DEFINE_string(benchmarks, "fillrandom",
              "Comma-separated list of benchmarks to run. Options:\n"
              "\tfillrandom             -- write N random values\n"
              "\tfillseq                -- write N values in sequential order\n"
              "\treadrandom             -- read N values in random order\n"
              "\treadseq                -- scan the DB\n"
              "\treadwrite              -- 1 thread writes while N - 1 threads "
              "do random\n"
              "\t                          reads\n"
              "\tseqreadwrite           -- 1 thread writes while N - 1 threads "
              "do scans\n");

DEFINE_string(memtablerep, "skiplist",
              "Which implementation of memtablerep to use. See "
              "include/memtablerep.h for\n"
              "  more details. Options:\n"
              "\tskiplist            -- backed by a skiplist\n"
              "\tvector              -- backed by an std::vector\n"
              "\thashskiplist        -- backed by a hash skip list\n"
              "\thashlinklist        -- backed by a hash linked list\n"
              "\tcuckoo              -- backed by a cuckoo hash table");

DEFINE_int64(bucket_count, 1000000,
             "bucket_count parameter to pass into NewHashSkiplistRepFactory or "
             "NewHashLinkListRepFactory");

DEFINE_int32(
    hashskiplist_height, 4,
    "skiplist_height parameter to pass into NewHashSkiplistRepFactory");

DEFINE_int32(
    hashskiplist_branching_factor, 4,
    "branching_factor parameter to pass into NewHashSkiplistRepFactory");

DEFINE_int32(
    huge_page_tlb_size, 0,
    "huge_page_tlb_size parameter to pass into NewHashLinkListRepFactory");

DEFINE_int32(bucket_entries_logging_threshold, 4096,
             "bucket_entries_logging_threshold parameter to pass into "
             "NewHashLinkListRepFactory");

DEFINE_bool(if_log_bucket_dist_when_flash, true,
            "if_log_bucket_dist_when_flash parameter to pass into "
            "NewHashLinkListRepFactory");

DEFINE_int32(
    threshold_use_skiplist, 256,
    "threshold_use_skiplist parameter to pass into NewHashLinkListRepFactory");

DEFINE_int64(
    write_buffer_size, 256,
    "write_buffer_size parameter to pass into NewHashCuckooRepFactory");

DEFINE_int64(
    average_data_size, 64,
    "average_data_size parameter to pass into NewHashCuckooRepFactory");

DEFINE_int64(
    hash_function_count, 4,
    "hash_function_count parameter to pass into NewHashCuckooRepFactory");

DEFINE_int32(
    num_threads, 1,
    "Number of concurrent threads to run. If the benchmark includes writes,\n"
    "then at most one thread will be a writer");

DEFINE_int32(num_operations, 1000000,
             "Number of operations to do for write and random read benchmarks");

DEFINE_int32(num_scans, 10,
             "Number of times for each thread to scan the memtablerep for "
             "sequential read "
             "benchmarks");

DEFINE_int32(item_size, 100, "Number of bytes each item should be");

DEFINE_int32(prefix_length, 8,
             "Prefix length to pass into NewFixedPrefixTransform");

/* VectorRep settings */
DEFINE_int64(vectorrep_count, 0,
             "Number of entries to reserve on VectorRep initialization");

DEFINE_int64(seed, 0,
             "Seed base for random number generators. "
             "When 0 it is deterministic.");

namespace rocksdb {

namespace {
struct CallbackVerifyArgs {
  bool found;
  LookupKey* key;
  MemTableRep* table;
  InternalKeyComparator* comparator;
};
}  // namespace

// Helper for quickly generating random data.
class RandomGenerator {
 private:
  std::string data_;
  unsigned int pos_;

 public:
  RandomGenerator() {
    Random rnd(301);
    auto size = (unsigned)std::max(1048576, FLAGS_item_size);
    test::RandomString(&rnd, size, &data_);
{'fraction_non_alphanumeric': 0.08736379277384822, 'fraction_numerical': 0.019212387688778438, 'mean_word_length': 2.836817015034837, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '625384', 'n_tokens_mistral': 3295, 'n_tokens_neox': 3042, 'n_words': 950}