<commit_before>// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include "db/version_set.h"

#include <algorithm>
#include <stdio.h>
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
#include "db/memtable.h"
#include "db/table_cache.h"
#include "leveldb/env.h"
#include "leveldb/table_builder.h"
#include "table/merger.h"
#include "table/two_level_iterator.h"
#include "util/coding.h"
#include "util/logging.h"

namespace leveldb {

static double MaxBytesForLevel(int level) {
  assert(level < leveldb::config::kNumLevels);
  static const double bytes[] = {10 * 1048576.0,
                                 10 * 1048576.0,
                                 100 * 1048576.0,
                                 1000 * 1048576.0,
                                 10000 * 1048576.0,
                                 100000 * 1048576.0,
                                 1000000 * 1048576.0};
  return bytes[level];
}

static uint64_t MaxFileSizeForLevel(int level) {
  assert(level < leveldb::config::kNumLevels);
  static const uint64_t bytes[] = {2 * 1048576,
                                   2 * 1048576,
                                   4 * 1048576,
                                   8 * 1048576,
                                   16 * 1048576,
                                   16 * 1048576,
                                   16 * 1048576};
  return bytes[level];
}

static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
  int64_t sum = 0;
  for (size_t i = 0; i < files.size(); i++) {
    sum += files[i]->file_size;
  }
  return sum;
}
<commit_msg>Go back to smaller files<commit_after>// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

#include "db/version_set.h"

#include <algorithm>
#include <stdio.h>
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
#include "db/memtable.h"
#include "db/table_cache.h"
#include "leveldb/env.h"
#include "leveldb/table_builder.h"
#include "table/merger.h"
#include "table/two_level_iterator.h"
#include "util/coding.h"
#include "util/logging.h"

namespace leveldb {

static double MaxBytesForLevel(int level) {
  assert(level < leveldb::config::kNumLevels);
  static const double bytes[] = {10 * 1048576.0,
                                 10 * 1048576.0,
                                 100 * 1048576.0,
                                 1000 * 1048576.0,
                                 10000 * 1048576.0,
                                 100000 * 1048576.0,
                                 1000000 * 1048576.0};
  return bytes[level];
}

static uint64_t MaxFileSizeForLevel(int level) {
  assert(level < leveldb::config::kNumLevels);
  static const uint64_t bytes[] = {2 * 1048576,
                                   2 * 1048576,
                                   2 * 1048576,
                                   2 * 1048576,
                                   2 * 1048576,
                                   2 * 1048576,
                                   2 * 1048576};
  return bytes[level];
}

static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
  int64_t sum = 0;
  for (size_t i = 0; i < files.size(); i++) {
    sum += files[i]->file_size;
  }
  return sum;
}

{'fraction_non_alphanumeric': 0.10956618464961068, 'fraction_numerical': 0.08704115684093437, 'mean_word_length': 1.7229371688115065, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2081714', 'n_tokens_mistral': 1326, 'n_tokens_neox': 1090, 'n_words': 284}