<commit_before>﻿/* mdbx_chk.c - memory-mapped database check tool */

/*
 * Copyright 2015-2018 <NAME> <<EMAIL>>
 * and other libmdbx authors: please see AUTHORS file.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>. */

#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif                          /* _MSC_VER (warnings) */

#include "../bits.h"

typedef struct flagbit {
  int bit;
  char *name;
} flagbit;

flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"},
                     {MDBX_INTEGERKEY, "integerkey"},
                     {MDBX_REVERSEKEY, "reversekey"},
                     {MDBX_DUPFIXED, "dupfixed"},
                     {MDBX_REVERSEDUP, "reversedup"},
                     {MDBX_INTEGERDUP, "integerdup"},
                     {0, NULL}};

#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"

static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
  (void)dwCtrlType;
  user_break = true;
  return true;
}

#else /* WINDOWS */

static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
  (void)sig;
  user_break = 1;
}

#endif /* !WINDOWS */

#define EXIT_INTERRUPTED (EXIT_FAILURE + 4)
#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3)
#define EXIT_FAILURE_MDB (EXIT_FAILURE + 2)
#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1)
#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE

typedef struct {
  const char *name;
  struct {
    uint64_t branch, large_count, large_volume, leaf;
    uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed;
    uint64_t total, empty, other;
  } pages;
  uint64_t payload_bytes;
  uint64_t lost_bytes;
} walk_dbi_t;

struct {
  walk_dbi_t dbi[MAX_DBI];
  short *pagemap;
  uint64_t total_payload_bytes;
  uint64_t pgcount;
} walk;

#define dbi_free walk.dbi[FREE_DBI]
#define dbi_main walk.dbi[MAIN_DBI]
#define dbi_meta walk.dbi[CORE_DBS]

uint64_t total_unused_bytes;
int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE;

MDBX_env *env;
MDBX_txn *txn;
MDBX_envinfo envinfo;
MDBX_stat envstat;
size_t maxkeysize, userdb_count, skipped_subdb;
uint64_t reclaimable_pages, gc_pages, lastpgno, unused_pages;
unsigned verbose, quiet;
const char *only_subdb;

struct problem {
  struct problem *pr_next;
  uint64_t count;
  const char *caption;
};

struct problem *problems_list;
uint64_t total_problems;

static void
#ifdef __GNUC__
    __attribute__((format(printf, 1, 2)))
#endif
    print(const char *msg, ...) {
  if (!quiet) {
    va_list args;

    fflush(stderr);
    va_start(args, msg);
    vfprintf(stdout, msg, args);
    va_end(args);
  }
}

static void
#ifdef __GNUC__
    __attribute__((format(printf, 1, 2)))
#endif
    error(const char *msg, ...) {
  total_problems++;

  if (!quiet) {
    va_list args;

    fflush(stdout);
    va_start(args, msg);
    fputs(" ! ", stderr);
    vfprintf(stderr, msg, args);
    va_end(args);
    fflush(NULL);
  }
}

static void pagemap_cleanup(void) {
  for (int i = CORE_DBS; ++i < MAX_DBI;) {
    if (walk.dbi[i].name) {
      free((void *)walk.dbi[i].name);
      walk.dbi[i].name = NULL;
    }
  }

  free(walk.pagemap);
  walk.pagemap = NULL;
}

static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
  static walk_dbi_t *last;

  if (last && strcmp(last->name, dbi_name) == 0)
    return last;

  walk_dbi_t *dbi = walk.dbi + CORE_DBS;
  for (dbi = walk.dbi + CORE_DBS; (++dbi)->name;) {
    if (strcmp(dbi->name, dbi_name) == 0)
      return last = dbi;
    if (dbi == walk.dbi + MAX_DBI)
      return NULL;
  }

  dbi->name = strdup(dbi_name);
  if (verbose > 1 && !silent) {
    print(" - found '%s' area\n", dbi_name);
    fflush(NULL);
  }

  return last = dbi;
}

static void
#ifdef __GNUC__
    __attribute__((format(printf, 4, 5)))
#endif
    problem_add(const char *object, uint64_t entry_number, const char *msg,
                const char *extra, ...) {
  total_problems++;

  if (!quiet) {
    int need_fflush = 0;
    struct problem *p;

    for (p = problems_list; p; p = p->pr_next)
      if (p->caption == msg)
        break;

    if (!p) {
      p = calloc(1, sizeof(*p));
      p->caption = msg;
      p->pr_next = problems_list;
      problems_list = p;
      need_fflush = 1;
    }

    p->count++;
    if (verbose > 1) {
      print("     %s #%" PRIu64 ": %s", object, entry_number, msg);
      if (extra) {
        va_list args;
        printf(" (");
        va_start(args, extra);
        vfprintf(stdout, extra, args);
        va_end(args);
        printf(")");
      }
      printf("\n");
      if (need_fflush)
        fflush(NULL);
    }
  }
}

static struct problem *problems_push(void) {
  struct problem *p = problems_list;
  problems_list = NULL;
  return p;
}

static uint64_t problems_pop(struct problem *list) {
  uint64_t count = 0;

  if (problems_list) {
    int i;

    print(" - problems: ");
    for (i = 0; problems_list; ++i) {
      struct problem *p = problems_list->pr_next;
      count += problems_list->count;
      print("%s%s (%" PRIu64 ")", i ? ", " : "", problems_list->caption,
            problems_list->count);
      free(problems_list);
      problems_list = p;
    }
    print("\n");
    fflush(NULL);
  }

  problems_list = list;
  return count;
}

static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx, int deep,
                     const char *dbi_name, size_t page_size,
                     MDBX_page_type_t pagetype, size_t nentries,
                     size_t payload_bytes, size_t header_bytes,
                     size_t unused_bytes) {
  (void)ctx;
  if (pagetype == MDBX_page_void)
    return MDBX_SUCCESS;

  walk_dbi_t fake, *dbi = &fake;
  if (deep > 0) {
    dbi = pagemap_lookup_dbi(dbi_name, false);
    if (!dbi)
      return MDBX_ENOMEM;
  } else if (deep == 0 && strcmp(dbi_name, dbi_main.name) == 0)
    dbi = &dbi_main;
  else if (deep == -1 && strcmp(dbi_name, dbi_free.name) == 0)
    dbi = &dbi_free;
  else if (deep == -2 && strcmp(dbi_name, dbi_meta.name) == 0)
    dbi = &dbi_meta;
  else
    problem_add("deep", deep, "unknown area", "%s", dbi_name);

  const uint64_t page_bytes = payload_bytes + header_bytes + unused_bytes;
  walk.pgcount += pgnumber;

  const char *pagetype_caption;
  switch (pagetype) {
  default:
    problem_add("page", pgno, "unknown page-type", "%u", (unsigned)pagetype);
    pagetype_caption = "unknown";
    dbi->pages.other += pgnumber;
    break;
  case MDBX_page_meta:
    pagetype_caption = "meta";
    dbi->pages.other += pgnumber;
    break;
  case MDBX_page_large:
    pagetype_caption = "large";
    dbi->pages.large_volume += pgnumber;
    dbi->pages.large_count += 1;
    break;
  case MDBX_page_branch:
    pagetype_caption = "branch";
    dbi->pages.branch += pgnumber;
    break;
  case MDBX_page_leaf:
    pagetype_caption = "leaf";
    dbi->pages.leaf += pgnumber;
    break;
  case MDBX_page_dupfixed_leaf:
    pagetype_caption = "leaf-dupfixed";
    dbi->pages.leaf_dupfixed += pgnumber;
    break;
  case MDBX_subpage_leaf:
    pagetype_caption = "subleaf-dupsort";
    dbi->pages.subleaf_dupsort += 1;
    break;
  case MDBX_subpage_dupfixed_leaf:
    pagetype_caption = "subleaf-dupfixed";
    dbi->pages.subleaf_dupfixed += 1;
    break;
  }

  if (pgnumber) {
    if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi_name) == 0)) {
      if (pgnumber == 1)
        print("     %s-page %" PRIu64, pagetype_caption, pgno);
      else
        print("     %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber);
      print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR
            ", unused %" PRIiPTR "\n",
            dbi_name, header_bytes, payload_bytes, unused_bytes);
    }
  }

  if (unused_bytes > page_size)
    problem_add("page", pgno, "illegal unused-bytes",
                "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0,
                unused_bytes, envstat.ms_psize);

  if (header_bytes < (int)sizeof(long) ||
      (size_t)header_bytes >= envstat.ms_psize - sizeof(long))
    problem_add("page", pgno, "illegal header-length",
                "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR,
                pagetype_caption, sizeof(long), header_bytes,
                envstat.ms_psize - sizeof(long));
  if (payload_bytes < 1) {
    if (nentries > 1) {
      problem_add("page", pgno, "zero size-of-entry",
                  "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
                  pagetype_caption, payload_bytes, nentries);
      /* if ((size_t)header_bytes + unused_bytes < page_size) {
        // LY: hush a misuse error
        page_bytes = page_size;
      } */
    } else {
      problem_add("page", pgno, "empty",
                  "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
                  pagetype_caption, payload_bytes, nentries);
      dbi->pages.empty += 1;
    }
  }

  if (pgnumber) {
    if (page_bytes != page_size) {
      problem_add("page", pgno, "misused",
                  "%s-page: %" PRIu64 " != %" PRIu64 " (%" PRIuPTR
                  "h + %" PRIuPTR "p + %" PRIuPTR "u)",
                  pagetype_caption, page_size, page_bytes, header_bytes,
                  payload_bytes, unused_bytes);
      if (page_size > page_bytes)
        dbi->lost_bytes += page_size - page_bytes;
    } else {
      dbi->payload_bytes += payload_bytes + header_bytes;
      walk.total_payload_bytes += payload_bytes + header_bytes;
    }
  }

  if (pgnumber) {
    do {
      if (pgno >= lastpgno)
        problem_add("page", pgno, "wrong page-no",
                    "%s-page: %" PRIu64 " > %" PRIu64, pagetype_caption, pgno,
                    lastpgno);
      else if (walk.pagemap[pgno])
        problem_add("page", pgno, "already used", "%s-page: by %s",
                    pagetype_caption, walk.dbi[walk.pagemap[pgno] - 1].name);
      else {
        walk.pagemap[pgno] = (short)(dbi - walk.dbi + 1);
        dbi->pages.total += 1;
      }
      ++pgno;
    } while (--pgnumber);
  }

  return user_break ? MDBX_EINTR : MDBX_SUCCESS;
}

typedef int(visitor)(const uint64_t record_number, const MDBX_val *key,
                     const MDBX_val *data);
static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
                      bool silent);

static int handle_userdb(const uint64_t record_number, const MDBX_val *key,
                         const MDBX_val *data) {
  (void)record_number;
  (void)key;
  (void)data;
  return MDBX_SUCCESS;
}

static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
                         const MDBX_val *data) {
  char *bad = "";
  pgno_t *iptr = data->iov_base;
  txnid_t txnid = *(txnid_t *)key->iov_base;

  if (key->iov_len != sizeof(txnid_t))
    problem_add("entry", record_number, "wrong txn-id size",
                "key-size %" PRIiPTR, key->iov_len);
  else if (txnid < 1 || txnid > envinfo.mi_recent_txnid)
    problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid);
  else {
    if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
      problem_add("entry", record_number, "wrong idl size", "%" PRIuPTR,
                  data->iov_len);
    size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0;
    if (number < 1 || number > MDBX_PNL_MAX)
      problem_add("entry", record_number, "wrong idl length", "%" PRIuPTR,
                  number);
    else if ((number + 1) * sizeof(pgno_t) > data->iov_len) {
      problem_add("entry", record_number, "trimmed idl",
                  "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
                  (number + 1) * sizeof(pgno_t), data->iov_len);
      number = data->iov_len / sizeof(pgno_t) - 1;
    } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
               /* LY: allow gap upto one page. it is ok
                * and better than shink-and-retry inside mdbx_update_gc() */
               envstat.ms_psize)
      problem_add("entry", record_number, "extra idl space",
                  "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)",
                  (number + 1) * sizeof(pgno_t), data->iov_len);

    gc_pages += number;
    if (envinfo.mi_latter_reader_txnid > txnid)
      reclaimable_pages += number;

    pgno_t prev =
        MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)envinfo.mi_last_pgno + 1;
    pgno_t span = 1;
    for (unsigned i = 0; i < number; ++i) {
      const pgno_t pgno = iptr[i];
      if (pgno < NUM_METAS || pgno > envinfo.mi_last_pgno)
        problem_add("entry", record_number, "wrong idl entry",
                    "%u < %" PRIaPGNO " < %" PRIu64, NUM_METAS, pgno,
                    envinfo.mi_last_pgno);
      else {
        if (MDBX_PNL_DISORDERED(prev, pgno)) {
          bad = " [bad sequence]";
          problem_add("entry", record_number, "bad sequence",
                      "%" PRIaPGNO " <> %" PRIaPGNO, prev, pgno);
        }
        if (walk.pagemap && walk.pagemap[pgno]) {
          if (walk.pagemap[pgno] > 0)
            problem_add("page", pgno, "already used", "by %s",
                        walk.dbi[walk.pagemap[pgno] - 1].name);
          else
            problem_add("page", pgno, "already listed in GC", nullptr);
          walk.pagemap[pgno] = -1;
        }
      }
      prev = pgno;<commit_msg>mdbx-chk: use txnid as record-number for GC.<commit_after>﻿/* mdbx_chk.c - memory-mapped database check tool */

/*
 * Copyright 2015-2018 <NAME> <<EMAIL>>
 * and other libmdbx authors: please see AUTHORS file.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted only as authorized by the OpenLDAP
 * Public License.
 *
 * A copy of this license is available in the file LICENSE in the
 * top-level directory of the distribution or, alternatively, at
 * <http://www.OpenLDAP.org/license.html>. */

#ifdef _MSC_VER
#if _MSC_VER > 1800
#pragma warning(disable : 4464) /* relative include path contains '..' */
#endif
#pragma warning(disable : 4996) /* The POSIX name is deprecated... */
#endif                          /* _MSC_VER (warnings) */

#include "../bits.h"

typedef struct flagbit {
  int bit;
  char *name;
} flagbit;

flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"},
                     {MDBX_INTEGERKEY, "integerkey"},
                     {MDBX_REVERSEKEY, "reversekey"},
                     {MDBX_DUPFIXED, "dupfixed"},
                     {MDBX_REVERSEDUP, "reversedup"},
                     {MDBX_INTEGERDUP, "integerdup"},
                     {0, NULL}};

#if defined(_WIN32) || defined(_WIN64)
#include "wingetopt.h"

static volatile BOOL user_break;
static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) {
  (void)dwCtrlType;
  user_break = true;
  return true;
}

#else /* WINDOWS */

static volatile sig_atomic_t user_break;
static void signal_handler(int sig) {
  (void)sig;
  user_break = 1;
}

#endif /* !WINDOWS */

#define EXIT_INTERRUPTED (EXIT_FAILURE + 4)
#define EXIT_FAILURE_SYS (EXIT_FAILURE + 3)
#define EXIT_FAILURE_MDB (EXIT_FAILURE + 2)
#define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1)
#define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE

typedef struct {
  const char *name;
  struct {
    uint64_t branch, large_count, large_volume, leaf;
    uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed;
    uint64_t total, empty, other;
  } pages;
  uint64_t payload_bytes;
  uint64_t lost_bytes;
} walk_dbi_t;

struct {
  walk_dbi_t dbi[MAX_DBI];
  short *pagemap;
  uint64_t total_payload_bytes;
  uint64_t pgcount;
} walk;

#define dbi_free walk.dbi[FREE_DBI]
#define dbi_main walk.dbi[MAIN_DBI]
#define dbi_meta walk.dbi[CORE_DBS]

uint64_t total_unused_bytes;
int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE;

MDBX_env *env;
MDBX_txn *txn;
MDBX_envinfo envinfo;
MDBX_stat envstat;
size_t maxkeysize, userdb_count, skipped_subdb;
uint64_t reclaimable_pages, gc_pages, lastpgno, unused_pages;
unsigned verbose, quiet;
const char *only_subdb;

struct problem {
  struct problem *pr_next;
  uint64_t count;
  const char *caption;
};

struct problem *problems_list;
uint64_t total_problems;

static void
#ifdef __GNUC__
    __attribute__((format(printf, 1, 2)))
#endif
    print(const char *msg, ...) {
  if (!quiet) {
    va_list args;

    fflush(stderr);
    va_start(args, msg);
    vfprintf(stdout, msg, args);
    va_end(args);
  }
}

static void
#ifdef __GNUC__
    __attribute__((format(printf, 1, 2)))
#endif
    error(const char *msg, ...) {
  total_problems++;

  if (!quiet) {
    va_list args;

    fflush(stdout);
    va_start(args, msg);
    fputs(" ! ", stderr);
    vfprintf(stderr, msg, args);
    va_end(args);
    fflush(NULL);
  }
}

static void pagemap_cleanup(void) {
  for (int i = CORE_DBS; ++i < MAX_DBI;) {
    if (walk.dbi[i].name) {
      free((void *)walk.dbi[i].name);
      walk.dbi[i].name = NULL;
    }
  }

  free(walk.pagemap);
  walk.pagemap = NULL;
}

static walk_dbi_t *pagemap_lookup_dbi(const char *dbi_name, bool silent) {
  static walk_dbi_t *last;

  if (last && strcmp(last->name, dbi_name) == 0)
    return last;

  walk_dbi_t *dbi = walk.dbi + CORE_DBS;
  for (dbi = walk.dbi + CORE_DBS; (++dbi)->name;) {
    if (strcmp(dbi->name, dbi_name) == 0)
      return last = dbi;
    if (dbi == walk.dbi + MAX_DBI)
      return NULL;
  }

  dbi->name = strdup(dbi_name);
  if (verbose > 1 && !silent) {
    print(" - found '%s' area\n", dbi_name);
    fflush(NULL);
  }

  return last = dbi;
}

static void
#ifdef __GNUC__
    __attribute__((format(printf, 4, 5)))
#endif
    problem_add(const char *object, uint64_t entry_number, const char *msg,
                const char *extra, ...) {
  total_problems++;

  if (!quiet) {
    int need_fflush = 0;
    struct problem *p;

    for (p = problems_list; p; p = p->pr_next)
      if (p->caption == msg)
        break;

    if (!p) {
      p = calloc(1, sizeof(*p));
      p->caption = msg;
      p->pr_next = problems_list;
      problems_list = p;
      need_fflush = 1;
    }

    p->count++;
    if (verbose > 1) {
      print("     %s #%" PRIu64 ": %s", object, entry_number, msg);
      if (extra) {
        va_list args;
        printf(" (");
        va_start(args, extra);
        vfprintf(stdout, extra, args);
        va_end(args);
        printf(")");
      }
      printf("\n");
      if (need_fflush)
        fflush(NULL);
    }
  }
}

static struct problem *problems_push(void) {
  struct problem *p = problems_list;
  problems_list = NULL;
  return p;
}

static uint64_t problems_pop(struct problem *list) {
  uint64_t count = 0;

  if (problems_list) {
    int i;

    print(" - problems: ");
    for (i = 0; problems_list; ++i) {
      struct problem *p = problems_list->pr_next;
      count += problems_list->count;
      print("%s%s (%" PRIu64 ")", i ? ", " : "", problems_list->caption,
            problems_list->count);
      free(problems_list);
      problems_list = p;
    }
    print("\n");
    fflush(NULL);
  }

  problems_list = list;
  return count;
}

static int pgvisitor(uint64_t pgno, unsigned pgnumber, void *ctx, int deep,
                     const char *dbi_name, size_t page_size,
                     MDBX_page_type_t pagetype, size_t nentries,
                     size_t payload_bytes, size_t header_bytes,
                     size_t unused_bytes) {
  (void)ctx;
  if (pagetype == MDBX_page_void)
    return MDBX_SUCCESS;

  walk_dbi_t fake, *dbi = &fake;
  if (deep > 0) {
    dbi = pagemap_lookup_dbi(dbi_name, false);
    if (!dbi)
      return MDBX_ENOMEM;
  } else if (deep == 0 && strcmp(dbi_name, dbi_main.name) == 0)
    dbi = &dbi_main;
  else if (deep == -1 && strcmp(dbi_name, dbi_free.name) == 0)
    dbi = &dbi_free;
  else if (deep == -2 && strcmp(dbi_name, dbi_meta.name) == 0)
    dbi = &dbi_meta;
  else
    problem_add("deep", deep, "unknown area", "%s", dbi_name);

  const uint64_t page_bytes = payload_bytes + header_bytes + unused_bytes;
  walk.pgcount += pgnumber;

  const char *pagetype_caption;
  switch (pagetype) {
  default:
    problem_add("page", pgno, "unknown page-type", "%u", (unsigned)pagetype);
    pagetype_caption = "unknown";
    dbi->pages.other += pgnumber;
    break;
  case MDBX_page_meta:
    pagetype_caption = "meta";
    dbi->pages.other += pgnumber;
    break;
  case MDBX_page_large:
    pagetype_caption = "large";
    dbi->pages.large_volume += pgnumber;
    dbi->pages.large_count += 1;
    break;
  case MDBX_page_branch:
    pagetype_caption = "branch";
    dbi->pages.branch += pgnumber;
    break;
  case MDBX_page_leaf:
    pagetype_caption = "leaf";
    dbi->pages.leaf += pgnumber;
    break;
  case MDBX_page_dupfixed_leaf:
    pagetype_caption = "leaf-dupfixed";
    dbi->pages.leaf_dupfixed += pgnumber;
    break;
  case MDBX_subpage_leaf:
    pagetype_caption = "subleaf-dupsort";
    dbi->pages.subleaf_dupsort += 1;
    break;
  case MDBX_subpage_dupfixed_leaf:
    pagetype_caption = "subleaf-dupfixed";
    dbi->pages.subleaf_dupfixed += 1;
    break;
  }

  if (pgnumber) {
    if (verbose > 3 && (!only_subdb || strcmp(only_subdb, dbi_name) == 0)) {
      if (pgnumber == 1)
        print("     %s-page %" PRIu64, pagetype_caption, pgno);
      else
        print("     %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber);
      print(" of %s: header %" PRIiPTR ", payload %" PRIiPTR
            ", unused %" PRIiPTR "\n",
            dbi_name, header_bytes, payload_bytes, unused_bytes);
    }
  }

  if (unused_bytes > page_size)
    problem_add("page", pgno, "illegal unused-bytes",
                "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0,
                unused_bytes, envstat.ms_psize);

  if (header_bytes < (int)sizeof(long) ||
      (size_t)header_bytes >= envstat.ms_psize - sizeof(long))
    problem_add("page", pgno, "illegal header-length",
                "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR,
                pagetype_caption, sizeof(long), header_bytes,
                envstat.ms_psize - sizeof(long));
  if (payload_bytes < 1) {
    if (nentries > 1) {
      problem_add("page", pgno, "zero size-of-entry",
                  "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
                  pagetype_caption, payload_bytes, nentries);
      /* if ((size_t)header_bytes + unused_bytes < page_size) {
        // LY: hush a misuse error
        page_bytes = page_size;
      } */
    } else {
      problem_add("page", pgno, "empty",
                  "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR " entries",
                  pagetype_caption, payload_bytes, nentries);
      dbi->pages.empty += 1;
    }
  }

  if (pgnumber) {
    if (page_bytes != page_size) {
      problem_add("page", pgno, "misused",
                  "%s-page: %" PRIu64 " != %" PRIu64 " (%" PRIuPTR
                  "h + %" PRIuPTR "p + %" PRIuPTR "u)",
                  pagetype_caption, page_size, page_bytes, header_bytes,
                  payload_bytes, unused_bytes);
      if (page_size > page_bytes)
        dbi->lost_bytes += page_size - page_bytes;
    } else {
      dbi->payload_bytes += payload_bytes + header_bytes;
      walk.total_payload_bytes += payload_bytes + header_bytes;
    }
  }

  if (pgnumber) {
    do {
      if (pgno >= lastpgno)
        problem_add("page", pgno, "wrong page-no",
                    "%s-page: %" PRIu64 " > %" PRIu64, pagetype_caption, pgno,
                    lastpgno);
      else if (walk.pagemap[pgno])
        problem_add("page", pgno, "already used", "%s-page: by %s",
                    pagetype_caption, walk.dbi[walk.pagemap[pgno] - 1].name);
      else {
        walk.pagemap[pgno] = (short)(dbi - walk.dbi + 1);
        dbi->pages.total += 1;
      }
      ++pgno;
    } while (--pgnumber);
  }

  return user_break ? MDBX_EINTR : MDBX_SUCCESS;
}

typedef int(visitor)(const uint64_t record_number, const MDBX_val *key,
                     const MDBX_val *data);
static int process_db(MDBX_dbi dbi_handle, char *dbi_name, visitor *handler,
                      bool silent);

static int handle_userdb(const uint64_t record_number, const MDBX_val *key,
                         const MDBX_val *data) {
  (void)record_number;
  (void)key;
  (void)data;
  return MDBX_SUCCESS;
}

static int handle_freedb(const uint64_t record_number, const MDBX_val *key,
                         const MDBX_val *data) {
  char *bad = "";
  pgno_t *iptr = data->iov_base;
  txnid_t txnid = *(txnid_t *)key->iov_base;

  if (key->iov_len != sizeof(txnid_t))
    problem_add("entry", record_number, "wrong txn-id size",
                "key-size %" PRIiPTR, key->iov_len);
  else if (txnid < 1 || txnid > envinfo.mi_recent_txnid)
    problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid);
  else {
    if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t))
      problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR, data->iov_len);
    size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0;
    if (number < 1 || number > MDBX_PNL_MAX)
      problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number);
    else if ((number + 1) * sizeof(pgno_t) > data->iov_len) {
      problem_add("entry", txnid, "trimmed idl",
                  "%" PRIuSIZE " > %" PRIuSIZE " (corruption)",
                  (number + 1) * sizeof(pgno_t), data->iov_len);
      number = data->iov_len / sizeof(pgno_t) - 1;
    } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >=
               /* LY: allow gap upto one page. it is ok
                * and better than shink-and-retry inside mdbx_update_gc() */
               envstat.ms_psize)
      problem_add("entry", txnid, "extra idl space",
                  "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)",
                  (number + 1) * sizeof(pgno_t), data->iov_len);

    gc_pages += number;
    if (envinfo.mi_latter_reader_txnid > txnid)
      reclaimable_pages += number;

    pgno_t prev =
        MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)envinfo.mi_last_pgno + 1;
    pgno_t span = 1;
    for (unsigned i = 0; i < number; ++i) {
      const pgno_t pgno = iptr[i];
      if (pgno < NUM_METAS || pgno > envinfo.mi_last_pgno)
        problem_add("entry", txnid, "wrong idl entry",
                    "%u < %" PRIaPGNO " < %" PRIu64, NUM_METAS, pgno,
                    envinfo.mi_last_pgno);
      else {
        if (MDBX_PNL_DISORDERED(prev, pgno)) {
          bad = " [bad sequence]";
          problem_add("entry", txnid, "bad sequence",
                      "%" PRIaPGNO " <> %" PRIaPGNO, prev, pgno);
        }
        if (walk.pagemap && walk.pagemap[pgno]) {
          if (walk.pagemap[pgno] > 0)
            problem_add("page", pgno, "already used", "by %s",
                        walk.dbi[walk.pagemap[pgno] - 1].name);
          else
            problem_add("page", pgno, "already listed in GC", nullptr);
          walk.pagemap[pgno] = -1;
        }
      }
      prev = pgno;
{'fraction_non_alphanumeric': 0.1343157740386378, 'fraction_numerical': 0.009971039994134682, 'mean_word_length': 2.5894736842105264, 'pattern_counts': {'":': 2, '<': 45, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2184983', 'n_tokens_mistral': 10392, 'n_tokens_neox': 9987, 'n_words': 2225}