<commit_before>#include <iostream>
#include <string>

#include <kcpolydb.h>

/* long estimate_a(long *pack) {
  return (
    4 + pack[0] * 8 +
    4 + 1 + pack[1] * 8 * 5 +
    4 + 1 + pack[2] * 8 * 5 + 3 * 11 * 1 +
    4 + 1 + pack[3] * 8 * 5 + 3 * 11 * 2 +
    4 + 1 + pack[4] * 8 * 5 + 3 * 11 * 4 +
    4 + 1 + pack[5] * 8 * 5 + 3 * 11 * 6
  );
}

long estimate_b(long *pack) {
  return (
    4 + pack[0] * 9 +
    4 + 1 + pack[1] * 9 * 5 +
    4 + 1 + pack[2] * 9 * 5 + 4 * 2 * 3 * 8 * 1 +
    4 + 1 + pack[3] * 9 * 5 + 4 * 2 * 3 * 8 * 2 +
    4 + 1 + pack[4] * 9 * 5 + 4 * 2 * 3 * 8 * 4 +
    4 + 1 + pack[5] * 9 * 5 + 4 * 2 * 3 * 8 * 6
  );
} */

int main(int argc, char **argv) {
    if (argc <= 1) {
      std::cout << "Usage: stat <dbfile.kct>" << std::endl;
      return 1;
    }

    kyotocabinet::TreeDB db;

    if (!db.open(argv[1], kyotocabinet::TreeDB::OREADER)) {
      std::cout << "Could not open database." << std::endl;
      return 2;
    }

    std::auto_ptr<kyotocabinet::TreeDB::Cursor> cur(db.cursor());
    cur->jump();

    std::string key, value;

    long pack[] = {0, 0, 0, 0, 0, 0, 0};
    long total = 0;

    std::cout << "Scanning ..." << std::endl;

    while (cur->get(&key, &value, true)) {
        total++;
        if (value.size() == 8) {
            pack[0]++;
        } else {
            pack[value.at(0)]++;
        }

        if (total % 50000 == 0) {
            std::cerr << ".";
        }
    }

    std::cerr << std::endl;

    for (int i = 0; i < 6; i++) {
        std::cout << "Pack format " << i << ": " << pack[i] << " nodes " << std::endl;
    }

    std::cout << "Unique positions: " << total << std::endl;

    /* std::cout << std::endl;

    std::cout << "Scheme A: " << estimate_a(pack) << " bytes" << std::endl;
    std::cout << "Scheme B: " << estimate_b(pack) << " bytes" << std::endl;
    std::cout << "B/A: " << ((double)estimate_b(pack)/estimate_a(pack)) << std::endl; */
<commit_msg>Also output the 6th pack format<commit_after>#include <iostream>
#include <string>

#include <kcpolydb.h>

/* long estimate_a(long *pack) {
  return (
    4 + pack[0] * 8 +
    4 + 1 + pack[1] * 8 * 5 +
    4 + 1 + pack[2] * 8 * 5 + 3 * 11 * 1 +
    4 + 1 + pack[3] * 8 * 5 + 3 * 11 * 2 +
    4 + 1 + pack[4] * 8 * 5 + 3 * 11 * 4 +
    4 + 1 + pack[5] * 8 * 5 + 3 * 11 * 6
  );
}

long estimate_b(long *pack) {
  return (
    4 + pack[0] * 9 +
    4 + 1 + pack[1] * 9 * 5 +
    4 + 1 + pack[2] * 9 * 5 + 4 * 2 * 3 * 8 * 1 +
    4 + 1 + pack[3] * 9 * 5 + 4 * 2 * 3 * 8 * 2 +
    4 + 1 + pack[4] * 9 * 5 + 4 * 2 * 3 * 8 * 4 +
    4 + 1 + pack[5] * 9 * 5 + 4 * 2 * 3 * 8 * 6
  );
} */

int main(int argc, char **argv) {
    if (argc <= 1) {
      std::cout << "Usage: stat <dbfile.kct>" << std::endl;
      return 1;
    }

    kyotocabinet::TreeDB db;

    if (!db.open(argv[1], kyotocabinet::TreeDB::OREADER)) {
      std::cout << "Could not open database." << std::endl;
      return 2;
    }

    std::auto_ptr<kyotocabinet::TreeDB::Cursor> cur(db.cursor());
    cur->jump();

    std::string key, value;

    long pack[] = {0, 0, 0, 0, 0, 0, 0};
    long total = 0;

    std::cout << "Scanning ..." << std::endl;

    while (cur->get(&key, &value, true)) {
        total++;
        if (value.size() == 8) {
            pack[0]++;
        } else {
            pack[value.at(0)]++;
        }

        if (total % 50000 == 0) {
            std::cerr << ".";
        }
    }

    std::cerr << std::endl;

    for (int i = 0; i < 7; i++) {
        std::cout << "Pack format " << i << ": " << pack[i] << " nodes " << std::endl;
    }

    std::cout << "Unique positions: " << total << std::endl;

    /* std::cout << std::endl;

    std::cout << "Scheme A: " << estimate_a(pack) << " bytes" << std::endl;
    std::cout << "Scheme B: " << estimate_b(pack) << " bytes" << std::endl;
    std::cout << "B/A: " << ((double)estimate_b(pack)/estimate_a(pack)) << std::endl; */

{'fraction_non_alphanumeric': 0.22432294328053143, 'fraction_numerical': 0.059018906489524785, 'mean_word_length': 1.9480421686746987, 'pattern_counts': {'":': 2, '<': 133, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '563664', 'n_tokens_mistral': 1757, 'n_tokens_neox': 1550, 'n_words': 417}