<commit_before>/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <libcouchstore/couch_db.h>
#include <snappy-c.h>
#include "bitfield.h"

#define SNAPPY_FLAG 128

static void printsb(sized_buf *sb)
{
    if (sb->buf == NULL) {
        printf("null\n");
        return;
    }
    printf("%.*s\n", (int) sb->size, sb->buf);
}

static int foldprint(Db *db, DocInfo *docinfo, void *ctx)
{
    int *count = (int *) ctx;
    Doc *doc;
    uint64_t cas;
    uint32_t expiry, flags;
    couchstore_open_doc_with_docinfo(db, docinfo, &doc, 0);
    printf("Doc seq: %"PRIu64"\n", docinfo->db_seq);
    printf("     id: ");
    printsb(&docinfo->id);
    printf("     rev: %"PRIu64"\n", docinfo->rev_seq);
    printf("     content_meta: %d\n", docinfo->content_meta);
    if (docinfo->rev_meta.size == 16) {
        cas = ntohll(*((uint64_t *)docinfo->rev_meta.buf));
        expiry = get_32(docinfo->rev_meta.buf + 8);
        flags = get_32(docinfo->rev_meta.buf + 12);
        printf("     cas: %"PRIu64", expiry: %"PRIu32", flags: %"PRIu32"\n", cas, expiry, flags);
    }
    if (docinfo->deleted) {
        printf("     doc deleted\n");
    }

    if (docinfo->content_meta & SNAPPY_FLAG) {
        size_t rlen;
        snappy_uncompressed_length(doc->data.buf, doc->data.size, &rlen);
        char *decbuf = (char *) malloc(rlen);
        size_t uncompr_len;
        snappy_uncompress(doc->data.buf, doc->data.size, decbuf, &uncompr_len);
        printf("     data: (snappy) %.*s\n", (int) uncompr_len, decbuf);
    } else {
        printf("     data: ");
        printsb(&doc->data);
    }<commit_msg>Fix dbdump to deal with deleted docs (no body)<commit_after>/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <libcouchstore/couch_db.h>
#include <snappy-c.h>
#include "bitfield.h"

#define SNAPPY_FLAG 128

static void printsb(sized_buf *sb)
{
    if (sb->buf == NULL) {
        printf("null\n");
        return;
    }
    printf("%.*s\n", (int) sb->size, sb->buf);
}

static int foldprint(Db *db, DocInfo *docinfo, void *ctx)
{
    int *count = (int *) ctx;
    Doc *doc = NULL;
    uint64_t cas;
    uint32_t expiry, flags;
    couchstore_open_doc_with_docinfo(db, docinfo, &doc, 0);
    printf("Doc seq: %"PRIu64"\n", docinfo->db_seq);
    printf("     id: ");
    printsb(&docinfo->id);
    printf("     rev: %"PRIu64"\n", docinfo->rev_seq);
    printf("     content_meta: %d\n", docinfo->content_meta);
    if (docinfo->rev_meta.size == 16) {
        cas = ntohll(*((uint64_t *)docinfo->rev_meta.buf));
        expiry = get_32(docinfo->rev_meta.buf + 8);
        flags = get_32(docinfo->rev_meta.buf + 12);
        printf("     cas: %"PRIu64", expiry: %"PRIu32", flags: %"PRIu32"\n", cas, expiry, flags);
    }
    if (docinfo->deleted) {
        printf("     doc deleted\n");
    }

    if (doc && docinfo->content_meta & SNAPPY_FLAG) {
        size_t rlen;
        snappy_uncompressed_length(doc->data.buf, doc->data.size, &rlen);
        char *decbuf = (char *) malloc(rlen);
        size_t uncompr_len;
        snappy_uncompress(doc->data.buf, doc->data.size, decbuf, &uncompr_len);
        printf("     data: (snappy) %.*s\n", (int) uncompr_len, decbuf);
    } else if(doc) {
        printf("     data: ");
        printsb(&doc->data);
    }
{'fraction_non_alphanumeric': 0.18194720408742548, 'fraction_numerical': 0.017598637524836785, 'mean_word_length': 2.8304347826086955, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3771629', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1353, 'n_words': 281}