<commit_before>/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <netinet/in.h>
#include <snappy-c.h>

#include "internal.h"
#include "crc32.h"
#include "util.h"

#define SIZE_BLOCK 4096

ssize_t total_read_len(off_t blockoffset, ssize_t finallen)
{
    ssize_t left;
    ssize_t add = 0;
    if (blockoffset == 0) {
        add++;
        blockoffset = 1;
    }

    left = SIZE_BLOCK - blockoffset;
    if (left >= finallen) {
        return finallen + add;
    } else {
        if ((finallen - left) % (SIZE_BLOCK - 1) != 0) {
            add++;
        }
        return finallen + add + ((finallen - left) / (SIZE_BLOCK - 1));
    }
}

static int remove_block_prefixes(char *buf, off_t offset, ssize_t len)
{
    off_t buf_pos = 0;
    off_t gap = 0;
    ssize_t remain_block;
    while (buf_pos + gap < len) {
        remain_block = SIZE_BLOCK - offset;

        if (offset == 0) {
            gap++;
        }

        if (remain_block > (len - gap - buf_pos)) {
            remain_block = len - gap - buf_pos;
        }

        if (offset == 0) {
            //printf("Move %d bytes <-- by %d, landing at %d\r\n", remain_block, gap, buf_pos);
            memmove(buf + buf_pos, buf + buf_pos + gap, remain_block);
            offset = 1;
        } else {
            buf_pos += remain_block;
            offset = 0;
        }
    }
    return len - gap;
}

// Sets *dst to returned buffer, returns end size.
// Increases pos by read len.
static int raw_read(Db *db, off_t *pos, ssize_t len, char **dst)
{
    off_t blockoffs = *pos % SIZE_BLOCK;
    ssize_t total = total_read_len(blockoffs, len);
    *dst = (char *) malloc(total);
    if (!*dst) {
        return -1;
    }
    ssize_t got_bytes = db->file_ops->pread(db, *dst, total, *pos);
    if (got_bytes <= 0) {
        goto fail;
    }

    *pos += got_bytes;
    return remove_block_prefixes(*dst, blockoffs, got_bytes);

fail:
    free(*dst);
    *dst = NULL;
    return -1;
}

static int pread_bin_int(Db *db, off_t pos, char **ret_ptr, int header)
{
    char *bufptr = NULL, *bufptr_rest = NULL, *newbufptr = NULL;
    int buf_len;
    uint32_t chunk_len, crc32 = 0;
    int skip = 0;
    int errcode = 0;
    buf_len = raw_read(db, &pos, 2 * SIZE_BLOCK - (pos % SIZE_BLOCK), &bufptr);
    if (buf_len == -1) {
        buf_len = raw_read(db, &pos, 4, &bufptr);
        error_unless(buf_len > 0, COUCHSTORE_ERROR_READ);
    }

    memcpy(&chunk_len, bufptr, 4);
    chunk_len = ntohl(chunk_len) & ~0x80000000;
    skip += 4;
    if (header) {
        chunk_len -= 4;    //Header len includes hash len.
    }
    memcpy(&crc32, bufptr + 4, 4);
    crc32 = ntohl(crc32);
    skip += 4;

    if (chunk_len == 0) {
        free(bufptr);
        *ret_ptr = NULL;
        return 0;
    }

    buf_len -= skip;
    memmove(bufptr, bufptr + skip, buf_len);<commit_msg>Use COUCH_BLOCK_SIZE instead of custom BLOCK_SIZE in couch_file_read.c<commit_after>/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
#include "config.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <netinet/in.h>
#include <snappy-c.h>

#include "internal.h"
#include "crc32.h"
#include "util.h"

ssize_t total_read_len(off_t blockoffset, ssize_t finallen)
{
    ssize_t left;
    ssize_t add = 0;
    if (blockoffset == 0) {
        add++;
        blockoffset = 1;
    }

    left = COUCH_BLOCK_SIZE - blockoffset;
    if (left >= finallen) {
        return finallen + add;
    } else {
        if ((finallen - left) % (COUCH_BLOCK_SIZE - 1) != 0) {
            add++;
        }
        return finallen + add + ((finallen - left) / (COUCH_BLOCK_SIZE - 1));
    }
}

static int remove_block_prefixes(char *buf, off_t offset, ssize_t len)
{
    off_t buf_pos = 0;
    off_t gap = 0;
    ssize_t remain_block;
    while (buf_pos + gap < len) {
        remain_block = COUCH_BLOCK_SIZE - offset;

        if (offset == 0) {
            gap++;
        }

        if (remain_block > (len - gap - buf_pos)) {
            remain_block = len - gap - buf_pos;
        }

        if (offset == 0) {
            //printf("Move %d bytes <-- by %d, landing at %d\r\n", remain_block, gap, buf_pos);
            memmove(buf + buf_pos, buf + buf_pos + gap, remain_block);
            offset = 1;
        } else {
            buf_pos += remain_block;
            offset = 0;
        }
    }
    return len - gap;
}

// Sets *dst to returned buffer, returns end size.
// Increases pos by read len.
static int raw_read(Db *db, off_t *pos, ssize_t len, char **dst)
{
    off_t blockoffs = *pos % COUCH_BLOCK_SIZE;
    ssize_t total = total_read_len(blockoffs, len);
    *dst = (char *) malloc(total);
    if (!*dst) {
        return -1;
    }
    ssize_t got_bytes = db->file_ops->pread(db, *dst, total, *pos);
    if (got_bytes <= 0) {
        goto fail;
    }

    *pos += got_bytes;
    return remove_block_prefixes(*dst, blockoffs, got_bytes);

fail:
    free(*dst);
    *dst = NULL;
    return -1;
}

static int pread_bin_int(Db *db, off_t pos, char **ret_ptr, int header)
{
    char *bufptr = NULL, *bufptr_rest = NULL, *newbufptr = NULL;
    int buf_len;
    uint32_t chunk_len, crc32 = 0;
    int skip = 0;
    int errcode = 0;
    buf_len = raw_read(db, &pos, 2 * COUCH_BLOCK_SIZE - (pos % COUCH_BLOCK_SIZE), &bufptr);
    if (buf_len == -1) {
        buf_len = raw_read(db, &pos, 4, &bufptr);
        error_unless(buf_len > 0, COUCHSTORE_ERROR_READ);
    }

    memcpy(&chunk_len, bufptr, 4);
    chunk_len = ntohl(chunk_len) & ~0x80000000;
    skip += 4;
    if (header) {
        chunk_len -= 4;    //Header len includes hash len.
    }
    memcpy(&crc32, bufptr + 4, 4);
    crc32 = ntohl(crc32);
    skip += 4;

    if (chunk_len == 0) {
        free(bufptr);
        *ret_ptr = NULL;
        return 0;
    }

    buf_len -= skip;
    memmove(bufptr, bufptr + skip, buf_len);
{'fraction_non_alphanumeric': 0.1402398401065956, 'fraction_numerical': 0.01832111925383078, 'mean_word_length': 2.21294810058855, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5134992', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2242, 'n_words': 575}