<commit_before>#include "includes.h"

struct memarena {
    char *buf;
    size_t buf_used, buf_size;
    size_t size_of_other_bufs; // the buf_size of all the other bufs.
    char **other_bufs;
    int n_other_bufs;
};

MEMARENA memarena_create (void) {
    MEMARENA MALLOC(result);  assert(result);
    result->buf_size = 1024;
    result->buf_used = 0;
    result->other_bufs = NULL;
    result->size_of_other_bufs = 0;
    result->n_other_bufs = 0;
    result->buf = toku_malloc(result->buf_size);  assert(result->buf);
    return result;
}

void memarena_clear (MEMARENA ma) {
    // Free the other bufs.
    int i;
    for (i=0; i<ma->n_other_bufs; i++) {
	toku_free(ma->other_bufs[i]);
	ma->other_bufs[i]=0;
    }
    ma->n_other_bufs=0;
    // But reuse the main buffer
    ma->buf_used = 0;
    ma->size_of_other_bufs = 0;
}

static size_t
round_to_page (size_t size) {
    const size_t PAGE_SIZE = 4096;
    const size_t result = PAGE_SIZE+((size-1)&~(PAGE_SIZE-1));
    assert(0==(result&(PAGE_SIZE-1))); // make sure it's aligned
    assert(result>=size);              // make sure it's not too small
    assert(size<result+PAGE_SIZE);     // make sure we didn't grow by more than a page.
    return result;
}

void* malloc_in_memarena (MEMARENA ma, size_t size) {
    if (ma->buf_size < ma->buf_used + size) {
	// The existing block isn't big enough.
	// Add the block to the vector of blocks.
	if (ma->buf) {
	    int old_n = ma->n_other_bufs;
	    REALLOC_N(old_n+1, ma->other_bufs);
	    assert(ma->other_bufs);
	    ma->other_bufs[old_n]=ma->buf;
	    ma->n_other_bufs = old_n+1;
	}
	// Make a new one
	{
	    size_t new_size = 2*ma->buf_size;
	    if (new_size<size) new_size=size;
	    new_size=round_to_page(new_size); // at least size, but round to the next page size
	    ma->buf = toku_malloc(new_size);
	    assert(ma->buf);
	    ma->buf_used = 0;
	    ma->buf_size = new_size;
	}
    }
    // allocate in the existing block.<commit_msg>Addresses #1577 Don't forget about buffers in memarena when one buffer gets full<commit_after>#include "includes.h"

struct memarena {
    char *buf;
    size_t buf_used, buf_size;
    size_t size_of_other_bufs; // the buf_size of all the other bufs.
    char **other_bufs;
    int n_other_bufs;
};

MEMARENA memarena_create (void) {
    MEMARENA MALLOC(result);  assert(result);
    result->buf_size = 1024;
    result->buf_used = 0;
    result->other_bufs = NULL;
    result->size_of_other_bufs = 0;
    result->n_other_bufs = 0;
    result->buf = toku_malloc(result->buf_size);  assert(result->buf);
    return result;
}

void memarena_clear (MEMARENA ma) {
    // Free the other bufs.
    int i;
    for (i=0; i<ma->n_other_bufs; i++) {
	toku_free(ma->other_bufs[i]);
	ma->other_bufs[i]=0;
    }
    ma->n_other_bufs=0;
    // But reuse the main buffer
    ma->buf_used = 0;
    ma->size_of_other_bufs = 0;
}

static size_t
round_to_page (size_t size) {
    const size_t PAGE_SIZE = 4096;
    const size_t result = PAGE_SIZE+((size-1)&~(PAGE_SIZE-1));
    assert(0==(result&(PAGE_SIZE-1))); // make sure it's aligned
    assert(result>=size);              // make sure it's not too small
    assert(size<result+PAGE_SIZE);     // make sure we didn't grow by more than a page.
    return result;
}

void* malloc_in_memarena (MEMARENA ma, size_t size) {
    if (ma->buf_size < ma->buf_used + size) {
	// The existing block isn't big enough.
	// Add the block to the vector of blocks.
	if (ma->buf) {
	    int old_n = ma->n_other_bufs;
	    REALLOC_N(old_n+1, ma->other_bufs);
	    assert(ma->other_bufs);
	    ma->other_bufs[old_n]=ma->buf;
	    ma->n_other_bufs = old_n+1;
            ma->size_of_other_bufs += ma->buf_size;
	}
	// Make a new one
	{
	    size_t new_size = 2*ma->buf_size;
	    if (new_size<size) new_size=size;
	    new_size=round_to_page(new_size); // at least size, but round to the next page size
	    ma->buf = toku_malloc(new_size);
	    assert(ma->buf);
	    ma->buf_used = 0;
	    ma->buf_size = new_size;
	}
    }
    // allocate in the existing block.
{'fraction_non_alphanumeric': 0.13288789903489234, 'fraction_numerical': 0.012868101954961643, 'mean_word_length': 2.9783464566929134, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7084280', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1546, 'n_words': 363}