<commit_before>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2010 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#if !defined(WIN32) && !defined(_WIN32)
#include <sys/time.h>
#include <dirent.h>
#include <unistd.h>
#endif

#include "libforestdb/forestdb.h"
#include "fdb_internal.h"
#include "filemgr.h"
#include "avltree.h"
#include "common.h"
#include "filemgr_ops.h"
#include "configuration.h"
#include "internal_types.h"
#include "compactor.h"
#include "wal.h"
#include "memleak.h"

#ifdef __DEBUG
#ifndef __DEBUG_CPT
    #undef DBG
    #undef DBGCMD
    #undef DBGSW
    #define DBG(...)
    #define DBGCMD(...)
    #define DBGSW(n, ...)
#endif
#endif

#define COMPACTOR_META_VERSION (1)
#define MAX_FNAMELEN (FDB_MAX_FILENAME_LEN)

// variables for initialization
static volatile uint8_t compactor_initialized = 0;
#ifdef SPIN_INITIALIZER
static spin_t cpt_lock = SPIN_INITIALIZER;
#else
static volatile unsigned int init_lock_status = 0;
static spin_t cpt_lock;
#endif

static thread_t compactor_tid;
static size_t sleep_duration = FDB_COMPACTOR_SLEEP_DURATION;

static mutex_t sync_mutex;
static thread_cond_t sync_cond;

typedef uint8_t compactor_status_t;
enum{
    CPT_IDLE = 0,
    CPT_WORKING = 1,
};
static compactor_status_t compactor_status;
static volatile uint8_t compactor_terminate_signal = 0;

static struct avl_tree openfiles;

// cursor of openfiles_elem that is currently being compacted.
// set to NULL if no file is being compacted.
static struct avl_node *target_cursor;

struct openfiles_elem {
    struct filemgr *file;
    fdb_config config;
    uint32_t register_count;
    bool compaction_flag; // set when the file is being compacted
    struct avl_node avl;
};

struct compactor_args {
    void *aux;
};

struct compactor_meta{
    uint32_t version;
    char filename[MAX_FNAMELEN];
    uint32_t crc;
};

#if !defined(WIN32) && !defined(_WIN32)
struct timespec convert_reltime_to_abstime(unsigned int ms) {
    struct timespec ts;
    struct timeval tp;
    uint64_t wakeup;

    memset(&ts, 0, sizeof(ts));

    /*
     * Unfortunately pthread_cond_timedwait doesn't support relative sleeps
     * so we need to convert back to an absolute time.
     */
    gettimeofday(&tp, NULL);
    wakeup = ((uint64_t)(tp.tv_sec) * 1000) + (tp.tv_usec / 1000) + ms;
    /* Round up for sub ms */
    if ((tp.tv_usec % 1000) > 499) {
        ++wakeup;
    }

    ts.tv_sec = wakeup / 1000;
    wakeup %= 1000;
    ts.tv_nsec = wakeup * 1000000;
    return ts;
}
#endif

// compares file names
int _compactor_cmp(struct avl_node *a, struct avl_node *b, void *aux)
{
    struct openfiles_elem *aa, *bb;
    aa = _get_entry(a, struct openfiles_elem, avl);
    bb = _get_entry(b, struct openfiles_elem, avl);
    return strcmp(aa->file->filename, bb->file->filename);
}

INLINE uint64_t _compactor_estimate_space(struct openfiles_elem *elem)
{
    bid_t dummy_bid;
    uint64_t dummy_64;
    uint64_t ret = 0;
    uint64_t datasize;
    uint64_t nlivenodes;
    size_t header_len;
    void *header_buf = NULL;

    header_buf = filemgr_fetch_header(elem->file, NULL, &header_len);
    if (header_len > 0) {
        char *compacted_filename;

        fdb_fetch_header(header_buf, &dummy_bid, &dummy_bid,
                         &dummy_64, &nlivenodes,
                         &datasize, &dummy_64,
                         &compacted_filename, NULL);

        ret += datasize;
        ret += nlivenodes * elem->config.blocksize;
        ret += wal_get_datasize(elem->file);
    }

    if (header_buf) {
        free(header_buf);
    }

    return ret;
}

// check if the compaction threshold is satisfied
INLINE int _compactor_is_threshold_satisfied(struct openfiles_elem *elem)
{
    uint64_t filesize;
    uint64_t active_data;
    int threshold;

    if (filemgr_is_rollback_on(elem->file)) {
        // do not perform compaction during rollback
        return 0;
    }

    threshold = elem->config.compaction_threshold;
    if (elem->config.compaction_mode == FDB_COMPACTION_AUTO &&
        threshold > 0 && !elem->compaction_flag)
        {
        filesize = filemgr_get_pos(elem->file);
        active_data = _compactor_estimate_space(elem);
        if (active_data == 0 ||
            filesize < elem->config.compaction_minimum_filesize) {
            return 0;
        }

        return ((filesize / 100.0 * threshold) < (filesize - active_data));
    } else {
        return 0;
    }
}

// return the location of '.'
INLINE int _compactor_prefix_len(char *filename)
{
    int i;
    int file_len = strlen(filename);
    int prefix_len = 0;
    // find the first '.'
    for (i=file_len-1; i>=0; --i){
        if (filename[i] == '.') {
            prefix_len = i+1;
            break;
        }
    }
    return prefix_len;
}

void _compactor_get_vfilename(char *filename, char *vfilename)<commit_msg>Do not trigger compaction when active_data >= filesize<commit_after>/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
/*
 *     Copyright 2010 Couchbase, Inc
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#if !defined(WIN32) && !defined(_WIN32)
#include <sys/time.h>
#include <dirent.h>
#include <unistd.h>
#endif

#include "libforestdb/forestdb.h"
#include "fdb_internal.h"
#include "filemgr.h"
#include "avltree.h"
#include "common.h"
#include "filemgr_ops.h"
#include "configuration.h"
#include "internal_types.h"
#include "compactor.h"
#include "wal.h"
#include "memleak.h"

#ifdef __DEBUG
#ifndef __DEBUG_CPT
    #undef DBG
    #undef DBGCMD
    #undef DBGSW
    #define DBG(...)
    #define DBGCMD(...)
    #define DBGSW(n, ...)
#endif
#endif

#define COMPACTOR_META_VERSION (1)
#define MAX_FNAMELEN (FDB_MAX_FILENAME_LEN)

// variables for initialization
static volatile uint8_t compactor_initialized = 0;
#ifdef SPIN_INITIALIZER
static spin_t cpt_lock = SPIN_INITIALIZER;
#else
static volatile unsigned int init_lock_status = 0;
static spin_t cpt_lock;
#endif

static thread_t compactor_tid;
static size_t sleep_duration = FDB_COMPACTOR_SLEEP_DURATION;

static mutex_t sync_mutex;
static thread_cond_t sync_cond;

typedef uint8_t compactor_status_t;
enum{
    CPT_IDLE = 0,
    CPT_WORKING = 1,
};
static compactor_status_t compactor_status;
static volatile uint8_t compactor_terminate_signal = 0;

static struct avl_tree openfiles;

// cursor of openfiles_elem that is currently being compacted.
// set to NULL if no file is being compacted.
static struct avl_node *target_cursor;

struct openfiles_elem {
    struct filemgr *file;
    fdb_config config;
    uint32_t register_count;
    bool compaction_flag; // set when the file is being compacted
    struct avl_node avl;
};

struct compactor_args {
    void *aux;
};

struct compactor_meta{
    uint32_t version;
    char filename[MAX_FNAMELEN];
    uint32_t crc;
};

#if !defined(WIN32) && !defined(_WIN32)
struct timespec convert_reltime_to_abstime(unsigned int ms) {
    struct timespec ts;
    struct timeval tp;
    uint64_t wakeup;

    memset(&ts, 0, sizeof(ts));

    /*
     * Unfortunately pthread_cond_timedwait doesn't support relative sleeps
     * so we need to convert back to an absolute time.
     */
    gettimeofday(&tp, NULL);
    wakeup = ((uint64_t)(tp.tv_sec) * 1000) + (tp.tv_usec / 1000) + ms;
    /* Round up for sub ms */
    if ((tp.tv_usec % 1000) > 499) {
        ++wakeup;
    }

    ts.tv_sec = wakeup / 1000;
    wakeup %= 1000;
    ts.tv_nsec = wakeup * 1000000;
    return ts;
}
#endif

// compares file names
int _compactor_cmp(struct avl_node *a, struct avl_node *b, void *aux)
{
    struct openfiles_elem *aa, *bb;
    aa = _get_entry(a, struct openfiles_elem, avl);
    bb = _get_entry(b, struct openfiles_elem, avl);
    return strcmp(aa->file->filename, bb->file->filename);
}

INLINE uint64_t _compactor_estimate_space(struct openfiles_elem *elem)
{
    bid_t dummy_bid;
    uint64_t dummy_64;
    uint64_t ret = 0;
    uint64_t datasize;
    uint64_t nlivenodes;
    size_t header_len;
    void *header_buf = NULL;

    header_buf = filemgr_fetch_header(elem->file, NULL, &header_len);
    if (header_len > 0) {
        char *compacted_filename;

        fdb_fetch_header(header_buf, &dummy_bid, &dummy_bid,
                         &dummy_64, &nlivenodes,
                         &datasize, &dummy_64,
                         &compacted_filename, NULL);

        ret += datasize;
        ret += nlivenodes * elem->config.blocksize;
        ret += wal_get_datasize(elem->file);
    }

    if (header_buf) {
        free(header_buf);
    }

    return ret;
}

// check if the compaction threshold is satisfied
INLINE int _compactor_is_threshold_satisfied(struct openfiles_elem *elem)
{
    uint64_t filesize;
    uint64_t active_data;
    int threshold;

    if (filemgr_is_rollback_on(elem->file)) {
        // do not perform compaction during rollback
        return 0;
    }

    threshold = elem->config.compaction_threshold;
    if (elem->config.compaction_mode == FDB_COMPACTION_AUTO &&
        threshold > 0 && !elem->compaction_flag)
        {
        filesize = filemgr_get_pos(elem->file);
        active_data = _compactor_estimate_space(elem);
        if (active_data == 0 || active_data >= filesize ||
            filesize < elem->config.compaction_minimum_filesize) {
            return 0;
        }

        return ((filesize / 100.0 * threshold) < (filesize - active_data));
    } else {
        return 0;
    }
}

// return the location of '.'
INLINE int _compactor_prefix_len(char *filename)
{
    int i;
    int file_len = strlen(filename);
    int prefix_len = 0;
    // find the first '.'
    for (i=file_len-1; i>=0; --i){
        if (filename[i] == '.') {
            prefix_len = i+1;
            break;
        }
    }
    return prefix_len;
}

void _compactor_get_vfilename(char *filename, char *vfilename)
{'fraction_non_alphanumeric': 0.10457987738910927, 'fraction_numerical': 0.01857194374323837, 'mean_word_length': 3.096381093057607, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4578501', 'n_tokens_mistral': 3949, 'n_tokens_neox': 3767, 'n_words': 1011}