<commit_before>/*
Copyright (c) 2012-2014 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include "ssdb_impl.h"
#include "../util/PTimer.h"
#ifdef USE_LEVELDB
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "leveldb/cache.h"
#include "leveldb/filter_policy.h"
#else
#include "rocksdb/options.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/cache.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/table.h"
#include "rocksdb/convenience.h"

#include "t_listener.h"
#define leveldb rocksdb
#endif

#include "iterator.h"
#include "ssdb_impl.h"

SSDBImpl::SSDBImpl()
	: bg_cv_(&mutex_bgtask_){
	ldb = NULL;
	this->bgtask_flag_ = true;
	expiration = NULL;
}

SSDBImpl::~SSDBImpl(){
    if(expiration){
        delete expiration;
    }

    this->stop();

    if(ldb){
        delete ldb;
    }
    log_info("SSDBImpl finalized");

#ifdef USE_LEVELDB
    //auto memory man in rocks
	if(options.block_cache){
		delete options.block_cache;
	}
	if(options.filter_policy){
		delete options.filter_policy;
	}
#endif
}

SSDB* SSDB::open(const Options &opt, const std::string &dir){
	SSDBImpl *ssdb = new SSDBImpl();
	ssdb->options.create_if_missing = true;
	ssdb->options.max_open_files = opt.max_open_files;
#ifdef USE_LEVELDB
	ssdb->options.filter_policy = leveldb::NewBloomFilterPolicy(10);
	ssdb->options.block_cache = leveldb::NewLRUCache(opt.cache_size * 1048576);
	ssdb->options.block_size = opt.block_size * 1024;
	ssdb->options.compaction_speed = opt.compaction_speed;
#else
	rocksdb::BlockBasedTableOptions op;

	//for spin disk
//	op.cache_index_and_filter_blocks = true;
//  	ssdb->options.optimize_filters_for_hits = true;
//	ssdb->options.skip_stats_update_on_db_open = true;
//	ssdb->options.level_compaction_dynamic_level_bytes = true;
//	ssdb->options.new_table_reader_for_compaction_inputs = true;
//	ssdb->options.compaction_readahead_size = 16 * 1024 * 1024;
//
//	ssdb->options.level0_file_num_compaction_trigger = 3; //start compaction<commit_msg>[性能优化] spop srandmember提高性能<commit_after>/*
Copyright (c) 2012-2014 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include "ssdb_impl.h"
#include "../util/PTimer.h"
#ifdef USE_LEVELDB
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "leveldb/cache.h"
#include "leveldb/filter_policy.h"
#else
#include "rocksdb/options.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/cache.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/table.h"
#include "rocksdb/convenience.h"

#include "t_listener.h"
#define leveldb rocksdb
#endif

#include "iterator.h"
#include "ssdb_impl.h"

SSDBImpl::SSDBImpl()
	: bg_cv_(&mutex_bgtask_){
	ldb = NULL;
	this->bgtask_flag_ = true;
	expiration = NULL;
}

SSDBImpl::~SSDBImpl(){
    if(expiration){
        delete expiration;
    }

    this->stop();

    if(ldb){
        delete ldb;
    }
    log_info("SSDBImpl finalized");

#ifdef USE_LEVELDB
    //auto memory man in rocks
	if(options.block_cache){
		delete options.block_cache;
	}
	if(options.filter_policy){
		delete options.filter_policy;
	}
#endif
}

SSDB* SSDB::open(const Options &opt, const std::string &dir){
	SSDBImpl *ssdb = new SSDBImpl();
	ssdb->options.create_if_missing = true;
	ssdb->options.max_open_files = opt.max_open_files;
#ifdef USE_LEVELDB
	ssdb->options.filter_policy = leveldb::NewBloomFilterPolicy(10);
	ssdb->options.block_cache = leveldb::NewLRUCache(opt.cache_size * 1048576);
	ssdb->options.block_size = opt.block_size * 1024;
	ssdb->options.compaction_speed = opt.compaction_speed;
#else
	rocksdb::BlockBasedTableOptions op;

	//for spin disk
//	op.cache_index_and_filter_blocks = true;
  	ssdb->options.optimize_filters_for_hits = true;
	ssdb->options.compaction_readahead_size = 4 * 1024 * 1024;
//	ssdb->options.skip_stats_update_on_db_open = true;
//	ssdb->options.level_compaction_dynamic_level_bytes = true;
//	ssdb->options.new_table_reader_for_compaction_inputs = true;
//
//	ssdb->options.level0_file_num_compaction_trigger = 3; //start compaction
{'fraction_non_alphanumeric': 0.1326360827192774, 'fraction_numerical': 0.015450439743285001, 'mean_word_length': 4.885314685314685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867632', 'n_tokens_mistral': 1644, 'n_tokens_neox': 1574, 'n_words': 303}