<commit_before>/*
Copyright (c) 2012-2014 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include "ssdb_impl.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/cache.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/table.h"

#include "iterator.h"
#include "t_kv.h"
#include "t_hash.h"
#include "t_zset.h"
#include "t_queue.h"

SSDBImpl::SSDBImpl(){
	ldb = NULL;
	binlogs = NULL;
}

SSDBImpl::~SSDBImpl(){
	if(binlogs){
		delete binlogs;
	}
	if(ldb){
		delete ldb;
	}
	/*if(options.block_cache){
		delete options.block_cache;
	}
	if(options.filter_policy){
		delete options.filter_policy;
	}*/
}

SSDB* SSDB::open(const Options &opt, const std::string &dir){
	SSDBImpl *ssdb = new SSDBImpl();
	ssdb->options.create_if_missing = true;
	ssdb->options.max_open_files = opt.max_open_files;

	rocksdb::BlockBasedTableOptions tableOptions;
	tableOptions.filter_policy = std::shared_ptr<const rocksdb::FilterPolicy>(rocksdb::NewBloomFilterPolicy(10));
	tableOptions.block_cache = rocksdb::NewLRUCache(opt.cache_size * 1048576);
	tableOptions.block_size = opt.block_size * 1024;
	ssdb->options.table_factory = std::shared_ptr<rocksdb::TableFactory>(rocksdb::NewBlockBasedTableFactory(tableOptions));
	/*ssdb->options.filter_policy = rocksdb::NewBloomFilterPolicy(10);

	ssdb->options.block_cache = rocksdb::NewLRUCache(opt.cache_size * 1048576);
	ssdb->options.block_size = opt.block_size * 1024;*/
	ssdb->options.write_buffer_size = opt.write_buffer_size * 1024 * 1024;
	/*ssdb->options.compaction_speed = opt.compaction_speed;*/
	if(opt.compression == "yes"){
		ssdb->options.compression = rocksdb::kSnappyCompression;
	}else{
		ssdb->options.compression = rocksdb::kNoCompression;
	}<commit_msg>fix file size to 32M<commit_after>/*
Copyright (c) 2012-2014 The SSDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
*/
#include "ssdb_impl.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/cache.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/table.h"

#include "iterator.h"
#include "t_kv.h"
#include "t_hash.h"
#include "t_zset.h"
#include "t_queue.h"

SSDBImpl::SSDBImpl(){
	ldb = NULL;
	binlogs = NULL;
}

SSDBImpl::~SSDBImpl(){
	if(binlogs){
		delete binlogs;
	}
	if(ldb){
		delete ldb;
	}
	/*if(options.block_cache){
		delete options.block_cache;
	}
	if(options.filter_policy){
		delete options.filter_policy;
	}*/
}

SSDB* SSDB::open(const Options &opt, const std::string &dir){
	SSDBImpl *ssdb = new SSDBImpl();
	ssdb->options.create_if_missing = true;
	ssdb->options.max_open_files = opt.max_open_files;

	rocksdb::BlockBasedTableOptions tableOptions;
	tableOptions.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
	tableOptions.block_cache = rocksdb::NewLRUCache(opt.cache_size * 1048576);
	tableOptions.block_size = opt.block_size * 1024;
	ssdb->options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(tableOptions));
	/*ssdb->options.filter_policy = rocksdb::NewBloomFilterPolicy(10);

	ssdb->options.block_cache = rocksdb::NewLRUCache(opt.cache_size * 1048576);
	ssdb->options.block_size = opt.block_size * 1024;*/
	ssdb->options.target_file_size_base = 32 * 1024 * 1024;
	ssdb->options.write_buffer_size = opt.write_buffer_size * 1024 * 1024;
	/*ssdb->options.compaction_speed = opt.compaction_speed;*/
	if(opt.compression == "yes"){
		ssdb->options.compression = rocksdb::kSnappyCompression;
	}else{
		ssdb->options.compression = rocksdb::kNoCompression;
	}
{'fraction_non_alphanumeric': 0.14131637168141592, 'fraction_numerical': 0.02654867256637168, 'mean_word_length': 5.902671755725191, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4475380', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1337, 'n_words': 241}