<commit_before>/*
 * Copyright 2008-2009 LinkedIn, Inc
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package voldemort.store.readonly;

import static java.lang.Math.floor;
import static java.lang.Math.log;
import static java.lang.Math.pow;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.log4j.Logger;

import voldemort.VoldemortException;
import voldemort.annotations.jmx.JmxGetter;
import voldemort.annotations.jmx.JmxOperation;
import voldemort.store.PersistenceFailureException;
import voldemort.store.StorageEngine;
import voldemort.store.StoreUtils;
import voldemort.utils.ByteArray;
import voldemort.utils.ByteUtils;
import voldemort.utils.ClosableIterator;
import voldemort.utils.Pair;
import voldemort.utils.Utils;
import voldemort.versioning.VectorClock;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;

import com.google.common.collect.Lists;

/**
 * A read-only store that fronts a big file
 * 
 * @author jay
 * 
 */
public class RandomAccessFileStore implements StorageEngine<ByteArray, byte[]> {

    private static Logger logger = Logger.getLogger(RandomAccessFileStore.class);

    private static final Comparator<Pair<ByteArray, Long>> KEYS_AND_VALUES_COMPARATOR = new Comparator<Pair<ByteArray, Long>>() {

        public int compare(Pair<ByteArray, Long> o1, Pair<ByteArray, Long> o2) {
            return o1.getSecond().compareTo(o2.getSecond());
        }
    };

    public static final int KEY_HASH_SIZE = 16;
    public static final int POSITION_SIZE = 8;
    public static final int INDEX_ENTRY_SIZE = KEY_HASH_SIZE + POSITION_SIZE;

    private final String name;
    private final long fdWaitTimeoutMs;
    private long indexFileSize;
    private final int numBackups;
    private final int numFileHandles;
    private final File storageDir;
    private final File dataFile;
    private final File indexFile;
    private final ReadWriteLock fileModificationLock;
    private final byte[][] keyCache;
    private final int maxCacheDepth;
    private final AtomicBoolean isOpen;

    private BlockingQueue<RandomAccessFile> indexFiles;
    private BlockingQueue<RandomAccessFile> dataFiles;

    /**
     * Create an instance of the store
     * 
     * @param name The name of the store
     * @param storageDir The directory in which the .data and .index files
     *        reside
     * @param numBackups The number of backups of these files to retain
     * @param numFileHandles The number of file descriptors to keep pooled for
     *        each file
     * @param fdWaitTimeoutMs The maximum time to wait to acquire a file handle
     * @param maxCacheSize The maximum size of the cache, in bytes. The actual
     *        size of the cache will be the largest power of two lower than this
     *        number
     */
    public RandomAccessFileStore(String name,
                                 File storageDir,
                                 int numBackups,
                                 int numFileHandles,
                                 long fdWaitTimeoutMs,
                                 long maxCacheSizeBytes) {
        this.storageDir = storageDir;
        this.numBackups = numBackups;
        this.indexFile = new File(storageDir, name + ".index");
        this.dataFile = new File(storageDir, name + ".data");
        this.name = Utils.notNull(name);
        this.fdWaitTimeoutMs = fdWaitTimeoutMs;
        this.dataFiles = new ArrayBlockingQueue<RandomAccessFile>(numFileHandles);
        this.indexFiles = new ArrayBlockingQueue<RandomAccessFile>(numFileHandles);
        this.numFileHandles = numFileHandles;
        /*
         * A lock that blocks reads during swap(), open(), and close()
         * operations
         */
        this.fileModificationLock = new ReentrantReadWriteLock();
        /*
         * The overhead for each cache element is the key size + 4 byte array
         * length + 12 byte object overhead + 8 bytes for a 64-bit reference to
         * the thing
         */
        int cacheElements = (int) floor(maxCacheSizeBytes / (KEY_HASH_SIZE + 24));
        this.maxCacheDepth = (int) floor(log(cacheElements) / log(2));
        this.keyCache = new byte[(int) pow(2, maxCacheDepth)][];
        this.isOpen = new AtomicBoolean(false);
        open();
    }

    /**
     * Open the store
     */
    public void open() {
        /* acquire modification lock */<commit_msg>Clarify byte calculation logic.<commit_after>/*
 * Copyright 2008-2009 LinkedIn, Inc
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package voldemort.store.readonly;

import static java.lang.Math.floor;
import static java.lang.Math.log;
import static java.lang.Math.pow;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.log4j.Logger;

import voldemort.VoldemortException;
import voldemort.annotations.jmx.JmxGetter;
import voldemort.annotations.jmx.JmxOperation;
import voldemort.store.PersistenceFailureException;
import voldemort.store.StorageEngine;
import voldemort.store.StoreUtils;
import voldemort.utils.ByteArray;
import voldemort.utils.ByteUtils;
import voldemort.utils.ClosableIterator;
import voldemort.utils.Pair;
import voldemort.utils.Utils;
import voldemort.versioning.VectorClock;
import voldemort.versioning.Version;
import voldemort.versioning.Versioned;

import com.google.common.collect.Lists;

/**
 * A read-only store that fronts a big file
 * 
 * @author jay
 * 
 */
public class RandomAccessFileStore implements StorageEngine<ByteArray, byte[]> {

    private static Logger logger = Logger.getLogger(RandomAccessFileStore.class);

    private static final Comparator<Pair<ByteArray, Long>> KEYS_AND_VALUES_COMPARATOR = new Comparator<Pair<ByteArray, Long>>() {

        public int compare(Pair<ByteArray, Long> o1, Pair<ByteArray, Long> o2) {
            return o1.getSecond().compareTo(o2.getSecond());
        }
    };

    public static final int KEY_HASH_SIZE = 16;
    public static final int POSITION_SIZE = 8;
    public static final int INDEX_ENTRY_SIZE = KEY_HASH_SIZE + POSITION_SIZE;

    /*
     * The overhead for each cache element is the key size + 4 byte array length
     * + 12 byte object overhead + 8 bytes for a 64-bit reference to the thing
     */
    public static final int MEMORY_OVERHEAD_PER_KEY = KEY_HASH_SIZE + 4 + 12 + 8;

    private final String name;
    private final long fdWaitTimeoutMs;
    private long indexFileSize;
    private final int numBackups;
    private final int numFileHandles;
    private final File storageDir;
    private final File dataFile;
    private final File indexFile;
    private final ReadWriteLock fileModificationLock;
    private final byte[][] keyCache;
    private final int maxCacheDepth;
    private final AtomicBoolean isOpen;

    private BlockingQueue<RandomAccessFile> indexFiles;
    private BlockingQueue<RandomAccessFile> dataFiles;

    /**
     * Create an instance of the store
     * 
     * @param name The name of the store
     * @param storageDir The directory in which the .data and .index files
     *        reside
     * @param numBackups The number of backups of these files to retain
     * @param numFileHandles The number of file descriptors to keep pooled for
     *        each file
     * @param fdWaitTimeoutMs The maximum time to wait to acquire a file handle
     * @param maxCacheSize The maximum size of the cache, in bytes. The actual
     *        size of the cache will be the largest power of two lower than this
     *        number
     */
    public RandomAccessFileStore(String name,
                                 File storageDir,
                                 int numBackups,
                                 int numFileHandles,
                                 long fdWaitTimeoutMs,
                                 long maxCacheSizeBytes) {
        this.storageDir = storageDir;
        this.numBackups = numBackups;
        this.indexFile = new File(storageDir, name + ".index");
        this.dataFile = new File(storageDir, name + ".data");
        this.name = Utils.notNull(name);
        this.fdWaitTimeoutMs = fdWaitTimeoutMs;
        this.dataFiles = new ArrayBlockingQueue<RandomAccessFile>(numFileHandles);
        this.indexFiles = new ArrayBlockingQueue<RandomAccessFile>(numFileHandles);
        this.numFileHandles = numFileHandles;
        /*
         * A lock that blocks reads during swap(), open(), and close()
         * operations
         */
        this.fileModificationLock = new ReentrantReadWriteLock();

        this.maxCacheDepth = calculateMaxCacheTreeDepth(maxCacheSizeBytes);
        this.keyCache = new byte[(int) pow(2, maxCacheDepth)][];
        this.isOpen = new AtomicBoolean(false);
        open();
    }

    /**
     * The max cache depth is the largest complete binary tree that will fit in
     * the given number of bytes.
     * 
     * @param maxCacheSizeBytes The number of bytes
     * @return The depth of the largest complete binary cache tree requiring
     *         those bytes.
     */
    private int calculateMaxCacheTreeDepth(long maxCacheSizeBytes) {
        if(maxCacheSizeBytes / MEMORY_OVERHEAD_PER_KEY > Integer.MAX_VALUE)
            throw new IllegalArgumentException("Maximum cache size is " + Integer.MAX_VALUE
                                               * MEMORY_OVERHEAD_PER_KEY + " bytes.");
        int cacheElements = (int) (maxCacheSizeBytes / MEMORY_OVERHEAD_PER_KEY);
        return (int) floor(log(cacheElements) / log(2));
    }

    /**
     * Open the store
     */
    public void open() {
        /* acquire modification lock */
{'fraction_non_alphanumeric': 0.07745684794865122, 'fraction_numerical': 0.005377743082661116, 'mean_word_length': 3.027244149493538, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3107308', 'n_tokens_mistral': 3220, 'n_tokens_neox': 3116, 'n_words': 991}