<commit_before>package org.cloudname.log.archiver;

import org.cloudname.log.pb.Timber;
import org.cloudname.log.recordstore.RecordWriter;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * This class takes care of writing log messages to a "slot".  A slot
 * can consist of one or more files.  Once a slot file reaches maximum
 * file size we begin to populate the next file.  Slot files are
 * numbered with an increasing sequence number.  This produces a sequence
 * of files of the form:
 *
 * <ul>
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-1
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-2
 *   <li> ...
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-10
 *   <li> ...
 * </ul>
 *
 * @author borud
 */
public class Slot {
    // When resuming an existing file we skip to the next sequence file if we have
    // already logged more than RESUME_LIMIT_PERCENT percent of maxSize
    private static final int RESUME_LIMIT_PERCENT = 90;

    // Some ad-hoc benchmarking suggests that even just 1k of buffer
    // space helps.  Not much to be gained from a larger buffer for a
    // single slot, but we bump the size a bit since there can be
    // several slots active at the same time to lower the number of
    // disk IO operations.
    private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 20 * 1024;

    private final String prefix;
    private final long maxSize;
    private final long resumeLimit;
    private final int outputBufferSize = DEFAULT_OUTPUT_BUFFER_SIZE;

    private File currentFile = null;
    private RecordWriter currentWriter = null;
    private boolean closed = false;

    // Write count for current slot file
    private int writeCount = 0;

    // Keeps track of number of bytes in file
    private long numBytesInFile = 0;

    /**
     * Create a Slot.
     *
     * @param prefix the prefix for the slot file
     * @param maxSize the maximum allowed file size in bytes for individual slot files
     */
    public Slot(final String prefix, final long maxSize) {
        this.prefix = prefix;
        this.maxSize = maxSize;

        resumeLimit = (maxSize * RESUME_LIMIT_PERCENT) / 100;
    }

    /**
     * @return the slot file path for a given timestamp.
     */
    private String nameForTimestamp(final long timestamp) {
        return prefix + "_" + timestamp;
    }

    /**
     * Check if the prospective filename exists with a suffix
     * indicating it has been compressed.
     *
     * @param file the file for which we want to check if there is
     *   a compressed version.
     * @return {@code true} if a compressed file of this slot exists,
     *   {@code false} otherwise.
     */
    private static boolean compressedSlotExists(final File file) {
        return new File(file.getAbsolutePath() + ".gz").exists()
            || new File(file.getAbsolutePath() + ".bz2").exists();

    }

    /**
     * Find the next slot file to write to.
     *
     * @return a File instance that points to the next slot file
     */
    private File findNextSlotFile() throws IOException {
        final File file = new File(nameForTimestamp(System.currentTimeMillis()));
        final File parentDir = file.getParentFile();

        // Make sure directory exists
        if (!parentDir.exists()) {
            parentDir.mkdirs();
        }
<commit_msg>It is important to check returnvalue of file.mkdirs() since it will return false when it fails.<commit_after>package org.cloudname.log.archiver;

import org.cloudname.log.pb.Timber;
import org.cloudname.log.recordstore.RecordWriter;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * This class takes care of writing log messages to a "slot".  A slot
 * can consist of one or more files.  Once a slot file reaches maximum
 * file size we begin to populate the next file.  Slot files are
 * numbered with an increasing sequence number.  This produces a sequence
 * of files of the form:
 *
 * <ul>
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-1
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-2
 *   <li> ...
 *   <li> &lt;slotDir&gt&lt;prefix&gt;-10
 *   <li> ...
 * </ul>
 *
 * @author borud
 */
public class Slot {
    // When resuming an existing file we skip to the next sequence file if we have
    // already logged more than RESUME_LIMIT_PERCENT percent of maxSize
    private static final int RESUME_LIMIT_PERCENT = 90;

    // Some ad-hoc benchmarking suggests that even just 1k of buffer
    // space helps.  Not much to be gained from a larger buffer for a
    // single slot, but we bump the size a bit since there can be
    // several slots active at the same time to lower the number of
    // disk IO operations.
    private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 20 * 1024;

    private final String prefix;
    private final long maxSize;
    private final long resumeLimit;
    private final int outputBufferSize = DEFAULT_OUTPUT_BUFFER_SIZE;

    private File currentFile = null;
    private RecordWriter currentWriter = null;
    private boolean closed = false;

    // Write count for current slot file
    private int writeCount = 0;

    // Keeps track of number of bytes in file
    private long numBytesInFile = 0;

    /**
     * Create a Slot.
     *
     * @param prefix the prefix for the slot file
     * @param maxSize the maximum allowed file size in bytes for individual slot files
     */
    public Slot(final String prefix, final long maxSize) {
        this.prefix = prefix;
        this.maxSize = maxSize;

        resumeLimit = (maxSize * RESUME_LIMIT_PERCENT) / 100;
    }

    /**
     * @return the slot file path for a given timestamp.
     */
    private String nameForTimestamp(final long timestamp) {
        return prefix + "_" + timestamp;
    }

    /**
     * Check if the prospective filename exists with a suffix
     * indicating it has been compressed.
     *
     * @param file the file for which we want to check if there is
     *   a compressed version.
     * @return {@code true} if a compressed file of this slot exists,
     *   {@code false} otherwise.
     */
    private static boolean compressedSlotExists(final File file) {
        return new File(file.getAbsolutePath() + ".gz").exists()
            || new File(file.getAbsolutePath() + ".bz2").exists();

    }

    /**
     * Find the next slot file to write to.
     *
     * @return a File instance that points to the next slot file
     */
    private File findNextSlotFile() throws IOException {
        final File file = new File(nameForTimestamp(System.currentTimeMillis()));
        final File parentDir = file.getParentFile();

        // Make sure directory exists
        if (!parentDir.exists()) {
            if (!parentDir.mkdirs()) {
                throw new ArchiverException("Could not create folder '"
                        + parentDir.getAbsolutePath() + "', hence slot-file logging will fail. " +
                        "Does your user have sufficient permissions to create folders?");
            }
        }

{'fraction_non_alphanumeric': 0.08514543914148545, 'fraction_numerical': 0.005365715899463428, 'mean_word_length': 2.619315278487481, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7245367', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1936, 'n_words': 783}