<commit_before>package com.github.lucene.store.database.index;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.zip.CRC32;
import java.util.zip.Checksum;

import org.apache.lucene.store.BufferedChecksum;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.lucene.store.database.DatabaseDirectory;
import com.github.lucene.store.database.DatabaseDirectoryException;
import com.github.lucene.store.database.handler.DatabaseDirectoryHandler;

public class DatabaseIndexOutput extends IndexOutput {

    private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseFileIndexOutput.class);
    private static final DatabaseDirectoryHandler handler = DatabaseDirectoryHandler.INSTANCE;

    private final String name;
    private final DatabaseDirectory directory;
    @SuppressWarnings("unused")
    private final IOContext context;

    private ByteArrayOutputStream baos = new ByteArrayOutputStream();
    private RandomAccessFile file;
    private File tempFile;
    private final Checksum digest = new BufferedChecksum(new CRC32());

    private long pos = 0;

    public DatabaseIndexOutput(final DatabaseDirectory directory, final String name, final IOContext context)
            throws DatabaseDirectoryException {
        super(name);
        this.directory = directory;
        this.name = name;
        this.context = context;
    }

    @Override
    public long getFilePointer() {
        LOGGER.trace("{}.getFilePointer()", this);
        return pos;
    }
<commit_msg>add documentation<commit_after>package com.github.lucene.store.database.index;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.zip.CRC32;
import java.util.zip.Checksum;

import org.apache.lucene.store.BufferedChecksum;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.github.lucene.store.database.DatabaseDirectory;
import com.github.lucene.store.database.DatabaseDirectoryException;
import com.github.lucene.store.database.config.DatabaseConfig;
import com.github.lucene.store.database.handler.DatabaseDirectoryHandler;

/**
 * An <code>IndexOutput</code> implementation that initially writes the data to
 * a memory buffer. Once it exceeds the configured threshold (
 * {@link DatabaseConfig#setThreshold(long)}, will start working with a
 * temporary file, releasing the previous buffer.
 *
 */
public class DatabaseIndexOutput extends IndexOutput {

    private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseFileIndexOutput.class);
    private static final DatabaseDirectoryHandler handler = DatabaseDirectoryHandler.INSTANCE;

    private final String name;
    private final DatabaseDirectory directory;
    @SuppressWarnings("unused")
    private final IOContext context;

    private ByteArrayOutputStream baos = new ByteArrayOutputStream();
    private RandomAccessFile file;
    private File tempFile;
    private final Checksum digest = new BufferedChecksum(new CRC32());

    private long pos = 0;

    public DatabaseIndexOutput(final DatabaseDirectory directory, final String name, final IOContext context)
            throws DatabaseDirectoryException {
        super(name);
        this.directory = directory;
        this.name = name;
        this.context = context;
    }

    @Override
    public long getFilePointer() {
        LOGGER.trace("{}.getFilePointer()", this);
        return pos;
    }

{'fraction_non_alphanumeric': 0.08287866978435958, 'fraction_numerical': 0.0036373083917900753, 'mean_word_length': 4.453257790368272, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5566124', 'n_tokens_mistral': 1030, 'n_tokens_neox': 1013, 'n_words': 260}