<commit_before>package com.onshape.cache.disk;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.UserDefinedFileAttributeView;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import java.util.function.Consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

import com.onshape.cache.DiskStore;
import com.onshape.cache.exception.CacheException;
import com.onshape.cache.exception.EntryNotFoundException;
import com.onshape.cache.metrics.MetricService;

@Service
public class DiskStoreImpl implements DiskStore, InitializingBean, HealthIndicator {
    private static final Logger LOG = LoggerFactory.getLogger(DiskStoreImpl.class);
    private static final String EXPIRE_ATTR = "e";
    private static final String LOST_FOUND = "lost+found";

    @Autowired
    private MetricService ms;

    /** Threshold beyond which memory mapping will be used for reading from and writing to files */
    @Value("${diskRoot}")
    private String root;

    private int rootNameCount;
    private boolean expirationSupported;

    @Override
    public void afterPropertiesSet() throws IOException {
        Path dir = Paths.get(root).toRealPath();
        root = dir.toString();
        rootNameCount = dir.getNameCount();

        LOG.info("Disk store root: {}", root);

        if (Files.notExists(dir)) {
            Files.createDirectories(dir);
        }
        if (!Files.isDirectory(dir)) {
            throw new IOException("Not a directory: " + dir);
        }

        FileStore fs = Files.getFileStore(dir);
        if (!fs.supportsFileAttributeView(UserDefinedFileAttributeView.class)) {
            expirationSupported = true;
        } else {
            LOG.error("File store '{}' does not support user file attributes. Expiration support is disabled", fs);
        }
    }

    @Override
    public ByteBuffer get(String key) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, key);
        if (Files.notExists(path)) {
            ms.increment("disk.get.miss");
            return null;
        }

        try {
            try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) {
                FileChannel fileChannel = raf.getChannel();
                ByteBuffer buffer = fileChannel.map(MapMode.READ_ONLY, 0, fileChannel.size());

                ms.reportMetrics("disk.get", start);
                return buffer;
            }
        }
        catch (IOException e) {
            LOG.error("Error reading file for entry: {}", key, e);
            throw new CacheException(e);
        }
    }

    @Override
    public List<String> list(String prefix) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, prefix);
        if (Files.notExists(path)) {
            ms.increment("disk.list.miss");
            throw new EntryNotFoundException();
        }

        try {
            List<String> keys = new ArrayList<>();
            Files.walk(path, 1)
                .filter((Path p) -> Files.isRegularFile(p))
                .forEach((Path p) -> keys.add(p.getFileName().toString()));

            ms.reportMetrics("disk.list", start);
            return keys;
        }
        catch (IOException e) {
            LOG.error("Failed to list directory: {}", prefix, e);
            throw new CacheException(e);
        }
    }

    @Async
    @Override
    public void removeAsync(String key) throws CacheException {
        long start = System.currentTimeMillis();
        try {
            Files.deleteIfExists(Paths.get(root, key));
            ms.reportMetrics("disk.delete", start);
        }
        catch (IOException e) {
            throw new CacheException(e);
        }
    }

    @Async
    @Override
    public void putAsync(String key, byte[] value, int expiresAtSecs) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, key);
        Path parent = path.getParent();
        if (Files.notExists(path.getParent())) {
            try {
                Files.createDirectories(parent);
            }
            catch (IOException e) {
                throw new CacheException(e);
            }
        }

        try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw")) {
            raf.setLength(0);
            raf.getChannel().write(ByteBuffer.wrap(value));
        }
        catch (IOException e) {
            try {
                Files.deleteIfExists(path);
            }
            catch (IOException ioe) {
                LOG.warn("Error deleting file for entry: {}", key, ioe);<commit_msg>Close file channel<commit_after>package com.onshape.cache.disk;

import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.UserDefinedFileAttributeView;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import java.util.function.Consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

import com.onshape.cache.DiskStore;
import com.onshape.cache.exception.CacheException;
import com.onshape.cache.exception.EntryNotFoundException;
import com.onshape.cache.metrics.MetricService;

@Service
public class DiskStoreImpl implements DiskStore, InitializingBean, HealthIndicator {
    private static final Logger LOG = LoggerFactory.getLogger(DiskStoreImpl.class);
    private static final String EXPIRE_ATTR = "e";
    private static final String LOST_FOUND = "lost+found";

    @Autowired
    private MetricService ms;

    /** Threshold beyond which memory mapping will be used for reading from and writing to files */
    @Value("${diskRoot}")
    private String root;

    private int rootNameCount;
    private boolean expirationSupported;

    @Override
    public void afterPropertiesSet() throws IOException {
        Path dir = Paths.get(root).toRealPath();
        root = dir.toString();
        rootNameCount = dir.getNameCount();

        LOG.info("Disk store root: {}", root);

        if (Files.notExists(dir)) {
            Files.createDirectories(dir);
        }
        if (!Files.isDirectory(dir)) {
            throw new IOException("Not a directory: " + dir);
        }

        FileStore fs = Files.getFileStore(dir);
        if (!fs.supportsFileAttributeView(UserDefinedFileAttributeView.class)) {
            expirationSupported = true;
        } else {
            LOG.error("File store '{}' does not support user file attributes. Expiration support is disabled", fs);
        }
    }

    @Override
    public ByteBuffer get(String key) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, key);
        if (Files.notExists(path)) {
            ms.increment("disk.get.miss");
            return null;
        }

        try {
            try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) {
                try (FileChannel fileChannel = raf.getChannel()) {
                    ByteBuffer buffer = fileChannel.map(MapMode.READ_ONLY, 0, fileChannel.size());

                    ms.reportMetrics("disk.get", start);
                    return buffer;
                }
            }
        }
        catch (IOException e) {
            LOG.error("Error reading file for entry: {}", key, e);
            throw new CacheException(e);
        }
    }

    @Override
    public List<String> list(String prefix) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, prefix);
        if (Files.notExists(path)) {
            ms.increment("disk.list.miss");
            throw new EntryNotFoundException();
        }

        try {
            List<String> keys = new ArrayList<>();
            Files.walk(path, 1)
                .filter((Path p) -> Files.isRegularFile(p))
                .forEach((Path p) -> keys.add(p.getFileName().toString()));

            ms.reportMetrics("disk.list", start);
            return keys;
        }
        catch (IOException e) {
            LOG.error("Failed to list directory: {}", prefix, e);
            throw new CacheException(e);
        }
    }

    @Async
    @Override
    public void removeAsync(String key) throws CacheException {
        long start = System.currentTimeMillis();
        try {
            Files.deleteIfExists(Paths.get(root, key));
            ms.reportMetrics("disk.delete", start);
        }
        catch (IOException e) {
            throw new CacheException(e);
        }
    }

    @Async
    @Override
    public void putAsync(String key, byte[] value, int expiresAtSecs) throws CacheException {
        long start = System.currentTimeMillis();
        Path path = Paths.get(root, key);
        Path parent = path.getParent();
        if (Files.notExists(path.getParent())) {
            try {
                Files.createDirectories(parent);
            }
            catch (IOException e) {
                throw new CacheException(e);
            }
        }

        try (RandomAccessFile raf = new RandomAccessFile(path.toFile(), "rw")) {
            try (FileChannel fileChannel = raf.getChannel()) {
                fileChannel.truncate(value.length);
                fileChannel.write(ByteBuffer.wrap(value));
            }
        }
        catch (IOException e) {
            try {
                Files.deleteIfExists(path);
            }
            catch (IOException ioe) {
                LOG.warn("Error deleting file for entry: {}", key, ioe);
{'fraction_non_alphanumeric': 0.10670302603800141, 'fraction_numerical': 0.0007916959887403237, 'mean_word_length': 2.5176361386138613, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5213606', 'n_tokens_mistral': 3053, 'n_tokens_neox': 3014, 'n_words': 708}