<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.filedistribution.maintenance;

import com.yahoo.io.IOUtils;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;

import static java.nio.file.Files.readAttributes;

/**
 * Removes file references not used since a configured time, but always keeps a certain number of file references
 * even when they are unused (unused is based on last access time for the file).
 *
 * @author hmusum
 */
public class FileDistributionCleanup {

    private static final Logger log = Logger.getLogger(FileDistributionCleanup.class.getName());
    private static final int numberToAlwaysKeep = 20;

    private final Clock clock;

    public FileDistributionCleanup(Clock clock) {
        this.clock = clock;
    }

    public List<String> deleteUnusedFileReferences(File fileReferencesPath,
                                                   Duration keepFileReferencesDuration,
                                                   Set<String> fileReferencesInUse) {
        return deleteUnusedFileReferences(fileReferencesPath,
                                          keepFileReferencesDuration,
                                          numberToAlwaysKeep,
                                          fileReferencesInUse);
    }

    public List<String> deleteUnusedFileReferences(File fileReferencesPath,
                                                   Duration keepFileReferencesDuration,
                                                   int numberToAlwaysKeep,
                                                   Set<String> fileReferencesInUse) {
        log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration);
        if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory");

        log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse);

        Stream<String> candidates = sortedUnusedFileReferences(fileReferencesPath.toPath(), fileReferencesInUse, keepFileReferencesDuration);
        List<String> fileReferencesDeleted = new ArrayList<>();
        // Do not delete the newest ones
        final AtomicInteger i = new AtomicInteger(0);
        candidates.forEach(fileReference -> {
            if (i.incrementAndGet() > numberToAlwaysKeep) {
                fileReferencesDeleted.add(fileReference);
                File file = new File(fileReferencesPath, fileReference);
                if (!IOUtils.recursiveDeleteDir(file))
                    log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
            }
        });
        return fileReferencesDeleted;
    }

    // Sorted, newest first
    private Stream<String> sortedUnusedFileReferences(Path fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) {
        Instant instant = clock.instant().minus(keepFileReferences);
        return getFileReferencesOnDisk(fileReferencesPath)
                .filter(fileReference -> !fileReferencesInUse.contains(fileReference))
                .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath.toFile(), fileReference), instant))
                .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath.toFile(), (String) a))).reversed());
    }

    private boolean isLastFileAccessBefore(File fileReference, Instant instant) {
        return lastAccessed(fileReference).isBefore(instant);
    }

    private Instant lastAccessed(File fileReference) {
        BasicFileAttributes fileAttributes;
        try {
            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
            return fileAttributes.lastAccessTime().toInstant();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static Stream<String> getFileReferencesOnDisk(Path directory) {
        try {
            return Files.list(directory).map(path -> path.toFile().getName());
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

}
<commit_msg>Use try-with-resources when getting file references in path<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.filedistribution.maintenance;

import com.yahoo.io.IOUtils;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;

import static java.nio.file.Files.readAttributes;

/**
 * Removes file references not used since a configured time, but always keeps a certain number of file references
 * even when they are unused (unused is based on last access time for the file).
 *
 * @author hmusum
 */
public class FileDistributionCleanup {

    private static final Logger log = Logger.getLogger(FileDistributionCleanup.class.getName());
    private static final int numberToAlwaysKeep = 20;

    private final Clock clock;

    public FileDistributionCleanup(Clock clock) {
        this.clock = clock;
    }

    public List<String> deleteUnusedFileReferences(File fileReferencesPath,
                                                   Duration keepFileReferencesDuration,
                                                   Set<String> fileReferencesInUse) {
        return deleteUnusedFileReferences(fileReferencesPath,
                                          keepFileReferencesDuration,
                                          numberToAlwaysKeep,
                                          fileReferencesInUse);
    }

    public List<String> deleteUnusedFileReferences(File fileReferencesDir,
                                                   Duration keepFileReferencesDuration,
                                                   int numberToAlwaysKeep,
                                                   Set<String> fileReferencesInUse) {
        if (!fileReferencesDir.isDirectory()) throw new RuntimeException(fileReferencesDir + " is not a directory");

        log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration +
                ", file references in use : " + fileReferencesInUse);
        List<String> fileReferencesDeleted = new ArrayList<>();
        Path fileReferencesPath = fileReferencesDir.toPath();
        try (Stream<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration)) {
            final AtomicInteger i = new AtomicInteger(0);
            candidates.forEach(fileReference -> {
                // Do not delete the newest ones
                if (i.incrementAndGet() > numberToAlwaysKeep) {
                    fileReferencesDeleted.add(fileReference);
                    File file = new File(fileReferencesDir, fileReference);
                    if (!IOUtils.recursiveDeleteDir(file))
                        log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath());
                }
            });
        }
        return fileReferencesDeleted;
    }

    // Sorted, newest first
    private Stream<String> sortedUnusedFileReferences(Path fileReferencesPath, Set<String> fileReferencesInUse, Duration keepFileReferences) {
        Instant instant = clock.instant().minus(keepFileReferences);
        return getFileReferencesOnDisk(fileReferencesPath)
                .filter(fileReference -> !fileReferencesInUse.contains(fileReference))
                .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath.toFile(), fileReference), instant))
                .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath.toFile(), (String) a))).reversed());
    }

    private boolean isLastFileAccessBefore(File fileReference, Instant instant) {
        return lastAccessed(fileReference).isBefore(instant);
    }

    private Instant lastAccessed(File fileReference) {
        BasicFileAttributes fileAttributes;
        try {
            fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class);
            return fileAttributes.lastAccessTime().toInstant();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static Stream<String> getFileReferencesOnDisk(Path directory) {
        try {
            return Files.list(directory).map(path -> path.toFile().getName());
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

}

{'fraction_non_alphanumeric': 0.08277830637488107, 'fraction_numerical': 0.001057194206575748, 'mean_word_length': 2.545727136431784, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4235958', 'n_tokens_mistral': 2327, 'n_tokens_neox': 2185, 'n_words': 552}