<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.filedistribution;

import com.yahoo.config.FileReference;
import com.yahoo.io.IOUtils;
import com.yahoo.text.Utf8;
import net.jpountz.xxhash.XXHash64;
import net.jpountz.xxhash.XXHashFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.time.Instant;
import java.util.logging.Level;
import java.util.logging.Logger;

public class FileDirectory  {

    private static final Logger log = Logger.getLogger(FileDirectory.class.getName());
    private final File root;

    public FileDirectory(File rootDir) {
        root = rootDir;
        try {
            ensureRootExist();
        } catch (IllegalArgumentException e) {
            log.log(Level.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.getMessage());
        }
    }

    private void ensureRootExist() {
        if (! root.exists()) {
            if ( ! root.mkdir()) {
                throw new IllegalArgumentException("Failed creating root dir '" + root.getAbsolutePath() + "'.");
            }
        } else if (!root.isDirectory()) {
            throw new IllegalArgumentException("'" + root.getAbsolutePath() + "' is not a directory");
        }
    }

    static private class Filter implements FilenameFilter {
        @Override
        public boolean accept(File dir, String name) {
            return !".".equals(name) && !"..".equals(name) ;
        }
    }

    String getPath(FileReference ref) {
        return root.getAbsolutePath() + "/" + ref.value();
    }

    public File getFile(FileReference reference) {
        ensureRootExist();
        File dir = new File(getPath(reference));
        if (!dir.exists())
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' does not exist.");
        if (!dir.isDirectory())
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory.");
        File [] files = dir.listFiles(new Filter());
        if (files == null || files.length == 0)
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain any files");
        return files[0];
    }

    File getRoot() {
        return root;
    }

    private Long computeHash(File file) throws IOException {
        XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
        if (file.isDirectory()) {
            return Files.walk(file.toPath(), 100).map(path -> {
                try {
                    log.log(Level.FINEST, () -> "Calculating hash for '" + path + "'");
                    return hash(path.toFile(), hasher);
                } catch (IOException e) {
                    log.log(Level.WARNING, "Failed getting hash from '" + path + "'");
                    return 0;
                }
            }).mapToLong(Number::longValue).sum();
        } else {
            return hash(file, hasher);
        }
    }

    private long hash(File file, XXHash64 hasher) throws IOException {
        byte[] wholeFile = file.isDirectory() ?  new byte[0] : IOUtils.readFileBytes(file);
        return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0));
    }

    public FileReference addFile(File source) throws IOException {
        Long hash = computeHash(source);
        verifyExistingFile(source, hash);
        FileReference fileReference = fileReferenceFromHash(hash);
        return addFile(source, fileReference);
    }

    // If there exists a directory for a file reference, but content does not have correct hash or the file we want to add
    // does not exist in the directory, delete it
    private void verifyExistingFile(File source, Long hashOfFileToBeAdded) throws IOException {
        FileReference fileReference = fileReferenceFromHash(hashOfFileToBeAdded);
        File destinationDir = destinationDir(fileReference);
        if (!destinationDir.exists()) return;

        File existingFile = destinationDir.toPath().resolve(source.getName()).toFile();
        if ( ! existingFile.exists() || ! computeHash(existingFile).equals(hashOfFileToBeAdded)) {
            log.log(Level.SEVERE, "Directory for file reference '" + fileReference.value() +
                    "' has content that does not match its hash, deleting everything in " +
                    destinationDir.getAbsolutePath());
            IOUtils.recursiveDeleteDir(destinationDir);
        } else {
            // Update last access time (used to keep track of when we can delete unused file references
            // so update when adding and it already exists)
            FileTime fileTime = FileTime.from(Instant.now());
            Files.setAttribute(destinationDir.toPath(), "basic:lastAccessTime", fileTime);
        }
    }

    private File destinationDir(FileReference fileReference) {
        return new File(root, fileReference.value());
    }

    private FileReference fileReferenceFromHash(Long hash) {
        return new FileReference(Long.toHexString(hash));
    }

    private FileReference addFile(File source, FileReference reference) {
        ensureRootExist();
        try {
            logfileInfo(source);
            File destinationDir = destinationDir(reference);
            Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing");
            File destination = new File(tempDestinationDir.toFile(), source.getName());
            if (!destinationDir.exists()) {
                destinationDir.mkdir();<commit_msg>Change log level and remove setting of last accessed time (not used anymore)<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.filedistribution;

import com.yahoo.config.FileReference;
import com.yahoo.io.IOUtils;
import com.yahoo.text.Utf8;
import net.jpountz.xxhash.XXHash64;
import net.jpountz.xxhash.XXHashFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.logging.Level;
import java.util.logging.Logger;

public class FileDirectory  {

    private static final Logger log = Logger.getLogger(FileDirectory.class.getName());
    private final File root;

    public FileDirectory(File rootDir) {
        root = rootDir;
        try {
            ensureRootExist();
        } catch (IllegalArgumentException e) {
            log.log(Level.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.getMessage());
        }
    }

    private void ensureRootExist() {
        if (! root.exists()) {
            if ( ! root.mkdir()) {
                throw new IllegalArgumentException("Failed creating root dir '" + root.getAbsolutePath() + "'.");
            }
        } else if (!root.isDirectory()) {
            throw new IllegalArgumentException("'" + root.getAbsolutePath() + "' is not a directory");
        }
    }

    static private class Filter implements FilenameFilter {
        @Override
        public boolean accept(File dir, String name) {
            return !".".equals(name) && !"..".equals(name) ;
        }
    }

    String getPath(FileReference ref) {
        return root.getAbsolutePath() + "/" + ref.value();
    }

    public File getFile(FileReference reference) {
        ensureRootExist();
        File dir = new File(getPath(reference));
        if (!dir.exists())
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' does not exist.");
        if (!dir.isDirectory())
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory.");
        File [] files = dir.listFiles(new Filter());
        if (files == null || files.length == 0)
            throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain any files");
        return files[0];
    }

    File getRoot() {
        return root;
    }

    private Long computeHash(File file) throws IOException {
        XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
        if (file.isDirectory()) {
            return Files.walk(file.toPath(), 100).map(path -> {
                try {
                    log.log(Level.FINEST, () -> "Calculating hash for '" + path + "'");
                    return hash(path.toFile(), hasher);
                } catch (IOException e) {
                    log.log(Level.WARNING, "Failed getting hash from '" + path + "'");
                    return 0;
                }
            }).mapToLong(Number::longValue).sum();
        } else {
            return hash(file, hasher);
        }
    }

    private long hash(File file, XXHash64 hasher) throws IOException {
        byte[] wholeFile = file.isDirectory() ?  new byte[0] : IOUtils.readFileBytes(file);
        return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0));
    }

    public FileReference addFile(File source) throws IOException {
        Long hash = computeHash(source);
        verifyExistingFile(source, hash);
        FileReference fileReference = fileReferenceFromHash(hash);
        return addFile(source, fileReference);
    }

    // If there exists a directory for a file reference, but content does not have correct hash or the file we want to add
    // does not exist in the directory, delete the directory
    private void verifyExistingFile(File source, Long hashOfFileToBeAdded) throws IOException {
        FileReference fileReference = fileReferenceFromHash(hashOfFileToBeAdded);
        File destinationDir = destinationDir(fileReference);
        if (!destinationDir.exists()) return;

        File existingFile = destinationDir.toPath().resolve(source.getName()).toFile();
        if ( ! existingFile.exists() || ! computeHash(existingFile).equals(hashOfFileToBeAdded)) {
            log.log(Level.WARNING, "Directory for file reference '" + fileReference.value() +
                    "' has content that does not match its hash, deleting everything in " +
                    destinationDir.getAbsolutePath());
            IOUtils.recursiveDeleteDir(destinationDir);
        }
    }

    private File destinationDir(FileReference fileReference) {
        return new File(root, fileReference.value());
    }

    private FileReference fileReferenceFromHash(Long hash) {
        return new FileReference(Long.toHexString(hash));
    }

    // Pre-condition: Destination dir does not exist
    private FileReference addFile(File source, FileReference reference) {
        ensureRootExist();
        try {
            logfileInfo(source);
            File destinationDir = destinationDir(reference);
            Path tempDestinationDir = Files.createTempDirectory(root.toPath(), "writing");
            File destination = new File(tempDestinationDir.toFile(), source.getName());
            if (!destinationDir.exists()) {
                destinationDir.mkdir();
{'fraction_non_alphanumeric': 0.11133467471495641, 'fraction_numerical': 0.00335345405767941, 'mean_word_length': 2.7196757093857187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3441447', 'n_tokens_mistral': 3112, 'n_tokens_neox': 3023, 'n_words': 829}