<commit_before>package org.yamcs.yarch;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

import org.yamcs.YConfiguration;
import org.yamcs.yarch.rocksdb.protobuf.Tablespace.BucketProperties;

/**
 * {@link BucketDatabase} implementation that maps objects to files on a file system.
 */
public class FileSystemBucketDatabase implements BucketDatabase {

    static final long MAX_BUCKET_SIZE = 100L * 1024 * 1024; // 100MB
    static final int MAX_NUM_OBJECTS_PER_BUCKET = 1000;

    private Path root;

    // Manually registered buckets (located outside of root)
    private Map<String, FileSystemBucket> additionalBuckets = new HashMap<>();

    public FileSystemBucketDatabase(String yamcsInstance) throws IOException {
        this(yamcsInstance, Collections.emptyMap());
    }

    public FileSystemBucketDatabase(String yamcsInstance, Map<String, Object> args) throws IOException {
        if (args.containsKey("dataDir")) {
            root = Paths.get(YConfiguration.getString(args, "dataDir"));
        } else {
            String dataDir = YarchDatabase.getDataDir();
            root = Paths.get(dataDir).resolve(yamcsInstance).resolve("fsbuckets");
        }
        Files.createDirectories(root);
    }

    @Override
    public FileSystemBucket createBucket(String bucketName) throws IOException {
        synchronized (additionalBuckets) {
            if (additionalBuckets.containsKey(bucketName)) {
                throw new IllegalArgumentException("Bucket already exists");
            }
        }

        Path path = root.resolve(bucketName);
        Files.createDirectory(path);
        return new FileSystemBucket(bucketName, path);
    }

    /**
     * Manually register a bucket based on a location on the file system.
     * <p>
     * It is not necessary to register buckets that exist in the standard location (directly under under dataDir).<commit_msg>avoid creating empty fsbuckets directories<commit_after>package org.yamcs.yarch;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

import org.yamcs.YConfiguration;
import org.yamcs.yarch.rocksdb.protobuf.Tablespace.BucketProperties;

/**
 * {@link BucketDatabase} implementation that maps objects to files on a file system.
 */
public class FileSystemBucketDatabase implements BucketDatabase {

    static final long MAX_BUCKET_SIZE = 100L * 1024 * 1024; // 100MB
    static final int MAX_NUM_OBJECTS_PER_BUCKET = 1000;

    private Path root;

    // Manually registered buckets (located outside of root)
    private Map<String, FileSystemBucket> additionalBuckets = new HashMap<>();

    public FileSystemBucketDatabase(String yamcsInstance) throws IOException {
        this(yamcsInstance, Collections.emptyMap());
    }

    public FileSystemBucketDatabase(String yamcsInstance, Map<String, Object> args) throws IOException {
        if (args.containsKey("dataDir")) {
            root = Paths.get(YConfiguration.getString(args, "dataDir"));
        } else {
            String dataDir = YarchDatabase.getDataDir();
            root = Paths.get(dataDir).resolve(yamcsInstance).resolve("fsbuckets");
        }
    }

    @Override
    public FileSystemBucket createBucket(String bucketName) throws IOException {
        synchronized (additionalBuckets) {
            if (additionalBuckets.containsKey(bucketName)) {
                throw new IllegalArgumentException("Bucket already exists");
            }
        }

        Path path = root.resolve(bucketName);
        Files.createDirectories(path);
        return new FileSystemBucket(bucketName, path);
    }

    /**
     * Manually register a bucket based on a location on the file system.
     * <p>
     * It is not necessary to register buckets that exist in the standard location (directly under under dataDir).
{'fraction_non_alphanumeric': 0.09236453201970443, 'fraction_numerical': 0.007389162561576354, 'mean_word_length': 3.6991321118611378, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2405927', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1324, 'n_words': 344}