<commit_before>package org.talend.dataprep.dataset.store.metadata.file;

import java.io.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.stream.Stream;

import javax.annotation.PostConstruct;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.dataset.exception.DataSetErrorCodes;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepository;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepositoryAdapter;
import org.talend.dataprep.exception.TDPException;
import org.talend.dataprep.exception.TDPExceptionContext;
import org.talend.dataprep.lock.DistributedLock;

/**
 * File system implementation of the DataSetMetadataRepository.
 *
 * Unfortunately, json serialization cannot be used because it's too much API oriented and the default jackson setup
 * ignores RowMetadata from DataSetMetadata. Hence, simple class serialization is performed.
 */
@Component
@ConditionalOnProperty(name = "dataset.metadata.store", havingValue = "file")
public class FileSystemDataSetMetadataRepository extends DataSetMetadataRepositoryAdapter {

    /** This class' logger. */
    private static final Logger LOG = LoggerFactory.getLogger(FileSystemDataSetMetadataRepository.class);

    /** Where to store the dataset metadata */
    @Value("${dataset.metadata.store.file.location}")
    private String storeLocation;

    @PostConstruct
    private void init() {
        getRootFolder().mkdirs();
    }

    /**
     * @see DataSetMetadataRepository#add(DataSetMetadata)
     */
    @Override
    public void add(DataSetMetadata metadata) {

        final File file = getFile(metadata.getId());

        try (ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(file))) {
            output.writeObject(metadata);
        } catch (IOException e) {
            LOG.error("Error saving {}", metadata, e);
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_STORE_DATASET_METADATA, e,
                    TDPExceptionContext.build().put("id", metadata.getId()));
        }
    }

    /**
     * @see DataSetMetadataRepository#get(String)
     */
    @Override
    public DataSetMetadata get(String id) {

        final File file = getFile(id);
        if (!file.exists()) {
            LOG.info("dataset #{} not found in file system", id);
            return null;
        }

        try (ObjectInputStream input = new ObjectInputStream(new FileInputStream(file))) {
            return (DataSetMetadata) input.readObject();
        } catch (ClassNotFoundException | IOException e) {
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_METADATA, e, TDPExceptionContext.build().put("id", id));
        }
    }

    /**
     * @see DataSetMetadataRepository#remove(String)
     */
    @Override
    public void remove(String id) {
        final File file = getFile(id);
        if (file.exists() && !isNFSSpecificFile(file)) {
            file.delete();
        }
        LOG.debug("metadata {} successfully deleted", id);
    }

    /**
     * @see DataSetMetadataRepository#size()
     */
    @Override
    public int size() {
        File folder = getRootFolder();
        int size = folder.list().length;
        LOG.debug("found {} datasets metadata", size);
        return size;
    }

    /**
     * @see DataSetMetadataRepository#list()
     */
    @Override
    public Iterable<DataSetMetadata> list() {
        File folder = getRootFolder();
        final File[] files = folder.listFiles();
        if (files == null) {
            return Collections.emptyList();
        } else {
            final Stream<DataSetMetadata> stream = Arrays.stream(files).map(f -> {

                try (ObjectInputStream input = new ObjectInputStream(new FileInputStream(f))) {
                    return (DataSetMetadata) input.readObject();
                } catch (ClassNotFoundException | IOException e) {
                    LOG.error("error reading metadata file {}", f.getAbsolutePath(), e);
                    throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_METADATA, e);
                }

            });
            return stream::iterator;
        }
    }

    /**
     * @see DataSetMetadataRepository#clear()
     */
    @Override
    public void clear() {
        // Remove all data set (but use lock for remaining asynchronous processes).
        for (DataSetMetadata metadata : list()) {
            final DistributedLock lock = createDatasetMetadataLock(metadata.getId());
            try {
                lock.lock();
                remove(metadata.getId());
            } finally {
                lock.unlock();
            }
        }
        LOG.debug("dataset metadata repository cleared.");
    }

    /**
     * Return the file that matches the given metadata id.
     * 
     * @param metadataId the metadata id.
     * @return the file where to read/write the metadata.
     */
    private File getFile(String metadataId) {
        return new File(storeLocation + "/metadata/" + metadataId);
    }

    /**
     * @return the metadata root folder.
     */
    private File getRootFolder() {
        return new File(storeLocation + "/metadata/");
    }

    /**
     * Specific OS NFS files must be not be deleted.
     * 
     * @param file the file to read.
     * @return True if the given belongs to the operating system and is NFS related (starts with ".nfs")
     */
    private boolean isNFSSpecificFile(File file) {
        return file.getName().startsWith(".nfs");
    }

}<commit_msg>TDP-381: remove the extra 'metadata' folder for storing dataset metadata filesystem repository<commit_after>package org.talend.dataprep.dataset.store.metadata.file;

import java.io.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.stream.Stream;

import javax.annotation.PostConstruct;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.dataset.exception.DataSetErrorCodes;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepository;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepositoryAdapter;
import org.talend.dataprep.exception.TDPException;
import org.talend.dataprep.exception.TDPExceptionContext;
import org.talend.dataprep.lock.DistributedLock;

/**
 * File system implementation of the DataSetMetadataRepository.
 *
 * Unfortunately, json serialization cannot be used because it's too much API oriented and the default jackson setup
 * ignores RowMetadata from DataSetMetadata. Hence, simple class serialization is performed.
 */
@Component
@ConditionalOnProperty(name = "dataset.metadata.store", havingValue = "file")
public class FileSystemDataSetMetadataRepository extends DataSetMetadataRepositoryAdapter {

    /** This class' logger. */
    private static final Logger LOG = LoggerFactory.getLogger(FileSystemDataSetMetadataRepository.class);

    /** Where to store the dataset metadata */
    @Value("${dataset.metadata.store.file.location}")
    private String storeLocation;

    @PostConstruct
    private void init() {
        getRootFolder().mkdirs();
    }

    /**
     * @see DataSetMetadataRepository#add(DataSetMetadata)
     */
    @Override
    public void add(DataSetMetadata metadata) {

        final File file = getFile(metadata.getId());

        try (ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(file))) {
            output.writeObject(metadata);
        } catch (IOException e) {
            LOG.error("Error saving {}", metadata, e);
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_STORE_DATASET_METADATA, e,
                    TDPExceptionContext.build().put("id", metadata.getId()));
        }
    }

    /**
     * @see DataSetMetadataRepository#get(String)
     */
    @Override
    public DataSetMetadata get(String id) {

        final File file = getFile(id);
        if (!file.exists()) {
            LOG.info("dataset #{} not found in file system", id);
            return null;
        }

        try (ObjectInputStream input = new ObjectInputStream(new FileInputStream(file))) {
            return (DataSetMetadata) input.readObject();
        } catch (ClassNotFoundException | IOException e) {
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_METADATA, e, TDPExceptionContext.build().put("id", id));
        }
    }

    /**
     * @see DataSetMetadataRepository#remove(String)
     */
    @Override
    public void remove(String id) {
        final File file = getFile(id);
        if (file.exists() && !isNFSSpecificFile(file)) {
            file.delete();
        }
        LOG.debug("metadata {} successfully deleted", id);
    }

    /**
     * @see DataSetMetadataRepository#size()
     */
    @Override
    public int size() {
        File folder = getRootFolder();
        int size = folder.list().length;
        LOG.debug("found {} datasets metadata", size);
        return size;
    }

    /**
     * @see DataSetMetadataRepository#list()
     */
    @Override
    public Iterable<DataSetMetadata> list() {
        File folder = getRootFolder();
        final File[] files = folder.listFiles();
        if (files == null) {
            return Collections.emptyList();
        } else {
            final Stream<DataSetMetadata> stream = Arrays.stream(files).map(f -> {

                try (ObjectInputStream input = new ObjectInputStream(new FileInputStream(f))) {
                    return (DataSetMetadata) input.readObject();
                } catch (ClassNotFoundException | IOException e) {
                    LOG.error("error reading metadata file {}", f.getAbsolutePath(), e);
                    throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_DATASET_METADATA, e);
                }

            });
            return stream::iterator;
        }
    }

    /**
     * @see DataSetMetadataRepository#clear()
     */
    @Override
    public void clear() {
        // Remove all data set (but use lock for remaining asynchronous processes).
        for (DataSetMetadata metadata : list()) {
            final DistributedLock lock = createDatasetMetadataLock(metadata.getId());
            try {
                lock.lock();
                remove(metadata.getId());
            } finally {
                lock.unlock();
            }
        }
        LOG.debug("dataset metadata repository cleared.");
    }

    /**
     * Return the file that matches the given metadata id.
     * 
     * @param metadataId the metadata id.
     * @return the file where to read/write the metadata.
     */
    private File getFile(String metadataId) {
        return new File(storeLocation + '/' + metadataId);
    }

    /**
     * @return the metadata root folder.
     */
    private File getRootFolder() {
        return new File(storeLocation + '/');
    }

    /**
     * Specific OS NFS files must be not be deleted.
     * 
     * @param file the file to read.
     * @return True if the given belongs to the operating system and is NFS related (starts with ".nfs")
     */
    private boolean isNFSSpecificFile(File file) {
        return file.getName().startsWith(".nfs");
    }

}
{'fraction_non_alphanumeric': 0.1055065513402415, 'fraction_numerical': 0.0005994690417059176, 'mean_word_length': 2.673482227115445, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1799861', 'n_tokens_mistral': 3073, 'n_tokens_neox': 3055, 'n_words': 761}