<commit_before>package com.thinkbiganalytics.kylo.catalog.file;

/*-
 * #%L
 * kylo-catalog-core
 * %%
 * Copyright (C) 2017 - 2018 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.thinkbiganalytics.kylo.catalog.api.KyloCatalogConstants;
import com.thinkbiganalytics.kylo.catalog.dataset.DataSetUtil;
import com.thinkbiganalytics.kylo.catalog.datasource.DataSourceUtil;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSet;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSetFile;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSource;
import com.thinkbiganalytics.kylo.catalog.spi.FileSystemProvider;
import com.thinkbiganalytics.kylo.util.HadoopClassLoader;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/**
 * Manages browsing and uploading files for data sets.
 */
@Component
public class CatalogFileManager {

    private static final Logger log = LoggerFactory.getLogger(CatalogFileManager.class);

    /**
     * Hadoop configuration with default values
     */
    @Nonnull
    private final Configuration defaultConf;

    /**
     * File system providers for listing buckets (or hosts) of Hadoop-compatible file systems
     */
    @Nullable
    private List<FileSystemProvider> fileSystemProviders;

    /**
     * Default group for uploaded files
     */
    @Nullable
    private String groupname;

    /**
     * Validates data set paths
     */
    @Nonnull
    private PathValidator pathValidator;

    /**
     * Default permissions for uploaded files
     */
    @Nullable
    private FsPermission permission;

    /**
     * Default owner for uploaded files
     */
    @Nullable
    private String username;

    /**
     * Constructs a {@code CatalogFileManager} using the specified Kylo data directory.
     */
    @Autowired
    public CatalogFileManager(@Nonnull final PathValidator pathValidator) {
        this.pathValidator = pathValidator;

        defaultConf = new Configuration();
        defaultConf.size();  // causes defaults to be loaded
        defaultConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");  // Spark uses file:/// as default FileSystem
    }

    /**
     * Sets the file system providers to use for listing files.
     */
    @Autowired
    public void setFileSystemProviders(@Nullable final List<FileSystemProvider> fileSystemProviders) {
        this.fileSystemProviders = fileSystemProviders;
    }

    /**
     * Sets the default group for uploaded files.
     */
    @Value("${catalog.uploads.group:#{null}}")
    public void setGroupname(@Nullable final String groupname) {
        this.groupname = groupname;
    }
<commit_msg>KYLO-2089 Annotate FileSystemProvider bean as optional<commit_after>package com.thinkbiganalytics.kylo.catalog.file;

/*-
 * #%L
 * kylo-catalog-core
 * %%
 * Copyright (C) 2017 - 2018 ThinkBig Analytics
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.thinkbiganalytics.kylo.catalog.api.KyloCatalogConstants;
import com.thinkbiganalytics.kylo.catalog.dataset.DataSetUtil;
import com.thinkbiganalytics.kylo.catalog.datasource.DataSourceUtil;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSet;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSetFile;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSetTemplate;
import com.thinkbiganalytics.kylo.catalog.rest.model.DataSource;
import com.thinkbiganalytics.kylo.catalog.spi.FileSystemProvider;
import com.thinkbiganalytics.kylo.util.HadoopClassLoader;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.file.AccessDeniedException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/**
 * Manages browsing and uploading files for data sets.
 */
@Component
public class CatalogFileManager {

    private static final Logger log = LoggerFactory.getLogger(CatalogFileManager.class);

    /**
     * Hadoop configuration with default values
     */
    @Nonnull
    private final Configuration defaultConf;

    /**
     * File system providers for listing buckets (or hosts) of Hadoop-compatible file systems
     */
    @Nullable
    private List<FileSystemProvider> fileSystemProviders;

    /**
     * Default group for uploaded files
     */
    @Nullable
    private String groupname;

    /**
     * Validates data set paths
     */
    @Nonnull
    private PathValidator pathValidator;

    /**
     * Default permissions for uploaded files
     */
    @Nullable
    private FsPermission permission;

    /**
     * Default owner for uploaded files
     */
    @Nullable
    private String username;

    /**
     * Constructs a {@code CatalogFileManager} using the specified Kylo data directory.
     */
    @Autowired
    public CatalogFileManager(@Nonnull final PathValidator pathValidator) {
        this.pathValidator = pathValidator;

        defaultConf = new Configuration();
        defaultConf.size();  // causes defaults to be loaded
        defaultConf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");  // Spark uses file:/// as default FileSystem
    }

    /**
     * Sets the file system providers to use for listing files.
     */
    @Autowired(required = false)
    public void setFileSystemProviders(@Nullable final List<FileSystemProvider> fileSystemProviders) {
        this.fileSystemProviders = fileSystemProviders;
    }

    /**
     * Sets the default group for uploaded files.
     */
    @Value("${catalog.uploads.group:#{null}}")
    public void setGroupname(@Nullable final String groupname) {
        this.groupname = groupname;
    }

{'fraction_non_alphanumeric': 0.09288514075305856, 'fraction_numerical': 0.003800926475828483, 'mean_word_length': 3.814179531160663, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572158', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2324, 'n_words': 618}