<commit_before>package org.embulk.output.hdfs.client;

import com.google.common.base.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.embulk.config.ConfigException;
import org.embulk.output.hdfs.HdfsFileOutputPlugin;
import org.embulk.spi.DataException;
import org.embulk.spi.Exec;
import org.embulk.spi.util.RetryExecutor;
import org.slf4j.Logger;

import java.io.File;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.util.List;
import java.util.Map;

public class HdfsClient
{
    public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task)
    {
        Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig());
        return new HdfsClient(conf, task.getDoas());
    }

    ;

    public static Configuration buildConfiguration(List<String> configFiles, Map<String, String> configs)
    {
        Configuration c = new Configuration();
        for (String configFile : configFiles) {
            File file = new File(configFile);
            try {
                c.addResource(file.toURI().toURL());
            }
            catch (MalformedURLException e) {
                throw new ConfigException(e);
            }
        }
        for (Map.Entry<String, String> config : configs.entrySet()) {
            c.set(config.getKey(), config.getValue());
        }
        return c;
    }

    private static Logger logger = Exec.getLogger(HdfsClient.class);
    private final Configuration conf;
    private final FileSystem fs;
    private final Optional<String> user;
    private final RetryExecutor re = RetryExecutor.retryExecutor()
            .withRetryLimit(3)
            .withMaxRetryWait(500)             // ms<commit_msg>Add UserGroupInformation#setConfiguration for kerberos authentication<commit_after>package org.embulk.output.hdfs.client;

import com.google.common.base.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.security.UserGroupInformation;
import org.embulk.config.ConfigException;
import org.embulk.output.hdfs.HdfsFileOutputPlugin;
import org.embulk.spi.DataException;
import org.embulk.spi.Exec;
import org.embulk.spi.util.RetryExecutor;
import org.slf4j.Logger;

import java.io.File;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.util.List;
import java.util.Map;

public class HdfsClient
{
    public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task)
    {
        Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig());
        return new HdfsClient(conf, task.getDoas());
    }

    ;

    public static Configuration buildConfiguration(List<String> configFiles, Map<String, String> configs)
    {
        Configuration c = new Configuration();
        for (String configFile : configFiles) {
            File file = new File(configFile);
            try {
                c.addResource(file.toURI().toURL());
            }
            catch (MalformedURLException e) {
                throw new ConfigException(e);
            }
        }
        for (Map.Entry<String, String> config : configs.entrySet()) {
            c.set(config.getKey(), config.getValue());
        }
        UserGroupInformation.setConfiguration(c);
        return c;
    }

    private static Logger logger = Exec.getLogger(HdfsClient.class);
    private final Configuration conf;
    private final FileSystem fs;
    private final Optional<String> user;
    private final RetryExecutor re = RetryExecutor.retryExecutor()
            .withRetryLimit(3)
            .withMaxRetryWait(500)             // ms
{'fraction_non_alphanumeric': 0.1024390243902439, 'fraction_numerical': 0.0024390243902439024, 'mean_word_length': 3.13824419778002, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6645308', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1184, 'n_words': 246}