<commit_before>/*
 * Copyright 2010 Ning, Inc.
 *
 * Ning licenses this file to you under the Apache License, version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.ning.metrics.collector.binder;

import com.google.inject.Inject;
import com.google.inject.Provider;
import com.ning.metrics.collector.binder.config.CollectorConfig;
import com.ning.serialization.ThriftEnvelopeSerialization;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.RawLocalFileSystem;

import java.io.IOException;
import java.net.URI;

public class FileSystemProvider implements Provider<FileSystem>
{
    private final FileSystem fileSystem;

    @Inject
    public FileSystemProvider(CollectorConfig collectorConfig) throws IOException
    {
        String hfsHost = collectorConfig.getHfsHost();
        if (!hfsHost.isEmpty()) {
            Configuration hadoopConfig = new Configuration();

            hadoopConfig.set("fs.default.name", hfsHost);
            hadoopConfig.setLong("dfs.block.size", collectorConfig.getHadoopBlockSize());
            hadoopConfig.set("hadoop.job.ugi", collectorConfig.getHadoopUgi());
            hadoopConfig.setStrings("io.serializations", ThriftEnvelopeSerialization.class.getName(), "org.apache.hadoop.io.serializer.WritableSerialization");

            fileSystem = FileSystem.get(hadoopConfig);
        }
        else {
            // Useful for testing in isolated environments
            FileSystem hadoopIsStupid = new RawLocalFileSystem();
            Configuration hadoopConfig = new Configuration();
            hadoopConfig.setStrings("io.serializations", ThriftEnvelopeSerialization.class.getName(), "org.apache.hadoop.io.serializer.WritableSerialization");
            hadoopIsStupid.initialize(URI.create("not://used"), hadoopConfig);

            fileSystem = new LocalFileSystem(hadoopIsStupid);
        }
    }

    public FileSystem get()
    {
        return fileSystem;
    }
}<commit_msg>hadoop: refactor HDFS connection<commit_after>/*
 * Copyright 2010 Ning, Inc.
 *
 * Ning licenses this file to you under the Apache License, version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.ning.metrics.collector.binder;

import com.google.inject.Inject;
import com.google.inject.Provider;
import com.ning.metrics.collector.binder.config.CollectorConfig;
import com.ning.serialization.ThriftEnvelopeSerialization;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;

import java.io.IOException;

public class FileSystemProvider implements Provider<FileSystem>
{
    private final FileSystem fileSystem;

    @Inject
    public FileSystemProvider(CollectorConfig collectorConfig) throws IOException
    {
        String hfsHost = collectorConfig.getHfsHost();
        Configuration hadoopConfig = new Configuration();

        if (hfsHost.isEmpty()) {
            // Local filesystem, for testing
            hadoopConfig.set("fs.default.name", "file:///");
        }
        else {
            hadoopConfig.set("fs.default.name", hfsHost);
        }

        hadoopConfig.setLong("dfs.block.size", collectorConfig.getHadoopBlockSize());
        hadoopConfig.set("hadoop.job.ugi", collectorConfig.getHadoopUgi());
        hadoopConfig.setStrings("io.serializations", ThriftEnvelopeSerialization.class.getName(), "org.apache.hadoop.io.serializer.WritableSerialization");

        fileSystem = FileSystem.get(hadoopConfig);
    }

    public FileSystem get()
    {
        return fileSystem;
    }
}
{'fraction_non_alphanumeric': 0.09488084730803177, 'fraction_numerical': 0.00353045013239188, 'mean_word_length': 3.5787878787878786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6080247', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1185, 'n_words': 327}