<commit_before>/**
 * Copyright 2012 Shopzilla.com
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  http://tech.shopzilla.com
 *
 */

package com.shopzilla.hadoop.testing;

import com.shopzilla.hadoop.HadoopREPL;
import com.shopzilla.hadoop.REPL;
import com.shopzilla.hadoop.testing.hdfs.DFSCluster;
import com.shopzilla.hadoop.testing.mapreduce.JobTracker;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * @author <NAME>
 * @since 9/5/12
 */
public class HadoopCLI {

    private static final String DEFAULT_CORE_SITE_LOCATION = "/tmp/core-site.xml";
    private static final String DEFAULT_MR_LOGS_LOCATION = "/tmp/minimrcluster/logs";

    private final File localRoot;
    private final File logDirectory = new File(DEFAULT_MR_LOGS_LOCATION);
    private final Configuration configuration;
    private DFSCluster dfsCluster;
    private JobTracker jobTracker;

    protected HadoopCLI() {
        this(null);
    }

    protected HadoopCLI(final File localRoot) {
        this.configuration = new Configuration();
        this.localRoot = localRoot;
        System.setProperty("hadoop.log.dir", logDirectory.getAbsolutePath());
    }

    @PostConstruct
    public void start(final File confFile) throws REPL.ExitSignal {
        try {
            dfsCluster = DFSCluster.builder()
                .usingConfiguration(configuration)
                .withInitialStructure(localRoot)
                .build()
                .start();

            jobTracker = JobTracker.builder()
                .withNameNode(dfsCluster.getFileSystem().getUri().toString())
                .build()
                .start();

            configuration.writeXml(new FileOutputStream(confFile));
            new HadoopREPL(configuration).loop("hadoop-in-a-box> ");
        } catch (final IOException ex) {
            throw new REPL.ExitSignal(1, ex.getMessage());
        }
    }

    @PreDestroy
    public void stop() {
        jobTracker.stop();
        dfsCluster.stop();
        FileUtils.deleteQuietly(logDirectory);
    }

    public static void main(final String[] args) {
        int exitCode = 0;
        HadoopCLI hadoopCLI = null;
        try {
            File localRoot = null;
            File configurationFile = new File(DEFAULT_CORE_SITE_LOCATION);
            if (args.length >= 1) {
                localRoot = new File(args[0]);
            }
            if (args.length == 2) {
                configurationFile = new File(args[1]);
            }
            if (args.length > 2) {
                throw new REPL.ExitSignal(1, "Usage: ./hdp /path/to/local/hdfs/root [HADOOP_CORE_SITE_FILE]");
            }
            hadoopCLI = new HadoopCLI(localRoot);
            hadoopCLI.start(configurationFile);
        } catch (final REPL.ExitSignal ex) {
            exitCode = ex.getExitCode();
            if (exitCode == 0) {
                System.out.println(ex.getMessage());
            } else {
                System.err.println(ex.getMessage());
            }
        } finally {
            if (hadoopCLI != null) {
                hadoopCLI.stop();
            }
        }
        System.exit(exitCode);
    }
}
<commit_msg>*Adding clean-up to core-site.xml file<commit_after>/**
 * Copyright 2012 Shopzilla.<EMAIL>
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 *  http://tech.shopzilla.com
 *
 */

package com.shopzilla.hadoop.testing;

import com.shopzilla.hadoop.HadoopREPL;
import com.shopzilla.hadoop.REPL;
import com.shopzilla.hadoop.testing.hdfs.DFSCluster;
import com.shopzilla.hadoop.testing.mapreduce.JobTracker;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * @author <NAME>
 * @since 9/5/12
 */
public class HadoopCLI {

    private static final String DEFAULT_CORE_SITE_LOCATION = "/tmp/core-site.xml";
    private static final String DEFAULT_MR_LOGS_LOCATION = "/tmp/minimrcluster/logs";

    private final File localRoot;
    private final File logDirectory = new File(DEFAULT_MR_LOGS_LOCATION);
    private final Configuration configuration;
    private final File configurationFile;
    private DFSCluster dfsCluster;
    private JobTracker jobTracker;

    protected HadoopCLI() {
        this(null, new File(DEFAULT_CORE_SITE_LOCATION));
    }

    protected HadoopCLI(final File localRoot, final File configurationFile) {
        this.configuration = new Configuration();
        this.configurationFile = configurationFile;
        this.localRoot = localRoot;
        System.setProperty("hadoop.log.dir", logDirectory.getAbsolutePath());
    }

    @PostConstruct
    public void start() throws REPL.ExitSignal {
        try {
            dfsCluster = DFSCluster.builder()
                .usingConfiguration(configuration)
                .withInitialStructure(localRoot)
                .build()
                .start();

            jobTracker = JobTracker.builder()
                .withNameNode(dfsCluster.getFileSystem().getUri().toString())
                .build()
                .start();

            configuration.writeXml(new FileOutputStream(configurationFile));
            new HadoopREPL(configuration).loop("hadoop-in-a-box> ");
        } catch (final IOException ex) {
            throw new REPL.ExitSignal(1, ex.getMessage());
        }
    }

    @PreDestroy
    public void stop() {
        jobTracker.stop();
        dfsCluster.stop();
        FileUtils.deleteQuietly(logDirectory);
        FileUtils.deleteQuietly(configurationFile);
    }

    public static void main(final String[] args) {
        int exitCode = 0;
        HadoopCLI hadoopCLI = null;
        try {
            File localRoot = null;
            File configurationFile = new File(DEFAULT_CORE_SITE_LOCATION);
            if (args.length >= 1) {
                localRoot = new File(args[0]);
            }
            if (args.length == 2) {
                configurationFile = new File(args[1]);
            }
            if (args.length > 2) {
                throw new REPL.ExitSignal(1, "Usage: ./hdp /path/to/local/hdfs/root [HADOOP_CORE_SITE_FILE]");
            }
            hadoopCLI = new HadoopCLI(localRoot, configurationFile);
            hadoopCLI.start();
        } catch (final REPL.ExitSignal ex) {
            exitCode = ex.getExitCode();
            if (exitCode == 0) {
                System.out.println(ex.getMessage());
            } else {
                System.err.println(ex.getMessage());
            }
        } finally {
            if (hadoopCLI != null) {
                hadoopCLI.stop();
            }
        }
        System.exit(exitCode);
    }
}

{'fraction_non_alphanumeric': 0.10076546618145313, 'fraction_numerical': 0.005270422888693688, 'mean_word_length': 2.329156223893066, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3402100', 'n_tokens_mistral': 2267, 'n_tokens_neox': 2175, 'n_words': 541}