<commit_before>
require 'rjb'

module HCC

    class Output

        attr_reader :cmd, :stdout, :stderr

        def initialize(cmd, stdout, stderr)
            @cmd    = cmd
            @stdout = stdout
            @stderr = stderr
        end

        def success?
            @stderr.empty?
        end

        def error?
            not @stderr.empty?
        end

    end

    class Shell

        def initialize(hadoop_home)

            @home = hadoop_home
            load_rjb()

        end

        # returns Output class
        def run(cmd)
            out, err = capture_output do
                @shell.run(cmd)
            end
            Output.new(cmd.join(" "), out, err)
        end

        def capture_output

            ord, owr = IO.pipe
            oldout = STDOUT.dup
            STDOUT.reopen owr

            erd, ewr = IO.pipe
            olderr = STDERR.dup
            STDERR.reopen ewr

            yield

            STDOUT.reopen oldout
            STDERR.reopen olderr

            owr.close
            ewr.close

            [ ord.read, erd.read ]

        ensure
            STDOUT.reopen oldout
            STDERR.reopen olderr
        end

        def load_rjb()

            if not ENV["JAVA_HOME"] then
                puts "JAVA_HOME not set! export or pass with --java_home"
                exit
            end<commit_msg>redirect stdout/stderr to a tempfile instead of a pipe<commit_after>
require 'rjb'
require 'tempfile'

module HCC

    class Output

        attr_reader :cmd, :stdout, :stderr

        def initialize(cmd, stdout, stderr)
            @cmd    = cmd
            @stdout = stdout
            @stderr = stderr
        end

        def success?
            @stderr.empty?
        end

        def error?
            not @stderr.empty?
        end

    end

    class Shell

        def initialize(hadoop_home)

            @home = hadoop_home
            load_rjb()

        end

        # returns Output class
        def run(cmd)
            out, err = capture_output do
                @shell.run(cmd)
            end
            Output.new(cmd.join(" "), out, err)
        end

        def capture_output

            newout = Tempfile.new("shell-")
            oldout = STDOUT.dup
            STDOUT.reopen newout

            newerr = Tempfile.new("shell-")
            olderr = STDERR.dup
            STDERR.reopen newerr

            yield

            STDOUT.reopen oldout
            STDERR.reopen olderr

            [ File.new(newout.path).read, File.new(newerr.path).read ]

        ensure
            STDOUT.reopen oldout
            STDERR.reopen olderr
        end

        def load_rjb()

            if not ENV["JAVA_HOME"] then
                puts "JAVA_HOME not set! export or pass with --java_home"
                exit
            end
{'fraction_non_alphanumeric': 0.06775452288045407, 'fraction_numerical': 0.0, 'mean_word_length': 1.1543162719633309, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5747980', 'n_tokens_mistral': 842, 'n_tokens_neox': 771, 'n_words': 211}