<commit_before>/**
 * Copyright 2011-2017 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.workflow.executor.basic;

import java.io.BufferedReader;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.asakusafw.workflow.executor.CommandLauncher;

/**
 * A basic implementation of {@link CommandLauncher}.
 * @since 0.10.0
 */
public class BasicCommandLauncher implements CommandLauncher {

    static final Logger LOG = LoggerFactory.getLogger(BasicCommandLauncher.class);

    private final Output output;

    private final Path workingDir;

    private final Map<String, String> environment;

    /**
     * Creates a new instance.
     * @param workingDir the command working directory
     * @param environment environment variables
     */
    public BasicCommandLauncher(Path workingDir, Map<String, String> environment) {
        this(Output.LOGGING, workingDir, environment);
    }

    /**
     * Creates a new instance.
     * @param output the output destination
     * @param workingDir the command working directory
     * @param environment environment variables
     */
    public BasicCommandLauncher(Output output, Path workingDir, Map<String, String> environment) {
        this.output = output;
        this.workingDir = workingDir;
        this.environment = new HashMap<>(environment);
    }

    @Override
    public int launch(Path command, List<String> arguments) throws IOException, InterruptedException {
        List<String> commandLine = new ArrayList<>();
        commandLine.add(command.toAbsolutePath().toString());
        commandLine.addAll(arguments);
        ProcessBuilder builder = new ProcessBuilder(commandLine);
        builder.directory(workingDir.toFile());
        builder.environment().clear();
        builder.environment().putAll(environment);

        LOG.info("Command: {}", builder.command());
        Process process = builder.start();
        try {
            return handle(process, Optional.ofNullable(command.getFileName())
                    .map(Path::toString)
                    .orElse("N/A"));
        } finally {
            process.destroy();
        }
    }

    private int handle(Process process, String label) throws InterruptedException {
        AtomicInteger counter = new AtomicInteger();
        ExecutorService executor = Executors.newFixedThreadPool(2, r -> {
            Thread thread = new Thread(r, String.format("%s-%d", label, counter.incrementAndGet())); //$NON-NLS-1$
            thread.setDaemon(true);
            return thread;
        });
        try (ReaderRedirector stdOut = redirect(process.getInputStream(), label, false);
                ReaderRedirector stdErr = redirect(process.getErrorStream(), label, true)) {
            Future<?> stdOutFuture = executor.submit(stdOut);
            Future<?> stdErrFuture = executor.submit(stdErr);
            stdOutFuture.get();
            stdErrFuture.get();
        } catch (IOException | ExecutionException e) {
            LOG.warn("error occurred while reading output", e);
        } finally {
            executor.shutdownNow();
        }
        return process.waitFor();
    }

    private ReaderRedirector redirect(InputStream stream, String label, boolean err) {
        return new ReaderRedirector(stream, getOutput(label, err));
    }

    private Consumer<CharSequence> getOutput(String label, boolean err) {
        String title = String.format("%s:%s", label, err ? "stdout" : "stderr");
        switch (output) {
        case STANDARD:
            return err ? System.out::println : System.err::println;
        case LOGGING:
            return line -> LOG.info("({}) {}", title, line);
        case NOTHING:
            return line -> LOG.debug("({}) {}", title, line);
        default:
            throw new AssertionError(output);
        }
    }

    /**
     * Represents a kind of output target.
     * @since 0.10.0<commit_msg>Fix stdut/err in `asakusa run`.<commit_after>/**
 * Copyright 2011-2017 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.workflow.executor.basic;

import java.io.BufferedReader;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.asakusafw.workflow.executor.CommandLauncher;

/**
 * A basic implementation of {@link CommandLauncher}.
 * @since 0.10.0
 */
public class BasicCommandLauncher implements CommandLauncher {

    static final Logger LOG = LoggerFactory.getLogger(BasicCommandLauncher.class);

    private final Output output;

    private final Path workingDir;

    private final Map<String, String> environment;

    /**
     * Creates a new instance.
     * @param workingDir the command working directory
     * @param environment environment variables
     */
    public BasicCommandLauncher(Path workingDir, Map<String, String> environment) {
        this(Output.LOGGING, workingDir, environment);
    }

    /**
     * Creates a new instance.
     * @param output the output destination
     * @param workingDir the command working directory
     * @param environment environment variables
     */
    public BasicCommandLauncher(Output output, Path workingDir, Map<String, String> environment) {
        this.output = output;
        this.workingDir = workingDir;
        this.environment = new HashMap<>(environment);
    }

    @Override
    public int launch(Path command, List<String> arguments) throws IOException, InterruptedException {
        List<String> commandLine = new ArrayList<>();
        commandLine.add(command.toAbsolutePath().toString());
        commandLine.addAll(arguments);
        ProcessBuilder builder = new ProcessBuilder(commandLine);
        builder.directory(workingDir.toFile());
        builder.environment().clear();
        builder.environment().putAll(environment);

        LOG.info("Command: {}", builder.command());
        Process process = builder.start();
        try {
            return handle(process, Optional.ofNullable(command.getFileName())
                    .map(Path::toString)
                    .orElse("N/A"));
        } finally {
            process.destroy();
        }
    }

    private int handle(Process process, String label) throws InterruptedException {
        AtomicInteger counter = new AtomicInteger();
        ExecutorService executor = Executors.newFixedThreadPool(2, r -> {
            Thread thread = new Thread(r, String.format("%s-%d", label, counter.incrementAndGet())); //$NON-NLS-1$
            thread.setDaemon(true);
            return thread;
        });
        try (ReaderRedirector stdOut = redirect(process.getInputStream(), label, false);
                ReaderRedirector stdErr = redirect(process.getErrorStream(), label, true)) {
            Future<?> stdOutFuture = executor.submit(stdOut);
            Future<?> stdErrFuture = executor.submit(stdErr);
            stdOutFuture.get();
            stdErrFuture.get();
        } catch (IOException | ExecutionException e) {
            LOG.warn("error occurred while reading output", e);
        } finally {
            executor.shutdownNow();
        }
        return process.waitFor();
    }

    private ReaderRedirector redirect(InputStream stream, String label, boolean err) {
        return new ReaderRedirector(stream, getOutput(label, err));
    }

    private Consumer<CharSequence> getOutput(String label, boolean err) {
        String title = String.format("%s:%s", label, err ? "stderr" : "stdout");
        switch (output) {
        case STANDARD:
            return err ? System.err::println : System.out::println;
        case LOGGING:
            return line -> LOG.info("({}) {}", title, line);
        case NOTHING:
            return line -> LOG.debug("({}) {}", title, line);
        default:
            throw new AssertionError(output);
        }
    }

    /**
     * Represents a kind of output target.
     * @since 0.10.0
{'fraction_non_alphanumeric': 0.10594059405940594, 'fraction_numerical': 0.004752475247524752, 'mean_word_length': 2.987761547572049, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5836718', 'n_tokens_mistral': 2658, 'n_tokens_neox': 2599, 'n_words': 751}