<commit_before>package net.morimekta.util.concurrent;

import net.morimekta.util.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

/**
 * Tests for the integration executor.
 */
@SuppressWarnings("ALL")
public class ProcessExecutorTest {
    @Rule
    public  TemporaryFolder tmp = new TemporaryFolder();

    private File integration;
    private ExecutorService executor;

    @Before
    public void setUp() throws IOException {
        executor = Executors.newFixedThreadPool(3);

        integration = tmp.newFile("integration.jar");
        try (OutputStream out = new FileOutputStream(integration)) {
            IOUtils.copy(getClass().getResourceAsStream("/integration.zip"),
                         out);
        }
    }

    @After
    public void tearDown() throws InterruptedException {
        executor.shutdownNow();
        if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
            fail("Oops");
        }
    }

    @Test
    public void testRun() throws IOException {
        File sh = tmp.newFile("tmp.sh");
        try (OutputStream out = new FileOutputStream(sh)) {
            out.write(("#!/bin/sh\n" +
                       "ls -1 $1\n").getBytes(UTF_8));
        }
        for (int i = 0; i < 100; ++i) {
            ProcessExecutor sut = new ProcessExecutor(
                    "sh", sh.getAbsolutePath(), tmp.getRoot().toString());

            assertThat("Run " + i, sut.call(), is(0));
            assertThat("Run " + i, sut.getOutput(), is("integration.jar\n" +
                                           "tmp.sh\n"));
            assertThat("Run " + i, sut.getError(), is(""));
        }
    }

    @Test
    public void testTimesOut() throws IOException {
        File sh = tmp.newFile("tmp.sh");
        try (OutputStream out = new FileOutputStream(sh)) {
            out.write(("#!/usr/bin/env bash\n" +
                       "sleep 10\n").getBytes(UTF_8));
        }
        ProcessExecutor sut = new ProcessExecutor(
                "sh", sh.getAbsolutePath(),
                "a'boo'b",
                "must be quoted");
        sut.setDeadlineMs(100);
<commit_msg>Upp the timeout on the 'ls' test. Travis keeps failing.<commit_after>package net.morimekta.util.concurrent;

import net.morimekta.util.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.hamcrest.CoreMatchers.endsWith;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

/**
 * Tests for the integration executor.
 */
@SuppressWarnings("ALL")
public class ProcessExecutorTest {
    @Rule
    public  TemporaryFolder tmp = new TemporaryFolder();

    private File integration;
    private ExecutorService executor;

    @Before
    public void setUp() throws IOException {
        executor = Executors.newFixedThreadPool(3);

        integration = tmp.newFile("integration.jar");
        try (OutputStream out = new FileOutputStream(integration)) {
            IOUtils.copy(getClass().getResourceAsStream("/integration.zip"),
                         out);
        }
    }

    @After
    public void tearDown() throws InterruptedException {
        executor.shutdownNow();
        if (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
            fail("Oops");
        }
    }

    @Test
    public void testRun() throws IOException {
        File sh = tmp.newFile("tmp.sh");
        try (OutputStream out = new FileOutputStream(sh)) {
            out.write(("#!/bin/sh\n" +
                       "ls -1 $1\n").getBytes(UTF_8));
        }
        for (int i = 0; i < 100; ++i) {
            // If the "root" directory is really cluttered on an overworked
            // machine using spin-disks (e.g. travis-ci.org), this may fail
            // without upping the timeouts.
            ProcessExecutor sut = new ProcessExecutor(
                    "sh", sh.getAbsolutePath(), tmp.getRoot().toString())
                    .setDeadlineMs(10000L)
                    .setDeadlineFlushMs(2000L);

            assertThat("Run " + i, sut.call(), is(0));
            assertThat("Run " + i, sut.getOutput(), is("integration.jar\n" +
                                           "tmp.sh\n"));
            assertThat("Run " + i, sut.getError(), is(""));
        }
    }

    @Test
    public void testTimesOut() throws IOException {
        File sh = tmp.newFile("tmp.sh");
        try (OutputStream out = new FileOutputStream(sh)) {
            out.write(("#!/usr/bin/env bash\n" +
                       "sleep 10\n").getBytes(UTF_8));
        }
        ProcessExecutor sut = new ProcessExecutor(
                "sh", sh.getAbsolutePath(),
                "a'boo'b",
                "must be quoted");
        sut.setDeadlineMs(100);

{'fraction_non_alphanumeric': 0.11705244652798125, 'fraction_numerical': 0.006592440668033987, 'mean_word_length': 2.6922660897782587, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5328002', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1940, 'n_words': 433}