<commit_before>package de.itemis.mosig.racecar.textconv;

import org.junit.After;
import org.junit.Assert;
import org.junit.Test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;


public class HtmlPagesConverterTest {

    private static final String PAGE_BREAK = "PAGE_BREAK";

    private Set<Path> pathsOfTestFiles = new HashSet<>();

    @After
    public void cleanupExistingTestFile() {
        pathsOfTestFiles.forEach(this::removeTestFile);
    }

    @Test
    public void shouldInsertOneBrForEachPageBreakEncounteredAndOneForTheLastLine() {
        HtmlPagesConverter underTest = prepareUnderTest("Hello", PAGE_BREAK, "World", PAGE_BREAK, "!");

        assertPageContents(underTest, "Hello<br />", "World<br />", "!<br />");
    }

    @Test
    public void shouldReturnFullFileContentsIfThereAreNoPageBreaks() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest("Hello World!");

        assertPageContents(underTest, "Hello World!<br />");
    }

    @Test
    public void shouldReturnEmptyStringOnEmptyFile() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest();

        assertPageContents(underTest, "");
    }

    @Test
    public void shouldReturnEmptyStringIfPageIsNegative() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest("");

        String result = underTest.getHtmlPage(-1);
        Assert.assertNull(getPage(underTest, -1));
    }

    @Test
    public void shouldReturnTwoEmptyPagesIfFileContainsOnePageBreakAndNothingElse() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest(PAGE_BREAK);

        assertPageContents(underTest, "", "");
    }

    @Test
    public void shouldReturnFourEmptyPagesIfFileContainsThreePageBreaks() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTestOld("only_multiple_page_breaks.txt");

        assertPageContents(underTest, "", "", "", "");
    }

    @Test
    public void shouldReturnFileNameIfFileExists() throws IOException {
        Path testFilePath = prepareTestFile("multiple_page_breaks.txt");
        String expectedResult = testFilePath.toString();
        HtmlPagesConverter underTest = new HtmlPagesConverter(expectedResult);

        String result = underTest.getFilename();

        Assert.assertEquals(expectedResult, result);
    }

    @Test
    public void shouldThrowIoExceptionIfFileDoesNotExist() {
        String nonExistingFileName = "this_file_does_not_exist";
        try {
            new HtmlPagesConverter(nonExistingFileName);
            Assert.fail();
        } catch (IOException e) {
            Assert.assertEquals(FileNotFoundException.class.getSimpleName(), e.getClass().getSimpleName());
            Assert.assertTrue(e.getMessage().contains(nonExistingFileName));
        }
    }

    /*
     * ##### Start private helper code #####
     */

    private void fail(String msg, Throwable cause) {
        Assert.fail(msg + " - " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
    }

    private void assertPageContents(HtmlPagesConverter underTest, String... contents) {
        for (int i = 0; i < contents.length; i++) {
            String expectedContent = contents[i];
            String actualContent = getPage(underTest, i);
            Assert.assertEquals(expectedContent, actualContent);
        }

        Assert.assertNull(getPage(underTest, contents.length));
    }

    private String getPage(HtmlPagesConverter underTest, int pageNbr) {
        String result = null;

        try {
            result = underTest.getHtmlPage(pageNbr);
        } catch (IOException e) {
            fail("Could not read from test instance", e);
        }

        return result;
    }

    private HtmlPagesConverter prepareUnderTest(String... fileContents) {
        return new HtmlPagesConverter(Arrays.asList(fileContents));
    }

    private HtmlPagesConverter prepareUnderTestOld(String resourceName) {
        String filePath = prepareTestFile(resourceName).toString();

        HtmlPagesConverter result = null;
        try {
            result = new HtmlPagesConverter(filePath);
        } catch (IOException e) {
            fail("Could not create instance of '" + HtmlPagesConverter.class.getSimpleName() + "'", e);
        }

        return result;
    }

    private Path prepareTestFile(String resourceName) {
        resourceName = "/" + resourceName.trim();
        Path result = null;

        try (InputStream srcFileStream = getClass().getResourceAsStream(resourceName)) {
            Assert.assertNotNull(srcFileStream);
            result = Files.createTempFile(this.getClass().getSimpleName(), "tmp");
            Files.copy(srcFileStream, result, StandardCopyOption.REPLACE_EXISTING);
        } catch (IOException e) {
            fail("Could not copy resource '" + resourceName + "' to temporary location", e);
        }

        pathsOfTestFiles.add(result);
        return result;
    }

    private void removeTestFile(Path path) {
        try {
            Files.deleteIfExists(path);
        } catch (IOException e) {
            System.err.println("WARN - Could not remove temporary file: " + path);
        }
    }
}<commit_msg>RED: Odd amount of page breaks is not processed correctly.<commit_after>package de.itemis.mosig.racecar.textconv;

import org.junit.After;
import org.junit.Assert;
import org.junit.Test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;


public class HtmlPagesConverterTest {

    private static final String PAGE_BREAK = "PAGE_BREAK";

    private Set<Path> pathsOfTestFiles = new HashSet<>();

    @After
    public void cleanupExistingTestFile() {
        pathsOfTestFiles.forEach(this::removeTestFile);
    }

    @Test
    public void shouldInsertOneBrForEachPageBreakEncounteredAndOneForTheLastLine() {
        HtmlPagesConverter underTest = prepareUnderTest("Hello", PAGE_BREAK, "World", PAGE_BREAK, "!");

        assertPageContents(underTest, "Hello<br />", "World<br />", "!<br />");
    }

    @Test
    public void shouldReturnFullFileContentsIfThereAreNoPageBreaks() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest("Hello World!");

        assertPageContents(underTest, "Hello World!<br />");
    }

    @Test
    public void shouldReturnEmptyStringOnEmptyFile() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest();

        assertPageContents(underTest, "");
    }

    @Test
    public void shouldReturnEmptyStringIfPageIsNegative() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest("");

        String result = underTest.getHtmlPage(-1);
        Assert.assertNull(getPage(underTest, -1));
    }

    @Test
    public void shouldReturnTwoEmptyPagesIfFileContainsOnePageBreakAndNothingElse() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest(PAGE_BREAK);

        assertPageContents(underTest, "", "");
    }

    @Test
    public void shouldReturnPageBreakCountPlus1EmptyPagesIfFileContainsOddNumberOfPageBreaksOnly() throws IOException {
        HtmlPagesConverter underTest = prepareUnderTest(PAGE_BREAK, PAGE_BREAK, PAGE_BREAK);

        assertPageContents(underTest, "", "", "", "");
    }

    @Test
    public void shouldReturnFileNameIfFileExists() throws IOException {
        Path testFilePath = prepareTestFile("multiple_page_breaks.txt");
        String expectedResult = testFilePath.toString();
        HtmlPagesConverter underTest = new HtmlPagesConverter(expectedResult);

        String result = underTest.getFilename();

        Assert.assertEquals(expectedResult, result);
    }

    @Test
    public void shouldThrowIoExceptionIfFileDoesNotExist() {
        String nonExistingFileName = "this_file_does_not_exist";
        try {
            new HtmlPagesConverter(nonExistingFileName);
            Assert.fail();
        } catch (IOException e) {
            Assert.assertEquals(FileNotFoundException.class.getSimpleName(), e.getClass().getSimpleName());
            Assert.assertTrue(e.getMessage().contains(nonExistingFileName));
        }
    }

    /*
     * ##### Start private helper code #####
     */

    private void fail(String msg, Throwable cause) {
        Assert.fail(msg + " - " + cause.getClass().getSimpleName() + ": " + cause.getMessage());
    }

    private void assertPageContents(HtmlPagesConverter underTest, String... contents) {
        for (int i = 0; i < contents.length; i++) {
            String expectedContent = contents[i];
            String actualContent = getPage(underTest, i);
            Assert.assertEquals(expectedContent, actualContent);
        }

        Assert.assertNull(getPage(underTest, contents.length));
    }

    private String getPage(HtmlPagesConverter underTest, int pageNbr) {
        String result = null;

        try {
            result = underTest.getHtmlPage(pageNbr);
        } catch (IOException e) {
            fail("Could not read from test instance", e);
        }

        return result;
    }

    private HtmlPagesConverter prepareUnderTest(String... fileContents) {
        return new HtmlPagesConverter(Arrays.asList(fileContents));
    }

    private HtmlPagesConverter prepareUnderTestOld(String resourceName) {
        String filePath = prepareTestFile(resourceName).toString();

        HtmlPagesConverter result = null;
        try {
            result = new HtmlPagesConverter(filePath);
        } catch (IOException e) {
            fail("Could not create instance of '" + HtmlPagesConverter.class.getSimpleName() + "'", e);
        }

        return result;
    }

    private Path prepareTestFile(String resourceName) {
        resourceName = "/" + resourceName.trim();
        Path result = null;

        try (InputStream srcFileStream = getClass().getResourceAsStream(resourceName)) {
            Assert.assertNotNull(srcFileStream);
            result = Files.createTempFile(this.getClass().getSimpleName(), "tmp");
            Files.copy(srcFileStream, result, StandardCopyOption.REPLACE_EXISTING);
        } catch (IOException e) {
            fail("Could not copy resource '" + resourceName + "' to temporary location", e);
        }

        pathsOfTestFiles.add(result);
        return result;
    }

    private void removeTestFile(Path path) {
        try {
            Files.deleteIfExists(path);
        } catch (IOException e) {
            System.err.println("WARN - Could not remove temporary file: " + path);
        }
    }
}
{'fraction_non_alphanumeric': 0.10209135938359934, 'fraction_numerical': 0.000642084021280499, 'mean_word_length': 3.0804640718562872, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1122571', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2830, 'n_words': 609}