<commit_before>package tdl.record.sourcecode;

import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import support.TestGeneratedSrcsFile;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertTrue;
import static support.TestUtils.writeFile;

public class ExportSegmentsCommandTest {

    @Rule
    public TemporaryFolder folder = new TemporaryFolder();

    @Rule
    public TestGeneratedSrcsFile recorder = new TestGeneratedSrcsFile(Arrays.asList(
            dst -> writeFile(dst, "test1.txt", "TEST1"), //key
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test2.txt", "TEST1TEST2"), //patch
            dst -> { //key
                writeFile(dst, "test2.txt", "TEST1TEST2");
                writeFile(dst, "subdir/test3.txt", "TEST3");
            },
            dst -> {/* Empty folder */ }, //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //key
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2") //key
    ), Arrays.asList("tag1", "x", "tag12", "tag3", "tag3"));

    @Test
    public void run() throws IOException {
        Path inputFilePath = recorder.getFilePath();

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 0);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("type KEY"));
            assertThat(info.get(0), containsString("tag1"));
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 1);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag1"));
            assertThat(info.get(1), containsString("x"));
            assertThat(info.get(1), containsString("type PATCH"));
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 2);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag1"));
            assertThat(info.get(1), containsString("tag x"));
            assertThat(info.get(1), containsString("type PATCH"));
            assertThat(info.get(1), containsString("tag x"));
            assertThat(info.get(2), containsString("tag tag12"));
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 3);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag3"));
            assertThat(info.get(0), containsString("type KEY"));
        }

        {
            Path newSrcsFilePath = exportTag(inputFilePath, "tag12");
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag12"));
            assertThat(info.get(0), containsString("type KEY"));
            assertThat(info.get(1), containsString("tag tag12"));
            assertThat(info.get(1), containsString("type PATCH"));
            assertThat(info.get(2), containsString("tag tag12"));
        }

        {
            Path newSrcsFilePath = exportTag(inputFilePath, "tag3");
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag3"));
            assertThat(info.get(0), containsString("type KEY"));
        }
    }

    private List<String> listInfoFrom(Path newFilePath) {
        ListCommand listCommand = new ListCommand();
        listCommand.inputFilePath = newFilePath.toString();
        listCommand.run();
        return listCommand.getGatheredInfo();
    }

    private Path exportTimestamp(Path inputFilePath, int timestamp) throws IOException {
        Path filePath = folder.newFile().toPath();
        ExportSegmentsCommand exportSegmentsCommand = new ExportSegmentsCommand(inputFilePath.toString(), filePath.toString(), timestamp);
        exportSegmentsCommand.run();
        return filePath;
    }

    private Path exportTag(Path inputFilePath, String tag) throws IOException {
        Path filePath = folder.newFile().toPath();
        ExportSegmentsCommand exportSegmentsCommand = new ExportSegmentsCommand(inputFilePath.toString(), filePath.toString(), tag);
        exportSegmentsCommand.run();
        return filePath;
    }
}
<commit_msg>More tests added to ExportSegmentsCommandTest to check for file contents after exporting segments<commit_after>package tdl.record.sourcecode;

import org.apache.commons.io.FileUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import support.TestGeneratedSrcsFile;
import tdl.record.sourcecode.snapshot.file.ToGitConverter;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static support.TestUtils.writeFile;

public class ExportSegmentsCommandTest {

    private static final boolean STOP_ON_ERRORS = true;

    @Rule
    public TemporaryFolder folder = new TemporaryFolder();

    @Rule
    public TestGeneratedSrcsFile recorder = new TestGeneratedSrcsFile(Arrays.asList(
            dst -> writeFile(dst, "test1.txt", "TEST1"), //key
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test2.txt", "TEST1TEST2"), //patch
            dst -> { //key
                writeFile(dst, "test2.txt", "TEST1TEST2");
                writeFile(dst, "subdir/test3.txt", "TEST3");
            },
            dst -> {/* Empty folder */ }, //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //key
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2"), //patch
            dst -> writeFile(dst, "test1.txt", "TEST1TEST2") //key
    ), Arrays.asList("tag1", "x", "tag12", "tag3", "tag3"));

    @Test
    public void run() throws IOException {
        Path inputFilePath = recorder.getFilePath();

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 0);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("type KEY"));
            assertThat(info.get(0), containsString("tag1"));

            verifyFileAndFileContents(newSrcsFilePath, "test1.txt", "TEST1");
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 1);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag1"));
            assertThat(info.get(1), containsString("x"));
            assertThat(info.get(1), containsString("type PATCH"));

            verifyFileAndFileContents(newSrcsFilePath, "test1.txt", "TEST1TEST2");
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 2);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag1"));
            assertThat(info.get(1), containsString("tag x"));
            assertThat(info.get(1), containsString("type PATCH"));
            assertThat(info.get(1), containsString("tag x"));
            assertThat(info.get(2), containsString("tag tag12"));

            verifyFileAndFileContents(newSrcsFilePath, "test2.txt", "TEST1TEST2");
        }

        {
            Path newSrcsFilePath = exportTimestamp(inputFilePath, 3);
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag3"));
            assertThat(info.get(0), containsString("type KEY"));

            verifyFileAndFileContents(newSrcsFilePath, "test2.txt", "TEST1TEST2");
            verifyFileAndFileContents(newSrcsFilePath, "subdir/test3.txt", "TEST3");
        }

        {
            Path newSrcsFilePath = exportTag(inputFilePath, "tag12");
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag12"));
            assertThat(info.get(0), containsString("type KEY"));
            assertThat(info.get(1), containsString("tag tag12"));
            assertThat(info.get(1), containsString("type PATCH"));
            assertThat(info.get(2), containsString("tag tag12"));

            verifyFileAndFileContents(newSrcsFilePath, "test2.txt", "TEST1TEST2");
        }

        {
            Path newSrcsFilePath = exportTag(inputFilePath, "tag3");
            assertTrue(newSrcsFilePath.toFile().exists());

            List<String> info = listInfoFrom(newSrcsFilePath);

            assertThat(info.size(), is(greaterThan(0)));
            assertThat(info.get(0), containsString("tag tag3"));
            assertThat(info.get(0), containsString("type KEY"));
            
            verifyFileAndFileContents(newSrcsFilePath, "test2.txt", "TEST1TEST2");
        }
    }

    private void verifyFileAndFileContents(Path newSrcsFilePath, String fileName, String fileContent) throws IOException {
        Path outputDir = folder.newFolder().toPath();
        ToGitConverter toGitConverter = new ToGitConverter(
                newSrcsFilePath,
                outputDir,
                System.out::println,
                STOP_ON_ERRORS);

        try {
            toGitConverter.convert();
            Path targetFile = concatenate(outputDir, "/" + fileName);
            assertThat(String.format("File %s does not exists or incorrect path/filename", fileName),
                    Files.exists(targetFile),
                    is(true));
            assertThat(String.format("File %s does not contain the expected string: %s", fileName, fileContent),
                    FileUtils.readFileToString(targetFile.toFile(), StandardCharsets.UTF_8),
                    is(fileContent));
        } catch (Exception e) {
            fail("Failed to convert the srcs file due to: " + e.getMessage());
        }
    }

    private Path concatenate(Path outputDir, String fileName) {
        return Paths.get(outputDir.toString() + fileName);
    }

    private List<String> listInfoFrom(Path newFilePath) {
        ListCommand listCommand = new ListCommand();
        listCommand.inputFilePath = newFilePath.toString();
        listCommand.run();
        return listCommand.getGatheredInfo();
    }

    private Path exportTimestamp(Path inputFilePath, int timestamp) throws IOException {
        Path filePath = folder.newFile().toPath();
        ExportSegmentsCommand exportSegmentsCommand = new ExportSegmentsCommand(inputFilePath.toString(), filePath.toString(), timestamp);
        exportSegmentsCommand.run();
        return filePath;
    }

    private Path exportTag(Path inputFilePath, String tag) throws IOException {
        Path filePath = folder.newFile().toPath();
        ExportSegmentsCommand exportSegmentsCommand = new ExportSegmentsCommand(inputFilePath.toString(), filePath.toString(), tag);
        exportSegmentsCommand.run();
        return filePath;
    }
}

{'fraction_non_alphanumeric': 0.1286865308604439, 'fraction_numerical': 0.013377926421404682, 'mean_word_length': 2.8191582002902758, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5712058', 'n_tokens_mistral': 3629, 'n_tokens_neox': 3575, 'n_words': 693}