<commit_before>
package info.freelibrary.util;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.custommonkey.xmlunit.XMLUnit;
import org.junit.BeforeClass;
import org.junit.Test;

import nu.xom.Attribute;
import nu.xom.Builder;
import nu.xom.Document;
import nu.xom.Element;
import nu.xom.Elements;
import nu.xom.tests.XOMTestCase;

/**
 * Tests for the {@link FileUtils} class.
 *
 * @author <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class FileUtilsTest {

    private final File TMP_DIR = new File(System.getProperty("java.io.tmpdir"));

    private static Document EXPECTED;

    /**
     * Sets up a directory structure used for testing.
     */
    @BeforeClass
    public static void setUpBeforeClass() {
        final SimpleDateFormat formatter = new SimpleDateFormat(FileUtils.DATE_FORMAT);
        final File tf0 = new File("src/test/resources/test_folder");
        final File tf1 = new File(tf0, "test_folder");
        final File tf1_1 = new File(tf1, "test_file1.txt");
        final File tf1_2 = new File(tf1, "test_file2.txt");
        final File tf2 = new File(tf0, "test_folder2");
        final File tf2_tf1 = new File(tf2, "test_folder");
        final File tf2_tf1_1 = new File(tf2_tf1, "test_file1.txt");
        final File tf2_1 = new File(tf2, "test_file1.txt");
        final File tf0_1 = new File(tf0, "test_file1.txt");

        final Element root = new Element("dir");
        root.addAttribute(new Attribute("path", tf0.getAbsolutePath()));
        root.addAttribute(new Attribute("modified", formatter.format(new Date(tf0.lastModified()))));
        root.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1 = new Element("dir");
        tFolder1.addAttribute(new Attribute("path", tf1.getAbsolutePath()));
        tFolder1.addAttribute(new Attribute("modified", formatter.format(new Date(tf1.lastModified()))));
        tFolder1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1File1 = new Element("file");
        tFolder1File1.addAttribute(new Attribute("path", tf1_1.getAbsolutePath()));
        tFolder1File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf1_1.lastModified()))));
        tFolder1File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1File2 = new Element("file");
        tFolder1File2.addAttribute(new Attribute("path", tf1_2.getAbsolutePath()));
        tFolder1File2.addAttribute(new Attribute("modified", formatter.format(new Date(tf1_2.lastModified()))));
        tFolder1File2.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2 = new Element("dir");
        tFolder2.addAttribute(new Attribute("path", tf2.getAbsolutePath()));
        tFolder2.addAttribute(new Attribute("modified", formatter.format(new Date(tf2.lastModified()))));
        tFolder2.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2tFolder1 = new Element("dir");
        tFolder2tFolder1.addAttribute(new Attribute("path", tf2_tf1.getAbsolutePath()));
        tFolder2tFolder1.addAttribute(new Attribute("modified", formatter.format(new Date(tf2_tf1.lastModified()))));
        tFolder2tFolder1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2tFolder1File1 = new Element("file");
        final Date tf2_tf1_1_date = new Date(tf2_tf1_1.lastModified());
        tFolder2tFolder1File1.addAttribute(new Attribute("path", tf2_tf1_1.getAbsolutePath()));
        tFolder2tFolder1File1.addAttribute(new Attribute("modified", formatter.format(tf2_tf1_1_date)));
        tFolder2tFolder1File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2File1 = new Element("file");
        tFolder2File1.addAttribute(new Attribute("path", tf2_1.getAbsolutePath()));
        tFolder2File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf2_1.lastModified()))));
        tFolder2File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder0File1 = new Element("file");
        tFolder0File1.addAttribute(new Attribute("path", tf0_1.getAbsolutePath()));
        tFolder0File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf0_1.lastModified()))));
        tFolder0File1.addAttribute(new Attribute("permissions", "rw"));

        EXPECTED = new Document(root);
        root.appendChild(tFolder1);
        tFolder1.appendChild(tFolder1File1);
        tFolder1.appendChild(tFolder1File2);
        tFolder2tFolder1.appendChild(tFolder2tFolder1File1);
        tFolder2.appendChild(tFolder2tFolder1);
        tFolder2.appendChild(tFolder2File1);
        root.appendChild(tFolder2);
        root.appendChild(tFolder0File1);
    }

    @Test
    public void testConstructorIsPrivate() throws NoSuchMethodException, IllegalAccessException,
            InvocationTargetException, InstantiationException {
        final Constructor<FileUtils> constructor = FileUtils.class.getDeclaredConstructor();
        assertTrue("Constructor should be private", Modifier.isPrivate(constructor.getModifiers()));
        constructor.setAccessible(true);
        constructor.newInstance();
    }

    @Test
    public void testToHashMapFilePath() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")), map.toString());
    }

    @Test
    public void testToHashMapFilePathPatternAll() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder", ".*");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")), map.toString());
    }

    @Test
    public void testToHashMapFilePathPatternTxt() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder", ".*\\.txt");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")), map.toString());
    }

    @Test(expected = FileNotFoundException.class)
    public void testToElementStringStringBooleanBad() throws FileNotFoundException {
        FileUtils.toElement("not_found", ".*", true);
    }

    @Test(expected = MalformedURLException.class)
    public void testToFileURLBad() throws MalformedURLException {
        FileUtils.toFile(new URL("http://example.org"));
    }

    @Test
    public void testToFileURL() throws MalformedURLException {
        final String path = new File("target/test-classes/test_folder-map.txt").getAbsolutePath();
        final URL url = new URL("file://" + path);
        final File testFile = FileUtils.toFile(url);

        assertTrue("Failed to test: " + url, testFile.exists());
    }

    @Test(expected = FileNotFoundException.class)
    public void testListFilesFileFilenameFilterBooleanString() throws FileNotFoundException {
        FileUtils.listFiles(new File("doesnotexist"), new RegexFileFilter(".*"), true, new String[] {});
    }

    @Test
    public void testListFilesFileFilenameFilterBooleanStringFileTarget() throws FileNotFoundException {
        assertEquals(1, FileUtils.listFiles(new File("target/test-classes/test_folder-map.txt"), new RegexFileFilter(
                ".*"), false, new String[] {}).length);
    }<commit_msg>Update maps to be sorted for testing repeatability<commit_after>
package info.freelibrary.util;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.custommonkey.xmlunit.XMLUnit;
import org.junit.BeforeClass;
import org.junit.Test;

import nu.xom.Attribute;
import nu.xom.Builder;
import nu.xom.Document;
import nu.xom.Element;
import nu.xom.Elements;
import nu.xom.tests.XOMTestCase;

/**
 * Tests for the {@link FileUtils} class.
 *
 * @author <a href="mailto:<EMAIL>"><EMAIL></a>
 */
public class FileUtilsTest {

    private final File TMP_DIR = new File(System.getProperty("java.io.tmpdir"));

    private static Document EXPECTED;

    /**
     * Sets up a directory structure used for testing.
     */
    @BeforeClass
    public static void setUpBeforeClass() {
        final SimpleDateFormat formatter = new SimpleDateFormat(FileUtils.DATE_FORMAT);
        final File tf0 = new File("src/test/resources/test_folder");
        final File tf1 = new File(tf0, "test_folder");
        final File tf1_1 = new File(tf1, "test_file1.txt");
        final File tf1_2 = new File(tf1, "test_file2.txt");
        final File tf2 = new File(tf0, "test_folder2");
        final File tf2_tf1 = new File(tf2, "test_folder");
        final File tf2_tf1_1 = new File(tf2_tf1, "test_file1.txt");
        final File tf2_1 = new File(tf2, "test_file1.txt");
        final File tf0_1 = new File(tf0, "test_file1.txt");

        final Element root = new Element("dir");
        root.addAttribute(new Attribute("path", tf0.getAbsolutePath()));
        root.addAttribute(new Attribute("modified", formatter.format(new Date(tf0.lastModified()))));
        root.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1 = new Element("dir");
        tFolder1.addAttribute(new Attribute("path", tf1.getAbsolutePath()));
        tFolder1.addAttribute(new Attribute("modified", formatter.format(new Date(tf1.lastModified()))));
        tFolder1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1File1 = new Element("file");
        tFolder1File1.addAttribute(new Attribute("path", tf1_1.getAbsolutePath()));
        tFolder1File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf1_1.lastModified()))));
        tFolder1File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder1File2 = new Element("file");
        tFolder1File2.addAttribute(new Attribute("path", tf1_2.getAbsolutePath()));
        tFolder1File2.addAttribute(new Attribute("modified", formatter.format(new Date(tf1_2.lastModified()))));
        tFolder1File2.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2 = new Element("dir");
        tFolder2.addAttribute(new Attribute("path", tf2.getAbsolutePath()));
        tFolder2.addAttribute(new Attribute("modified", formatter.format(new Date(tf2.lastModified()))));
        tFolder2.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2tFolder1 = new Element("dir");
        tFolder2tFolder1.addAttribute(new Attribute("path", tf2_tf1.getAbsolutePath()));
        tFolder2tFolder1.addAttribute(new Attribute("modified", formatter.format(new Date(tf2_tf1.lastModified()))));
        tFolder2tFolder1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2tFolder1File1 = new Element("file");
        final Date tf2_tf1_1_date = new Date(tf2_tf1_1.lastModified());
        tFolder2tFolder1File1.addAttribute(new Attribute("path", tf2_tf1_1.getAbsolutePath()));
        tFolder2tFolder1File1.addAttribute(new Attribute("modified", formatter.format(tf2_tf1_1_date)));
        tFolder2tFolder1File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder2File1 = new Element("file");
        tFolder2File1.addAttribute(new Attribute("path", tf2_1.getAbsolutePath()));
        tFolder2File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf2_1.lastModified()))));
        tFolder2File1.addAttribute(new Attribute("permissions", "rw"));

        final Element tFolder0File1 = new Element("file");
        tFolder0File1.addAttribute(new Attribute("path", tf0_1.getAbsolutePath()));
        tFolder0File1.addAttribute(new Attribute("modified", formatter.format(new Date(tf0_1.lastModified()))));
        tFolder0File1.addAttribute(new Attribute("permissions", "rw"));

        EXPECTED = new Document(root);
        root.appendChild(tFolder1);
        tFolder1.appendChild(tFolder1File1);
        tFolder1.appendChild(tFolder1File2);
        tFolder2tFolder1.appendChild(tFolder2tFolder1File1);
        tFolder2.appendChild(tFolder2tFolder1);
        tFolder2.appendChild(tFolder2File1);
        root.appendChild(tFolder2);
        root.appendChild(tFolder0File1);
    }

    @Test
    public void testConstructorIsPrivate() throws NoSuchMethodException, IllegalAccessException,
            InvocationTargetException, InstantiationException {
        final Constructor<FileUtils> constructor = FileUtils.class.getDeclaredConstructor();
        assertTrue("Constructor should be private", Modifier.isPrivate(constructor.getModifiers()));
        constructor.setAccessible(true);
        constructor.newInstance();
    }

    @Test
    public void testToHashMapFilePath() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")),
                new TreeMap<String, List<String>>(map).toString());
    }

    @Test
    public void testToHashMapFilePathPatternAll() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder", ".*");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")),
                new TreeMap<String, List<String>>(map).toString());
    }

    @Test
    public void testToHashMapFilePathPatternTxt() throws IOException {
        final Map<String, List<String>> map = FileUtils.toHashMap("src/test/resources/test_folder", ".*\\.txt");

        assertEquals(StringUtils.read(new File("target/test-classes/test_folder-map.txt")),
                new TreeMap<String, List<String>>(map).toString());
    }

    @Test(expected = FileNotFoundException.class)
    public void testToElementStringStringBooleanBad() throws FileNotFoundException {
        FileUtils.toElement("not_found", ".*", true);
    }

    @Test(expected = MalformedURLException.class)
    public void testToFileURLBad() throws MalformedURLException {
        FileUtils.toFile(new URL("http://example.org"));
    }

    @Test
    public void testToFileURL() throws MalformedURLException {
        final String path = new File("target/test-classes/test_folder-map.txt").getAbsolutePath();
        final URL url = new URL("file://" + path);
        final File testFile = FileUtils.toFile(url);

        assertTrue("Failed to test: " + url, testFile.exists());
    }

    @Test(expected = FileNotFoundException.class)
    public void testListFilesFileFilenameFilterBooleanString() throws FileNotFoundException {
        FileUtils.listFiles(new File("doesnotexist"), new RegexFileFilter(".*"), true, new String[] {});
    }

    @Test
    public void testListFilesFileFilenameFilterBooleanStringFileTarget() throws FileNotFoundException {
        assertEquals(1, FileUtils.listFiles(new File("target/test-classes/test_folder-map.txt"), new RegexFileFilter(
                ".*"), false, new String[] {}).length);
    }
{'fraction_non_alphanumeric': 0.12511013215859032, 'fraction_numerical': 0.019005663939584645, 'mean_word_length': 4.152723735408561, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1458069', 'n_tokens_mistral': 4584, 'n_tokens_neox': 4547, 'n_words': 881}