<commit_before>package com.github.onsdigital.util;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;

import org.junit.Test;

import com.google.gson.JsonIOException;
import com.google.gson.JsonSyntaxException;

public class LoadIndexHelperTest {
	private final static String FILE_NAME = "index";
	private final static String FILE_EXTENSION = ".html";
	private final static String DATA_JSON_FILE_NAME = "data.json";
	private final static String CONTENT_TYPE = "bulletins";
	private final static String DELIMITTER = "/";
	private final static String RESOURCE_FILE_PATH = "src/main/resources/files";
	private final static String TAXONOMY_PATH = "/sample-taxonomy/";
	private final static String PRE_BULLETINS_PATH = RESOURCE_FILE_PATH + TAXONOMY_PATH;
	private final static String CONTENT_TYPE_TEST_FILE = PRE_BULLETINS_PATH + CONTENT_TYPE + DELIMITTER + FILE_NAME + FILE_EXTENSION;
	private final static String HOME_PAGE_TEST_FILE = Paths.get(PRE_BULLETINS_PATH + DATA_JSON_FILE_NAME).toAbsolutePath().toString();
	private final static String CONTENT_TYPE_URL = TAXONOMY_PATH + CONTENT_TYPE + DELIMITTER + FILE_NAME + FILE_EXTENSION;
	private final static String HOME_PAGE_URL = TAXONOMY_PATH;

	@Test
	public void testGetFileNames() throws IOException {
		List<String> fileNames = LoadIndexHelper.getAbsoluteFilePaths();
		assertFalse("Lookup should return some files", fileNames.isEmpty());
	}

	@Test
	public void testGetDocumentMapForContentType() throws JsonIOException, JsonSyntaxException, IOException {
		Map<String, String> documentMap = LoadIndexHelper.getDocumentMap(CONTENT_TYPE_TEST_FILE);
		assertEquals("url should math file structure", documentMap.get("url"), CONTENT_TYPE_URL);
		assertEquals("type should be bulletins", documentMap.get("type"), "bulletins");
		assertEquals("title should be data.json", documentMap.get("title"), "index.html");
		assertTrue("tags should contain subdirs", documentMap.get("tags").contains("sample-taxonomy"));
	}

	@Test
	public void testGetDocumentMapForHomePage() throws JsonIOException, JsonSyntaxException, IOException {
		Map<String, String> documentMap = LoadIndexHelper.getDocumentMap(HOME_PAGE_TEST_FILE);
		assertEquals("url should math file structure", documentMap.get("url"), HOME_PAGE_URL);
		assertEquals("type should be bulletins", documentMap.get("type"), "home");
		assertEquals("title should be uksectoraccounts", documentMap.get("title"), "Government, Public Sector and Taxes");
		assertFalse("tags should not contain content type subdir", documentMap.get("tags").contains("CONTENT_TYPE"));
	}
}<commit_msg>Tweaks to get more info on why tests are failing on Travis.<commit_after>package com.github.onsdigital.util;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;

import org.junit.Test;

import com.google.gson.JsonIOException;
import com.google.gson.JsonSyntaxException;

public class LoadIndexHelperTest {
	private final static String FILE_NAME = "index";
	private final static String FILE_EXTENSION = ".html";
	private final static String DATA_JSON_FILE_NAME = "data.json";
	private final static String CONTENT_TYPE = "bulletins";
	private final static String DELIMITTER = "/";
	private final static String RESOURCE_FILE_PATH = "src/main/resources/files";
	private final static String TAXONOMY_PATH = "/sample-taxonomy/";
	private final static String PRE_BULLETINS_PATH = RESOURCE_FILE_PATH
			+ TAXONOMY_PATH;
	private final static String CONTENT_TYPE_TEST_FILE = PRE_BULLETINS_PATH
			+ CONTENT_TYPE + DELIMITTER + FILE_NAME + FILE_EXTENSION;
	private final static String HOME_PAGE_TEST_FILE = Paths
			.get(PRE_BULLETINS_PATH + DATA_JSON_FILE_NAME).toAbsolutePath()
			.toString();
	private final static String CONTENT_TYPE_URL = TAXONOMY_PATH + CONTENT_TYPE
			+ DELIMITTER + FILE_NAME + FILE_EXTENSION;
	private final static String HOME_PAGE_URL = TAXONOMY_PATH;

	@Test
	public void testGetFileNames() throws IOException {
		List<String> fileNames = LoadIndexHelper.getAbsoluteFilePaths();
		System.out.println("Tut siise is: " + fileNames.size());
		assertFalse("Lookup should return some files", fileNames.isEmpty());
	}

	@Test
	public void testGetDocumentMapForContentType() throws JsonIOException,
			JsonSyntaxException, IOException {
		Map<String, String> documentMap = LoadIndexHelper
				.getDocumentMap(CONTENT_TYPE_TEST_FILE);
		assertEquals("url should math file structure", documentMap.get("url"),
				CONTENT_TYPE_URL);
		assertEquals("type should be bulletins", documentMap.get("type"),
				"bulletins");
		assertEquals("title should be data.json", documentMap.get("title"),
				"index.html");
		assertTrue("tags should contain subdirs", documentMap.get("tags")
				.contains("sample-taxonomy"));
	}

	@Test
	public void testGetDocumentMapForHomePage() throws JsonIOException,
			JsonSyntaxException, IOException {
		Map<String, String> documentMap = LoadIndexHelper
				.getDocumentMap(HOME_PAGE_TEST_FILE);
		assertEquals("url should math file structure", documentMap.get("url"),
				HOME_PAGE_URL);
		assertEquals("type should be bulletins", documentMap.get("type"),
				"home");
		assertEquals("title should be uksectoraccounts",
				documentMap.get("title"), "Government, Public Sector and Taxes");
		assertFalse("tags should not contain content type subdir", documentMap
				.get("tags").contains("CONTENT_TYPE"));
	}
}
{'fraction_non_alphanumeric': 0.10003516174402251, 'fraction_numerical': 0.0, 'mean_word_length': 6.247133757961784, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5711774', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1647, 'n_words': 427}