<commit_before>package org.searsia.index;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.logging.ConsoleHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

import org.searsia.Hit;
import org.searsia.SearchResult;
import org.searsia.index.SearchResultIndex;

/**
 *  Tests Searsia's hits  index
 */
public class SearchResultIndexTest {

    private static final Logger LOGGER = Logger.getLogger("org.searsia");
    private static final String PATH  = "target/index-test";
    private static final String INDEX = "test";
    private static SearchResultIndex index;
      
    @BeforeClass
    public static void setUp() throws Exception {
    	Handler[] handlers = LOGGER.getHandlers();
    	for(Handler handler : handlers) {
    	  LOGGER.removeHandler(handler);
    	}
    	LOGGER.addHandler(new ConsoleHandler());
    	LOGGER.setLevel(Level.SEVERE);
    	index = new SearchResultIndex(PATH, INDEX, 10);
        SearchResult result = readFile("exampleSearchResult.json");
        index.offer(result);
        index.flush();
    }


    private static SearchResult readFile(String fileString) throws IOException {
        SearchResult result = new SearchResult();       
        String s, jsonString = "";       // TODO: Does the following file name work in Windows?
        BufferedReader reader = new BufferedReader(new FileReader("src/test/resources/" + fileString)); 
        try {
            while ((s = reader.readLine()) != null) {
            	jsonString += s;
            }
        }
        finally {
            reader.close();
        }
        JSONObject json = new JSONObject(jsonString);
        JSONArray hits = json.getJSONArray("hits");<commit_msg>fixes UTF-8 test<commit_after>package org.searsia.index;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.logging.ConsoleHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

import org.searsia.Hit;
import org.searsia.SearchResult;
import org.searsia.index.SearchResultIndex;

/**
 *  Tests Searsia's hits  index
 */
public class SearchResultIndexTest {

    private static final Logger LOGGER = Logger.getLogger("org.searsia");
    private static final String PATH  = "target/index-test";
    private static final String INDEX = "test";
    private static SearchResultIndex index;
      
    @BeforeClass
    public static void setUp() throws Exception {
    	Handler[] handlers = LOGGER.getHandlers();
    	for(Handler handler : handlers) {
    	  LOGGER.removeHandler(handler);
    	}
    	LOGGER.addHandler(new ConsoleHandler());
    	LOGGER.setLevel(Level.SEVERE);
    	index = new SearchResultIndex(PATH, INDEX, 10);
        SearchResult result = readFile("exampleSearchResult.json");
        index.offer(result);
        index.flush();
    }


    private static SearchResult readFile(String fileString) throws IOException {
        SearchResult result = new SearchResult();       
        String s, jsonString = "";       // TODO: Does the following file name work in Windows?
        BufferedReader reader = new BufferedReader(new FileReader("src/test/resources/" + fileString, StandardCharsets.UTF_8));
        try {
            while ((s = reader.readLine()) != null) {
            	jsonString += s;
            }
        }
        finally {
            reader.close();
        }
        JSONObject json = new JSONObject(jsonString);
        JSONArray hits = json.getJSONArray("hits");
{'fraction_non_alphanumeric': 0.1034394250513347, 'fraction_numerical': 0.001540041067761807, 'mean_word_length': 3.0934873949579833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2012000', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1099, 'n_words': 270}