<commit_before>import org.junit.Test;

import java.io.*;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toList;

// TODO: This isn't really about unit tests, just a way of driving out some parsing and output of data
public class PetitionDataParserTest {
    @Test
    public void showOrderedByCountryAndOrderedByConstituency() throws IOException {
//        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json");
        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("prevent-trump-state-visit-data.json");

        String jsonString;
        try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {
            jsonString = buffer.lines().collect(Collectors.joining("\n"));
        }

        PetitionDataParser parser = new PetitionDataParser();
        PetitionData petitionData = parser.parse(jsonString);

        Map<String, Integer> countByCountry = petitionData.getCountByCountry();

        List<Map.Entry<String, Integer>> sorted = countByCountry.entrySet().stream().sorted(
                Comparator.comparing(Map.Entry::getValue)
        ).collect(toList());

        System.out.println("Countries:\n");
        sorted.forEach(
                e -> System.out.println(e.getKey() + ": " + e.getValue())
        );

        Map<String, Integer> countByConstituency = petitionData.getCountByConstituency();

        List<Map.Entry<String, Integer>> sortedByConstituency = countByConstituency.entrySet().stream().sorted(
                Comparator.comparing(Map.Entry::getValue)
        ).collect(toList());
        System.out.println("\n\n");

        System.out.println("Constituencies: ");
        sortedByConstituency.forEach(
                e -> System.out.println(e.getKey() + ": " + e.getValue())
        );

        System.out.println("Total count:  " + petitionData.calculateTotal());
    }

    @Test
    public void testDeltaByCountry() throws Exception {
        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json");
        InputStream streamBots = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data-including-bots.json");

        PetitionData countsByCountry = dataFromStream(stream);

        PetitionData countsByCountryWithBots = dataFromStream(streamBots);

        DataDeltaChecker checker = new DataDeltaChecker();
        checker.spotTheCountryDifferences(countsByCountryWithBots, countsByCountry);
    }

    @Test
    public void testDeltaByConstituency() throws Exception {
        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json");
        InputStream streamBots = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data-including-bots.json");

        PetitionData data = dataFromStream(stream);

        PetitionData dataWithBots = dataFromStream(streamBots);

        DataDeltaChecker checker = new DataDeltaChecker();
        checker.spotTheConstituencyyDifferences(data, dataWithBots);
    }

    private PetitionData dataFromStream(InputStream stream) throws IOException {
        String jsonString;
        try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {
            jsonString = buffer.lines().collect(Collectors.joining("\n"));
        }

        PetitionDataParser parser = new PetitionDataParser();
        return parser.parse(jsonString);
    }
}<commit_msg>Tweaking test code to clean up resources.<commit_after>import org.junit.Test;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toList;

// TODO: This isn't really about unit tests, just a way of driving out some parsing and output of data
public class PetitionDataParserTest {
    @Test
    public void showOrderedByCountryAndOrderedByConstituency() throws IOException {
//        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json");
        InputStream stream = PetitionDataParserTest.class.getResourceAsStream("prevent-trump-state-visit-data.json");

        String jsonString;
        try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
            jsonString = buffer.lines().collect(Collectors.joining("\n"));
        }

        PetitionDataParser parser = new PetitionDataParser();
        PetitionData petitionData = parser.parse(jsonString);

        Map<String, Integer> countByCountry = petitionData.getCountByCountry();

        List<Map.Entry<String, Integer>> sorted = countByCountry.entrySet().stream().sorted(
                Comparator.comparing(Map.Entry::getValue)
        ).collect(toList());

        System.out.println("Countries:\n");
        sorted.forEach(
                e -> System.out.println(e.getKey() + ": " + e.getValue())
        );

        Map<String, Integer> countByConstituency = petitionData.getCountByConstituency();

        List<Map.Entry<String, Integer>> sortedByConstituency = countByConstituency.entrySet().stream().sorted(
                Comparator.comparing(Map.Entry::getValue)
        ).collect(toList());
        System.out.println("\n\n");

        System.out.println("Constituencies: ");
        sortedByConstituency.forEach(
                e -> System.out.println(e.getKey() + ": " + e.getValue())
        );

        System.out.println("Total count:  " + petitionData.calculateTotal());
    }

    @Test
    public void testDeltaByCountry() throws Exception {
        PetitionData countsByCountry = null;
        try (InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json")) {
            countsByCountry = dataFromStream(stream);
        }

        PetitionData countsByCountryWithBots = null;
        try (InputStream streamBots = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data-including-bots.json")) {
            countsByCountryWithBots = dataFromStream(streamBots);
        }

        DataDeltaChecker checker = new DataDeltaChecker();
        checker.spotTheCountryDifferences(countsByCountryWithBots, countsByCountry);
    }

    @Test
    public void testDeltaByConstituency() throws Exception {
        PetitionData data = null;
        try (InputStream stream = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data.json")) {
            data = dataFromStream(stream);
        }

        PetitionData dataWithBots = null;
        try (InputStream streamBots = PetitionDataParserTest.class.getResourceAsStream("2nd-referendum-data-including-bots.json")) {
            dataWithBots = dataFromStream(streamBots);
        }

        DataDeltaChecker checker = new DataDeltaChecker();
        checker.spotTheConstituencyyDifferences(data, dataWithBots);
    }

    private PetitionData dataFromStream(InputStream stream) throws IOException {
        String jsonString;
        try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
            jsonString = buffer.lines().collect(Collectors.joining("\n"));
        }

        PetitionDataParser parser = new PetitionDataParser();
        return parser.parse(jsonString);
    }
}
{'fraction_non_alphanumeric': 0.10560923242082662, 'fraction_numerical': 0.001610305958132045, 'mean_word_length': 3.386698057680989, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551691', 'n_tokens_mistral': 2048, 'n_tokens_neox': 1989, 'n_words': 362}