<commit_before>package no.deichman.services.search;

import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import org.apache.jena.query.ResultSet;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.binding.Binding;

import java.io.Reader;
import java.text.Collator;
import java.text.ParseException;
import java.text.RuleBasedCollator;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newLinkedList;

/**
 * Responsibility: holds an alphabetical list of things with names.
 */
public class NameIndexer {
    private LinkedList<NameEntry> alphabeticalList;
    private Collator coll;
    private static final String SORTING_RULES = ""
            + ", '.' < a, á, à, ã, ä, A, Á, À, Ã, Ä < b, B < c, C < ð, Ð < d, D < e, é, è, ê, ë, E, É, È, Ê, Ë < f, F "
            + "< g, G < h, H < i, í, ï, I, Í, Ï"
            + "< j, J < k, K < l, L < m, M < n, ñ, N, Ñ < o, ó, ò, ô, O, Ó, Ò, Ô < p, P < q, Q < r, R"
            + "< s, S < t, T < u, U, ü, Ü < v, V < w, W < x, X < y, Y < z, Z"
            + "< \u00E6, \u00C6, ä, Ä"
            + "< \u00F8, \u00D8, ö, Ö"
            + "< \u00E5 = a\u030A,"
            + "  \u00C5 = A\u030A;"
            + "  aa, AA, Aa";

    public NameIndexer() {
        try {
            coll = new RuleBasedCollator(SORTING_RULES);
        } catch (ParseException e) {
            throw new RuntimeException(e);
        }
        coll.setStrength(Collator.PRIMARY);
        alphabeticalList = newLinkedList();
    }

    public NameIndexer(Reader reader) {
        this();
        GsonBuilder gsonBuilder = new GsonBuilder();
        JsonElement jsonElement = gsonBuilder.create().fromJson(reader, JsonElement.class);
        jsonElement.getAsJsonObject().get("results").getAsJsonObject().getAsJsonArray("bindings").forEach(e -> {<commit_msg>services: fix sorting rules for å and Aa DEICH-506<commit_after>package no.deichman.services.search;

import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import org.apache.jena.query.ResultSet;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.binding.Binding;

import java.io.Reader;
import java.text.Collator;
import java.text.ParseException;
import java.text.RuleBasedCollator;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Lists.newLinkedList;

/**
 * Responsibility: holds an alphabetical list of things with names.
 */
public class NameIndexer {
    private LinkedList<NameEntry> alphabeticalList;
    private Collator coll;
    private static final String SORTING_RULES = ""
            + ", '.' < a, á, à, ã, ä, A, Á, À, Ã, Ä < b, B < c, C < ð, Ð < d, D < e, é, è, ê, ë, E, É, È, Ê, Ë < f, F "
            + "< g, G < h, H < i, í, ï, I, Í, Ï"
            + "< j, J < k, K < l, L < m, M < n, ñ, N, Ñ < o, ó, ò, ô, O, Ó, Ò, Ô < p, P < q, Q < r, R"
            + "< s, S < t, T < u, U, ü, Ü < v, V < w, W < x, X < y, Y < z, Z"
            + "< æ, ä, Æ, Ä"
            + "< ø, ö, Ø, Ö"
            + "< å = a\u030A = aa, "
            + "  Å = A\u030A = Aa";

    public NameIndexer() {
        try {
            coll = new RuleBasedCollator(SORTING_RULES);
        } catch (ParseException e) {
            throw new RuntimeException(e);
        }
        coll.setStrength(Collator.PRIMARY);
        alphabeticalList = newLinkedList();
    }

    public NameIndexer(Reader reader) {
        this();
        GsonBuilder gsonBuilder = new GsonBuilder();
        JsonElement jsonElement = gsonBuilder.create().fromJson(reader, JsonElement.class);
        jsonElement.getAsJsonObject().get("results").getAsJsonObject().getAsJsonArray("bindings").forEach(e -> {
{'fraction_non_alphanumeric': 0.14494949494949494, 'fraction_numerical': 0.008333333333333333, 'mean_word_length': 2.5716862037871957, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '681049', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1412, 'n_words': 375}