<commit_before>package org.oscii.lex;

import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import gnu.trove.THashMap;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.concordance.AlignedCorpus;

import java.io.*;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;

/**
 * A map from expressions to meanings.
 */
public class Lexicon {
    // language -> degraded text -> matching expressions -> meanings
    private Map<String, PatriciaTrie<Map<Expression, Meanings>>> index = new PatriciaTrie<>();
    private final boolean skipIdentity;
    private int numRemoved = 0;

    private final static Logger log = LogManager.getLogger(Lexicon.class);

    public Lexicon() {
        this.skipIdentity = false;
    }

    public Lexicon(boolean skipIdentity) {
        this.skipIdentity = skipIdentity;
    }

    /* Construction */

    public void add(Meaning meaning) {
        if (meaning.translations.size() == 0 && meaning.definitions.size() == 0) {
            return;
        }

        if (skipIdentity) {
            // remove identity translations
            for (Iterator<Translation> iterator = meaning.translations.iterator(); iterator.hasNext(); ) {
                Translation t = iterator.next();
                if (t.translation.text.equals(meaning.expression.text)) {
                    iterator.remove();
                    ++numRemoved;
                }
            }
        }

        Expression expression = meaning.expression;
        if (!index.containsKey(expression.language)) {
            index.put(expression.language, new PatriciaTrie<>());
        }
        String key = expression.degraded_text;
        Map<Expression, Meanings> entries = index.get(expression.language).get(key);
        if (entries == null) {
            entries = new THashMap<>(1);
            index.get(expression.language).put(key, entries);
        }
        Meanings meanings = entries.get(expression);
        if (meanings == null) {
            meanings = new Meanings(expression);
            entries.put(expression, meanings);
        }
        meanings.add(meaning);
    }

    private void forEachMeanings(Consumer<Meanings> fn) {
        index.values().stream().forEach(trie -> trie.values().forEach(map -> map.values().forEach(fn)));
    }

    /*
     * Add translation frequency information from a corpus.
     */
    public void addFrequencies(AlignedCorpus corpus) {
        log.info("Computing translation frequencies");
        forEachMeanings(ms -> {
            ms.meanings.parallelStream().forEach(m -> setTranslationFrequencies(m, corpus));
            ms.meanings.sort(Order.byMaxTranslationFrequency);
        });
    }

    public void addScores(AlignedCorpus corpus) {<commit_msg>add lexicon merging<commit_after>package org.oscii.lex;

import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import gnu.trove.THashMap;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.concordance.AlignedCorpus;

import java.io.*;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;

import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;

/**
 * A map from expressions to meanings.
 */
public class Lexicon {
    // language -> degraded text -> matching expressions -> meanings
    private Map<String, PatriciaTrie<Map<Expression, Meanings>>> index = new PatriciaTrie<>();
    private final boolean skipIdentity;
    private int numRemoved = 0;

    private final static Logger log = LogManager.getLogger(Lexicon.class);

    public Lexicon() {
        this.skipIdentity = false;
    }

    public Lexicon(boolean skipIdentity) {
        this.skipIdentity = skipIdentity;
    }

    /* Construction */

    public void add(Meaning meaning) {
        if (meaning.translations.size() == 0 && meaning.definitions.size() == 0) {
            return;
        }

        if (skipIdentity) {
            // remove identity translations
            for (Iterator<Translation> iterator = meaning.translations.iterator(); iterator.hasNext(); ) {
                Translation t = iterator.next();
                if (t.translation.text.equals(meaning.expression.text)) {
                    iterator.remove();
                    ++numRemoved;
                }
            }
        }

        Expression expression = meaning.expression;
        if (!index.containsKey(expression.language)) {
            index.put(expression.language, new PatriciaTrie<>());
        }
        String key = expression.degraded_text;
        Map<Expression, Meanings> entries = index.get(expression.language).get(key);
        if (entries == null) {
            entries = new THashMap<>(1);
            index.get(expression.language).put(key, entries);
        }
        Meanings meanings = entries.get(expression);
        if (meanings == null) {
            meanings = new Meanings(expression);
            entries.put(expression, meanings);
        }
        meanings.add(meaning);
    }

    public void merge(Lexicon otherLex) {
        log.info("Merging lexicons");
        otherLex.forEachMeanings(ms -> ms.meanings.stream().forEach(m -> this.add(m)));
    }

    public void forEachMeanings(Consumer<Meanings> fn) {
        index.values().stream().forEach(trie -> trie.values().forEach(map -> map.values().forEach(fn)));
    }

    /*
     * Add translation frequency information from a corpus.
     */
    public void addFrequencies(AlignedCorpus corpus) {
        log.info("Computing translation frequencies");
        forEachMeanings(ms -> {
            ms.meanings.parallelStream().forEach(m -> setTranslationFrequencies(m, corpus));
            ms.meanings.sort(Order.byMaxTranslationFrequency);
        });
    }

    public void addScores(AlignedCorpus corpus) {
{'fraction_non_alphanumeric': 0.11777777777777777, 'fraction_numerical': 0.0022222222222222222, 'mean_word_length': 2.6485234510712217, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6258377', 'n_tokens_mistral': 1804, 'n_tokens_neox': 1700, 'n_words': 355}