// Path: src/main/java/org/oscii/concordance/AlignedCorpus.java
// public abstract class AlignedCorpus {
//     static ParallelFiles paths(String path, String sourceLanguage, String targetLanguage) {
//         Function<String, Path> p = ext -> Paths.get(String.format("%s.%s-%s.%s", path, sourceLanguage, targetLanguage, ext));
//         return new ParallelFiles(p.apply(sourceLanguage), p.apply(targetLanguage), p.apply("align"));
//     }
// 
//     public boolean exists(String path, String sourceLanguage, String targetLanguage) {
//         return paths(path, sourceLanguage, targetLanguage).stream().allMatch(Files::exists);
//     }
// 
//     /*
//      * Read parallel files.
//      */
//     public abstract void read(String path, String sourceLanguage, String targetLanguage, int max) throws IOException;
// 
//     /*
//      * Return a function that takes phrases in another language and returns translation frequencies.
//      */
//     public abstract Function<Expression, Double> translationFrequencies(Expression source);
// 
//     /*
//      * Score a meaning for the purpose of ranking.
//      */
//     public void scoreMeaning(Meaning m) {};
// 
//     // Always return 0.0
//     static Double zeroFrequency(Expression e) {
//         return 0.0;
//     }
// 
//     // Return count / total
//     Function<Expression, Double> normalizeByLanguage(final Map<String, Map<String, Long>> counts) {
//         final Map<String, Long> totals = sumCounts(counts);
//         return target -> {
//             Map<String, Long> byTarget = counts.get(target.language);
//             Long count = byTarget == null ? null : byTarget.get(target.text);
//             if (count != null && count > 0) {
//                 long total = totals.get(target.language);
//                 return 1.0 * count / total;
//             }
//             return 0.0;
//         };
//     }
// 
//     /*
//      * Sum counts for each word by target language.
//      */
//     private Map<String, Long> sumCounts(Map<String, Map<String, Long>> counts) {
//         Function<Map<String, Long>, Long> sumValues =
//                 c -> c.values().stream().mapToLong(x -> x).sum();
//         return mapValues(counts, sumValues);
//     }
// 
//     /*
//      * Map values of a map, maintaining keys.
//      */
//     static <K, T, U> Map<K, U> mapValues(Map<K, T> m, Function<T, U> f) {
//         return m.entrySet().stream().collect(toMap(
//                 Map.Entry::getKey,
//                 kv -> f.apply(kv.getValue()),
//                 (a, b) -> a,
//                 THashMap::new));
//     }
// 
//     /*
//      * Return examples for a phrase translated into a target language.
//      */
//     public List<SentenceExample> examples(String query, String source, String target, int max, int memoryId, boolean bLimit) {
//       return examples(query, source, target, null, max, memoryId, bLimit);
//     }
// 
//     
//     /*
//      * Return examples for a phrase translated into a target language.
//      */
//     public abstract List<SentenceExample> examples(String query, String source, String target, String systemId, int max, int memoryId, boolean bLimit);
// 
//     public void tally() {
//     }
// 
//     static class ParallelFiles {
//         Path sourceSentences;
//         Path targetSentences;
//         Path alignments;
// 
//         ParallelFiles(Path sourceSentences, Path targetSentences, Path alignments) {
//             this.sourceSentences = sourceSentences;
//             this.targetSentences = targetSentences;
//             this.alignments = alignments;
//         }
// 
//         Stream<Path> stream() {
//             return Arrays.asList(new Path[]{sourceSentences, targetSentences, alignments}).stream();
//         }
//     }
// }


import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import gnu.trove.THashMap;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Strings;
import org.oscii.concordance.AlignedCorpus;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
        if (!index.containsKey(language)) {
            return Collections.EMPTY_LIST;
        }
        return index.get(language).get(expression.degraded_text).get(expression).meanings;
    }

    public static String degrade(String query) {
      Objects.requireNonNull(query);
      // TODO(denero) Unicode normalize, remove non-alpha, & normalize diacritics
      String[] tokens = query.trim().split("\\s+");
      String newQuery = String.join(" ", tokens).toLowerCase();
      return newQuery;
    }

    // Prefer to return translations with parts of speech
    private static Translation pickTranslation(List<Translation> ts) {
        return ts.stream()
                .filter(t -> !t.pos.isEmpty()).findFirst()
                .orElse(ts.iterator().next());
    }

    public List<Translation> translate(String query, String source, String target) {
        return translate(query, source, target, null);
    }

    public List<Translation> translate(String query, String source, String target, AlignedCorpus corpus) {
        List<Meaning> all = lookup(query, source);
        if (corpus != null) {
            all.forEach(m -> setTranslationFrequencies(m, corpus));
        }
// Path: src/main/java/org/oscii/concordance/AlignedCorpus.java
// public abstract class AlignedCorpus {
//     static ParallelFiles paths(String path, String sourceLanguage, String targetLanguage) {
//         Function<String, Path> p = ext -> Paths.get(String.format("%s.%s-%s.%s", path, sourceLanguage, targetLanguage, ext));
//         return new ParallelFiles(p.apply(sourceLanguage), p.apply(targetLanguage), p.apply("align"));
//     }
// 
//     public boolean exists(String path, String sourceLanguage, String targetLanguage) {
//         return paths(path, sourceLanguage, targetLanguage).stream().allMatch(Files::exists);
//     }
// 
//     /*
//      * Read parallel files.
//      */
//     public abstract void read(String path, String sourceLanguage, String targetLanguage, int max) throws IOException;
// 
//     /*
//      * Return a function that takes phrases in another language and returns translation frequencies.
//      */
//     public abstract Function<Expression, Double> translationFrequencies(Expression source);
// 
//     /*
//      * Score a meaning for the purpose of ranking.
//      */
//     public void scoreMeaning(Meaning m) {};
// 
//     // Always return 0.0
//     static Double zeroFrequency(Expression e) {
//         return 0.0;
//     }
// 
//     // Return count / total
//     Function<Expression, Double> normalizeByLanguage(final Map<String, Map<String, Long>> counts) {
//         final Map<String, Long> totals = sumCounts(counts);
//         return target -> {
//             Map<String, Long> byTarget = counts.get(target.language);
//             Long count = byTarget == null ? null : byTarget.get(target.text);
//             if (count != null && count > 0) {
//                 long total = totals.get(target.language);
//                 return 1.0 * count / total;
//             }
//             return 0.0;
//         };
//     }
// 
//     /*
//      * Sum counts for each word by target language.
//      */
//     private Map<String, Long> sumCounts(Map<String, Map<String, Long>> counts) {
//         Function<Map<String, Long>, Long> sumValues =
//                 c -> c.values().stream().mapToLong(x -> x).sum();
//         return mapValues(counts, sumValues);
//     }
// 
//     /*
//      * Map values of a map, maintaining keys.
//      */
//     static <K, T, U> Map<K, U> mapValues(Map<K, T> m, Function<T, U> f) {
//         return m.entrySet().stream().collect(toMap(
//                 Map.Entry::getKey,
//                 kv -> f.apply(kv.getValue()),
//                 (a, b) -> a,
//                 THashMap::new));
//     }
// 
//     /*
//      * Return examples for a phrase translated into a target language.
//      */
//     public List<SentenceExample> examples(String query, String source, String target, int max, int memoryId, boolean bLimit) {
//       return examples(query, source, target, null, max, memoryId, bLimit);
//     }
// 
//     
//     /*
//      * Return examples for a phrase translated into a target language.
//      */
//     public abstract List<SentenceExample> examples(String query, String source, String target, String systemId, int max, int memoryId, boolean bLimit);
// 
//     public void tally() {
//     }
// 
//     static class ParallelFiles {
//         Path sourceSentences;
//         Path targetSentences;
//         Path alignments;
// 
//         ParallelFiles(Path sourceSentences, Path targetSentences, Path alignments) {
//             this.sourceSentences = sourceSentences;
//             this.targetSentences = targetSentences;
//             this.alignments = alignments;
//         }
// 
//         Stream<Path> stream() {
//             return Arrays.asList(new Path[]{sourceSentences, targetSentences, alignments}).stream();
//         }
//     }
// }


// Path: src/main/java/org/oscii/lex/Lexicon.java
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import gnu.trove.THashMap;
import org.apache.commons.collections4.trie.PatriciaTrie;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Strings;
import org.oscii.concordance.AlignedCorpus;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Stream;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;

        if (!index.containsKey(language)) {
            return Collections.EMPTY_LIST;
        }
        return index.get(language).get(expression.degraded_text).get(expression).meanings;
    }

    public static String degrade(String query) {
      Objects.requireNonNull(query);
      // TODO(denero) Unicode normalize, remove non-alpha, & normalize diacritics
      String[] tokens = query.trim().split("\\s+");
      String newQuery = String.join(" ", tokens).toLowerCase();
      return newQuery;
    }

    // Prefer to return translations with parts of speech
    private static Translation pickTranslation(List<Translation> ts) {
        return ts.stream()
                .filter(t -> !t.pos.isEmpty()).findFirst()
                .orElse(ts.iterator().next());
    }

    public List<Translation> translate(String query, String source, String target) {
        return translate(query, source, target, null);
    }

    public List<Translation> translate(String query, String source, String target, AlignedCorpus corpus) {
        List<Meaning> all = lookup(query, source);
        if (corpus != null) {
            all.forEach(m -> setTranslationFrequencies(m, corpus));
        }
        List<Translation> translations = all.stream()