<commit_before>package morphology;

import zemberek.core.turkish.PrimaryPos;
import zemberek.core.turkish.RootAttribute;
import zemberek.core.turkish.SecondaryPos;
import zemberek.morphology.lexicon.DictionaryItem;
import zemberek.morphology.parser.MorphParse;
import zemberek.morphology.parser.tr.TurkishWordParserGenerator;

import java.io.IOException;
import java.util.List;


public class AddNewDictionaryItem {

    static void addProperNoun() throws IOException {
        TurkishWordParserGenerator parserGenerator = TurkishWordParserGenerator.createWithDefaults();
        String input = "Meydan'a";

        List<MorphParse> before = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " before adding lemma :");
        printResults(before);

        // This must be called. So that parser forgets the old results. Due to a small bug,
        // parserGenerator.invalidateCache(input); does not work.
        parserGenerator.invalidateAllCache();

        DictionaryItem item =
                new DictionaryItem("Meydan", "meydan", "meydan", PrimaryPos.Noun, SecondaryPos.ProperNoun);
        parserGenerator.getGraph().addDictionaryItem(item);

        List<MorphParse> after = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " after adding lemma :");
        printResults(after);
    }

    private static void addVerb() throws IOException {
        TurkishWordParserGenerator parserGenerator = TurkishWordParserGenerator.createWithDefaults();
        String input = "tweetleyeyazdım";
        List<MorphParse> before = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " before adding lemma `tweetlemek` = " + before);
        DictionaryItem item =
                new DictionaryItem("tweetlemek", "tweetle", "tivitle", PrimaryPos.Verb, SecondaryPos.None);
        parserGenerator.getGraph().addDictionaryItem(item);
        parserGenerator.invalidateCache(input);
        List<MorphParse> after = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " after adding lemma `tweetlemek` = " + after);
    }

    private static void printResults(List<MorphParse> before) {
        int i = 1;
        for (MorphParse morphParse : before) {
            String str = morphParse.formatLong();
            if (morphParse.dictionaryItem.attributes.contains(RootAttribute.Runtime)) {
                str = str + " (Generated by UnidentifiedTokenParser)";
            }
            System.out.println(i + " - " + str);
            i++;
        }
    }

    public static void main(String[] args) throws IOException {
        //addVerb();
        addProperNoun();
    }
}<commit_msg>add more examples for new dictionary items.<commit_after>package morphology;

import zemberek.core.turkish.PrimaryPos;
import zemberek.core.turkish.RootAttribute;
import zemberek.core.turkish.SecondaryPos;
import zemberek.morphology.lexicon.DictionaryItem;
import zemberek.morphology.parser.MorphParse;
import zemberek.morphology.parser.tr.TurkishWordParserGenerator;

import java.io.IOException;
import java.util.List;


public class AddNewDictionaryItem {

    TurkishWordParserGenerator parserGenerator;

    public AddNewDictionaryItem(TurkishWordParserGenerator parserGenerator) {
        this.parserGenerator = parserGenerator;
    }

    void addNewDictionaryItemTest(String input, DictionaryItem newItem) throws IOException {

        List<MorphParse> before = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " before adding lemma :");
        printResults(before);

        // This must be called. So that parser forgets the old results. Due to a small bug,
        // parserGenerator.invalidateCache(input); does not work for proper nouns.
        parserGenerator.invalidateAllCache();

        parserGenerator.getGraph().addDictionaryItem(newItem);

        List<MorphParse> after = parserGenerator.parse(input);
        System.out.println("Parses for " + input + " after adding lemma :");
        printResults(after);
    }

    private void printResults(List<MorphParse> before) {
        int i = 1;
        for (MorphParse morphParse : before) {
            String str = morphParse.formatLong();
            if (morphParse.dictionaryItem.attributes.contains(RootAttribute.Runtime)) {
                str = str + " (Generated by UnidentifiedTokenParser)";
            }
            System.out.println(i + " - " + str);
            i++;
        }
    }

    public static void main(String[] args) throws IOException {

        AddNewDictionaryItem app = new AddNewDictionaryItem(TurkishWordParserGenerator.createWithDefaults());

        System.out.println("Proper Noun Test - 1 :");
        app.addNewDictionaryItemTest("Meydan'a",
                new DictionaryItem("Meydan", "meydan", "meydan", PrimaryPos.Noun, SecondaryPos.ProperNoun));
        System.out.println();

        System.out.println("Proper Noun Test - 2 :");
        app.addNewDictionaryItemTest("Meeeydan'a",
                new DictionaryItem("Meeeydan", "meeeydan", "meeeydan", PrimaryPos.Noun, SecondaryPos.ProperNoun));
        System.out.println();

        System.out.println("Verb Test : ");
        app.addNewDictionaryItemTest("tweetleyeyazdım",
                new DictionaryItem("tweetlemek", "tweetle", "tivitle", PrimaryPos.Verb, SecondaryPos.None));

    }
}
{'fraction_non_alphanumeric': 0.10387683175663143, 'fraction_numerical': 0.0007419773696902244, 'mean_word_length': 3.186335403726708, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7053360', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1478, 'n_words': 314}