<commit_before>#!/usr/bin/perl -w
#
# Convert the Breton lexicon from apertium into a lexicon suitable for
# the LanguageTool grammar checker.
#
# LanguageTool POS tags for Breton are more or less similar to French tags to
# keep it simple.  This makes it easier to maintain grammar for # both French
# and Breton without too much to remember.
#
# POS tags for LanguageTool simplify POS tags present in apertium.
# Simpler POS tags make it easier to write regular expression in
# LanguageTool, but information can be lost in the conversion.
#
# How to use this script:
#
# 1) Download apertium Breton dictionary:
#    $ svn co https://apertium.svn.sourceforge.net/svnroot/apertium/trunk/apertium-br-fr
# 2) Install apertium tools:
#    $ sudo apt-get install lttoolbox
# 3) Run the script:
#    $ cd apertium-br-fr/
#    $ ./create-lexicon.pl
# 4) Use morfologik-stemming to create the LanguageTool dictionary:
#    See http://languagetool.wikidot.com/developing-a-tagger-dictionary
#    $ java -jar morfologik-stemming-nodict-1.4.0.jar tab2morph -i apertium-br-fr.br.dix-LT.txt -o output.txt
#    $ java -jar morfologik-stemming-nodict-1.4.0.jar fsa_build -i output.txt -o breton.dict
#
# Author: <NAME> <<EMAIL>>

use strict;

my $dic_in  = 'apertium-br-fr.br.dix';
my $dic_out = "$dic_in-LT.txt";
my $dic_err = "$dic_in-LT.err";

# List of plural masculine nouns of persons for which it matters to know
# whether they are persons or not for the mutation after articile "ar".
# Those are unfortunately not tagged in the Apertium dictionary.
# So we enhance tagging here to be able to detect some incorrect mutations
# after the article ar/an/al.
# Only plural words that have a first letter which can mutate need to
# be listed here. So "studerien" for example does not need to be
# listed for example since s.* word don't mutate.
# This list is far from being complete. The more words the more
# mutation errors can be detected. But missing words should not
# cause false positives.
# Case matters!
my @anv_lies_tud = (
  # plural              softening         reinforcing     spirant
  "baraerien",          "varaerien",      "paraerien",
  "barzhed",            "varzhed",        "parzhed",
  "beleien",            "veleien",        "peleien",
  "beajourien",         "veajourien",     "peajourien",
  "breudeur",           "vreudeur",       "preudeur",
  "bugale",             "vugale",         "pugale",
  "butuner",            "vutuner",        "putuner",
  "dañserien",                            "tañserien",
  "gwazed",             "wazed",          "kwazed",
  "Gallaoued",          "C’hallaoued",    "Kallaoued",
  "genaoueien",         "c’henaouien",    "kenaouien",
  "goved",              "c’hoved",        "koved",
  "gwazed",             "wazed",          "kwazed",
  "Gwenediz",           "Wenediz",        "Kwenediz",
  "gwerzherien",        "werzherien",     "kwerzherien",
  "gwiaderien",         "wiaderien",      "kwiaderien",
  "gwiaderion",         "wiaderion",      "kwiaderion",
  "kabitened",          "gabitened",      "c’habitened",
  "kamaraded",          "gamaraded",      "c’hamaraded",
  "kariaded",           "gariaded",       "c’hariaded",
  "kazetennerien",      "gazetennerien",  "c’hazetennerien",
  "keginerien",         "geginerien",     "c’heginerien",
  "kelennerien",        "gelennerien",    "c’helennerien",
  "kemenerien",         "gemenerien",     "c’hemenerien",
  "kenlabourerien",     "genlabourerien", "c’henlabourerien",
  "kenwezherien",       "genwezherien",   "c’henwezherien",
  "kendirvi",           "gendirvi",       "c’hendirvi",
  "kereon",             "gereon",         "c’hereon",
  "kigerien",           "gigerien",       "c’higerien",
  "klañvdiourien",      "glañvdiourien",  "c’hlañvdiourien",
  "konversañted",       "gonversañted",   "c’honversañted",
  "krennarded",         "grennarded",     "c’hrennarded",
  "kristenien",         "gristenien",     "c’hristenien",
  "kristenion",         "gristenion",     "c’hristenion",
  "maered",             "vaered",
  "marc’hadourien",     "varc’hadourien",
  "martoloded",         "vartoloded",
  "medisined",          "vedisined",
  "mevelien",           "vevelien",
  "mezerien",           "vezeien",
  "mibien",             "vibien",
  "micherourien",       "vicherourien",
  "mignoned",           "vignoned",
  "militaerion",        "vilitaerion",
  "milinerien",         "vilinerien",
  "milinerien",         "vilinerien",
  "milvezeien",         "vilvezeien",
  "mistri",             "vistri",
  "mistri-skol",        "vistri-skol",
  "paeroned",           "baeroned",                        "faeroned",
  "paotred",            "baotred",                         "faotred",
  "paotred-al-lizhiri", "baotred-al-lizhiri",              "faotred-al-lizhiri",
  "perc’henned",        "berc’henned",                     "ferc’henned",
  "perukennerien",      "berukennerien",                   "ferukennerien",
  "perukennerion",      "berukennerion",                   "ferukennerion",
  "personed",           "bersoned",                        "fersoned",
  "pesketaerien",       "besketaerien",                    "fesketaerien",
  "poliserien",         "boliserien",                      "foliserien",
  "priñsed",            "briñsed",                         "friñsed",
  "toerien",            "doerien",                         "zoerien",
  "touristed",          "douristed",                       "zouristed",
  "tredanerien",        "dredanerien",                     "zredanerien",
  "tredanerion",        "dredanerion",                     "zredanerion",
  "tud",                "dud",                             "zud",
);
my %anv_lies_tud = map { $_ => 0 } @anv_lies_tud;

open(LT_EXPAND, "lt-expand $dic_in |") or die "can't fork lt-expand: $!\n";
open(OUT, "> $dic_out") or die "can't open $dic_out: $!\n";
open(ERR, "> $dic_err") or die "can't open $dic_err: $!\n";<commit_msg>[br] fixed typo in script.<commit_after>#!/usr/bin/perl -w
#
# Convert the Breton lexicon from apertium into a lexicon suitable for
# the LanguageTool grammar checker.
#
# LanguageTool POS tags for Breton are more or less similar to French tags to
# keep it simple.  This makes it easier to maintain grammar for # both French
# and Breton without too much to remember.
#
# POS tags for LanguageTool simplify POS tags present in apertium.
# Simpler POS tags make it easier to write regular expression in
# LanguageTool, but information can be lost in the conversion.
#
# How to use this script:
#
# 1) Download apertium Breton dictionary:
#    $ svn co https://apertium.svn.sourceforge.net/svnroot/apertium/trunk/apertium-br-fr
# 2) Install apertium tools:
#    $ sudo apt-get install lttoolbox
# 3) Run the script:
#    $ cd apertium-br-fr/
#    $ ./create-lexicon.pl
# 4) Use morfologik-stemming to create the LanguageTool dictionary:
#    See http://languagetool.wikidot.com/developing-a-tagger-dictionary
#    $ java -jar morfologik-stemming-nodict-1.4.0.jar tab2morph -i apertium-br-fr.br.dix-LT.txt -o output.txt
#    $ java -jar morfologik-stemming-nodict-1.4.0.jar fsa_build -i output.txt -o breton.dict
#
# Author: <NAME> <<EMAIL>>

use strict;

my $dic_in  = 'apertium-br-fr.br.dix';
my $dic_out = "$dic_in-LT.txt";
my $dic_err = "$dic_in-LT.err";

# List of plural masculine nouns of persons for which it matters to know
# whether they are persons or not for the mutation after articile "ar".
# Those are unfortunately not tagged in the Apertium dictionary.
# So we enhance tagging here to be able to detect some incorrect mutations
# after the article ar/an/al.
# Only plural words that have a first letter which can mutate need to
# be listed here. So "studerien" for example does not need to be
# listed for example since s.* word don't mutate.
# This list is far from being complete. The more words the more
# mutation errors can be detected. But missing words should not
# cause false positives.
# Case matters!
my @anv_lies_tud = (
  # plural              softening         reinforcing     spirant
  "baraerien",          "varaerien",      "paraerien",
  "barzhed",            "varzhed",        "parzhed",
  "beleien",            "veleien",        "peleien",
  "beajourien",         "veajourien",     "peajourien",
  "breudeur",           "vreudeur",       "preudeur",
  "bugale",             "vugale",         "pugale",
  "butuner",            "vutuner",        "putuner",
  "dañserien",                            "tañserien",
  "gwazed",             "wazed",          "kwazed",
  "Gallaoued",          "C’hallaoued",    "Kallaoued",
  "genaoueien",         "c’henaouien",    "kenaouien",
  "goved",              "c’hoved",        "koved",
  "gwazed",             "wazed",          "kwazed",
  "Gwenediz",           "Wenediz",        "Kwenediz",
  "gwerzherien",        "werzherien",     "kwerzherien",
  "gwiaderien",         "wiaderien",      "kwiaderien",
  "gwiaderion",         "wiaderion",      "kwiaderion",
  "kabitened",          "gabitened",      "c’habitened",
  "kamaraded",          "gamaraded",      "c’hamaraded",
  "kariaded",           "gariaded",       "c’hariaded",
  "kazetennerien",      "gazetennerien",  "c’hazetennerien",
  "keginerien",         "geginerien",     "c’heginerien",
  "kelennerien",        "gelennerien",    "c’helennerien",
  "kemenerien",         "gemenerien",     "c’hemenerien",
  "kenlabourerien",     "genlabourerien", "c’henlabourerien",
  "kenwezherien",       "genwezherien",   "c’henwezherien",
  "kendirvi",           "gendirvi",       "c’hendirvi",
  "kereon",             "gereon",         "c’hereon",
  "kigerien",           "gigerien",       "c’higerien",
  "klañvdiourien",      "glañvdiourien",  "c’hlañvdiourien",
  "konversañted",       "gonversañted",   "c’honversañted",
  "krennarded",         "grennarded",     "c’hrennarded",
  "kristenien",         "gristenien",     "c’hristenien",
  "kristenion",         "gristenion",     "c’hristenion",
  "maered",             "vaered",
  "marc’hadourien",     "varc’hadourien",
  "martoloded",         "vartoloded",
  "medisined",          "vedisined",
  "mevelien",           "vevelien",
  "mezeien",            "vezeien",
  "mibien",             "vibien",
  "micherourien",       "vicherourien",
  "mignoned",           "vignoned",
  "militaerion",        "vilitaerion",
  "milinerien",         "vilinerien",
  "milinerien",         "vilinerien",
  "milvezeien",         "vilvezeien",
  "mistri",             "vistri",
  "mistri-skol",        "vistri-skol",
  "paeroned",           "baeroned",                        "faeroned",
  "paotred",            "baotred",                         "faotred",
  "paotred-al-lizhiri", "baotred-al-lizhiri",              "faotred-al-lizhiri",
  "perc’henned",        "berc’henned",                     "ferc’henned",
  "perukennerien",      "berukennerien",                   "ferukennerien",
  "perukennerion",      "berukennerion",                   "ferukennerion",
  "personed",           "bersoned",                        "fersoned",
  "pesketaerien",       "besketaerien",                    "fesketaerien",
  "poliserien",         "boliserien",                      "foliserien",
  "priñsed",            "briñsed",                         "friñsed",
  "toerien",            "doerien",                         "zoerien",
  "touristed",          "douristed",                       "zouristed",
  "tredanerien",        "dredanerien",                     "zredanerien",
  "tredanerion",        "dredanerion",                     "zredanerion",
  "tud",                "dud",                             "zud",
);
my %anv_lies_tud = map { $_ => 0 } @anv_lies_tud;

open(LT_EXPAND, "lt-expand $dic_in |") or die "can't fork lt-expand: $!\n";
open(OUT, "> $dic_out") or die "can't open $dic_out: $!\n";
open(ERR, "> $dic_err") or die "can't open $dic_err: $!\n";
{'fraction_non_alphanumeric': 0.13599864762065758, 'fraction_numerical': 0.002028569013608317, 'mean_word_length': 2.1746713174134693, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2887054', 'n_tokens_mistral': 4268, 'n_tokens_neox': 4122, 'n_words': 947}