<commit_before># Sinhala pronunciation rules.
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# First, delete ZWNJ and ZWJ to simplify further processing.
\u200C → ;
\u200D → ;

# Insert a schwa after every consonant that is not followed by a dependent vowel
# or virama.
::Null;
([\u0D9A-\u0DC6]) } [^\u0DCA-\u0DDF] → $1 ə;

# Pronunciation rules proper.
::Null;

# fප is an alternative spelling of ෆ.
# This occurs e.g. in ඩේවිඩ් කොපර්fපීල්ඩ් (<NAME>)
# [see http://bradshawofthefuture.blogspot.com/2013/02/f.html].
[Ff]ප → f;

# zස is seemingly the only way to unambiguously indicate a voiced /z/ sound.
# This occurs in e.g. ඇල්zසයිම' රෝගය (Alzheimer's disease)
# [see https://si.wikipedia.org/wiki/ඇල්zසයිම%27_රෝගය]
# or in zසීබ්‍රා (zebra) [see https://si.wikipedia.org/wiki/‍zසීබ්‍රා].
[Zz]ස → z;

ං → ŋ;
ඃ → h;

අ → a;
ආ → aː;
ඇ → æ;
ඈ → æː;
ඉ → i;
ඊ → iː;
උ → u;
ඌ → uː;
ඍ → ri;
ඎ → ruː;
ඏ → ilu;
ඐ → iluː;
එ → e;
ඒ → eː;
ඓ → ai̯;
ඔ → o;
ඕ → oː;
ඖ → ou̯;

ක → k;
ඛ → k;
ග → g;
ඝ → g;
ඞ → ŋ;
ඟ → ⁿg;
ච → c;
ඡ → c;
ජ → ɟ;
ඣ → ɟ;
ඤ → ɲ;
ඥ → ɟɲ;  # TODO: check if this is correct
ඦ → ɲɟ;  # TODO: check if this is correct
ට → ʈ;
ඨ → ʈ;
ඩ → ɖ;
ඪ → ɖ;
ණ → n;
ඬ → ⁿɖ;
ත → t;
ථ → t;
ද → d;
ධ → d;
න → n;
ඳ → ⁿd;
ප → p;
ඵ → p;
බ → b;
භ → b;
ම → m;
ඹ → ᵐb;
ය → j;
ර → r;
ල → l;
ව → w;
ශ → ʃ;
ෂ → ʃ;
ස → s;
හ → h;
ළ → l;
ෆ → f;

\u0DCA → ;  # delete virama

ා → aː;
ැ → æ;
ෑ → æː;
\u0DD2 → i;
\u0DD3 → iː;
\u0DD4 → u;
\u0DD6 → uː;
ෘ → ru;
ෙ → e;
ේ → eː;
ෛ → ai̯;
ො → o;
ෝ → oː;
ෞ → ou̯;
ෟ → lu;
ෲ → ruː;
ෳ → luː;

# Heuristics for turning /ə/ into /a/.
# Based on Wasala, Weerasinghe & Gamage, COLING/ACL 2006.

$c=[k g ŋ {ⁿg} c ɟ ɲ ʈ ɖ {ⁿɖ} t d n {ⁿd} p b m {ᵐb} j r l w ʃ s z h f];

# Rule #1
::Null;
[$] sv    { ə       → ə;  # exception (a)
[$] k     { ə } r   → ə;  # exception (b)
[$] $c    { ə } [$] → ə;  # exception (c)
[$] $c $c { ə       → a;
[$] $c    { ə       → a;

# Rule #2
::Null;
$c r { ə } $c → a;  # clause (a) and (b)
$c r { a } h  → a;  # clause (d), exception
$c r { a } $c → ə;  # clause (c)

# Rule #3
# The paper is unclear about what this rule means. The interpretation here
# assumes that "preceded" in the paper is a typo and should be read "followed".
::Null;
[a e æ o ə] h { ə → a;

# Rules #4 through #7
::Null;
    ə } $c $c      → a;  # Rule #4
    ə } [rbɖʈ] [$] → ə;  # Rule #5 exception
    ə } $c     [$] → a;  # Rule #5
    ə } ji     [$] → a;  # Rule #6
k { ə } [rl] u     → a;  # Rule #7

# Rule #8
# Note that the paper doesn't say explicitly that this rule should be
# anchored at the beginning of a word, but the remarks before the rules
# seem to imply this.
::Null;
[$] k { a } l[aeo]ːj   → ə;  # Typo in paper: /j/ was /y/.
[$] k { a } le[mh][ui] → ə;
[$] k { alə } h[ui]    → əle;
[$] k { a } lə         → ə;

# Diphthongs
::Null;
[i {iː} e {eː} æ {æː} o {oː} a {aː}] { wu → u̯;
[u {uː} e {eː} æ {æː} o {oː} a {aː}] { ji → i̯;<commit_msg>Generalize word boundaries in Sinhala pronunciation rules.<commit_after># Sinhala pronunciation rules.
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# First, delete ZWNJ and ZWJ to simplify further processing.
\u200C → ;
\u200D → ;

# Insert a schwa after every consonant that is not followed by a dependent vowel
# or virama.
::Null;
([\u0D9A-\u0DC6]) } [^\u0DCA-\u0DDF] → $1 ə;

# Pronunciation rules proper.
::Null;

# fප is an alternative spelling of ෆ.
# This occurs e.g. in ඩේවිඩ් කොපර්fපීල්ඩ් (<NAME>)
# [see http://bradshawofthefuture.blogspot.com/2013/02/f.html].
[Ff]ප → f;

# zස is seemingly the only way to unambiguously indicate a voiced /z/ sound.
# This occurs in e.g. ඇල්zසයිම' රෝගය (Alzheimer's disease)
# [see https://si.wikipedia.org/wiki/ඇල්zසයිම%27_රෝගය]
# or in zසීබ්‍රා (zebra) [see https://si.wikipedia.org/wiki/‍zසීබ්‍රා].
[Zz]ස → z;

ං → ŋ;
ඃ → h;

අ → a;
ආ → aː;
ඇ → æ;
ඈ → æː;
ඉ → i;
ඊ → iː;
උ → u;
ඌ → uː;
ඍ → ri;
ඎ → ruː;
ඏ → ilu;
ඐ → iluː;
එ → e;
ඒ → eː;
ඓ → ai̯;
ඔ → o;
ඕ → oː;
ඖ → ou̯;

ක → k;
ඛ → k;
ග → g;
ඝ → g;
ඞ → ŋ;
ඟ → ⁿg;
ච → c;
ඡ → c;
ජ → ɟ;
ඣ → ɟ;
ඤ → ɲ;
ඥ → ɟɲ;  # TODO: check if this is correct
ඦ → ɲɟ;  # TODO: check if this is correct
ට → ʈ;
ඨ → ʈ;
ඩ → ɖ;
ඪ → ɖ;
ණ → n;
ඬ → ⁿɖ;
ත → t;
ථ → t;
ද → d;
ධ → d;
න → n;
ඳ → ⁿd;
ප → p;
ඵ → p;
බ → b;
භ → b;
ම → m;
ඹ → ᵐb;
ය → j;
ර → r;
ල → l;
ව → w;
ශ → ʃ;
ෂ → ʃ;
ස → s;
හ → h;
ළ → l;
ෆ → f;

\u0DCA → ;  # delete virama

ා → aː;
ැ → æ;
ෑ → æː;
\u0DD2 → i;
\u0DD3 → iː;
\u0DD4 → u;
\u0DD6 → uː;
ෘ → ru;
ෙ → e;
ේ → eː;
ෛ → ai̯;
ො → o;
ෝ → oː;
ෞ → ou̯;
ෟ → lu;
ෲ → ruː;
ෳ → luː;

# Heuristics for turning /ə/ into /a/.
# Based on Wasala, Weerasinghe & Gamage, COLING/ACL 2006.

$c=[k g ŋ {ⁿg} c ɟ ɲ ʈ ɖ {ⁿɖ} t d n {ⁿd} p b m {ᵐb} j r l w ʃ s z h f];

$s=[:^L:];

# Rule #1
::Null;
$s sv    { ə      → ə;  # exception (a)
$s k     { ə } r  → ə;  # exception (b)
$s $c    { ə } $s → ə;  # exception (c)
$s $c $c { ə      → a;
$s $c    { ə      → a;

# Rule #2
::Null;
$c r { ə } $c → a;  # clause (a) and (b)
$c r { a } h  → a;  # clause (d), exception
$c r { a } $c → ə;  # clause (c)

# Rule #3
# The paper is unclear about what this rule means. The interpretation here
# assumes that "preceded" in the paper is a typo and should be read "followed".
::Null;
[a e æ o ə] h { ə → a;

# Rules #4 through #7
::Null;
    ə } $c $c     → a;  # Rule #4
    ə } [rbɖʈ] $s → ə;  # Rule #5 exception
    ə } $c     $s → a;  # Rule #5
    ə } ji     $s → a;  # Rule #6
k { ə } [rl] u    → a;  # Rule #7

# Rule #8
# Note that the paper doesn't say explicitly that this rule should be
# anchored at the beginning of a word, but the remarks before the rules
# seem to imply this.
::Null;
$s k { a } l[aeo]ːj   → ə;  # Typo in paper: /j/ was /y/.
$s k { a } le[mh][ui] → ə;
$s k { alə } h[ui]    → əle;
$s k { a } lə         → ə;

# Diphthongs
::Null;
[i {iː} e {eː} æ {æː} o {oː} a {aː}] { wu → u̯;
[u {uː} e {eː} æ {æː} o {oː} a {aː}] { ji → i̯;
{'fraction_non_alphanumeric': 0.17848870874348582, 'fraction_numerical': 0.015344528083381586, 'mean_word_length': 2.4964574898785425, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313960', 'n_tokens_mistral': 3726, 'n_tokens_neox': 3863, 'n_words': 1405}