<commit_before>package Lingua::LO::NLP::Romanize::IPA;
use strict;
use warnings;
use 5.012000;
use utf8;
use feature qw/ unicode_strings say /;
use charnames qw/ :full lao /;
use version 0.77; our $VERSION = version->declare('v1.0.0');
use Carp;
use Lingua::LO::NLP::Analyze;
use parent 'Lingua::LO::NLP::Romanize::PCGN';

=encoding utf8

=head1 NAME

Lingua::LO::NLP::Romanize::IPA - Convert Lao syllables to the International Phonetic Alphabet

=head1 FUNCTION

This class is not supposed to be used directly. Rather use
L<Lingua::LO::NLP::Romanize> as a factory:

    my $o = Lingua::LO::NLP::Romanize->new(variant => 'IPA');

=cut

my %CONSONANTS = (
   'ກ'  => 'k', # TODO: split /g/+/k/ here?
   'ຂ'  => [qw/ kʰ k /],
   'ຄ'  => [qw/ kʰ k /],
   'ງ'  => 'ŋ',
   'ຈ'  => [qw/ tɕ t /],
   'ສ'  => [qw/ s t /],
   'ຊ'  => [qw/ s t /],
   'ຍ'  => 'ɲ',
   'ດ'  => [qw/ d t /],
   'ຕ'  => 't',
   'ຖ'  => [qw/ tʰ t /],
   'ທ'  => [qw/ tʰ t /],
   'ນ'  => 'n',
   'ບ'  => [qw/ b p /],
   'ປ'  => 'p',
   'ຜ'  => 'pʰ',
   'ຝ'  => [qw/ f p /],
   'ພ'  => [qw/ pʰ p /],
   'ຟ'  => [qw/ f p /],
   'ມ'  => 'm',
   'ຢ'  => 'j',
   'ລ'  => [qw/ l n /],
   "\N{LAO SEMIVOWEL SIGN LO}"  => 'l',
   'ວ'  => [qw/ ʋ w /],
   'ຫ'  => 'h',
   'ອ'  => 'ʔ',
   'ຮ'  => 'h',
   'ຣ'  => [qw/ r n /],   # TODO l?
   'ໜ'  => 'n',
   'ໝ'  => 'm',
   'ຫຼ'  => 'l',
   'ຫຍ' => 'ɲ',
   'ຫນ' => 'n',
   'ຫມ' => 'm',
   'ຫຣ' => 'r',   # TODO l?
   'ຫລ' => 'l',
   'ຫວ' => 'w', # TODO ʋ?
);

my %VOWELS = (
    ### Monophthongs
    'Xະ'   => 'a',
    'Xັ'    => 'a',
    'Xາ'   => 'aː',
    'Xາວ'  => 'aːo',

    'Xິ'    => 'i',
    'Xີ'    => 'iː',
    'Xິວ'   => 'iu', # TODO correct?
    'Xີວ'   => 'iːu', # TODO correct?

    'Xຶ'    => 'ɯ',
    'Xື'    => 'ɯː',

    'Xຸ'    => 'u',
    'Xູ'    => 'uː',

    'ເXະ'  => 'e',
    'ເXັ'   => 'e',
    'ເX'   => 'eː',

    'ແXະ'  => 'ɛ',
    'ແXັ'   => 'ɛ',
    'ແX'   => 'ɛː',
    'ແXວ'  => 'ɛːo',

    'ໂXະ'  => 'o',
    'Xົ'    => 'o',
    'ໂX'   => 'oː',
    'ໂXຍ'  => 'oːi', # TODO correct?
    'Xອຍ'  => 'oːi',

    'ເXາະ' => 'ɔ',
    'Xັອ'   => 'ɔ',
    'Xໍ'    => 'ɔː',
    'Xອ'   => 'ɔː',

    'ເXິ'   => 'ɤ',
    'ເXີ'   => 'ɤː',
    'ເXື'   => 'ɯːə', # TODO correct?

    'ເXັຍ'  => 'iə',
    'Xັຽ'   => 'iə',
    'ເXຍ'  => 'iːə',
    'Xຽ'   => 'iːə',
    'Xຽວ'  => 'iːəo', # TODO correct?

    'ເXີຍ'  => 'ɤːi',
    'ເXິຍ'  => 'ɤi',

    'ເXຶອ'  => 'ɯə',
    'ເXືອ'  => 'ɯːə',
    'ເXືອຍ' => 'ɯːəi',

    'Xົວະ'  => 'uə',
    'Xົວ'   => 'uːə',
    'Xວ'   => 'uːə',
    'Xວຍ'  => 'uːəi',

    'ໄX'   => 'aj',
    'ໃX'   => 'aj',
    'Xາຍ'  => 'aːj',
    'Xັຍ'   => 'aj',
<commit_msg>add glottal stop symbol in IPA for ະ<commit_after>package Lingua::LO::NLP::Romanize::IPA;
use strict;
use warnings;
use 5.012000;
use utf8;
use feature qw/ unicode_strings say /;
use charnames qw/ :full lao /;
use version 0.77; our $VERSION = version->declare('v1.0.0');
use Carp;
use Lingua::LO::NLP::Analyze;
use parent 'Lingua::LO::NLP::Romanize::PCGN';

=encoding utf8

=head1 NAME

Lingua::LO::NLP::Romanize::IPA - Convert Lao syllables to the International Phonetic Alphabet

=head1 FUNCTION

This class is not supposed to be used directly. Rather use
L<Lingua::LO::NLP::Romanize> as a factory:

    my $o = Lingua::LO::NLP::Romanize->new(variant => 'IPA');

=cut

my %CONSONANTS = (
   'ກ'  => 'k', # TODO: split /g/+/k/ here?
   'ຂ'  => [qw/ kʰ k /],
   'ຄ'  => [qw/ kʰ k /],
   'ງ'  => 'ŋ',
   'ຈ'  => [qw/ tɕ t /],
   'ສ'  => [qw/ s t /],
   'ຊ'  => [qw/ s t /],
   'ຍ'  => 'ɲ',
   'ດ'  => [qw/ d t /],
   'ຕ'  => 't',
   'ຖ'  => [qw/ tʰ t /],
   'ທ'  => [qw/ tʰ t /],
   'ນ'  => 'n',
   'ບ'  => [qw/ b p /],
   'ປ'  => 'p',
   'ຜ'  => 'pʰ',
   'ຝ'  => [qw/ f p /],
   'ພ'  => [qw/ pʰ p /],
   'ຟ'  => [qw/ f p /],
   'ມ'  => 'm',
   'ຢ'  => 'j',
   'ລ'  => [qw/ l n /],
   "\N{LAO SEMIVOWEL SIGN LO}"  => 'l',
   'ວ'  => [qw/ ʋ w /],
   'ຫ'  => 'h',
   'ອ'  => 'ʔ',
   'ຮ'  => 'h',
   'ຣ'  => [qw/ r n /],   # TODO l?
   'ໜ'  => 'n',
   'ໝ'  => 'm',
   'ຫຼ'  => 'l',
   'ຫຍ' => 'ɲ',
   'ຫນ' => 'n',
   'ຫມ' => 'm',
   'ຫຣ' => 'r',   # TODO l?
   'ຫລ' => 'l',
   'ຫວ' => 'w', # TODO ʋ?
);

my %VOWELS = (
    ### Monophthongs
    'Xະ'   => 'aʔ',
    'Xັ'    => 'a',
    'Xາ'   => 'aː',
    'Xາວ'  => 'aːo',

    'Xິ'    => 'i',
    'Xີ'    => 'iː',
    'Xິວ'   => 'iu', # TODO correct?
    'Xີວ'   => 'iːu', # TODO correct?

    'Xຶ'    => 'ɯ',
    'Xື'    => 'ɯː',

    'Xຸ'    => 'u',
    'Xູ'    => 'uː',

    'ເXະ'  => 'eʔ',
    'ເXັ'   => 'e',
    'ເX'   => 'eː',

    'ແXະ'  => 'ɛʔ',
    'ແXັ'   => 'ɛ',
    'ແX'   => 'ɛː',
    'ແXວ'  => 'ɛːo',

    'ໂXະ'  => 'oʔ',
    'Xົ'    => 'o',
    'ໂX'   => 'oː',
    'ໂXຍ'  => 'oːi', # TODO correct?
    'Xອຍ'  => 'oːi',

    'ເXາະ' => 'ɔʔ',
    'Xັອ'   => 'ɔ',
    'Xໍ'    => 'ɔː',
    'Xອ'   => 'ɔː',

    'ເXິ'   => 'ɤ',
    'ເXີ'   => 'ɤː',
    'ເXື'   => 'ɯːə', # TODO correct?

    'ເXັຍ'  => 'iə',
    'Xັຽ'   => 'iə',
    'ເXຍ'  => 'iːə',
    'Xຽ'   => 'iːə',
    'Xຽວ'  => 'iːəo', # TODO correct?

    'ເXີຍ'  => 'ɤːi',
    'ເXິຍ'  => 'ɤi',

    'ເXຶອ'  => 'ɯə',
    'ເXືອ'  => 'ɯːə',
    'ເXືອຍ' => 'ɯːəi',

    'Xົວະ'  => 'uəʔ',
    'Xົວ'   => 'uːə',
    'Xວ'   => 'uːə',
    'Xວຍ'  => 'uːəi',

    'ໄX'   => 'aj',
    'ໃX'   => 'aj',
    'Xາຍ'  => 'aːj',
    'Xັຍ'   => 'aj',

{'fraction_non_alphanumeric': 0.3016144349477683, 'fraction_numerical': 0.006457739791073124, 'mean_word_length': 1.8342303552206674, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '907920', 'n_tokens_mistral': 3408, 'n_tokens_neox': 3265, 'n_words': 603}