<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- 

#
# Copyright 2013, 2014, 2016 <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# interactive curses lexicon editor
#

import os
import sys
import logging
import traceback
import curses
import curses.textpad
import locale
import codecs

from optparse import OptionParser

import utils
from speech_transcripts import Transcripts
from speech_lexicon import Lexicon, ipa2xsampa, xsampa2ipa
from speech_tokenizer import tokenize
from speech_sequitur import sequitur_gen_ipa
from tts_client import TTSClient

#
# Lex Editor
#

def lex_paint_main():

    global stdscr, lex_token, lex, lex_entry, lex_tokens, lex_cur_token, lex_gen

    stdscr.clear()

    my, mx = stdscr.getmaxyx()

    for x in range(mx):
        stdscr.insstr(   0, x, ' ', curses.A_REVERSE)
        stdscr.insstr(my-2, x, ' ', curses.A_REVERSE)
        stdscr.insstr(my-1, x, ' ', curses.A_REVERSE)

    stdscr.insstr(0,     0, "%3d/%3d" % (lex_cur_token, len(lex_tokens)), curses.A_REVERSE )
    stdscr.insstr(0, mx-15, "Lexicon Editor", curses.A_REVERSE )

    stdscr.insstr(4, 2, ("Token       : %s" % lex_token).encode('utf8'))
    stdscr.insstr(5, 2, ("IPA         : %s" % lex_entry['ipa']).encode('utf8'))

    cy = 6
    for engine in sorted(lex_gen):
        stdscr.insstr(cy, 2, ("%-11s : %s" % (engine, lex_gen[engine])).encode('utf8'))
        cy += 1

    cy += 1

    if lex_token in lex:

        m = lex.get_multi(lex_token)
        for k in m:
            stdscr.insstr(cy, 4, ("%s [%s]" % (k, m[k]['ipa'])).encode('utf8'))
            cy += 1

    else:
        stdscr.insstr(10, 4, "NEW TOKEN")

    stdscr.insstr(my-2, 0, "SPEAK  P:de-unitsel  O:de-hsmm                   I:fr-hsmm   U:en-hsmm", curses.A_REVERSE )
    stdscr.insstr(my-1, 0, "GEN    G:de-mary     H:de-espeak  J:de-sequitur  K:fr-mary   L:en-mary", curses.A_REVERSE )
    stdscr.insstr(my-2, mx-40, "                                 N:Next ", curses.A_REVERSE )
    stdscr.insstr(my-1, mx-40, "           E:Edit  T:Token       Q:Quit ", curses.A_REVERSE )
    stdscr.refresh()

def lex_gen_ipa (locale, engine, voice, speak=False):

    global tts

    if engine == 'sequitur':
        ipas = sequitur_gen_ipa (lex_base)
    
    else:
        tts.set_locale (locale)
        tts.set_engine (engine)
        tts.set_voice  (voice)
        ipas = tts.gen_ipa (lex_base)

    if speak:
        tts.set_locale ('de')
        tts.set_engine ('mary')
        tts.set_voice  ('bits3')
        tts.say_ipa(ipas)

    return ipas


def lex_set_token(token):

    global lex, lex_token, lex_entry, lex_base, lex_gen

    tts.set_locale ('de')
    tts.set_engine ('mary')
    tts.set_voice ('bits3')

    lex_token = token
    lex_base  = token.split('_')[0]

    if lex_token in lex:

        lex_entry = lex[lex_token]

    else:

        ipas = tts.gen_ipa (lex_base)
        lex_entry = {'ipa': ipas}
        lex[lex_token] = lex_entry

    ipas = lex_entry['ipa']
    tts.say_ipa(ipas)

    lex_gen['de-mary']     = lex_gen_ipa('de', 'mary',     'bits3')
    lex_gen['de-espeak']   = lex_gen_ipa('de', 'espeak',   'de')
    lex_gen['de-sequitur'] = lex_gen_ipa('de', 'sequitur', 'de')


logging.basicConfig(level=logging.DEBUG)
# logging.basicConfig(level=logging.INFO)

#
# command line
#<commit_msg>make sequitur default ipa generator for new entries<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- 

#
# Copyright 2013, 2014, 2016 <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
# 
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# interactive curses lexicon editor
#

import os
import sys
import logging
import traceback
import curses
import curses.textpad
import locale
import codecs

from optparse import OptionParser

import utils
from speech_transcripts import Transcripts
from speech_lexicon import Lexicon, ipa2xsampa, xsampa2ipa
from speech_tokenizer import tokenize
from speech_sequitur import sequitur_gen_ipa
from tts_client import TTSClient

#
# Lex Editor
#

def lex_paint_main():

    global stdscr, lex_token, lex, lex_entry, lex_tokens, lex_cur_token, lex_gen

    stdscr.clear()

    my, mx = stdscr.getmaxyx()

    for x in range(mx):
        stdscr.insstr(   0, x, ' ', curses.A_REVERSE)
        stdscr.insstr(my-2, x, ' ', curses.A_REVERSE)
        stdscr.insstr(my-1, x, ' ', curses.A_REVERSE)

    stdscr.insstr(0,     0, "%3d/%3d" % (lex_cur_token, len(lex_tokens)), curses.A_REVERSE )
    stdscr.insstr(0, mx-15, "Lexicon Editor", curses.A_REVERSE )

    stdscr.insstr(4, 2, ("Token       : %s" % lex_token).encode('utf8'))
    stdscr.insstr(5, 2, ("IPA         : %s" % lex_entry['ipa']).encode('utf8'))

    cy = 6
    for engine in sorted(lex_gen):
        stdscr.insstr(cy, 2, ("%-11s : %s" % (engine, lex_gen[engine])).encode('utf8'))
        cy += 1

    cy += 1

    if lex_token in lex:

        m = lex.get_multi(lex_token)
        for k in m:
            stdscr.insstr(cy, 4, ("%s [%s]" % (k, m[k]['ipa'])).encode('utf8'))
            cy += 1

    else:
        stdscr.insstr(10, 4, "NEW TOKEN")

    stdscr.insstr(my-2, 0, "SPEAK  P:de-unitsel  O:de-hsmm                   I:fr-hsmm   U:en-hsmm", curses.A_REVERSE )
    stdscr.insstr(my-1, 0, "GEN    G:de-mary     H:de-espeak  J:de-sequitur  K:fr-mary   L:en-mary", curses.A_REVERSE )
    stdscr.insstr(my-2, mx-40, "                                 N:Next ", curses.A_REVERSE )
    stdscr.insstr(my-1, mx-40, "           E:Edit  T:Token       Q:Quit ", curses.A_REVERSE )
    stdscr.refresh()

def lex_gen_ipa (locale, engine, voice, speak=False):

    global tts

    if engine == 'sequitur':
        ipas = sequitur_gen_ipa (lex_base)
    
    else:
        tts.set_locale (locale)
        tts.set_engine (engine)
        tts.set_voice  (voice)
        ipas = tts.gen_ipa (lex_base)

    if speak:
        tts.set_locale ('de')
        tts.set_engine ('mary')
        tts.set_voice  ('bits3')
        tts.say_ipa(ipas)

    return ipas


def lex_set_token(token):

    global lex, lex_token, lex_entry, lex_base, lex_gen

    lex_token = token
    lex_base  = token.split('_')[0]

    if lex_token in lex:

        lex_entry = lex[lex_token]

    else:

        ipas = lex_gen_ipa('de', 'sequitur', 'de')
        lex_entry = {'ipa': ipas}
        lex[lex_token] = lex_entry


    ipas = lex_entry['ipa']

    tts.set_locale ('de')
    tts.set_engine ('mary')
    tts.set_voice ('bits3')
    tts.say_ipa(ipas)

    lex_gen['de-mary']     = lex_gen_ipa('de', 'mary',     'bits3')
    lex_gen['de-espeak']   = lex_gen_ipa('de', 'espeak',   'de')
    lex_gen['de-sequitur'] = lex_gen_ipa('de', 'sequitur', 'de')


logging.basicConfig(level=logging.DEBUG)
# logging.basicConfig(level=logging.INFO)

#
# command line
#
{'fraction_non_alphanumeric': 0.12572656052565073, 'fraction_numerical': 0.01491028556987617, 'mean_word_length': 2.83664566165778, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2836832', 'n_tokens_mistral': 3052, 'n_tokens_neox': 2867, 'n_words': 736}