<commit_before>#!/usr/bin/python
#
# Recognizes status dictation or free dictation speech files (select using the
# -l option.
#
# Arguments:
#   model-directory
#   speech-directory
#   output-file
#   -a acoustic-model [basename]
#   -l language-model [status | free]
#   -s language-model-scale
#
# Example:
#   rec_hammaspuhe.py /share/puhe/hammaspuhe/models \
#     /share/puhe/hammaspuhe/audio/free_complete output.csv \
#     -a speecon_test_ml_12.10.2011_20 -l free

import time
import string
import sys
import os
import subprocess
import re
import tempfile
import filecmp
import gzip
import math
from optparse import OptionParser, OptionValueError


# Set your decoder swig path in here!
sys.path.append(os.path.dirname(sys.argv[0]) + "/src/swig");
#sys.path.append('/home/jpylkkon/decoder_new/decoder/src/swig/');

import Decoder

##################################################
# From Wikibooks
# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance#Python
#

def levenshtein_l(s1, s2):
    if len(s1) < len(s2):
        return levenshtein_l(s2, s1)
    if not s1:
        return len(s2)

    previous_row = xrange(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
            deletions = current_row[j] + 1       # than s2
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row

    return previous_row[-1]

def levenshtein_w(s1, s2):
	words1 = s1.split(None)
	words2 = s2.split(None)

	count1 = len(words1)
	count2 = len(words2)

	if count1 < count2:
		return levenshtein_w(s2, s1)
	if not s1:
		return count2

	previous_row = xrange(count2 + 1)
	for i, w1 in enumerate(words1):
		current_row = [i + 1]
		for j, w2 in enumerate(words2):
			insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
			deletions = current_row[j] + 1       # than s2
			substitutions = previous_row[j] + (w1 != w2)
			current_row.append(min(insertions, deletions, substitutions))
		previous_row = current_row

	return previous_row[-1]

def natural_sorted(list):
	convert = lambda text: int(text) if text.isdigit() else text.lower()
	alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
	return sorted(list, key=alphanum_key)

def log(x):
	#return math.log(x)
	return math.log10(x)

def exp(x):
	#return math.exp(x)
	return math.pow(10, x)

def logprobsum(x, y):
	return y + log(1 + exp(x - y))

def logprobmul(x, y):
	return x + y

##################################################
# Initialize
#

parser = OptionParser()
parser.add_option('-a', '--acoustic-model',
				  action='store', type='string', dest='am', default='speecon_test_ml_12.10.2011_20')
parser.add_option('-l', '--language-model',
				  action='store', type='string', dest='lm', default='status')
parser.add_option('-s', '--language-model-scale',
				  action='store', type='int', dest='lm_scale', default=30)
parser.add_option('-b', '--beam',
				  action='store', type='int', dest='beam', default=300)

(options, args) = parser.parse_args()
if len(args) != 3:
	parser.error("incorrect number of arguments")

model_directory = args[0]
speech_directory = args[1]
output_file = args[2]

akupath = os.path.dirname(sys.argv[0]) + "/../aku"

ac_model = model_directory + "/" + options.am
hmms = ac_model + ".ph"
dur = ac_model + ".dur"

if options.lm == 'free':
	lexicon = model_directory + "/FreeDictationLM.lex"
	ngram = model_directory + "/FreeDictationLM.6gram.bin"
	lookahead_ngram = model_directory + "/FreeDictationLM.2gram.bin"
	morph_model = True
	generate_word_graph = False
elif options.lm == 'status':
	lexicon = model_directory + "/StatusDictationLM.lex"
	ngram = model_directory + "/StatusDictationLM.3gram.bin"
	lookahead_ngram = model_directory + "/StatusDictationLM.2gram.bin"
	morph_model = False
	generate_word_graph = True
else:
	raise OptionValueError("language-model has to be either 'free' or 'status'")

global_beam = options.beam
lm_scale = options.lm_scale

print 'AM:', ac_model
print 'Lexicon:', lexicon
print 'LM:', ngram
print 'Lookahead LM:', lookahead_ngram
print 'Generate word graph:', generate_word_graph
print 'Beam:', global_beam
print 'LM scale factor:', lm_scale


##################################################
# Generate LNA files for recognition
#

recipe_file = tempfile.NamedTemporaryFile()

for wav_file in os.listdir(speech_directory):
	if not wav_file.endswith('.wav'):
		continue;
	audio = speech_directory + "/" + wav_file
	lna = wav_file[:-4] + ".lna"
	if not os.path.exists(speech_directory + "/" + lna):
		recipe_file.write("audio=" + audio + " lna=" + lna + "\n")

# Don't close the temporary file yet, or it will get deleted.
recipe_file.flush()

print "Computing state probabilities."
command = [akupath + "/phone_probs", \
		"-b", ac_model, \
 		"-c", ac_model + ".cfg", \
 		"-r", recipe_file.name, \
		"-o", speech_directory]
#command = [akupath + "/phone_probs", \
#		"-b", ac_model, \
#		"-c", ac_model + ".cfg", \
#		"-r", recipe_file.name, \
#		"-C", ac_model + ".gcl", \
#		"-o", speech_directory, \
#		"--eval-ming", "0.50"]
try:
	result = subprocess.check_call(command)
except subprocess.CalledProcessError as e:
	print "phone_probs returned a non-zero exit status."
	sys.exit(-1)

recipe_file.close()


##################################################
# Recognize
#

t = Decoder.Toolbox()

t.select_decoder(0)

if morph_model:
	t.set_silence_is_word(1)
else:<commit_msg>Gaussian approximation.<commit_after>#!/usr/bin/python
#
# Recognizes status dictation or free dictation speech files (select using the
# -l option.
#
# Arguments:
#   model-directory
#   speech-directory
#   output-file
#   -a acoustic-model [basename]
#   -l language-model [status | free]
#   -s language-model-scale
#
# Example:
#   rec_hammaspuhe.py /share/puhe/hammaspuhe/models \
#     /share/puhe/hammaspuhe/audio/free_complete output.csv \
#     -a speecon_test_ml_12.10.2011_20 -l free

import time
import string
import sys
import os
import subprocess
import re
import tempfile
import filecmp
import gzip
import math
from optparse import OptionParser, OptionValueError


# Set your decoder swig path in here!
sys.path.append(os.path.dirname(sys.argv[0]) + "/src/swig");
#sys.path.append('/home/jpylkkon/decoder_new/decoder/src/swig/');

import Decoder

##################################################
# From Wikibooks
# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance#Python
#

def levenshtein_l(s1, s2):
    if len(s1) < len(s2):
        return levenshtein_l(s2, s1)
    if not s1:
        return len(s2)

    previous_row = xrange(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
            deletions = current_row[j] + 1       # than s2
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row

    return previous_row[-1]

def levenshtein_w(s1, s2):
	words1 = s1.split(None)
	words2 = s2.split(None)

	count1 = len(words1)
	count2 = len(words2)

	if count1 < count2:
		return levenshtein_w(s2, s1)
	if not s1:
		return count2

	previous_row = xrange(count2 + 1)
	for i, w1 in enumerate(words1):
		current_row = [i + 1]
		for j, w2 in enumerate(words2):
			insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
			deletions = current_row[j] + 1       # than s2
			substitutions = previous_row[j] + (w1 != w2)
			current_row.append(min(insertions, deletions, substitutions))
		previous_row = current_row

	return previous_row[-1]

def natural_sorted(list):
	convert = lambda text: int(text) if text.isdigit() else text.lower()
	alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
	return sorted(list, key=alphanum_key)

def log(x):
	#return math.log(x)
	return math.log10(x)

def exp(x):
	#return math.exp(x)
	return math.pow(10, x)

def logprobsum(x, y):
	return y + log(1 + exp(x - y))

def logprobmul(x, y):
	return x + y

##################################################
# Initialize
#

parser = OptionParser()
parser.add_option('-a', '--acoustic-model',
				  action='store', type='string', dest='am', default='speecon_test_ml_12.10.2011_20')
parser.add_option('-l', '--language-model',
				  action='store', type='string', dest='lm', default='status')
parser.add_option('-s', '--language-model-scale',
				  action='store', type='int', dest='lm_scale', default=30)
parser.add_option('-b', '--beam',
				  action='store', type='int', dest='beam', default=300)

(options, args) = parser.parse_args()
if len(args) != 3:
	parser.error("incorrect number of arguments")

model_directory = args[0]
speech_directory = args[1]
output_file = args[2]

akupath = os.path.dirname(sys.argv[0]) + "/../aku"
timepath = '/usr/bin/time'

ac_model = model_directory + "/" + options.am
hmms = ac_model + ".ph"
dur = ac_model + ".dur"

if options.lm == 'free':
	lexicon = model_directory + "/FreeDictationLM.lex"
	ngram = model_directory + "/FreeDictationLM.6gram.bin"
	lookahead_ngram = model_directory + "/FreeDictationLM.2gram.bin"
	morph_model = True
	generate_word_graph = False
elif options.lm == 'status':
	lexicon = model_directory + "/StatusDictationLM.lex"
	ngram = model_directory + "/StatusDictationLM.3gram.bin"
	lookahead_ngram = model_directory + "/StatusDictationLM.2gram.bin"
	morph_model = False
	generate_word_graph = True
else:
	raise OptionValueError("language-model has to be either 'free' or 'status'")

global_beam = options.beam
lm_scale = options.lm_scale

print 'AM:', ac_model
print 'Lexicon:', lexicon
print 'LM:', ngram
print 'Lookahead LM:', lookahead_ngram
print 'Generate word graph:', generate_word_graph
print 'Beam:', global_beam
print 'LM scale factor:', lm_scale


##################################################
# Generate LNA files for recognition
#

recipe_file = tempfile.NamedTemporaryFile()

for wav_file in os.listdir(speech_directory):
	if not wav_file.endswith('.wav'):
		continue;
	audio = speech_directory + "/" + wav_file
	lna = wav_file[:-4] + ".lna"
	if not os.path.exists(speech_directory + "/" + lna):
		recipe_file.write("audio=" + audio + " lna=" + lna + "\n")

# Don't close the temporary file yet, or it will get deleted.
recipe_file.flush()

print "Computing state probabilities."
if False:
	# No optimization.
	command = [timepath, \
			"-f", "CPU seconds used: %e", \
			akupath + "/phone_probs", \
			"-b", ac_model, \
			"-c", ac_model + ".cfg", \
			"-r", recipe_file.name, \
			"-o", speech_directory, \
			"-i", "1"]
else:
	# Approximate Gaussian by cluster centers.
	command = [timepath, \
			"-f", "CPU seconds used: %e", \
			akupath + "/phone_probs", \
			"-b", ac_model, \
			"-c", ac_model + ".cfg", \
			"-r", recipe_file.name, \
			"-C", ac_model + ".gcl", \
			"-o", speech_directory, \
			"--eval-ming", "0.25", \
			"-i", "1"]
try:
	result = subprocess.check_call(command)
except subprocess.CalledProcessError as e:
	print "phone_probs returned a non-zero exit status."
	sys.exit(-1)

recipe_file.close()


##################################################
# Recognize
#

t = Decoder.Toolbox()

t.select_decoder(0)

if morph_model:
	t.set_silence_is_word(1)
else:
{'fraction_non_alphanumeric': 0.1732283464566929, 'fraction_numerical': 0.01780212256076686, 'mean_word_length': 4.095944177932839, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7174900', 'n_tokens_mistral': 4449, 'n_tokens_neox': 4174, 'n_words': 994}