<commit_before>#!/usr/bin/python
#
# Recognizes cariological status dictation, or free dictation (select using the
# lm variable). Takes the model directory, speech directory, and output file as
# arguments, e.g.
#   ./cariology_test.py /share/puhe/hammas /share/puhe/audio/hammas2011/andre050711 output.csv

import time
import string
import sys
import os
import subprocess
import re
import tempfile
import filecmp
import gzip
import math

# Set your decoder swig path in here!
sys.path.append(os.path.dirname(sys.argv[0]) + "/src/swig");
#sys.path.append('/home/jpylkkon/decoder_new/decoder/src/swig/');

import Decoder

##################################################
# From Wikibooks
# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance#Python
#

def levenshtein_l(s1, s2):
    if len(s1) < len(s2):
        return levenshtein_l(s2, s1)
    if not s1:
        return len(s2)

    previous_row = xrange(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
            deletions = current_row[j] + 1       # than s2
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row

    return previous_row[-1]

def levenshtein_w(s1, s2):
	words1 = s1.split(None)
	words2 = s2.split(None)
	
	count1 = len(words1)
	count2 = len(words2)

	if count1 < count2:
		return levenshtein_w(s2, s1)
	if not s1:
		return count2

	previous_row = xrange(count2 + 1)
	for i, w1 in enumerate(words1):
		current_row = [i + 1]
		for j, w2 in enumerate(words2):
			insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
			deletions = current_row[j] + 1       # than s2
			substitutions = previous_row[j] + (w1 != w2)
			current_row.append(min(insertions, deletions, substitutions))
		previous_row = current_row

	return previous_row[-1]

def natural_sorted(list): 
	convert = lambda text: int(text) if text.isdigit() else text.lower() 
	alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] 
	return sorted(list, key = alphanum_key)


##################################################
# Initialize
#

akupath = os.path.dirname(sys.argv[0]) + "/../aku"

am = 'speechdat'

if am == 'speechdat':
	# 8 kHz telephone line
	ac_model = sys.argv[1] + "/speechdat_gain5000_occ300_23.2.2009_22";
else:
	# 16 kHz acoustic model
	ac_model = sys.argv[1] + "/test_mfcc_noisy_trained";
hmms = ac_model + ".ph"
dur = ac_model + ".dur"

speech_directory = sys.argv[2]

#lm = 'FreeDictation'
#lm = 'StatusDictation'
lm = 'StatusDictationEven'
#lm = ''

if lm == 'FreeDictation':
	lexicon = "/share/puhe/hammas/FreeDictationLexicon.lex"
	ngram = "/share/puhe/hammas/FreeDictationLM.6gram.bin"
	lookahead_ngram = "/share/puhe/hammas/FreeDictationLM.2gram.bin"
	morph_model = True
elif lm == 'StatusDictation':
	lexicon = "/share/puhe/hammas/StatusDictationLexicon.lex"
	ngram = "/share/puhe/hammas/StatusDictationLM.3gram.bin"
	lookahead_ngram = "/share/puhe/hammas/StatusDictationLM.2gram.bin"
	morph_model = False
elif lm == 'StatusDictationEven':
	lexicon = "/share/puhe/hammas/StatusDictationLexicon.lex"
	ngram = "/share/puhe/hammas/StatusDictationLM.even.3gram.bin"
	lookahead_ngram = "/share/puhe/hammas/StatusDictationLM.even.2gram.bin"
	morph_model = False
else:
	lexicon = "/share/work/jpylkkon/bin_lm/morph19k.lex"
	ngram = "/share/work/jpylkkon/bin_lm/morph19k_D20E10_varigram.bin"
	lookahead_ngram = "/share/work/jpylkkon/bin_lm/morph19k_2gram.bin"
	morph_model = True<commit_msg>Fixed paths.<commit_after>#!/usr/bin/python
#
# Recognizes cariological status dictation, or free dictation (select using the
# lm variable). Takes the model directory, speech directory, and output file as
# arguments, e.g.
#   ./cariology_test.py /share/puhe/hammas2011 /share/puhe/audio/hammas2011/andre050711 output.csv

import time
import string
import sys
import os
import subprocess
import re
import tempfile
import filecmp
import gzip
import math

# Set your decoder swig path in here!
sys.path.append(os.path.dirname(sys.argv[0]) + "/src/swig");
#sys.path.append('/home/jpylkkon/decoder_new/decoder/src/swig/');

import Decoder

##################################################
# From Wikibooks
# http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/Levenshtein_distance#Python
#

def levenshtein_l(s1, s2):
    if len(s1) < len(s2):
        return levenshtein_l(s2, s1)
    if not s1:
        return len(s2)

    previous_row = xrange(len(s2) + 1)
    for i, c1 in enumerate(s1):
        current_row = [i + 1]
        for j, c2 in enumerate(s2):
            insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
            deletions = current_row[j] + 1       # than s2
            substitutions = previous_row[j] + (c1 != c2)
            current_row.append(min(insertions, deletions, substitutions))
        previous_row = current_row

    return previous_row[-1]

def levenshtein_w(s1, s2):
	words1 = s1.split(None)
	words2 = s2.split(None)
	
	count1 = len(words1)
	count2 = len(words2)

	if count1 < count2:
		return levenshtein_w(s2, s1)
	if not s1:
		return count2

	previous_row = xrange(count2 + 1)
	for i, w1 in enumerate(words1):
		current_row = [i + 1]
		for j, w2 in enumerate(words2):
			insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer
			deletions = current_row[j] + 1       # than s2
			substitutions = previous_row[j] + (w1 != w2)
			current_row.append(min(insertions, deletions, substitutions))
		previous_row = current_row

	return previous_row[-1]

def natural_sorted(list): 
	convert = lambda text: int(text) if text.isdigit() else text.lower() 
	alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] 
	return sorted(list, key = alphanum_key)


##################################################
# Initialize
#

akupath = os.path.dirname(sys.argv[0]) + "/../aku"

am = 'speechdat'

if am == 'speechdat':
	# 8 kHz telephone line
	ac_model = sys.argv[1] + "/speechdat_gain5000_occ300_23.2.2009_22";
else:
	# 16 kHz acoustic model
	ac_model = sys.argv[1] + "/test_mfcc_noisy_trained";
hmms = ac_model + ".ph"
dur = ac_model + ".dur"

speech_directory = sys.argv[2]

#lm = 'FreeDictation'
lm = 'StatusDictation'
#lm = ''

if lm == 'FreeDictation':
	lexicon = sys.argv[1] + "/FreeDictationLexicon.lex"
	ngram = sys.argv[1] + "/FreeDictationLM.6gram.bin"
	lookahead_ngram = sys.argv[1] + "/FreeDictationLM.2gram.bin"
	morph_model = True
elif lm == 'StatusDictation':
	lexicon = sys.argv[1] + "/StatusDictationLexicon.lex"
	ngram = sys.argv[1] + "/StatusDictationLM.3gram.bin"
	lookahead_ngram = sys.argv[1] + "/StatusDictationLM.2gram.bin"
	morph_model = False
else:
	lexicon = "/share/work/jpylkkon/bin_lm/morph19k.lex"
	ngram = "/share/work/jpylkkon/bin_lm/morph19k_D20E10_varigram.bin"
	lookahead_ngram = "/share/work/jpylkkon/bin_lm/morph19k_2gram.bin"
	morph_model = True
{'fraction_non_alphanumeric': 0.14961937716262977, 'fraction_numerical': 0.03072664359861592, 'mean_word_length': 4.063770147161878, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4394263', 'n_tokens_mistral': 2858, 'n_tokens_neox': 2671, 'n_words': 593}