<commit_before>/*
 * Copyright 1999-2013 Carnegie Mellon University.
 * Portions Copyright 2004 Sun Microsystems, Inc.
 * Portions Copyright 2004 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 *
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL
 * WARRANTIES.
 *
 */
package edu.cmu.sphinx.demo.aligner;

import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.io.Files.asCharSource;

import java.io.File;
import java.net.URL;

import edu.cmu.sphinx.alignment.SpeechAligner;
import edu.cmu.sphinx.result.WordResult;


/**
 * This class demonstrates how to align audio to existing transcription and receive word timestamps.
 *
 * <br/>
 * In order to initialize the aligner you need to specify several data files which might be
 * downloaded from the CMUSphinx website. There should be an acoustic model for your lanaguage, a
 * dictionary an optional G2P model to convert word strings to pronunciation. <br/>
 * Currently the audio must have specific format (16khz, 16bit, mono), but in the future other
 * formats will be supported. <br/>
 * Text should be a clean text in lower case. It should be cleaned from punctuation marks, numbers
 * and other non-speakable things. In the future automatic cleanup will be supported.
 */
public class AlignerDemo {
    private static final String ACOUSTIC_MODEL_PATH =
            "resource:/edu/cmu/sphinx/models/acoustic/wsj";
    private static final String DICTIONARY_PATH =
            "resource:/edu/cmu/sphinx/models/acoustic/wsj/dict/cmudict.0.6d";
    private static final String TEXT = "one zero zero zero one nine oh two "
            + "one oh zero one eight zero three";

    public static void main(String args[]) throws Exception {
        URL audioUrl;
        String transcript;
        if (args.length > 1) {
            audioUrl = new File(args[0]).toURI().toURL();
            transcript = asCharSource(new File(args[1]), UTF_8).read();
        } else {
            audioUrl = AlignerDemo.class.getResource("10001-90210-01803.wav");
            transcript = TEXT;
        }
        String acousticModelPath =
                (args.length > 2) ? args[2] : ACOUSTIC_MODEL_PATH;
        String dictionaryPath = (args.length > 3) ? args[3] : DICTIONARY_PATH;
        String g2pPath = (args.length > 4) ? args[4] : null;
        SpeechAligner aligner =
                new SpeechAligner(acousticModelPath, dictionaryPath, g2pPath);

        for (WordResult result : aligner.align(audioUrl, transcript)) {
            System.out.format("%-25s [%s]\n",
                              result.getWord().getSpelling(),
                              result.getTimeFrame());
        }
    }
}<commit_msg>Diff-like output of the Aligner demo.<commit_after>/*
 * Copyright 1999-2013 Carnegie Mellon University.
 * Portions Copyright 2004 Sun Microsystems, Inc.
 * Portions Copyright 2004 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 *
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL
 * WARRANTIES.
 *
 */
package edu.cmu.sphinx.demo.aligner;

import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.collect.Lists.transform;
import static com.google.common.io.Files.asCharSource;
import static edu.cmu.sphinx.result.WordResults.toSpelling;

import java.io.File;
import java.net.URL;
import java.util.List;

import edu.cmu.sphinx.alignment.*;
import edu.cmu.sphinx.result.WordResult;


/**
 * This class demonstrates how to align audio to existing transcription and receive word timestamps.
 *
 * <br/>
 * In order to initialize the aligner you need to specify several data files which might be
 * downloaded from the CMUSphinx website. There should be an acoustic model for your lanaguage, a
 * dictionary an optional G2P model to convert word strings to pronunciation. <br/>
 * Currently the audio must have specific format (16khz, 16bit, mono), but in the future other
 * formats will be supported. <br/>
 * Text should be a clean text in lower case. It should be cleaned from punctuation marks, numbers
 * and other non-speakable things. In the future automatic cleanup will be supported.
 */
public class AlignerDemo {
    private static final String ACOUSTIC_MODEL_PATH =
            "resource:/edu/cmu/sphinx/models/acoustic/wsj";
    private static final String DICTIONARY_PATH =
            "resource:/edu/cmu/sphinx/models/acoustic/wsj/dict/cmudict.0.6d";
    private static final String TEXT = "one zero zero zero one nine oh two "
            + "one oh zero one eight zero three";

    public static void main(String args[]) throws Exception {
        URL audioUrl;
        String transcript;
        if (args.length > 1) {
            audioUrl = new File(args[0]).toURI().toURL();
            transcript = asCharSource(new File(args[1]), UTF_8).read();
        } else {
            audioUrl = AlignerDemo.class.getResource("10001-90210-01803.wav");
            transcript = TEXT;
        }
        String acousticModelPath =
                (args.length > 2) ? args[2] : ACOUSTIC_MODEL_PATH;
        String dictionaryPath = (args.length > 3) ? args[3] : DICTIONARY_PATH;
        String g2pPath = (args.length > 4) ? args[4] : null;
        SpeechAligner aligner =
                new SpeechAligner(acousticModelPath, dictionaryPath, g2pPath);

        WordTokenizer tokenizer = new EnglishWordTokenizer();
        List<WordResult> results = aligner.align(audioUrl, transcript);
        LongTextAligner textAligner =
                new LongTextAligner(transform(results, toSpelling()), 1);
        List<String> words = tokenizer.getWords(transcript);
        int[] aid = textAligner.align(words);

        int lastId = -1;
        for (int i = 0; i < aid.length; ++i) {
            if (aid[i] == -1) {
                System.out.format("- %s\n", words.get(i));
            } else {
                if (aid[i] - lastId > 1) {
                    for (WordResult result : results.subList(lastId + 1,
                            aid[i])) {
                        System.out.format("+ %-25s [%s]\n", result.getWord()
                                .getSpelling(), result.getTimeFrame());
                    }
                }
                System.out.format("  %-25s [%s]\n", results.get(aid[i])
                        .getWord().getSpelling(), results.get(aid[i])
                        .getTimeFrame());
                lastId = aid[i];
            }
        }

        if (results.size() - lastId > 1) {
            for (WordResult result : results.subList(lastId, results.size())) {
                System.out.format("+ %-25s [%s]\n", result.getWord()
                        .getSpelling(), result.getTimeFrame());
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.10150158821830782, 'fraction_numerical': 0.016892867455963037, 'mean_word_length': 2.5002526528549773, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6722640', 'n_tokens_mistral': 2109, 'n_tokens_neox': 1942, 'n_words': 611}