<commit_before>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */
package edu.cmu.sphinx.util;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.StringTokenizer;

/**
 * Implements a portion of the NIST align/scoring algorithm to compare
 * a reference string to a hypothesis string.  It only keeps track of
 * substitutions, insertions, and deletions.
 */
public class NISTAlign {

    /* Constants that help with the align.  The following are
     * used in the backtrace table and backtrace list.
     */
    final static int OK = 0;
    final static int SUBSTITUTION = 1;
    final static int INSERTION = 2;
    final static int DELETION = 3;

    /* Constants that help with the align.  The following are
     * used to create the penalty table.
     */
    final static int MAX_PENALTY = 1000000;
    final static int SUBSTITUTION_PENALTY = 100;
    final static int INSERTION_PENALTY = 75;
    final static int DELETION_PENALTY = 75;

    /* Used for padding out aligned strings.
     */
    final static String STARS  =
        "********************************************";
    final static String SPACES =
        "                                            ";
    final static String HRULE =
        "============================================"
        + "================================";
    
    /**
     * Totals over the life of this class.  These can be reset to 0
     * with a call to resetTotals.
     */
    static int totalSentences;
    static int totalSentencesWithErrors;
    static int totalSentencesWithSubtitutions;
    static int totalSentencesWithInsertions;
    static int totalSentencesWithDeletions;
    static int totalReferenceWords;
    static int totalHypothesisWords;
    static int totalAlignedWords;
    static int totalWordsCorrect;
    static int totalSubstitutions;
    static int totalInsertions;
    static int totalDeletions;
    
    /**
     * Error values for one call to 'align'
     */
    int substitutions;
    int insertions;
    int deletions;
    int correct;

    /**
     * The raw reference string.  Updated with each call to 'align'.
     */
    String rawReference;

    /**
     * The reference annotation; typically the name of the
     * audio file for the reference string.  This is an optional
     * part of the rawReference string.  If it is included, it
     * is appended to the end of the string in parentheses.
     * Updated with each call to 'align'.
     */
    String referenceAnnotation;
    
    /**
     * Ordered list of words from rawReference after the annotation
     * has been removed.  Updated with each call to 'align'.
     */
    LinkedList referenceWords;

    /**
     * Aligned list of words from rawReference.  Created in
     * alignWords.  Updated with each call to 'align'.
     */
    LinkedList alignedReferenceWords;

    /**
     * The raw hypothesis string.  Updated with each call to
     * 'align'.
     */
    String rawHypothesis;

    /**
     * Ordered list of words from rawHypothesis after the annotation
     * has been removed.  Updated with each call to 'align'.
     */
    LinkedList hypothesisWords;

    /**
     * Aligned list of words from rawHypothesis.  Created in
     * alignWords.  Updated with each call to 'align'.
     */
    LinkedList alignedHypothesisWords;

    /**
     * Helpers to create percentage strings.
     */
    DecimalFormat percentageFormat = new DecimalFormat("##0.0%");
    
    
    private boolean showResults;
    private boolean showAlignedResults;

    /**
     * Creates a new NISTAlign object.
     */
    public NISTAlign(boolean showResults, boolean showAlignedResults) {
        this.showResults = showResults;
        this.showAlignedResults = showAlignedResults;
        resetTotals();
    }

    /**
     * Reset the total insertions, deletions, and substitutions
     * counts for this class.
     */
    static public void resetTotals() {
        totalSentences = 0;
        totalSentencesWithErrors = 0;
        totalSentencesWithSubtitutions = 0;
        totalSentencesWithInsertions = 0;
        totalSentencesWithDeletions = 0;
        totalReferenceWords = 0;
        totalHypothesisWords = 0;
        totalAlignedWords = 0;
        totalWordsCorrect = 0;<commit_msg>Made the static member data be non-static, so different recognizers can have there own aligner with out collisions on the summary data.<commit_after>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electronic Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */
package edu.cmu.sphinx.util;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.text.DecimalFormat;
import java.util.LinkedList;
import java.util.ListIterator;
import java.util.StringTokenizer;

/**
 * Implements a portion of the NIST align/scoring algorithm to compare
 * a reference string to a hypothesis string.  It only keeps track of
 * substitutions, insertions, and deletions.
 */
public class NISTAlign {

    /* Constants that help with the align.  The following are
     * used in the backtrace table and backtrace list.
     */
    final static int OK = 0;
    final static int SUBSTITUTION = 1;
    final static int INSERTION = 2;
    final static int DELETION = 3;

    /* Constants that help with the align.  The following are
     * used to create the penalty table.
     */
    final static int MAX_PENALTY = 1000000;
    final static int SUBSTITUTION_PENALTY = 100;
    final static int INSERTION_PENALTY = 75;
    final static int DELETION_PENALTY = 75;

    /* Used for padding out aligned strings.
     */
    final static String STARS  =
        "********************************************";
    final static String SPACES =
        "                                            ";
    final static String HRULE =
        "============================================"
        + "================================";
    
    /**
     * Totals over the life of this class.  These can be reset to 0
     * with a call to resetTotals.
     */
    private int totalSentences;
    private int totalSentencesWithErrors;
    private int totalSentencesWithSubtitutions;
    private int totalSentencesWithInsertions;
    private int totalSentencesWithDeletions;
    private int totalReferenceWords;
    private int totalHypothesisWords;
    private int totalAlignedWords;
    private int totalWordsCorrect;
    private int totalSubstitutions;
    private int totalInsertions;
    private int totalDeletions;
    
    /**
     * Error values for one call to 'align'
     */
    private int substitutions;
    private int insertions;
    private int deletions;
    private int correct;

    /**
     * The raw reference string.  Updated with each call to 'align'.
     */
    private String rawReference;

    /**
     * The reference annotation; typically the name of the
     * audio file for the reference string.  This is an optional
     * part of the rawReference string.  If it is included, it
     * is appended to the end of the string in parentheses.
     * Updated with each call to 'align'.
     */
    private String referenceAnnotation;
    
    /**
     * Ordered list of words from rawReference after the annotation
     * has been removed.  Updated with each call to 'align'.
     */
    private LinkedList referenceWords;

    /**
     * Aligned list of words from rawReference.  Created in
     * alignWords.  Updated with each call to 'align'.
     */
    private LinkedList alignedReferenceWords;

    /**
     * The raw hypothesis string.  Updated with each call to
     * 'align'.
     */
    private String rawHypothesis;

    /**
     * Ordered list of words from rawHypothesis after the annotation
     * has been removed.  Updated with each call to 'align'.
     */
    private LinkedList hypothesisWords;

    /**
     * Aligned list of words from rawHypothesis.  Created in
     * alignWords.  Updated with each call to 'align'.
     */
    private LinkedList alignedHypothesisWords;

    /**
     * Helpers to create percentage strings.
     */
    static DecimalFormat percentageFormat = new DecimalFormat("##0.0%");
    
    
    private boolean showResults;
    private boolean showAlignedResults;

    /**
     * Creates a new NISTAlign object.
     */
    public NISTAlign(boolean showResults, boolean showAlignedResults) {
        this.showResults = showResults;
        this.showAlignedResults = showAlignedResults;
        resetTotals();
    }

    /**
     * Reset the total insertions, deletions, and substitutions
     * counts for this class.
     */
     public void resetTotals() {
        totalSentences = 0;
        totalSentencesWithErrors = 0;
        totalSentencesWithSubtitutions = 0;
        totalSentencesWithInsertions = 0;
        totalSentencesWithDeletions = 0;
        totalReferenceWords = 0;
        totalHypothesisWords = 0;
        totalAlignedWords = 0;
        totalWordsCorrect = 0;
{'fraction_non_alphanumeric': 0.08976345418861688, 'fraction_numerical': 0.009503150500981303, 'mean_word_length': 2.618086696562033, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6940354', 'n_tokens_mistral': 2742, 'n_tokens_neox': 2485, 'n_words': 949}