<commit_before>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */


package edu.cmu.sphinx.frontend.endpoint;

import java.util.LinkedList;
import java.util.List;

import edu.cmu.sphinx.frontend.BaseDataProcessor;
import edu.cmu.sphinx.frontend.Data;
import edu.cmu.sphinx.frontend.DataEndSignal;
import edu.cmu.sphinx.frontend.DataProcessingException;
import edu.cmu.sphinx.frontend.DataStartSignal;
import edu.cmu.sphinx.frontend.DoubleData;
import edu.cmu.sphinx.frontend.Signal;
import edu.cmu.sphinx.util.props.PropertyException;
import edu.cmu.sphinx.util.props.PropertySheet;
import edu.cmu.sphinx.util.props.PropertyType;
import edu.cmu.sphinx.util.props.Registry;


/**
 * Given a sequence of Data, filters out the non-speech regions.
 * The sequence of Data should have the speech and non-speech regions
 * marked out by the SpeechStartSignal and SpeechEndSignal, using
 * the {@link SpeechMarker SpeechMarker}. Such a sequence of Data 
 * for an utterance should look like one of the following two:
 * <p>
 * <b>Case 1: Only one speech region</b>
 * <p>In the first case, the data stream has only one speech region:
 * <p><img src="doc-files/one-region.gif">
 * <br><i>Figure 1: A data stream with only one speech region</i>.
 * <p>After filtering, the non-speech regions are removed, and becomes:
 * <p><img src="doc-files/one-region-filtered.gif">
 * <br><i>Figure 2: A data stream with only on speech region 
 * after filtering.</i>
 * <p>
 * <br><b>Case 2: Multiple speech regions</b>
 * <p>
 * We will use the example of a data stream with two speech regions
 * to illustrate the case of a data stream with multiple speech regions:
 * <p><img src="doc-files/two-regions.gif">
 * <br><i>Figure 3: A data stream with two speech regions.</i>
 * <p>
 * This case is more complicated than one speech region.
 * The property <b>mergeSpeechSegments</b> is very important
 * in controlling the behavior of this filter. This property determines
 * whether individual speech regions (and the non-speech regions between
 * them) in an utterance should be merged into one big region, or
 * whether the individual speech regions should be converted into
 * individual utterances.
 * If <b>mergeSpeechSegments</b> is set to true (the default),
 * all the Data from the first SpeechStartSignal to the last SpeechEndSignal
 * will be considered as one Utterance, and enclosed by a pair of
 * DataStartSignal and DataEndSignal. All non-speech
 * regions, as well as all SpeechStartSignals and SpeechEndSignals,
 * are removed from the stream. This gives:
 * <p>
 * <img src="doc-files/two-regions-merge.gif">
 * <br><i>Figure 4: A data stream with two speech regions after filtering,
 * when <b>mergeSpeechSegments</b> is set to <b>true</b>. Note that all
 * SpeechStartSignals and SpeechEndSignals are removed.</i>
 * <p>
 * On the other hand, if <b>mergeSpeechSegments</b> is set to
 * false, then each speech region will become its own data stream.
 * Pictorially, our data stream with two speech regions becomes:
 * <p><img src="doc-files/two-regions-nonmerge.gif">
 * <br><i>Figure 5: A data stream with two speech regions after filtering,
 * when <b>mergeSpeechSegments</b> is set to <b>false</b>.</i>
 * <p>
 * That is, the SpeechStartSignal replaced by DataStartSignal, 
 * the SpeechEndSignal replaced by DataEndSignal, and the non-speech
 * regions are removed.<commit_msg>Corrected the javadoc. The property 'mergeSpeechSegments' is false by default, not true.<commit_after>/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */


package edu.cmu.sphinx.frontend.endpoint;

import java.util.LinkedList;
import java.util.List;

import edu.cmu.sphinx.frontend.BaseDataProcessor;
import edu.cmu.sphinx.frontend.Data;
import edu.cmu.sphinx.frontend.DataEndSignal;
import edu.cmu.sphinx.frontend.DataProcessingException;
import edu.cmu.sphinx.frontend.DataStartSignal;
import edu.cmu.sphinx.frontend.DoubleData;
import edu.cmu.sphinx.frontend.Signal;
import edu.cmu.sphinx.util.props.PropertyException;
import edu.cmu.sphinx.util.props.PropertySheet;
import edu.cmu.sphinx.util.props.PropertyType;
import edu.cmu.sphinx.util.props.Registry;


/**
 * Given a sequence of Data, filters out the non-speech regions.
 * The sequence of Data should have the speech and non-speech regions
 * marked out by the SpeechStartSignal and SpeechEndSignal, using
 * the {@link SpeechMarker SpeechMarker}. Such a sequence of Data 
 * for an utterance should look like one of the following two:
 * <p>
 * <b>Case 1: Only one speech region</b>
 * <p>In the first case, the data stream has only one speech region:
 * <p><img src="doc-files/one-region.gif">
 * <br><i>Figure 1: A data stream with only one speech region</i>.
 * <p>After filtering, the non-speech regions are removed, and becomes:
 * <p><img src="doc-files/one-region-filtered.gif">
 * <br><i>Figure 2: A data stream with only on speech region 
 * after filtering.</i>
 * <p>
 * <br><b>Case 2: Multiple speech regions</b>
 * <p>
 * We will use the example of a data stream with two speech regions
 * to illustrate the case of a data stream with multiple speech regions:
 * <p><img src="doc-files/two-regions.gif">
 * <br><i>Figure 3: A data stream with two speech regions.</i>
 * <p>
 * This case is more complicated than one speech region.
 * The property <b>mergeSpeechSegments</b> is very important
 * in controlling the behavior of this filter. This property determines
 * whether individual speech regions (and the non-speech regions between
 * them) in an utterance should be merged into one big region, or
 * whether the individual speech regions should be converted into
 * individual utterances.
 * If <b>mergeSpeechSegments</b> is set to true,
 * all the Data from the first SpeechStartSignal to the last SpeechEndSignal
 * will be considered as one Utterance, and enclosed by a pair of
 * DataStartSignal and DataEndSignal. All non-speech
 * regions, as well as all SpeechStartSignals and SpeechEndSignals,
 * are removed from the stream. This gives:
 * <p>
 * <img src="doc-files/two-regions-merge.gif">
 * <br><i>Figure 4: A data stream with two speech regions after filtering,
 * when <b>mergeSpeechSegments</b> is set to <b>true</b>. Note that all
 * SpeechStartSignals and SpeechEndSignals are removed.</i>
 * <p>
 * On the other hand, if <b>mergeSpeechSegments</b> is set to false 
 * (the default), then each speech region will become its own data stream.
 * Pictorially, our data stream with two speech regions becomes:
 * <p><img src="doc-files/two-regions-nonmerge.gif">
 * <br><i>Figure 5: A data stream with two speech regions after filtering,
 * when <b>mergeSpeechSegments</b> is set to <b>false</b>.</i>
 * <p>
 * That is, the SpeechStartSignal replaced by DataStartSignal, 
 * the SpeechEndSignal replaced by DataEndSignal, and the non-speech
 * regions are removed.
{'fraction_non_alphanumeric': 0.09845456967759127, 'fraction_numerical': 0.0061284305888622435, 'mean_word_length': 4.381362007168459, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 107, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1214654', 'n_tokens_mistral': 2375, 'n_tokens_neox': 2230, 'n_words': 864}