<commit_before>/*        __
 *        \ \
 *   _   _ \ \  ______
 *  | | | | > \(  __  )
 *  | |_| |/ ^ \| || |
 *  | ._,_/_/ \_\_||_|
 *  | |
 *  |_|
 * 
 * ----------------------------------------------------------------------------
 * "THE BEER-WARE LICENSE" (Revision 42):
 * <rob ∂ CLABS dot CC> wrote this file. As long as you retain this notice you
 * can do whatever you want with this stuff. If we meet some day, and you think
 * this stuff is worth it, you can buy me a beer in return.
 * ----------------------------------------------------------------------------
 */
package cc.clabs.stratosphere.mlp.utils;

import cc.clabs.stratosphere.mlp.types.PactSentence;
import cc.clabs.stratosphere.mlp.types.PactWord;
import java.util.ArrayList;
import java.util.List;


/**
 *
 * @author rob
 */
public class SentenceUtils {
    
    public final static String WILDCARD = "*";
        
    public static PactSentence replaceAllByPattern( PactSentence sentence, String regex, String withTag ) {
        PactSentence result = new PactSentence();
        for ( PactWord word: sentence ) {
            if ( word.getWord().matches( regex ) )
                word.setTag( withTag );
            result.add( word );
        }
        return result;
    }
    
    public static PactSentence replaceAllByTag( PactSentence sentence, String tag, String regex, String replacement ) {
        PactSentence result = new PactSentence();
        for ( PactWord word : sentence ) {
            // skip other words than those with a specific tag
            if ( word.getTag().equals( tag ) ) {
                String text = word.getWord();
                text = text.replaceAll( regex, replacement );
                word.setWord( text );
            }
            result.add( word );
        }
        return result;
    }

    /**
     * Joins a set of PactWords, described by a pattern,
     * together with a given tag into a new PactWord within
     * the sentence. A pattern is a string containing tags
     * and wildcards where every part is separated by a space
     * character.
     * 
     * @param sentence  the sentence<commit_msg>no need to create new sentences<commit_after>/*        __
 *        \ \
 *   _   _ \ \  ______
 *  | | | | > \(  __  )
 *  | |_| |/ ^ \| || |
 *  | ._,_/_/ \_\_||_|
 *  | |
 *  |_|
 * 
 * ----------------------------------------------------------------------------
 * "THE BEER-WARE LICENSE" (Revision 42):
 * <rob ∂ CLABS dot CC> wrote this file. As long as you retain this notice you
 * can do whatever you want with this stuff. If we meet some day, and you think
 * this stuff is worth it, you can buy me a beer in return.
 * ----------------------------------------------------------------------------
 */
package cc.clabs.stratosphere.mlp.utils;

import cc.clabs.stratosphere.mlp.types.PactSentence;
import cc.clabs.stratosphere.mlp.types.PactWord;
import java.util.ArrayList;
import java.util.List;


/**
 *
 * @author rob
 */
public class SentenceUtils {
    
    public final static String WILDCARD = "*";
        
    public static PactSentence replaceAllByPattern( PactSentence sentence, String regex, String withTag ) {
        for ( PactWord word: sentence )
            if ( word.getWord().matches( regex ) )
                word.setTag( withTag );
        return sentence;
    }
    
    public static PactSentence replaceAllByTag( PactSentence sentence, String tag, String regex, String replacement ) {
        for ( PactWord word : sentence )
            // skip other words than those with a specific tag
            if ( word.getTag().equals( tag ) ) {
                String text = word.getWord();
                text = text.replaceAll( regex, replacement );
                word.setWord( text );
            }
        return sentence;
    }

    /**
     * Joins a set of PactWords, described by a pattern,
     * together with a given tag into a new PactWord within
     * the sentence. A pattern is a string containing tags
     * and wildcards where every part is separated by a space
     * character.
     * 
     * @param sentence  the sentence
{'fraction_non_alphanumeric': 0.16845180136319376, 'fraction_numerical': 0.0009737098344693282, 'mean_word_length': 1.6441441441441442, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5526992', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1125, 'n_words': 364}