<commit_before>/**
 * 
 * @author chilang
 * Created 2003-10-18, 21:29:00.
 */
package com.chilang.util;

import java.util.Map;
import java.util.LinkedList;

public class StringUtils {
    /**
     * Check if token contains only digits
     * @param token
     * @return <code>true</code> iff token is contains only digits
     */
    public static boolean isAllDigits(String token) {

        for (int i=0; i<token.length(); i++) {
            if (!Character.isDigit(token.charAt(i)))
                return false;
        }
        return true;
    }

    /**
     * Check if token contains only letters
     * @param token
     * @return <code>true</code> iff token is contains only letters
     */
    public static boolean isAllLetters(String token) {

        for (int i=0; i<token.length(); i++) {
            if (!Character.isLetter(token.charAt(i)))
                return false;
        }
        return true;
    }

    /**
     * Calculate the degree of ovelap (how much of second set is contained in the first set)
     * between two set" of words : i.e. = |w1 * w2|/w1
     * @param words the larger set of words
     * @param subset
     * @return
     */
    public static double overlap(String[] words, String[] subset) {
        //brute force version
        int overlap = 0;
        for (int i = 0; i < subset.length; i++) {
            for (int j = 0; j < words.length; j++) {
                if (subset[i].equalsIgnoreCase(words[j]))
                    overlap++;
            }
        }
        return (double)overlap/words.length;
    }

    public static String getQueryNameFromFilePath(String filepath) {
        return filepath.substring(filepath.lastIndexOf("\\")+1, filepath.lastIndexOf(".")).replaceAll("_", " ").toLowerCase();
    }

    public static Object getCarrotParameter(Map params, Object key) {
        LinkedList values = (LinkedList)params.get(key);
        if (values != null) {
            return values.get(0);
        } else {
            return null;
        }

    }


    /**
     * Trim given string from leading/trailing character c
     * @param s String to be trimmed
     * @param c character to be removed
     * @return trimmed String
     */
    public static String trim(String s, char c) {
        StringBuffer b = new StringBuffer(s);
        if (b.charAt(0) == c)
            b.deleteCharAt(0);
        if (b.charAt(b.length()-1) == c)
            b.deleteCharAt(b.length() -1);
        return b.toString();
    }

    public static String join(String[] ss, String joiner) {
        StringBuffer b = new StringBuffer();
        String sep = "";
        for (int i = 0, l = ss.length; i <l; i++) {
            b.append(sep + ss[i]);
            sep = joiner;
        }
        return b.toString();
    }
}
<commit_msg>string index out of bounds exception fixed.<commit_after>/**
 * 
 * @author chilang
 * Created 2003-10-18, 21:29:00.
 */
package com.chilang.util;

import java.util.Map;
import java.util.LinkedList;

public class StringUtils {
    /**
     * Check if token contains only digits
     * @param token
     * @return <code>true</code> iff token is contains only digits
     */
    public static boolean isAllDigits(String token) {

        for (int i=0; i<token.length(); i++) {
            if (!Character.isDigit(token.charAt(i)))
                return false;
        }
        return true;
    }

    /**
     * Check if token contains only letters
     * @param token
     * @return <code>true</code> iff token is contains only letters
     */
    public static boolean isAllLetters(String token) {

        for (int i=0; i<token.length(); i++) {
            if (!Character.isLetter(token.charAt(i)))
                return false;
        }
        return true;
    }

    /**
     * Calculate the degree of ovelap (how much of second set is contained in the first set)
     * between two set" of words : i.e. = |w1 * w2|/w1
     * @param words the larger set of words
     * @param subset
     * @return
     */
    public static double overlap(String[] words, String[] subset) {
        //brute force version
        int overlap = 0;
        for (int i = 0; i < subset.length; i++) {
            for (int j = 0; j < words.length; j++) {
                if (subset[i].equalsIgnoreCase(words[j]))
                    overlap++;
            }
        }
        return (double)overlap/words.length;
    }

    public static String getQueryNameFromFilePath(String filepath) {
        return filepath.substring(filepath.lastIndexOf("\\")+1, filepath.lastIndexOf(".")).replaceAll("_", " ").toLowerCase();
    }

    public static Object getCarrotParameter(Map params, Object key) {
        LinkedList values = (LinkedList)params.get(key);
        if (values != null) {
            return values.get(0);
        } else {
            return null;
        }

    }


    /**
     * Trim given string from leading/trailing character c
     * @param s String to be trimmed
     * @param c character to be removed
     * @return trimmed String
     */
    public static String trim(String s, char c) {
        StringBuffer b = new StringBuffer(s);
        if (b.length() > 0 && b.charAt(0) == c)
            b.deleteCharAt(0);
        if (b.length() > 0 && b.charAt(b.length()-1) == c)
            b.deleteCharAt(b.length() -1);
        return b.toString();
    }

    public static String join(String[] ss, String joiner) {
        StringBuffer b = new StringBuffer();
        String sep = "";
        for (int i = 0, l = ss.length; i <l; i++) {
            b.append(sep + ss[i]);
            sep = joiner;
        }
        return b.toString();
    }
}

{'fraction_non_alphanumeric': 0.12575595873354678, 'fraction_numerical': 0.010672358591248666, 'mean_word_length': 1.9195223260643821, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2895121', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1658, 'n_words': 498}