<commit_before>package dk.in2isoft.commons.lang;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;

import com.google.common.collect.Lists;
import com.google.gson.Gson;

import dk.in2isoft.onlineobjects.core.Pair;

public class Strings {

	public static final Object ISO_8859_1 = "ISO-8859-1";
	public static final String UTF8 = "UTF-8";

	public static final String DEGREE = "\u00B0";
	public static final String RIGHT_SINGLE_QUOTE = "\u2019";
	public static final String DOUBLE_APOSTROPHE = "\u02EE";

	public static final String UNICODE_AA_LARGE = "\u00C5";
	public static final String UNICODE_AA = "\u00E5";
	
	public static final String UNICODE_AE_LARGE = "\u00C6";
	public static final String UNICODE_AE = "\u00E6";
	
	public static final String UNICODE_OE_LARGE = "\u00D8";
	public static final String UNICODE_OE = "\u00F8";
	
	public static final String UNICODE_TRADEMARK = "\u2122";
	public static final String UNICODE_REGISTERED_TRADEMARK = "\u00AE";
	
	
	public static final String[] ALPHABETH = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z",Strings.UNICODE_AE,Strings.UNICODE_OE,Strings.UNICODE_AA};
	public static final String TRIANGLE_HEADED_RIGHTWARDS_ARROW = "\u279D";
	
	public static final Object RIGHTWARDS_ARROW = "\u2192";
	
	// private static Logger log = Logger.getLogger(LangUtil.class);

	public static String concatWords(String first, String second) {
		if (first == null && second == null) {
			return null;
		} else if (first != null && second == null) {
			return first;
		} else if (first == null && second != null) {
			return second;
		} else {
			return first + " " + second;
		}
	}
	
	public static String deAccent(String str) {
		if (str==null) {
			return null;
		}
	    String nfdNormalizedString = Normalizer.normalize(str, Normalizer.Form.NFD); 
	    Pattern pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
	    return pattern.matcher(nfdNormalizedString).replaceAll("");
	}
	
	/**
	 * For use when indexing - has no other usage
	 * @param text
	 * @return
	 */
	public static String getAlphabethStartLetter(String text) {
		if (text==null) {
			return "none";
		}
		text = text.trim();
		if (text.length()==0) {
			return "none";
		}
		String letter = text.substring(0, 1).toLowerCase();
		if (Strings.contains(letter, ALPHABETH)) {
			return letter;
		}
		letter = deAccent(letter);
		if (Strings.contains(letter, ALPHABETH)) {
			return letter;
		}
		if (StringUtils.isNumeric(letter)) {
			return "number";
		}
		return "other";
	}

	public static String concatWords(String... words) {
		StringBuilder str = new StringBuilder();
		for (int i = 0; i < words.length; i++) {
			if (words[i] != null) {
				String word = words[i].trim();
				if (word.length() > 0) {
					if (str.length() > 0) {
						str.append(" ");
					}
					str.append(word);
				}
			}
		}
		return str.toString();
	}

	public static String implode(String[] array, String delimiter) {
		if (array.length == 0) {
			return "";
		} else if (array.length == 1) {
			return array[0];
		} else {
			StringBuilder str = new StringBuilder();
			for (int i = 0; i < array.length; i++) {
				if (i > 0)
					str.append(delimiter);
				str.append(array[i]);
			}
			return str.toString();
		}
	}

	public static String generateRandomString(int n) {
		char[] pw = new char[n];
		int c = 'A';
		int r1 = 0;
		for (int i = 0; i < n; i++) {
			r1 = (int) (Math.random() * 3);
			switch (r1) {
			case 0:
				c = '0' + (int) (Math.random() * 10);
				break;
			case 1:
				c = 'a' + (int) (Math.random() * 26);
				break;
			case 2:
				c = 'A' + (int) (Math.random() * 26);
				break;
			}
			pw[i] = (char) c;
		}
		return new String(pw);
	}
	
	public static boolean isDefined(String[] words) {
		return words != null && words.length > 0;
	}

	public static boolean isNotBlank(String str) {
		return StringUtils.isNotBlank(str);
	}
	
	public static boolean isBlank(String str) {
		return StringUtils.isBlank(str);
	}
	
	public static String fallback(String str, String defaultValue) {
		if (Strings.isBlank(str)) {
			return defaultValue;
		}
		return str;
	}

	public static String[] getWords(String query) {
		if (Strings.isBlank(query)) {<commit_msg>Fixed parameter name<commit_after>package dk.in2isoft.commons.lang;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;

import com.google.common.collect.Lists;
import com.google.gson.Gson;

import dk.in2isoft.onlineobjects.core.Pair;

public class Strings {

	public static final Object ISO_8859_1 = "ISO-8859-1";
	public static final String UTF8 = "UTF-8";

	public static final String DEGREE = "\u00B0";
	public static final String RIGHT_SINGLE_QUOTE = "\u2019";
	public static final String DOUBLE_APOSTROPHE = "\u02EE";

	public static final String UNICODE_AA_LARGE = "\u00C5";
	public static final String UNICODE_AA = "\u00E5";
	
	public static final String UNICODE_AE_LARGE = "\u00C6";
	public static final String UNICODE_AE = "\u00E6";
	
	public static final String UNICODE_OE_LARGE = "\u00D8";
	public static final String UNICODE_OE = "\u00F8";
	
	public static final String UNICODE_TRADEMARK = "\u2122";
	public static final String UNICODE_REGISTERED_TRADEMARK = "\u00AE";
	
	
	public static final String[] ALPHABETH = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z",Strings.UNICODE_AE,Strings.UNICODE_OE,Strings.UNICODE_AA};
	public static final String TRIANGLE_HEADED_RIGHTWARDS_ARROW = "\u279D";
	
	public static final Object RIGHTWARDS_ARROW = "\u2192";
	
	// private static Logger log = Logger.getLogger(LangUtil.class);

	public static String concatWords(String first, String second) {
		if (first == null && second == null) {
			return null;
		} else if (first != null && second == null) {
			return first;
		} else if (first == null && second != null) {
			return second;
		} else {
			return first + " " + second;
		}
	}
	
	public static String deAccent(String str) {
		if (str==null) {
			return null;
		}
	    String nfdNormalizedString = Normalizer.normalize(str, Normalizer.Form.NFD); 
	    Pattern pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
	    return pattern.matcher(nfdNormalizedString).replaceAll("");
	}
	
	/**
	 * For use when indexing - has no other usage
	 * @param text
	 * @return
	 */
	public static String getAlphabethStartLetter(String text) {
		if (text==null) {
			return "none";
		}
		text = text.trim();
		if (text.length()==0) {
			return "none";
		}
		String letter = text.substring(0, 1).toLowerCase();
		if (Strings.contains(letter, ALPHABETH)) {
			return letter;
		}
		letter = deAccent(letter);
		if (Strings.contains(letter, ALPHABETH)) {
			return letter;
		}
		if (StringUtils.isNumeric(letter)) {
			return "number";
		}
		return "other";
	}

	public static String concatWords(String... words) {
		StringBuilder str = new StringBuilder();
		for (int i = 0; i < words.length; i++) {
			if (words[i] != null) {
				String word = words[i].trim();
				if (word.length() > 0) {
					if (str.length() > 0) {
						str.append(" ");
					}
					str.append(word);
				}
			}
		}
		return str.toString();
	}

	public static String implode(String[] array, String delimiter) {
		if (array.length == 0) {
			return "";
		} else if (array.length == 1) {
			return array[0];
		} else {
			StringBuilder str = new StringBuilder();
			for (int i = 0; i < array.length; i++) {
				if (i > 0)
					str.append(delimiter);
				str.append(array[i]);
			}
			return str.toString();
		}
	}

	public static String generateRandomString(int n) {
		char[] pw = new char[n];
		int c = 'A';
		int r1 = 0;
		for (int i = 0; i < n; i++) {
			r1 = (int) (Math.random() * 3);
			switch (r1) {
			case 0:
				c = '0' + (int) (Math.random() * 10);
				break;
			case 1:
				c = 'a' + (int) (Math.random() * 26);
				break;
			case 2:
				c = 'A' + (int) (Math.random() * 26);
				break;
			}
			pw[i] = (char) c;
		}
		return new String(pw);
	}
	
	public static boolean isDefined(String[] strings) {
		return strings != null && strings.length > 0;
	}

	public static boolean isNotBlank(String str) {
		return StringUtils.isNotBlank(str);
	}
	
	public static boolean isBlank(String str) {
		return StringUtils.isBlank(str);
	}
	
	public static String fallback(String str, String defaultValue) {
		if (Strings.isBlank(str)) {
			return defaultValue;
		}
		return str;
	}

	public static String[] getWords(String query) {
		if (Strings.isBlank(query)) {
{'fraction_non_alphanumeric': 0.14884125775214885, 'fraction_numerical': 0.017843542596017843, 'mean_word_length': 3.6307304785894208, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5319955', 'n_tokens_mistral': 3693, 'n_tokens_neox': 3194, 'n_words': 814}