<commit_before>package uk.ac.cam.ch.wwmm.opsin;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

/**Static routines for string manipulation.
 * This is a specially tailored version of StringTools as found in OSCAR for use in OPSIN
 *
 * @author ptc24/dl387
 *
 */
public final class StringTools {

	private static final Pattern MATCH_WHITESPACE = Pattern.compile("\\s+");

	/**Converts a list of characters into a string.
	 *
	 * @param l A list of characters.
	 * @return The corresponding string.
	 */
	public static String charListToString(List<Character> l) {
		StringBuffer sb = new StringBuffer();
		for(char c : l) {
			sb.append(c);
		}
		return sb.toString();
	}

	/**
	 * Converts a list of strings into a single string delimited by the given separator
	 *
	 * @param l A list of strings.
	 * @return The corresponding string.
	 */
	public static String stringListToString(List<String> l, String separator) {
		StringBuffer sb = new StringBuffer();
		for(int i=0;i<l.size();i++) {
			sb.append(l.get(i));
			if(separator != null && i < l.size()-1) sb.append(separator);
		}
		return sb.toString();
	}

	/**Converts a string to a list of characters.
	 *
	 * @param s A string.
	 * @return The corresponding list of characters.
	 */
	public static List<Character> stringToList(String s) {
		List<Character> cl = new ArrayList<Character>();
		for(int i=0;i<s.length();i++) {
			cl.add(s.charAt(i));
		}
		return cl;
	}

	/**Produce repetitions of a string. Eg. HelloWorld * 2 = HelloWorldHelloWorld.
	 *
	 * @param s The string to multiply.
	 * @param n The number of times to multiply it.
	 * @return The multiplied string.
	 */
	public static String multiplyString(String s, int n) {
		StringBuffer sb = new StringBuffer();
		for(int i=n-1;i>=0;i--) {
			sb.append(s);
		}
		return sb.toString();
	}

	/**Joins an array of strings into a single string.
	 *
	 * @param stringArray The strings to join together.
	 * @param separator The separator to use.
	 * @return The resulting string.
	 */
	public static String arrayToString(String [] stringArray, String separator) {
		StringBuffer sb = new StringBuffer();
		for(int i=0;i<stringArray.length-1;i++) {
			sb.append(stringArray[i]);
			sb.append(separator);
		}
		sb.append(stringArray[stringArray.length-1]);
		return sb.toString();
	}

	/**Converts a unicode string into ISO-8859-1, converting greek letters
	 * to their names, and difficult characters to underscore.
	 *
	 * @param s The string to convert.
	 * @return The converted string.
	 * @throws PreProcessingException
	 */
	public static String convertNonAsciiAndNormaliseRepresentation(String s) throws PreProcessingException {
		s = MATCH_WHITESPACE.matcher(s).replaceAll(" ");//normalise white space
		StringBuilder sb = new StringBuilder(s);
		for(int i=0;i<sb.length();i++) {
			char c = sb.charAt(i);
			if(c >= 128) {
				sb.replace(i, i+1, getReplacementForNonASCIIChar(c));//replace non ascii characters with hard coded ascii strings
			}
			else if (c ==96){
				sb.replace(i, i+1, "'");//replace back ticks with apostrophe
			}
			else if (c ==34){
				sb.replace(i, i+1, "''");//replace quotation mark with two primes
			}
		}
		return sb.toString();
	}

    private static String getReplacementForNonASCIIChar(char c) throws PreProcessingException {
        switch (c) {
            case '\u03b1': return "alpha";//greeks
            case '\u03b2': return "beta";
            case '\u03b3': return "gamma";
            case '\u03b4': return "delta";
            case '\u03b5': return "epsilon";
            case '\u03b6': return "zeta";
            case '\u03b7': return "eta";
            case '\u03b8': return "theta";
            case '\u03b9': return "iota";
            case '\u03ba': return "kappa";
            case '\u03bb': return "lambda";
            case '\u03bc': return "mu";
            case '\u03bd': return "nu";
            case '\u03be': return "xi";
            case '\u03bf': return "omicron";
            case '\u03c0': return "pi";
            case '\u03c1': return "rho";
            case '\u03c2': return "stigma";
            case '\u03c3': return "sigma";
            case '\u03c4': return "tau";
            case '\u03c5': return "upsilon";
            case '\u03c6': return "phi";
            case '\u03c7': return "chi";
            case '\u03c8': return "psi";
            case '\u03c9': return "omega";

            case '\u00B1': return "+-";//plus minus symbol
            case '\u2213': return "-+";
         
            case '\u2018': return "'";//quotation marks and primes (map to apostrophe/s)
            case '\u2019': return "'";
            case '\u201B': return "'";
            case '\u2032': return "'";//primes
            case '\u2033': return "''";
            case '\u2034': return "'''";
            case '\u2057': return "''''";
            case '\u2035': return "'";//back primes
            case '\u2036': return "''";
            case '\u2037': return "'''";
            case '\u00B4': return "'";//accents
            case '\u02CA': return "'";
            case '\u0301': return "'";
            case '\u02DD': return "''";
            case '\u030B': return "''";

            case '\u2010': return "-";//dashes, hyphens and the minus sign
            case '\u2011': return "-";
            case '\u2012': return "-";
            case '\u2013': return "-";
            case '\u2014': return "-";
            case '\u2015': return "-";
            case '\u2212': return "-";
            
            case '\u00A0': return " ";//Non-breaking spaces
            case '\u2007': return " ";
            case '\u202F': return " ";
            
            case '\u200b': return "";//zero width space
            case '\u200d': return "";//zero width joiner
<commit_msg>Added some common ligatures to the recognised unicode characters<commit_after>package uk.ac.cam.ch.wwmm.opsin;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;

/**Static routines for string manipulation.
 * This is a specially tailored version of StringTools as found in OSCAR for use in OPSIN
 *
 * @author ptc24/dl387
 *
 */
public final class StringTools {

	private static final Pattern MATCH_WHITESPACE = Pattern.compile("\\s+");

	/**Converts a list of characters into a string.
	 *
	 * @param l A list of characters.
	 * @return The corresponding string.
	 */
	public static String charListToString(List<Character> l) {
		StringBuffer sb = new StringBuffer();
		for(char c : l) {
			sb.append(c);
		}
		return sb.toString();
	}

	/**
	 * Converts a list of strings into a single string delimited by the given separator
	 *
	 * @param l A list of strings.
	 * @return The corresponding string.
	 */
	public static String stringListToString(List<String> l, String separator) {
		StringBuffer sb = new StringBuffer();
		for(int i=0;i<l.size();i++) {
			sb.append(l.get(i));
			if(separator != null && i < l.size()-1) sb.append(separator);
		}
		return sb.toString();
	}

	/**Converts a string to a list of characters.
	 *
	 * @param s A string.
	 * @return The corresponding list of characters.
	 */
	public static List<Character> stringToList(String s) {
		List<Character> cl = new ArrayList<Character>();
		for(int i=0;i<s.length();i++) {
			cl.add(s.charAt(i));
		}
		return cl;
	}

	/**Produce repetitions of a string. Eg. HelloWorld * 2 = HelloWorldHelloWorld.
	 *
	 * @param s The string to multiply.
	 * @param n The number of times to multiply it.
	 * @return The multiplied string.
	 */
	public static String multiplyString(String s, int n) {
		StringBuffer sb = new StringBuffer();
		for(int i=n-1;i>=0;i--) {
			sb.append(s);
		}
		return sb.toString();
	}

	/**Joins an array of strings into a single string.
	 *
	 * @param stringArray The strings to join together.
	 * @param separator The separator to use.
	 * @return The resulting string.
	 */
	public static String arrayToString(String [] stringArray, String separator) {
		StringBuffer sb = new StringBuffer();
		for(int i=0;i<stringArray.length-1;i++) {
			sb.append(stringArray[i]);
			sb.append(separator);
		}
		sb.append(stringArray[stringArray.length-1]);
		return sb.toString();
	}

	/**Converts a unicode string into ISO-8859-1, converting greek letters
	 * to their names, and difficult characters to underscore.
	 *
	 * @param s The string to convert.
	 * @return The converted string.
	 * @throws PreProcessingException
	 */
	public static String convertNonAsciiAndNormaliseRepresentation(String s) throws PreProcessingException {
		s = MATCH_WHITESPACE.matcher(s).replaceAll(" ");//normalise white space
		StringBuilder sb = new StringBuilder(s);
		for(int i=0;i<sb.length();i++) {
			char c = sb.charAt(i);
			if(c >= 128) {
				sb.replace(i, i+1, getReplacementForNonASCIIChar(c));//replace non ascii characters with hard coded ascii strings
			}
			else if (c ==96){
				sb.replace(i, i+1, "'");//replace back ticks with apostrophe
			}
			else if (c ==34){
				sb.replace(i, i+1, "''");//replace quotation mark with two primes
			}
		}
		return sb.toString();
	}

    private static String getReplacementForNonASCIIChar(char c) throws PreProcessingException {
        switch (c) {
            case '\u03b1': return "alpha";//greeks
            case '\u03b2': return "beta";
            case '\u03b3': return "gamma";
            case '\u03b4': return "delta";
            case '\u03b5': return "epsilon";
            case '\u03b6': return "zeta";
            case '\u03b7': return "eta";
            case '\u03b8': return "theta";
            case '\u03b9': return "iota";
            case '\u03ba': return "kappa";
            case '\u03bb': return "lambda";
            case '\u03bc': return "mu";
            case '\u03bd': return "nu";
            case '\u03be': return "xi";
            case '\u03bf': return "omicron";
            case '\u03c0': return "pi";
            case '\u03c1': return "rho";
            case '\u03c2': return "stigma";
            case '\u03c3': return "sigma";
            case '\u03c4': return "tau";
            case '\u03c5': return "upsilon";
            case '\u03c6': return "phi";
            case '\u03c7': return "chi";
            case '\u03c8': return "psi";
            case '\u03c9': return "omega";

            case '\u00B1': return "+-";//plus minus symbol
            case '\u2213': return "-+";
            
            case '\u00C6': return "ae";//common ligatures
            case '\u00E6': return "ae";
            case '\u0152': return "oe";
            case '\u0153': return "oe";
            case '\u0132': return "ij";
            case '\u0133': return "ij";
            case '\u1D6B': return "ue";
            case '\uFB00': return "ff";
            case '\uFB01': return "fi";
            case '\uFB02': return "fl";
            case '\uFB03': return "ffi";
            case '\uFB04': return "ffl";
            case '\uFB06': return "st";
            
            case '\u2018': return "'";//quotation marks and primes (map to apostrophe/s)
            case '\u2019': return "'";
            case '\u201B': return "'";
            case '\u2032': return "'";//primes
            case '\u2033': return "''";
            case '\u2034': return "'''";
            case '\u2057': return "''''";
            case '\u2035': return "'";//back primes
            case '\u2036': return "''";
            case '\u2037': return "'''";
            case '\u00B4': return "'";//accents
            case '\u02CA': return "'";
            case '\u0301': return "'";
            case '\u02DD': return "''";
            case '\u030B': return "''";

            case '\u2010': return "-";//dashes, hyphens and the minus sign
            case '\u2011': return "-";
            case '\u2012': return "-";
            case '\u2013': return "-";
            case '\u2014': return "-";
            case '\u2015': return "-";
            case '\u2212': return "-";
            
            case '\u00A0': return " ";//Non-breaking spaces
            case '\u2007': return " ";
            case '\u202F': return " ";
            
            case '\u200b': return "";//zero width space
            case '\u200d': return "";//zero width joiner

{'fraction_non_alphanumeric': 0.14878666557725304, 'fraction_numerical': 0.03611406160634039, 'mean_word_length': 2.287671232876712, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5561223', 'n_tokens_mistral': 4165, 'n_tokens_neox': 3675, 'n_words': 1167}