<commit_before>package org.elasticsearch.index.analysis;

import java.io.*;
import java.util.*;

/**
 * some parts of code copied from:http://code.google.com/p/java-zhconverter/
 */
public class STConverter {

    private Properties charMap = new Properties();
    private Properties revCharMap = new Properties();
    private Set conflictingSets  = new HashSet();

    public STConverter(){


        InputStream file1 = null;
        file1 = this.getClass().getResourceAsStream("/t2s.properties");
        InputStreamReader is = null;
        try {
            is = new InputStreamReader(file1, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        if (is != null) {
            BufferedReader reader = null;
            try {
                reader = new BufferedReader(is);
                charMap.load(reader);
            } catch (FileNotFoundException e) {
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    if (reader != null)
                        reader.close();
                    if (is != null)
                        is.close();
                } catch (IOException e) {
                }
            }
        }
        initializeHelper();
    }

    private void initializeHelper() {

        Map stringPossibilities = new HashMap();
        Iterator iter = charMap.keySet().iterator();
        while (iter.hasNext()) {

            //fill revmap
            String key = (String) iter.next();
            revCharMap.put(charMap.get(key),key);

            if (key.length() >= 1) {

                for (int i = 0; i < (key.length()); i++) {
                    String keySubstring = key.substring(0, i + 1);
                    if (stringPossibilities.containsKey(keySubstring)) {
                        Integer integer = (Integer)(stringPossibilities.get(keySubstring));
                        stringPossibilities.put(keySubstring, new Integer(
                                integer.intValue() + 1));

                    } else {
                        stringPossibilities.put(keySubstring, new Integer(1));
                    }

                }
            }
        }

        iter = stringPossibilities.keySet().iterator();
        while (iter.hasNext()) {
            String key = (String) iter.next();
            if (((Integer)(stringPossibilities.get(key))).intValue() > 1) {
                conflictingSets.add(key);
            }
        }
    }

    public String convert(STConvertType type,String in) {
           Map map=charMap;
        if(type== STConvertType.simple2traditional){
            map=revCharMap;
        }

        StringBuilder target = new StringBuilder();
        StringBuilder source = new StringBuilder();

        for (int i = 0; i < in.length(); i++) {

            char c = in.charAt(i);
            String key = "" + c;
            source.append(key);

            if (conflictingSets.contains(source.toString())) {
            } else if (map.containsKey(source.toString())) {
                target.append(map.get(source.toString()));
                source.setLength(0);
            } else {
                CharSequence sequence = source.subSequence(0, source.length()-1);
                source.delete(0, source.length()-1);
                mapping(map, target, new StringBuilder(sequence));
            }
        }

        mapping(map, target, source);

        return target.toString();
    }

    private static STConverter instance=new STConverter();

    public static STConverter getInstance(){
        if(instance==null){instance = new STConverter();}
        return instance;
    }

    public String convert(String text, STConvertType converterType) {
        return getInstance().convert(converterType,text);
    }

    private void mapping(Map map, StringBuilder outString, StringBuilder stackString) {
        while (stackString.length() > 0){
            if (map.containsKey(stackString.toString())) {
                outString.append(map.get(stackString.toString()));
                stackString.setLength(0);

            } else {
                outString.append("" + stackString.charAt(0));
                stackString.delete(0, 1);
            }

        }
    }

}
<commit_msg>squid:S2325  'private' methods that don't access instance data should be 'static' squid:S1148  Throwable.printStackTrace(...) should not be called findbugs:DM_NUMBER_CTOR  Performance - Method invokes inefficient Number constructor; use static valueOf instead<commit_after>package org.elasticsearch.index.analysis;

import java.io.*;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;


/**
 * some parts of code copied from:http://code.google.com/p/java-zhconverter/
 */
public class STConverter {
    
    private static final Logger LOGGER = Logger.getLogger(STConverter.class.getName());

    private Properties charMap = new Properties();
    private Properties revCharMap = new Properties();
    private Set conflictingSets  = new HashSet();

    public STConverter(){


        InputStream file1 = null;
        file1 = this.getClass().getResourceAsStream("/t2s.properties");
        InputStreamReader is = null;
        try {
            is = new InputStreamReader(file1, "UTF-8");
        } catch (UnsupportedEncodingException e) {
            LOGGER.log(Level.SEVERE, "Unsupported character encoding " + e.getMessage(), e);
        }
        if (is != null) {
            BufferedReader reader = null;
            try {
                reader = new BufferedReader(is);
                charMap.load(reader);
            } catch (FileNotFoundException e) {
            } catch (IOException e) {
                LOGGER.log(Level.SEVERE, "IOException in loading charMap: " + e.getMessage(), e);
            } finally {
                try {
                    if (reader != null)
                        reader.close();
                    if (is != null)
                        is.close();
                } catch (IOException e) {
                }
            }
        }
        initializeHelper();
    }

    private void initializeHelper() {

        Map stringPossibilities = new HashMap();
        Iterator iter = charMap.keySet().iterator();
        while (iter.hasNext()) {

            //fill revmap
            String key = (String) iter.next();
            revCharMap.put(charMap.get(key),key);

            if (key.length() >= 1) {

                for (int i = 0; i < (key.length()); i++) {
                    String keySubstring = key.substring(0, i + 1);
                    if (stringPossibilities.containsKey(keySubstring)) {
                        Integer integer = (Integer)(stringPossibilities.get(keySubstring));
                        stringPossibilities.put(keySubstring, new Integer(Integer.valueOf(integer) + 1));

                    } else {
                        stringPossibilities.put(keySubstring, new Integer(1));
                    }

                }
            }
        }

        iter = stringPossibilities.keySet().iterator();
        while (iter.hasNext()) {
            String key = (String) iter.next();
            int value = Integer.valueOf( (Integer)stringPossibilities.get(key) );
            if (value > 1) {
                conflictingSets.add(key);
            }
        }
    }

    public String convert(STConvertType type,String in) {
           Map map=charMap;
        if(type== STConvertType.simple2traditional){
            map=revCharMap;
        }

        StringBuilder target = new StringBuilder();
        StringBuilder source = new StringBuilder();

        for (int i = 0; i < in.length(); i++) {

            char c = in.charAt(i);
            String key = "" + c;
            source.append(key);

            if (conflictingSets.contains(source.toString())) {
            } else if (map.containsKey(source.toString())) {
                target.append(map.get(source.toString()));
                source.setLength(0);
            } else {
                CharSequence sequence = source.subSequence(0, source.length()-1);
                source.delete(0, source.length()-1);
                mapping(map, target, new StringBuilder(sequence));
            }
        }

        mapping(map, target, source);

        return target.toString();
    }

    private static STConverter instance=new STConverter();

    public static STConverter getInstance(){
        if(instance==null){instance = new STConverter();}
        return instance;
    }

    public String convert(String text, STConvertType converterType) {
        return getInstance().convert(converterType,text);
    }

    private static void mapping(Map map, StringBuilder outString, StringBuilder stackString) {
        while (stackString.length() > 0){
            if (map.containsKey(stackString.toString())) {
                outString.append(map.get(stackString.toString()));
                stackString.setLength(0);

            } else {
                outString.append("" + stackString.charAt(0));
                stackString.delete(0, 1);
            }

        }
    }

}

{'fraction_non_alphanumeric': 0.11403030633380573, 'fraction_numerical': 0.006104872996838548, 'mean_word_length': 1.7401433691756272, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3227110', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2264, 'n_words': 528}