<commit_before>package io.github.eb4j.ext;

import io.github.eb4j.EBException;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.StringTokenizer;


/**
 * UnicodeMap class to search alternative unicode for extfont codepoint.
 */
public class UnicodeMap {

    private HashMap<Integer, String> unicodeMap = new HashMap<>();

    /**
     * Constructor.
     *
     * @param title target dictionary title
     * @param dir target dictionary directory
     * @throws EBException when unicode map does not exist.
     */
    public UnicodeMap(final String title, final File dir) throws EBException {
        boolean loaded = false;
        if (dir != null) {
            if (!dir.isDirectory()) {
                throw new EBException(EBException.DIR_NOT_FOUND, dir.getPath());
            }
            if (!dir.canRead()) {
                throw new EBException(EBException.CANT_READ_DIR, dir.getPath());
            }
            File [] listFile = dir.listFiles();
            if (listFile != null) {
                try {
                    for (File file : listFile) {
                        if (file.isFile()) {
                            if (file.getName().toLowerCase().endsWith(".map")) {
                                loadMap(file);
                                loaded = true;
                                break;
                            }
                        }
                    }
                } catch (EBException ignored) {
                }
            }
        }
        if (!loaded) {
            if (isUnno(title)) {
                setUnnoMap();
            } else {
                throw new EBException(EBException.CANT_FIND_UNICODEMAP);
            }
        }
    }

    public String get(final int extCode) {
        return unicodeMap.get(extCode);
    }

    private void loadMap(final File file) throws EBException {
        StringTokenizer st;
        try {
            BufferedReader tsvFile = new BufferedReader(new InputStreamReader(new FileInputStream(file),
                    Charset.forName("SHIFT-JIS")));
            String dataRow = tsvFile.readLine();
            while (dataRow != null) {
                List<String> dataArray = new ArrayList<>() ;
                st = new StringTokenizer(dataRow, "\t");
                while (st.hasMoreElements()) {
                    dataArray.add(st.nextElement().toString());
                }
                if (dataArray.size() >= 2) {
                    String key = dataArray.get(0);
                    if (!key.startsWith("#")) {
                        String val = dataArray.get(1);
                        if ((key.startsWith("h") || key.startsWith("z"))) {
                            Integer keyNum = Integer.parseInt(key.substring(1, 5), 16);
                            if (val.startsWith("u")) {
                                if (val.contains(",")) {
                                    StringBuilder sb = new StringBuilder();
                                    for (String item : val.split(",")) {
                                        if (item.startsWith("u")) {
                                            sb.append(Character.toChars(Integer.valueOf(item.substring(1, 5), 16)));
                                        }
                                    }
                                    unicodeMap.put(keyNum, sb.toString());
                                } else {
                                    String uniValue = new String(Character.toChars(Integer.valueOf(val.substring(1, 5), 16)));
                                    unicodeMap.put(keyNum, uniValue);
                                }
                            } else if (val.startsWith("-") && dataArray.size() == 3) {
                                String uniValue = dataArray.get(2);
                                unicodeMap.put(keyNum, uniValue);<commit_msg>Improve coding style<commit_after>package io.github.eb4j.ext;

import io.github.eb4j.EBException;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.StringTokenizer;


/**
 * UnicodeMap class to search alternative unicode for extfont codepoint.
 */
public class UnicodeMap {

    private HashMap<Integer, String> unicodeMap = new HashMap<>();

    /**
     * Constructor.
     *
     * @param title target dictionary title
     * @param dir target dictionary directory
     * @throws EBException when unicode map does not exist.
     */
    public UnicodeMap(final String title, final File dir) throws EBException {
        boolean loaded = false;
        if (dir != null) {
            if (!dir.isDirectory()) {
                throw new EBException(EBException.DIR_NOT_FOUND, dir.getPath());
            }
            if (!dir.canRead()) {
                throw new EBException(EBException.CANT_READ_DIR, dir.getPath());
            }
            File[] listFile = dir.listFiles();
            if (listFile != null) {
                try {
                    for (File file : listFile) {
                        if (file.isFile()) {
                            if (file.getName().toLowerCase().endsWith(".map")) {
                                loadMap(file);
                                loaded = true;
                                break;
                            }
                        }
                    }
                } catch (EBException ignored) {
                }
            }
        }
        if (!loaded) {
            if (isUnno(title)) {
                setUnnoMap();
            } else {
                throw new EBException(EBException.CANT_FIND_UNICODEMAP);
            }
        }
    }

    /**
     * Get Unicode character code from map.
     * @param extCode to query
     * @return unicode string
     */
    public String get(final int extCode) {
        return unicodeMap.get(extCode);
    }

    private void loadMap(final File file) throws EBException {
        StringTokenizer st;
        try {
            BufferedReader tsvFile = new BufferedReader(new InputStreamReader(new FileInputStream(file),
                    Charset.forName("SHIFT-JIS")));
            String dataRow = tsvFile.readLine();
            while (dataRow != null) {
                List<String> dataArray = new ArrayList<>();
                st = new StringTokenizer(dataRow, "\t");
                while (st.hasMoreElements()) {
                    dataArray.add(st.nextElement().toString());
                }
                if (dataArray.size() >= 2) {
                    String key = dataArray.get(0);
                    if (!key.startsWith("#")) {
                        String val = dataArray.get(1);
                        if ((key.startsWith("h") || key.startsWith("z"))) {
                            Integer keyNum = Integer.parseInt(key.substring(1, 5), 16);
                            if (val.startsWith("u")) {
                                if (val.contains(",")) {
                                    StringBuilder sb = new StringBuilder();
                                    for (String item : val.split(",")) {
                                        if (item.startsWith("u")) {
                                            sb.append(Character.toChars(
                                                    Integer.valueOf(item.substring(1, 5), 16)));
                                        }
                                    }
                                    unicodeMap.put(keyNum, sb.toString());
                                } else {
                                    String uniValue = new String(Character.toChars(
                                            Integer.valueOf(val.substring(1, 5), 16)));
                                    unicodeMap.put(keyNum, uniValue);
                                }
                            } else if (val.startsWith("-") && dataArray.size() == 3) {
                                String uniValue = dataArray.get(2);
                                unicodeMap.put(keyNum, uniValue);
{'fraction_non_alphanumeric': 0.10416666666666667, 'fraction_numerical': 0.004523809523809524, 'mean_word_length': 1.227200424178155, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '971759', 'n_tokens_mistral': 2101, 'n_tokens_neox': 1977, 'n_words': 450}