<commit_before>package com.coccoc;

import java.util.ArrayList;
import java.util.List;

/**
 * Rewrite class com.coccoc.Tokenizer for Elasticsearch integration.
 *
 * @author duydo, CocCoc team
 */
public class Tokenizer {
    public static final String TOKENIZER_SHARED_LIB_NAME = "coccoc_tokenizer_jni";

    static {
        System.loadLibrary(TOKENIZER_SHARED_LIB_NAME);
    }


    public enum TokenizeOption {
        NORMAL(0),
        HOST(1),
        URL(2);

        private final int value;

        TokenizeOption(int value) {
            this.value = value;
        }

        public int value() {
            return value;
        }
    }

    public static final String SPACE = " ";
    public static final String UNDERSCORE = "_";
    public static final String COMMA = ",";
    public static final String DOT = ".";


    private static String dictPath = null;

    private static final class Loader {
        private static final Tokenizer INSTANCE = get();

        private Loader() {
        }

        private static Tokenizer get() {
            return new Tokenizer(dictPath);
        }
    }

    public static Tokenizer getInstance(String dictPath) {
        Tokenizer.dictPath = dictPath;
        return Loader.INSTANCE;
    }

    private Tokenizer(String dictPath) {
        int status = initialize(dictPath);
        if (0 > status) {
            throw new RuntimeException(String.format("Cannot initialize Tokenizer: %s", dictPath));
        }
    }

    public List<Token> segment(String text, TokenizeOption option, boolean keepPunctuation) {
        return segment(text, false, option.value(), keepPunctuation);
    }

    private List<Token> segment(String text, boolean forTransforming, int tokenizeOption, boolean keepPunctuation) {
        if (text == null) {
            throw new IllegalArgumentException("text is null");
        }
        long resPointer = segmentPointer(text, forTransforming, tokenizeOption, keepPunctuation);

        final List<Token> res = new ArrayList<>();
        // Positions from JNI implementation .cpp file
        long normalizedStringPointer = Unsafe.UNSAFE.getLong(resPointer + 8);
        int rangesSize = (int) Unsafe.UNSAFE.getLong(resPointer + 8 * 2);
        long rangesDataPointer = Unsafe.UNSAFE.getLong(resPointer + 8 * 3);

        int spacePositionsSize = (int) Unsafe.UNSAFE.getLong(resPointer + 8 * 5);
        long spacePositionsDataPointer = Unsafe.UNSAFE.getLong(resPointer + 8 * 6);
        int[] spacePositions = new int[spacePositionsSize + 1];
        for (int i = 0; i < spacePositionsSize; ++i) {
            spacePositions[i] = Unsafe.UNSAFE.getInt(spacePositionsDataPointer + i * 4);
        }
        spacePositions[spacePositionsSize] = -1;

        int tokenSize = 4 * 6;
        for (int i = 0, spacePos = 0; i < rangesSize; ++i) {
            // Positions of UNSAFE values are calculated from {struct Token} in tokenizer.hpp
            int originalStartPos = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 8);
            int originalEndPos = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 12);
            int type = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 16);
            int segType = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 20);

            // Build substring from UNSAFE array of codepoints
            final StringBuilder sb = new StringBuilder();
            for (int j = originalStartPos; j < originalEndPos; ++j) {
                sb.appendCodePoint(text.charAt(j));
            }
            res.add(new Token(segType == 1 ? sb.toString().replace(COMMA, DOT) : sb.toString(),
                    Token.Type.fromInt(type), Token.SegType.fromInt(segType), originalStartPos, originalEndPos));
        }
        if (forTransforming && tokenizeOption == TokenizeOption.NORMAL.value()) {
            res.add(Token.FULL_STOP);
        }
        freeMemory(resPointer);
        return res;
    }<commit_msg>Removed unused variables in segment method<commit_after>package com.coccoc;

import java.util.ArrayList;
import java.util.List;

/**
 * Rewrite class com.coccoc.Tokenizer for Elasticsearch integration.
 *
 * @author duydo, CocCoc team
 */
public class Tokenizer {
    public static final String TOKENIZER_SHARED_LIB_NAME = "coccoc_tokenizer_jni";

    static {
        System.loadLibrary(TOKENIZER_SHARED_LIB_NAME);
    }


    public enum TokenizeOption {
        NORMAL(0),
        HOST(1),
        URL(2);

        private final int value;

        TokenizeOption(int value) {
            this.value = value;
        }

        public int value() {
            return value;
        }
    }

    public static final String SPACE = " ";
    public static final String UNDERSCORE = "_";
    public static final String COMMA = ",";
    public static final String DOT = ".";


    private static String dictPath = null;

    private static final class Loader {
        private static final Tokenizer INSTANCE = get();

        private Loader() {
        }

        private static Tokenizer get() {
            return new Tokenizer(dictPath);
        }
    }

    public static Tokenizer getInstance(String dictPath) {
        Tokenizer.dictPath = dictPath;
        return Loader.INSTANCE;
    }

    private Tokenizer(String dictPath) {
        int status = initialize(dictPath);
        if (0 > status) {
            throw new RuntimeException(String.format("Cannot initialize Tokenizer: %s", dictPath));
        }
    }

    public List<Token> segment(String text, TokenizeOption option, boolean keepPunctuation) {
        return segment(text, false, option.value(), keepPunctuation);
    }

    private List<Token> segment(String text, boolean forTransforming, int tokenizeOption, boolean keepPunctuation) {
        if (text == null) {
            throw new IllegalArgumentException("text is null");
        }
        long resPointer = segmentPointer(text, forTransforming, tokenizeOption, keepPunctuation);

        final List<Token> res = new ArrayList<>();
        // Positions from JNI implementation .cpp file
        int rangesSize = (int) Unsafe.UNSAFE.getLong(resPointer + 8 * 2);
        long rangesDataPointer = Unsafe.UNSAFE.getLong(resPointer + 8 * 3);
        int tokenSize = 4 * 6;
        for (int i = 0, spacePos = 0; i < rangesSize; ++i) {
            // Positions of UNSAFE values are calculated from {struct Token} in tokenizer.hpp
            int originalStartPos = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 8);
            int originalEndPos = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 12);
            int type = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 16);
            int segType = Unsafe.UNSAFE.getInt(rangesDataPointer + i * tokenSize + 20);

            // Build substring from UNSAFE array of codepoints
            final StringBuilder sb = new StringBuilder();
            for (int j = originalStartPos; j < originalEndPos; ++j) {
                sb.appendCodePoint(text.charAt(j));
            }
            res.add(new Token(segType == 1 ? sb.toString().replace(COMMA, DOT) : sb.toString(),
                    Token.Type.fromInt(type), Token.SegType.fromInt(segType), originalStartPos, originalEndPos));
        }
        if (forTransforming && tokenizeOption == TokenizeOption.NORMAL.value()) {
            res.add(Token.FULL_STOP);
        }
        freeMemory(resPointer);
        return res;
    }
{'fraction_non_alphanumeric': 0.09734156820622986, 'fraction_numerical': 0.006578947368421052, 'mean_word_length': 2.4550092764378477, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '809307', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2078, 'n_words': 560}