<commit_before>package net.reduls.gomoku.dic;

import java.io.DataInputStream;
import net.reduls.gomoku.util.Misc;

public final class SurfaceId {
    private static final int idOffset;
    private static final int nodeCount;
    private static final long[] nodes;
    
    static {
        {
            DataInputStream in = Misc.openDictionaryDataAsDIS("surface-id.bin");
            
            nodeCount = Misc.readInt(in);
            nodes = new long[nodeCount];
            for(int i=0; i < nodeCount; i++)
                nodes[i] = Misc.readLong(in);
            Misc.close(in);
        }
        {
            DataInputStream in = Misc.openDictionaryDataAsDIS("category.bin");
            idOffset = Misc.readInt(in);
            Misc.close(in);
        }
    }
 
    public static void eachCommonPrefix(String text, int start, WordDic.Callback fn) {
        int node = 0;
        int id = idOffset;
        
        for(int i=start;; i++) {
            if(isTerminal(node))
                WordDic.eachViterbiNode(fn, id++, start, i-start, false);
            <commit_msg>不要なフィールドを削除<commit_after>package net.reduls.gomoku.dic;

import java.io.DataInputStream;
import net.reduls.gomoku.util.Misc;

public final class SurfaceId {
    private static final int idOffset;
    private static final long[] nodes;
    
    static {
        {
            DataInputStream in = Misc.openDictionaryDataAsDIS("surface-id.bin");
            
            final int nodeCount = Misc.readInt(in);
            nodes = new long[nodeCount];
            for(int i=0; i < nodeCount; i++)
                nodes[i] = Misc.readLong(in);
            Misc.close(in);
        }
        {
            DataInputStream in = Misc.openDictionaryDataAsDIS("category.bin");
            idOffset = Misc.readInt(in);
            Misc.close(in);
        }
    }
 
    public static void eachCommonPrefix(String text, int start, WordDic.Callback fn) {
        int node = 0;
        int id = idOffset;
        
        for(int i=start;; i++) {
            if(isTerminal(node))
                WordDic.eachViterbiNode(fn, id++, start, i-start, false);
            
{'fraction_non_alphanumeric': 0.10627340823970037, 'fraction_numerical': 0.0018726591760299626, 'mean_word_length': 1.8723118279569892, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6640085', 'n_tokens_mistral': 650, 'n_tokens_neox': 635, 'n_words': 141}