<commit_before>/*
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.trie4j.doublearray;

import java.io.Externalizable;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.trie4j.AbstractTermIdTrie;
import org.trie4j.Node;
import org.trie4j.TermIdNode;
import org.trie4j.TermIdTrie;
import org.trie4j.Trie;
import org.trie4j.bv.Rank1OnlySuccinctBitVector;
import org.trie4j.bv.SuccinctBitVector;
import org.trie4j.doublearray.DoubleArray.TermNodeListener;
import org.trie4j.tail.FastTailCharIterator;
import org.trie4j.tail.TailBuilder;
import org.trie4j.tail.TailCharIterator;
import org.trie4j.tail.TailUtil;
import org.trie4j.tail.builder.SuffixTrieTailBuilder;
import org.trie4j.util.BitSet;
import org.trie4j.util.FastBitSet;
import org.trie4j.util.Pair;

public class TailDoubleArray extends AbstractTermIdTrie implements TermIdTrie, Externalizable{
	public static interface TermNodeListener{
		void listen(Node node, int nodeIndex);
	}

	public TailDoubleArray(){
	}

	public TailDoubleArray(Trie orig){
		this(orig, new SuffixTrieTailBuilder());
	}

	public TailDoubleArray(Trie orig, TailBuilder tb){
		this(orig, tb, new TermNodeListener(){
			@Override
			public void listen(Node node, int nodeIndex) {
			}
		});
	}

	public TailDoubleArray(Trie orig, TailBuilder tb, TermNodeListener listener){
		size = orig.size();
		int as = size;
		if(as <= 1) as = 2;
		base = new int[as];
		Arrays.fill(base, BASE_EMPTY);
		check = new int[as];
		Arrays.fill(check, -1);
		tail = new int[as];
		Arrays.fill(tail, -1);
		Arrays.fill(charToCode, (char)0);

		FastBitSet bs = new FastBitSet(65536);
		build(orig.getRoot(), 0, tb, bs, listener);
		term = new Rank1OnlySuccinctBitVector(bs.getBytes(), bs.size());
		tails = tb.getTails();
	}

	@Override
	public int size() {
		return size;
	}

	@Override
	public TermIdNode getRoot() {
		return new TailDoubleArrayNode(0);
	}

	public int[] getBase(){
		return base;
	}

	public int[] getCheck(){
		return check;
	}

	public BitSet getTerm() {
		return term;
	}

	private class TailDoubleArrayNode implements TermIdNode{
		public TailDoubleArrayNode(int nodeId){
			this.nodeId = nodeId;
		}

		public TailDoubleArrayNode(int nodeId, char firstChar){
			this.nodeId = nodeId;
			this.firstChar = firstChar;
		}

		@Override
		public boolean isTerminate() {
			return term.get(nodeId);
		}

		public int getNodeId() {
			return nodeId;
		}

		@Override
		public int getTermId() {
			return term.get(nodeId) ? term.rank1(nodeId) - 1 : -1;
		}

		@Override
		public char[] getLetters() {
			StringBuilder ret = new StringBuilder();
			ret.append(firstChar);
			TailUtil.appendChars(tails, tail[nodeId],ret);
			return ret.toString().toCharArray();
		}

		@Override
		public TermIdNode[] getChildren() {
			List<TermIdNode> ret = new ArrayList<TermIdNode>();
			int b = base[nodeId];
			for(char c : chars){
				int code = charToCode[c];
				int nid = b + code;
				if(nid >= 0 && nid < check.length && check[nid] == nodeId){
					ret.add(new TailDoubleArrayNode(nid, c));
				}
			}
			return ret.toArray(emptyNodes);<commit_msg>remove unused import and method.<commit_after>/*
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.trie4j.doublearray;

import java.io.Externalizable;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.trie4j.AbstractTermIdTrie;
import org.trie4j.Node;
import org.trie4j.TermIdNode;
import org.trie4j.TermIdTrie;
import org.trie4j.Trie;
import org.trie4j.bv.Rank1OnlySuccinctBitVector;
import org.trie4j.bv.SuccinctBitVector;
import org.trie4j.tail.FastTailCharIterator;
import org.trie4j.tail.TailBuilder;
import org.trie4j.tail.TailCharIterator;
import org.trie4j.tail.TailUtil;
import org.trie4j.tail.builder.SuffixTrieTailBuilder;
import org.trie4j.util.BitSet;
import org.trie4j.util.FastBitSet;
import org.trie4j.util.Pair;

public class TailDoubleArray extends AbstractTermIdTrie implements TermIdTrie, Externalizable{
	public static interface TermNodeListener{
		void listen(Node node, int nodeIndex);
	}

	public TailDoubleArray(){
	}

	public TailDoubleArray(Trie orig){
		this(orig, new SuffixTrieTailBuilder());
	}

	public TailDoubleArray(Trie orig, TailBuilder tb){
		this(orig, tb, new TermNodeListener(){
			@Override
			public void listen(Node node, int nodeIndex) {
			}
		});
	}

	public TailDoubleArray(Trie orig, TailBuilder tb, TermNodeListener listener){
		size = orig.size();
		int as = size;
		if(as <= 1) as = 2;
		base = new int[as];
		Arrays.fill(base, BASE_EMPTY);
		check = new int[as];
		Arrays.fill(check, -1);
		tail = new int[as];
		Arrays.fill(tail, -1);
		Arrays.fill(charToCode, (char)0);

		FastBitSet bs = new FastBitSet(65536);
		build(orig.getRoot(), 0, tb, bs, listener);
		term = new Rank1OnlySuccinctBitVector(bs.getBytes(), bs.size());
		tails = tb.getTails();
	}

	@Override
	public int size() {
		return size;
	}

	@Override
	public TermIdNode getRoot() {
		return new TailDoubleArrayNode(0);
	}

	public int[] getBase(){
		return base;
	}

	public int[] getCheck(){
		return check;
	}

	public BitSet getTerm() {
		return term;
	}

	private class TailDoubleArrayNode implements TermIdNode{
		public TailDoubleArrayNode(int nodeId){
			this.nodeId = nodeId;
		}

		public TailDoubleArrayNode(int nodeId, char firstChar){
			this.nodeId = nodeId;
			this.firstChar = firstChar;
		}

		@Override
		public boolean isTerminate() {
			return term.get(nodeId);
		}

		@Override
		public int getTermId() {
			return term.get(nodeId) ? term.rank1(nodeId) - 1 : -1;
		}

		@Override
		public char[] getLetters() {
			StringBuilder ret = new StringBuilder();
			ret.append(firstChar);
			TailUtil.appendChars(tails, tail[nodeId],ret);
			return ret.toString().toCharArray();
		}

		@Override
		public TermIdNode[] getChildren() {
			List<TermIdNode> ret = new ArrayList<TermIdNode>();
			int b = base[nodeId];
			for(char c : chars){
				int code = charToCode[c];
				int nid = b + code;
				if(nid >= 0 && nid < check.length && check[nid] == nodeId){
					ret.add(new TailDoubleArrayNode(nid, c));
				}
			}
			return ret.toArray(emptyNodes);
{'fraction_non_alphanumeric': 0.11399283685315549, 'fraction_numerical': 0.010497715203161664, 'mean_word_length': 4.338167435728411, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4935817', 'n_tokens_mistral': 2981, 'n_tokens_neox': 2733, 'n_words': 710}