<commit_before>/*
 * Copyright 2013-2014 eBay Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.kylinolap.dict;

import org.apache.hadoop.hbase.util.Bytes;

/**
 * @author yangli9
 * 
 */
public class NumberDictionary<T> extends TrieDictionary<T> {

    public static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = 16;

    // encode a number into an order preserving byte sequence
    // for positives -- padding '0'
    // for negatives -- '-' sign, padding '9', invert digits, and terminate by
    // ';'
    static class NumberBytesCodec {

        byte[] buf = new byte[MAX_DIGITS_BEFORE_DECIMAL_POINT * 2];
        int bufOffset = 0;
        int bufLen = 0;

        void encodeNumber(byte[] value, int offset, int len) {
            if (len == 0) {
                bufOffset = 0;
                bufLen = 0;
                return;
            }

            boolean negative = value[offset] == '-';

            // terminate negative ';'
            int start = buf.length - len;
            int end = buf.length;
            if (negative) {
                start--;
                end--;
                buf[end] = ';';
            }

            // copy & find decimal point
            int decimalPoint = end;
            for (int i = start, j = offset; i < end; i++, j++) {
                buf[i] = value[j];
                if (buf[i] == '.' && i < decimalPoint) {
                    decimalPoint = i;
                }
            }
            // remove '-' sign
            if (negative) {
                start++;
            }

            // prepend '0'
            int nZeroPadding = MAX_DIGITS_BEFORE_DECIMAL_POINT - (decimalPoint - start);
            if (nZeroPadding < 0 || nZeroPadding + 1 > start)
                throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len));
            for (int i = 0; i < nZeroPadding; i++) {
                buf[--start] = '0';
            }

            // consider negative
            if (negative) {
                buf[--start] = '-';
                for (int i = start + 1; i < buf.length; i++) {
                    int c = buf[i];
                    if (c >= '0' && c <= '9') {
                        buf[i] = (byte) ('9' - (c - '0'));
                    }
                }
            } else {
                buf[--start] = '0';
            }

            bufOffset = start;
            bufLen = buf.length - start;
        }

        int decodeNumber(byte[] returnValue, int offset) {
            if (bufLen == 0) {<commit_msg>more error msg in NumberDictionary<commit_after>/*
 * Copyright 2013-2014 eBay Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.kylinolap.dict;

import org.apache.hadoop.hbase.util.Bytes;

/**
 * @author yangli9
 * 
 */
public class NumberDictionary<T> extends TrieDictionary<T> {

    public static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = 16;

    // encode a number into an order preserving byte sequence
    // for positives -- padding '0'
    // for negatives -- '-' sign, padding '9', invert digits, and terminate by ';'
    static class NumberBytesCodec {

        byte[] buf = new byte[MAX_DIGITS_BEFORE_DECIMAL_POINT * 2];
        int bufOffset = 0;
        int bufLen = 0;

        void encodeNumber(byte[] value, int offset, int len) {
            if (len == 0) {
                bufOffset = 0;
                bufLen = 0;
                return;
            }

            if (len > buf.length) {
                throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Internal buffer is only " + buf.length + " bytes");
            }

            boolean negative = value[offset] == '-';

            // terminate negative ';'
            int start = buf.length - len;
            int end = buf.length;
            if (negative) {
                start--;
                end--;
                buf[end] = ';';
            }

            // copy & find decimal point
            int decimalPoint = end;
            for (int i = start, j = offset; i < end; i++, j++) {
                buf[i] = value[j];
                if (buf[i] == '.' && i < decimalPoint) {
                    decimalPoint = i;
                }
            }
            // remove '-' sign
            if (negative) {
                start++;
            }

            // prepend '0'
            int nZeroPadding = MAX_DIGITS_BEFORE_DECIMAL_POINT - (decimalPoint - start);
            if (nZeroPadding < 0 || nZeroPadding + 1 > start)
                throw new IllegalArgumentException("Too many digits for NumberDictionary: " + Bytes.toString(value, offset, len) + ". Expect " + MAX_DIGITS_BEFORE_DECIMAL_POINT + " digits before decimal point at max.");
            for (int i = 0; i < nZeroPadding; i++) {
                buf[--start] = '0';
            }

            // consider negative
            if (negative) {
                buf[--start] = '-';
                for (int i = start + 1; i < buf.length; i++) {
                    int c = buf[i];
                    if (c >= '0' && c <= '9') {
                        buf[i] = (byte) ('9' - (c - '0'));
                    }
                }
            } else {
                buf[--start] = '0';
            }

            bufOffset = start;
            bufLen = buf.length - start;
        }

        int decodeNumber(byte[] returnValue, int offset) {
            if (bufLen == 0) {
{'fraction_non_alphanumeric': 0.11284403669724771, 'fraction_numerical': 0.010703363914373088, 'mean_word_length': 1.5264580919273851, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3633070', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1723, 'n_words': 579}