<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2012 VoltDB Inc.
 *
 * VoltDB is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * VoltDB is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;

import com.google.common.base.Charsets;

/**
 * Encode and decode strings and byte arrays to/from hexidecimal
 * string values. This was originally added so binary values could
 * be added to the VoltDB catalogs.
 *
 */
public class Encoder {
    private static final int caseDiff = ('a' - 'A');
    /**
     *
     * @param data A binary array of bytes.
     * @return A hex-encoded string with double length.
     */
    public static String hexEncode(byte[] data) {
        if (data == null)
            return null;

        StringBuilder sb = new StringBuilder();
        for (byte b : data) {
            // hex encoding same way as java.net.URLEncoder.
            char ch = Character.forDigit((b >> 4) & 0xF, 16);
            // to uppercase
            if (Character.isLetter(ch)) {
                ch -= caseDiff;
            }
            sb.append(ch);
            ch = Character.forDigit(b & 0xF, 16);
            if (Character.isLetter(ch)) {
                ch -= caseDiff;
            }
            sb.append(ch);
        }
        return sb.toString();
    }

    /**
     *
     * @param string A (latin) string to be hex encoded.
     * @return The double-length hex encoded string.
     */
    public static String hexEncode(String string) {
        // this will need to be less "western" in the future
        return hexEncode(string.getBytes(Charsets.ISO_8859_1));
    }

    /**
     *
     * @param hexString An (even-length) hexidecimal string to be decoded.
     * @return The binary byte array value for the string (half length).
     */
    public static byte[] hexDecode(String hexString) {
        if ((hexString.length() % 2) != 0)
            throw new RuntimeException("String is not properly hex-encoded.");

        hexString = hexString.toUpperCase();
        byte[] retval = new byte[hexString.length() / 2];
        for (int i = 0; i < retval.length; i++) {
            int value = Integer.parseInt(hexString.substring(2 * i, 2 * i + 2), 16);
            if ((value < 0) || (value > 255))
                throw new RuntimeException("String is not properly hex-encoded.");
            retval[i] = (byte) value;
        }
        return retval;
    }

    /**
     *
     * @param hexString A string of hexidecimal chars of even length.
     * @return A string value de-hexed from the input.
     */
    public static String hexDecodeToString(String hexString) {
        byte[] decodedValue = hexDecode(hexString);
        return new String(decodedValue, Charsets.ISO_8859_1);
    }

    public static boolean isHexEncodedString(String hexString) {
        if ((hexString.length() % 2) != 0)
            return false;
        try {
            for (int i = 0; i < hexString.length(); i++) {
                int value = Integer.parseInt(hexString.substring(i, i + 1), 16);
                if ((value < 0) || (value > 15)) return false;
            }
        } catch (NumberFormatException e) {
            return false;
        }
        return true;
    }

    public static String compressAndBase64Encode(String string) {
        try {
            byte[] inBytes = string.getBytes(Charsets.UTF_8);
            ByteArrayOutputStream baos = new ByteArrayOutputStream((int)(string.length() * 0.7));
            DeflaterOutputStream dos = new DeflaterOutputStream(baos);
            dos.write(inBytes);
            dos.close();
            byte[] outBytes = baos.toByteArray();
            return Base64.encodeToString(outBytes, false);
        }
        catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static String decodeBase64AndDecompress(String string) {
        byte bytes[] = Base64.decode(string);
        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
        InflaterInputStream dis = new InflaterInputStream(bais);

        byte buffer[] = new byte[1024 * 8];
        int length = 0;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        try {
            while ( (length = dis.read( buffer )) >= 0) {
                baos.write(buffer, 0, length);
            }<commit_msg>Switch to fast version of the base64 decoding function.<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2012 VoltDB Inc.
 *
 * VoltDB is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * VoltDB is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.utils;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;

import com.google.common.base.Charsets;

/**
 * Encode and decode strings and byte arrays to/from hexidecimal
 * string values. This was originally added so binary values could
 * be added to the VoltDB catalogs.
 *
 */
public class Encoder {
    private static final int caseDiff = ('a' - 'A');
    /**
     *
     * @param data A binary array of bytes.
     * @return A hex-encoded string with double length.
     */
    public static String hexEncode(byte[] data) {
        if (data == null)
            return null;

        StringBuilder sb = new StringBuilder();
        for (byte b : data) {
            // hex encoding same way as java.net.URLEncoder.
            char ch = Character.forDigit((b >> 4) & 0xF, 16);
            // to uppercase
            if (Character.isLetter(ch)) {
                ch -= caseDiff;
            }
            sb.append(ch);
            ch = Character.forDigit(b & 0xF, 16);
            if (Character.isLetter(ch)) {
                ch -= caseDiff;
            }
            sb.append(ch);
        }
        return sb.toString();
    }

    /**
     *
     * @param string A (latin) string to be hex encoded.
     * @return The double-length hex encoded string.
     */
    public static String hexEncode(String string) {
        // this will need to be less "western" in the future
        return hexEncode(string.getBytes(Charsets.ISO_8859_1));
    }

    /**
     *
     * @param hexString An (even-length) hexidecimal string to be decoded.
     * @return The binary byte array value for the string (half length).
     */
    public static byte[] hexDecode(String hexString) {
        if ((hexString.length() % 2) != 0)
            throw new RuntimeException("String is not properly hex-encoded.");

        hexString = hexString.toUpperCase();
        byte[] retval = new byte[hexString.length() / 2];
        for (int i = 0; i < retval.length; i++) {
            int value = Integer.parseInt(hexString.substring(2 * i, 2 * i + 2), 16);
            if ((value < 0) || (value > 255))
                throw new RuntimeException("String is not properly hex-encoded.");
            retval[i] = (byte) value;
        }
        return retval;
    }

    /**
     *
     * @param hexString A string of hexidecimal chars of even length.
     * @return A string value de-hexed from the input.
     */
    public static String hexDecodeToString(String hexString) {
        byte[] decodedValue = hexDecode(hexString);
        return new String(decodedValue, Charsets.ISO_8859_1);
    }

    public static boolean isHexEncodedString(String hexString) {
        if ((hexString.length() % 2) != 0)
            return false;
        try {
            for (int i = 0; i < hexString.length(); i++) {
                int value = Integer.parseInt(hexString.substring(i, i + 1), 16);
                if ((value < 0) || (value > 15)) return false;
            }
        } catch (NumberFormatException e) {
            return false;
        }
        return true;
    }

    public static String compressAndBase64Encode(String string) {
        try {
            byte[] inBytes = string.getBytes(Charsets.UTF_8);
            ByteArrayOutputStream baos = new ByteArrayOutputStream((int)(string.length() * 0.7));
            DeflaterOutputStream dos = new DeflaterOutputStream(baos);
            dos.write(inBytes);
            dos.close();
            byte[] outBytes = baos.toByteArray();
            return Base64.encodeToString(outBytes, false);
        }
        catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static String decodeBase64AndDecompress(String string) {
        byte bytes[] = Base64.decodeFast(string);
        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
        InflaterInputStream dis = new InflaterInputStream(bais);

        byte buffer[] = new byte[1024 * 8];
        int length = 0;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        try {
            while ( (length = dis.read( buffer )) >= 0) {
                baos.write(buffer, 0, length);
            }
{'fraction_non_alphanumeric': 0.1014910536779324, 'fraction_numerical': 0.013518886679920477, 'mean_word_length': 2.1778269109286166, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3509065', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2707, 'n_words': 932}