<commit_before>/*
 * Copyright (c) 2018, DB Systel GmbH
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 * 
 * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 * 
 * 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Author: <NAME>, DB Systel GmbH
 *
 * Changes: 
 *     2018-08-16: V1.0.0: Created. fhs
 *     2018-08-20: V1.1.0: Expand valuation of 2 to 4 byte compressed numbers. fhs
 */
package dbsnumberlib;

import java.util.Arrays;

/**
 * Converts integers from and to an unsigned packed byte array
 * 
 * @author FrankSchwab
 * @version 1.1.0
 */
public class PackedUnsignedInteger {

   private final static int START_TWO_BYTE_VALUE = 0x40;
   private final static int START_THREE_BYTE_VALUE = 0x4000;
   private final static int START_FOUR_BYTE_VALUE = 0x400000;
   private final static int START_TOO_LARGE_VALUE = 0x40000000;
   
   /**
    * Convert an integer into a packed decimal byte array
    * 
    * Valid integers range from 0 to 1077936127.
    * All other numbers throw an IllegalArgumentException
    * 
    * @param aNumber Integer to convert
    * @return Packed decimal byte array with integer as value 
    * @throws IllegalArgumentException
    */
   public static byte[] fromInteger(final int aNumber) throws IllegalArgumentException {
      byte[] result;
      int intermediateNumber;
      
      if (aNumber >= 0)
         if (aNumber <= 0x3f) {
            result = new byte[1];
            result[0] = (byte) (aNumber & 0x3f);
         } else if (aNumber < (START_THREE_BYTE_VALUE + START_TWO_BYTE_VALUE)) {
            result = new byte[2];
            intermediateNumber = aNumber - START_TWO_BYTE_VALUE;
            result[1] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[0] = (byte) (0x40 | (intermediateNumber & 0xff));
         } else if (aNumber < (START_FOUR_BYTE_VALUE + START_THREE_BYTE_VALUE)) {
            result = new byte[3];
            intermediateNumber = aNumber - START_THREE_BYTE_VALUE;
            result[2] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[1] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[0] = (byte) (0x80 | (intermediateNumber & 0xff));
         } else if (aNumber < (START_TOO_LARGE_VALUE + START_FOUR_BYTE_VALUE)) {
            result = new byte[4];
            intermediateNumber = aNumber - START_FOUR_BYTE_VALUE;
            result[3] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[2] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[1] = (byte) (intermediateNumber & 0xff);
            intermediateNumber >>>= 8;
            result[0] = (byte) (0xc0 | (intermediateNumber & 0xff));
         } else
            throw new IllegalArgumentException();
      else
         throw new IllegalArgumentException();

      return result;
   }

   /**
    * Get expected length of packed decimal byte array from first byte
    * 
    * @param firstByteOfPackedNumber First byte of packed decimal integer
    * @return Expected length (1 to 4)
    */
   public static int getExpectedLength(final byte firstByteOfPackedNumber) {
      return ((firstByteOfPackedNumber >>> 6) & 0x03) + 1;
   }

   /**
    * Convert a packed decimal byte array itno an integer
    * 
    * @param packedNumber Packed decimal byte array
    * @return Converted integer
    * @throws IllegalArgumentException
    */
   public static int toInteger(final byte[] packedNumber) throws IllegalArgumentException {
      int result = 0;

      final int expectedLength = getExpectedLength(packedNumber[0]);

      if (expectedLength == packedNumber.length)
         switch (expectedLength) {
            case 1:
               result = (int) (packedNumber[0] & 0x3f);
               break;

            case 2:
               result = (int) (((packedNumber[0] & 0x3f) << 8) | (packedNumber[1] & 0xff)) + START_TWO_BYTE_VALUE;
               break;

            case 3:
               result = (int) (((packedNumber[0] & 0x3f) << 16) | ((packedNumber[1] & 0xff) << 8) | (packedNumber[2] & 0xff)) + START_THREE_BYTE_VALUE;
               break;

            case 4:
               result = (int) (((packedNumber[0] & 0x3f) << 24) | ((packedNumber[1] & 0xff) << 16) | ((packedNumber[2] & 0xff) << 8) | (packedNumber[3] & 0xff)) + START_FOUR_BYTE_VALUE;
               break;

            default:
               throw new IllegalArgumentException();
         }
      else
         throw new IllegalArgumentException();

      return result;<commit_msg>Better layout in PackedUnsignedInteger class<commit_after>/*
 * Copyright (c) 2018, DB Systel GmbH
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 * 
 * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 * 
 * 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Author: <NAME>, DB Systel GmbH
 *
 * Changes: 
 *     2018-08-16: V1.0.0: Created. fhs
 *     2018-08-20: V1.1.0: Expand valuation of 2 to 4 byte compressed numbers. fhs
 */
package dbsnumberlib;

import java.util.Arrays;

/**
 * Converts integers from and to an unsigned packed byte array
 * 
 * @author FrankSchwab
 * @version 1.1.0
 */
public class PackedUnsignedInteger {

   private final static int START_TWO_BYTE_VALUE = 0x40;
   private final static int START_THREE_BYTE_VALUE = 0x4000;
   private final static int START_FOUR_BYTE_VALUE = 0x400000;
   private final static int START_TOO_LARGE_VALUE = 0x40000000;
   
   /**
    * Convert an integer into a packed decimal byte array
    * 
    * Valid integers range from 0 to 1077936127.
    * All other numbers throw an IllegalArgumentException
    * 
    * @param aNumber Integer to convert
    * @return Packed decimal byte array with integer as value 
    * @throws IllegalArgumentException
    */
   public static byte[] fromInteger(final int aNumber) throws IllegalArgumentException {
      byte[] result;
      int intermediateNumber;
      
      if (aNumber >= 0)
         if (aNumber <= 0x3f) {
            result = new byte[1];
            result[0] = (byte) (aNumber & 0x3f);
         } else if (aNumber < (START_THREE_BYTE_VALUE + START_TWO_BYTE_VALUE)) {
            result = new byte[2];
            intermediateNumber = aNumber - START_TWO_BYTE_VALUE;

            result[1] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[0] = (byte) (0x40 | (intermediateNumber & 0xff));
         } else if (aNumber < (START_FOUR_BYTE_VALUE + START_THREE_BYTE_VALUE)) {
            result = new byte[3];
            intermediateNumber = aNumber - START_THREE_BYTE_VALUE;

            result[2] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[1] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[0] = (byte) (0x80 | (intermediateNumber & 0xff));
         } else if (aNumber < (START_TOO_LARGE_VALUE + START_FOUR_BYTE_VALUE)) {
            result = new byte[4];
            intermediateNumber = aNumber - START_FOUR_BYTE_VALUE;

            result[3] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[2] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[1] = (byte) (intermediateNumber & 0xff);

            intermediateNumber >>>= 8;
            result[0] = (byte) (0xc0 | (intermediateNumber & 0xff));
         } else
            throw new IllegalArgumentException();
      else
         throw new IllegalArgumentException();

      return result;
   }

   /**
    * Get expected length of packed decimal byte array from first byte
    * 
    * @param firstByteOfPackedNumber First byte of packed decimal integer
    * @return Expected length (1 to 4)
    */
   public static int getExpectedLength(final byte firstByteOfPackedNumber) {
      return ((firstByteOfPackedNumber >>> 6) & 0x03) + 1;
   }

   /**
    * Convert a packed decimal byte array into an integer
    * 
    * @param packedNumber Packed decimal byte array
    * @return Converted integer
    * @throws IllegalArgumentException
    */
   public static int toInteger(final byte[] packedNumber) throws IllegalArgumentException {
      int result = 0;

      final int expectedLength = getExpectedLength(packedNumber[0]);

      if (expectedLength == packedNumber.length)
         switch (expectedLength) {
            case 1:
               result = (int) (packedNumber[0] & 0x3f);
               break;

            case 2:
               result = (int) (((packedNumber[0] & 0x3f) << 8) |
                                (packedNumber[1] & 0xff)) +
                              START_TWO_BYTE_VALUE;
               break;

            case 3:
               result = (int) (((packedNumber[0] & 0x3f) << 16) |
                               ((packedNumber[1] & 0xff) << 8) |
                                (packedNumber[2] & 0xff)) +
                              START_THREE_BYTE_VALUE;
               break;

            case 4:
               result = (int) (((packedNumber[0] & 0x3f) << 24) |
                               ((packedNumber[1] & 0xff) << 16) |
                               ((packedNumber[2] & 0xff) << 8) |
                                (packedNumber[3] & 0xff)) +
                              START_FOUR_BYTE_VALUE;
               break;

            default:
               throw new IllegalArgumentException();
         }
      else
         throw new IllegalArgumentException();

      return result;
{'fraction_non_alphanumeric': 0.08503706117129592, 'fraction_numerical': 0.02557628085037061, 'mean_word_length': 2.519059902550874, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1550701', 'n_tokens_mistral': 3596, 'n_tokens_neox': 3225, 'n_words': 1173}