<commit_before>/*
Copyright 2011-2017 <NAME>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kanzi.entropy;

import kanzi.EntropyEncoder;
import kanzi.OutputBitStream;


// Based on Order 0 range coder by <NAME> itself derived from the algorithm
// described by <NAME> in his seminal article in 1979.
// [<NAME> on the Data Recording Conference, Southampton, 1979]
// Optimized for speed.

// Not thread safe
public final class RangeEncoder implements EntropyEncoder
{
    private static final long TOP_RANGE    = 0x0FFFFFFFFFFFFFFFL;
    private static final long BOTTOM_RANGE = 0x000000000000FFFFL;
    private static final long RANGE_MASK   = 0x0FFFFFFF00000000L;
    private static final int DEFAULT_CHUNK_SIZE = 1 << 16; // 64 KB by default
    private static final int DEFAULT_LOG_RANGE = 13;


    private long low;
    private long range;
    private final int[] alphabet;
    private final int[] freqs;
    private final long[] cumFreqs;
    private final EntropyUtils eu;
    private final OutputBitStream bitstream;
    private final int chunkSize;
    private final int logRange;
    private int shift;
    
    
    public RangeEncoder(OutputBitStream bitstream)
    {
       this(bitstream, DEFAULT_CHUNK_SIZE, DEFAULT_LOG_RANGE);
    }
    
    
    // The chunk size indicates how many bytes are encoded (per block) before 
    // resetting the frequency stats. 0 means that frequencies calculated at the
    // beginning of the block apply to the whole block.
    // The default chunk size is 65536 bytes.
    public RangeEncoder(OutputBitStream bs, int chunkSize, int logRange)
    {
      if (bs == null)
         throw new NullPointerException("Invalid null bitstream parameter");

      if ((chunkSize != 0) && (chunkSize < 1024))
         throw new IllegalArgumentException("The chunk size must be at least 1024");

      if (chunkSize > 1<<30)
         throw new IllegalArgumentException("The chunk size must be at most 2^30");

      if ((logRange < 8) || (logRange > 16))
         throw new IllegalArgumentException("Invalid range parameter: "+logRange+" (must be in [8..16])");

      this.bitstream = bs;
      this.alphabet = new int[256];
      this.freqs = new int[256];
      this.cumFreqs = new long[257];
      this.logRange = logRange;
      this.chunkSize = chunkSize;
      this.eu = new EntropyUtils();
    }

    
   private int updateFrequencies(int[] frequencies, int size, int lr)
   {
      if ((frequencies == null) || (frequencies.length != 256))
         return -1;

      int alphabetSize = this.eu.normalizeFrequencies(frequencies, this.alphabet, size, 1<<lr);
      
      if (alphabetSize > 0)
      {
         this.cumFreqs[0] = 0;

         // Create histogram of frequencies scaled to 'range'
         for (int i=0; i<256; i++)
            this.cumFreqs[i+1] = this.cumFreqs[i] + frequencies[i];
      }
          
      this.encodeHeader(alphabetSize, this.alphabet, frequencies, lr);
      return alphabetSize;
   }

   
   protected boolean encodeHeader(int alphabetSize, int[] alphabet, int[] frequencies, int lr)
   {
      EntropyUtils.encodeAlphabet(this.bitstream, alphabet, alphabetSize);

      if (alphabetSize == 0)
         return true;

      this.bitstream.writeBits(lr-8, 3); // logRange
      int inc = (alphabetSize > 64) ? 16 : 8;
      int llr = 3;

      while (1<<llr <= lr)
         llr++;

      // Encode all frequencies (but the first one) by chunks of size 'inc'
      for (int i=1; i<alphabetSize; i+=inc)
      {
         int max = 0;
         int logMax = 1;
         final int endj = (i+inc < alphabetSize) ? i + inc : alphabetSize;

         // Search for max frequency log size in next chunk
         for (int j=i; j<endj; j++)
         {
            if (frequencies[alphabet[j]] > max)<commit_msg>Improve encoding of distribution.<commit_after>/*
Copyright 2011-2017 <NAME>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kanzi.entropy;

import kanzi.EntropyEncoder;
import kanzi.OutputBitStream;


// Based on Order 0 range coder by <NAME> itself derived from the algorithm
// described by G.N.N Martin in his seminal article in 1979.
// [G.N.N. Martin on the Data Recording Conference, Southampton, 1979]
// Optimized for speed.

// Not thread safe
public final class RangeEncoder implements EntropyEncoder
{
    private static final long TOP_RANGE    = 0x0FFFFFFFFFFFFFFFL;
    private static final long BOTTOM_RANGE = 0x000000000000FFFFL;
    private static final long RANGE_MASK   = 0x0FFFFFFF00000000L;
    private static final int DEFAULT_CHUNK_SIZE = 1 << 16; // 64 KB by default
    private static final int DEFAULT_LOG_RANGE = 13;


    private long low;
    private long range;
    private final int[] alphabet;
    private final int[] freqs;
    private final long[] cumFreqs;
    private final EntropyUtils eu;
    private final OutputBitStream bitstream;
    private final int chunkSize;
    private final int logRange;
    private int shift;
    
    
    public RangeEncoder(OutputBitStream bitstream)
    {
       this(bitstream, DEFAULT_CHUNK_SIZE, DEFAULT_LOG_RANGE);
    }
    
    
    // The chunk size indicates how many bytes are encoded (per block) before 
    // resetting the frequency stats. 0 means that frequencies calculated at the
    // beginning of the block apply to the whole block.
    // The default chunk size is 65536 bytes.
    public RangeEncoder(OutputBitStream bs, int chunkSize, int logRange)
    {
      if (bs == null)
         throw new NullPointerException("Invalid null bitstream parameter");

      if ((chunkSize != 0) && (chunkSize < 1024))
         throw new IllegalArgumentException("The chunk size must be at least 1024");

      if (chunkSize > 1<<30)
         throw new IllegalArgumentException("The chunk size must be at most 2^30");

      if ((logRange < 8) || (logRange > 16))
         throw new IllegalArgumentException("Invalid range parameter: "+logRange+" (must be in [8..16])");

      this.bitstream = bs;
      this.alphabet = new int[256];
      this.freqs = new int[256];
      this.cumFreqs = new long[257];
      this.logRange = logRange;
      this.chunkSize = chunkSize;
      this.eu = new EntropyUtils();
    }

    
   private int updateFrequencies(int[] frequencies, int size, int lr)
   {
      if ((frequencies == null) || (frequencies.length != 256))
         return -1;

      int alphabetSize = this.eu.normalizeFrequencies(frequencies, this.alphabet, size, 1<<lr);
      
      if (alphabetSize > 0)
      {
         this.cumFreqs[0] = 0;

         // Create histogram of frequencies scaled to 'range'
         for (int i=0; i<256; i++)
            this.cumFreqs[i+1] = this.cumFreqs[i] + frequencies[i];
      }
          
      this.encodeHeader(alphabetSize, this.alphabet, frequencies, lr);
      return alphabetSize;
   }

   
   protected boolean encodeHeader(int alphabetSize, int[] alphabet, int[] frequencies, int lr)
   {
      EntropyUtils.encodeAlphabet(this.bitstream, alphabet, alphabetSize);

      if (alphabetSize == 0)
         return true;

      this.bitstream.writeBits(lr-8, 3); // logRange
      int inc = (alphabetSize > 64) ? 6 : 4;
      int llr = 3;

      while (1<<llr <= lr)
         llr++;

      // Encode all frequencies (but the first one) by chunks of size 'inc'
      for (int i=1; i<alphabetSize; i+=inc)
      {
         int max = 0;
         int logMax = 1;
         final int endj = (i+inc < alphabetSize) ? i + inc : alphabetSize;

         // Search for max frequency log size in next chunk
         for (int j=i; j<endj; j++)
         {
            if (frequencies[alphabet[j]] > max)
{'fraction_non_alphanumeric': 0.08978832286578274, 'fraction_numerical': 0.026634100953710164, 'mean_word_length': 2.7484742807323452, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3308159', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2423, 'n_words': 910}