<commit_before>#![allow(dead_code, unused_imports)]
use super::command::{Command, ComputeDistanceCode, InitCommand, GetInsertLengthCode, GetCopyLengthCode, CombineLengthCodes, PrefixEncodeCopyDistance, CommandCopyLen, BrotliDistanceParams};
use super::backward_references::{BrotliEncoderParams, kHashMul32,kHashMul64, kHashMul64Long, BrotliHasherParams, kInvalidMatch, kDistanceCacheIndex, kDistanceCacheOffset, AnyHasher};
use super::dictionary_hash::kStaticDictionaryHash;
use super::static_dict::{BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, FindMatchLengthWithLimit};
use super::static_dict::{BrotliDictionary, kBrotliEncDictionary, BrotliFindAllStaticDictionaryMatches};
use super::literal_cost::BrotliEstimateBitCostsForLiterals;
use super::constants::{kInsExtra, kCopyExtra};
use super::super::alloc;
use super::super::alloc::{SliceWrapper, SliceWrapperMut, Allocator};
use super::util::{Log2FloorNonZero, brotli_max_size_t,FastLog2, floatX};
use super::hash_to_binary_tree::{InitBackwardMatch, BackwardMatch, BackwardMatchMut, StoreAndFindMatchesH10, Allocable, H10Params, H10, ZopfliNode, Union1, kInfinity};
use core;

const BROTLI_WINDOW_GAP:usize = 16;
const BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN:usize = 37;

/*
static kBrotliMinWindowBits: i32 = 10i32;

static kBrotliMaxWindowBits: i32 = 24i32;

static kInvalidMatch: u32 = 0xfffffffu32;

static kCutoffTransformsCount: u32 = 10u32;

static kCutoffTransforms: u64 = 0x71b520au64 << 32i32 | 0xda2d3200u32 as (u64);

pub static kHashMul32: u32 = 0x1e35a7bdu32;

pub static kHashMul64: u64 = 0x1e35a7bdu64 << 32i32 | 0x1e35a7bdu64;

pub static kHashMul64Long: u64 = 0x1fe35a7bu32 as (u64) << 32i32 | 0xd3579bd3u32 as (u64);

*/
pub const BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE:usize = 544;
pub const BROTLI_NUM_LITERAL_SYMBOLS:usize = 256;
pub const BROTLI_NUM_COMMAND_SYMBOLS:usize = 704;



pub fn BrotliInitZopfliNodes(
    array : &mut [ZopfliNode], length : usize
) {
    let stub = ZopfliNode::default();
    let mut i : usize;
    i = 0usize;
    while i < length {
        array[(i as (usize)) ]= stub;
        i = i.wrapping_add(1 as (usize));
    }
}



fn ZopfliNodeCopyLength(
    xself : & ZopfliNode
) -> u32 {
    (*xself).length & 0x1ffffffu32
}

fn ZopfliNodeCopyDistance(
    xself : & ZopfliNode
) -> u32 {
    (*xself).distance
}

fn ZopfliNodeLengthCode(
    xself : & ZopfliNode
) -> u32 {
    let modifier : u32 = (*xself).length >> 25i32;
    ZopfliNodeCopyLength(xself).wrapping_add(9u32).wrapping_sub(
        modifier
    )
}

fn brotli_min_size_t(
    a : usize, b : usize
) -> usize {
    if a < b { a } else { b }
}

fn ZopfliNodeDistanceCode(
    xself : & ZopfliNode
) -> u32 {
    let short_code : u32 = (*xself).dcode_insert_length >> 27i32;
    if short_code == 0u32 {
        ZopfliNodeCopyDistance(xself).wrapping_add(
            16u32
        ).wrapping_sub(
            1u32
        )
    } else {
        short_code.wrapping_sub(1u32)
    }
}


pub fn BrotliZopfliCreateCommands(
    num_bytes : usize,
    block_start : usize,
    max_backward_limit : usize,
    nodes : & [ZopfliNode],
    dist_cache : &mut [i32],
    last_insert_len : &mut usize,
    params : &BrotliEncoderParams,
    commands : &mut [Command],
    num_literals : &mut usize) {
    let mut pos : usize = 0usize;
    let mut offset : u32 = match (nodes[(0usize)]).u { Union1::next(off) => off, _ => 0};
    let mut i : usize;
    let gap : usize = 0usize;
    i = 0usize;
    while offset != !(0u32) {
        {
            let next
                : &ZopfliNode
                = &nodes[(
                        pos.wrapping_add(offset as (usize)) as (usize)
                    ) ];
            let copy_length
                : usize
                = ZopfliNodeCopyLength(next) as (usize);
            let mut insert_length
                : usize
                = ((*next).dcode_insert_length & 0x7ffffffu32) as (usize);
            pos = pos.wrapping_add(insert_length);
            offset = match (*next).u { Union1::next(off) => off, _ => 0};
            if i == 0usize {
                insert_length = insert_length.wrapping_add(*last_insert_len);
                *last_insert_len = 0usize;
            }
            {
                let distance : usize = ZopfliNodeCopyDistance(next) as (usize);
                let len_code : usize = ZopfliNodeLengthCode(next) as (usize);
                let max_distance
                    : usize
                    = brotli_min_size_t(
                          block_start.wrapping_add(pos),
                          max_backward_limit
                      );
                let is_dictionary
                    : i32
                    = if !!(distance > max_distance.wrapping_add(gap)) {
                          1i32
                      } else {
                          0i32
                      };
                let dist_code
                    : usize
                    = ZopfliNodeDistanceCode(next) as (usize);
                InitCommand(
                    &mut commands[(i as (usize)) ],
                    &(*params).dist ,
                    insert_length,
                    copy_length,
                    (len_code as (i32) - copy_length as (i32)) as usize,
                    dist_code
                );
                if is_dictionary == 0 && (dist_code > 0usize) {
                    dist_cache[(3usize) ]= dist_cache[(
                                                               2usize
                                                           )];
                    dist_cache[(2usize) ]= dist_cache[(
                                                               1usize
                                                           )];
                    dist_cache[(1usize) ]= dist_cache[(
                                                               0usize
                                                           )];
                    dist_cache[(0usize) ]= distance as (i32);
                }
            }
            *num_literals = (*num_literals).wrapping_add(insert_length);
            pos = pos.wrapping_add(copy_length);
        }
        i = i.wrapping_add(1 as (usize));
    }
    *last_insert_len = (*last_insert_len).wrapping_add(
                           num_bytes.wrapping_sub(pos)
                       );
}

<commit_msg>use older variant on InitCommand that subtracts inputs<commit_after>#![allow(dead_code, unused_imports)]
use super::command::{Command, ComputeDistanceCode, InitCommand, GetInsertLengthCode, GetCopyLengthCode, CombineLengthCodes, PrefixEncodeCopyDistance, CommandCopyLen, BrotliDistanceParams};
use super::backward_references::{BrotliEncoderParams, kHashMul32,kHashMul64, kHashMul64Long, BrotliHasherParams, kInvalidMatch, kDistanceCacheIndex, kDistanceCacheOffset, AnyHasher};
use super::dictionary_hash::kStaticDictionaryHash;
use super::static_dict::{BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, FindMatchLengthWithLimit};
use super::static_dict::{BrotliDictionary, kBrotliEncDictionary, BrotliFindAllStaticDictionaryMatches};
use super::literal_cost::BrotliEstimateBitCostsForLiterals;
use super::constants::{kInsExtra, kCopyExtra};
use super::super::alloc;
use super::super::alloc::{SliceWrapper, SliceWrapperMut, Allocator};
use super::util::{Log2FloorNonZero, brotli_max_size_t,FastLog2, floatX};
use super::hash_to_binary_tree::{InitBackwardMatch, BackwardMatch, BackwardMatchMut, StoreAndFindMatchesH10, Allocable, H10Params, H10, ZopfliNode, Union1, kInfinity};
use core;

const BROTLI_WINDOW_GAP:usize = 16;
const BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN:usize = 37;

/*
static kBrotliMinWindowBits: i32 = 10i32;

static kBrotliMaxWindowBits: i32 = 24i32;

static kInvalidMatch: u32 = 0xfffffffu32;

static kCutoffTransformsCount: u32 = 10u32;

static kCutoffTransforms: u64 = 0x71b520au64 << 32i32 | 0xda2d3200u32 as (u64);

pub static kHashMul32: u32 = 0x1e35a7bdu32;

pub static kHashMul64: u64 = 0x1e35a7bdu64 << 32i32 | 0x1e35a7bdu64;

pub static kHashMul64Long: u64 = 0x1fe35a7bu32 as (u64) << 32i32 | 0xd3579bd3u32 as (u64);

*/
pub const BROTLI_MAX_EFFECTIVE_DISTANCE_ALPHABET_SIZE:usize = 544;
pub const BROTLI_NUM_LITERAL_SYMBOLS:usize = 256;
pub const BROTLI_NUM_COMMAND_SYMBOLS:usize = 704;



pub fn BrotliInitZopfliNodes(
    array : &mut [ZopfliNode], length : usize
) {
    let stub = ZopfliNode::default();
    let mut i : usize;
    i = 0usize;
    while i < length {
        array[(i as (usize)) ]= stub;
        i = i.wrapping_add(1 as (usize));
    }
}



fn ZopfliNodeCopyLength(
    xself : & ZopfliNode
) -> u32 {
    (*xself).length & 0x1ffffffu32
}

fn ZopfliNodeCopyDistance(
    xself : & ZopfliNode
) -> u32 {
    (*xself).distance
}

fn ZopfliNodeLengthCode(
    xself : & ZopfliNode
) -> u32 {
    let modifier : u32 = (*xself).length >> 25i32;
    ZopfliNodeCopyLength(xself).wrapping_add(9u32).wrapping_sub(
        modifier
    )
}

fn brotli_min_size_t(
    a : usize, b : usize
) -> usize {
    if a < b { a } else { b }
}

fn ZopfliNodeDistanceCode(
    xself : & ZopfliNode
) -> u32 {
    let short_code : u32 = (*xself).dcode_insert_length >> 27i32;
    if short_code == 0u32 {
        ZopfliNodeCopyDistance(xself).wrapping_add(
            16u32
        ).wrapping_sub(
            1u32
        )
    } else {
        short_code.wrapping_sub(1u32)
    }
}


pub fn BrotliZopfliCreateCommands(
    num_bytes : usize,
    block_start : usize,
    max_backward_limit : usize,
    nodes : & [ZopfliNode],
    dist_cache : &mut [i32],
    last_insert_len : &mut usize,
    params : &BrotliEncoderParams,
    commands : &mut [Command],
    num_literals : &mut usize) {
    let mut pos : usize = 0usize;
    let mut offset : u32 = match (nodes[(0usize)]).u { Union1::next(off) => off, _ => 0};
    let mut i : usize;
    let gap : usize = 0usize;
    i = 0usize;
    while offset != !(0u32) {
        {
            let next
                : &ZopfliNode
                = &nodes[(
                        pos.wrapping_add(offset as (usize)) as (usize)
                    ) ];
            let copy_length
                : usize
                = ZopfliNodeCopyLength(next) as (usize);
            let mut insert_length
                : usize
                = ((*next).dcode_insert_length & 0x7ffffffu32) as (usize);
            pos = pos.wrapping_add(insert_length);
            offset = match (*next).u { Union1::next(off) => off, _ => 0};
            if i == 0usize {
                insert_length = insert_length.wrapping_add(*last_insert_len);
                *last_insert_len = 0usize;
            }
            {
                let distance : usize = ZopfliNodeCopyDistance(next) as (usize);
                let len_code : usize = ZopfliNodeLengthCode(next) as (usize);
                let max_distance
                    : usize
                    = brotli_min_size_t(
                          block_start.wrapping_add(pos),
                          max_backward_limit
                      );
                let is_dictionary
                    : i32
                    = if !!(distance > max_distance.wrapping_add(gap)) {
                          1i32
                      } else {
                          0i32
                      };
                let dist_code
                    : usize
                    = ZopfliNodeDistanceCode(next) as (usize);
                InitCommand(
                    &mut commands[(i as (usize)) ],
                    &(*params).dist ,
                    insert_length,
                    copy_length,
                    len_code,
                    dist_code
                );
                if is_dictionary == 0 && (dist_code > 0usize) {
                    dist_cache[(3usize) ]= dist_cache[(
                                                               2usize
                                                           )];
                    dist_cache[(2usize) ]= dist_cache[(
                                                               1usize
                                                           )];
                    dist_cache[(1usize) ]= dist_cache[(
                                                               0usize
                                                           )];
                    dist_cache[(0usize) ]= distance as (i32);
                }
            }
            *num_literals = (*num_literals).wrapping_add(insert_length);
            pos = pos.wrapping_add(copy_length);
        }
        i = i.wrapping_add(1 as (usize));
    }
    *last_insert_len = (*last_insert_len).wrapping_add(
                           num_bytes.wrapping_sub(pos)
                       );
}


{'fraction_non_alphanumeric': 0.09566246056782335, 'fraction_numerical': 0.03533123028391167, 'mean_word_length': 1.8644680370454032, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '649164', 'n_tokens_mistral': 4324, 'n_tokens_neox': 4002, 'n_words': 756}