<commit_before>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

struct LZMAConstants {
    static let topValue: Int = 1 << 24
    static let numBitModelTotalBits: Int = 11
    static let numMoveBits: Int = 5
    static let probInitValue: Int = ((1 << numBitModelTotalBits) / 2)
    static let numPosBitsMax: Int = 4
    static let numStates: Int = 12
    static let numLenToPosStates: Int = 4
    static let numAlignBits: Int = 4
    static let startPosModelIndex: Int = 4
    static let endPosModelIndex: Int = 14
    static let numFullDistances: Int = (1 << (endPosModelIndex >> 1))
    static let matchMinLen: Int = 2
    // LZMAConstants.numStates << LZMAConstants.numPosBitsMax = 192
}

class LZMADecoder {

    private var pointerData: DataWithPointer

    private var lc: UInt8 = 0
    private var lp: UInt8 = 0
    private var pb: UInt8 = 0
    private var dictionarySize: Int = 0

    private var rangeDecoder: LZMARangeDecoder = LZMARangeDecoder()
    private var posSlotDecoder: [LZMABitTreeDecoder] = []
    private var alignDecoder: LZMABitTreeDecoder
    private var lenDecoder: LZMALenDecoder
    private var repLenDecoder: LZMALenDecoder

    /**
     For literal decoding we need `1 << (lc + lp)` amount of tables.
     Each table contains 0x300 probabilities.
     */
    private var literalProbs: [[Int]] = []

    /**
     Array with all probabilities:

     - 0..<192: isMatch
     - 193..<205: isRep
     - 205..<217: isRepG0
     - 217..<229: isRepG1
     - 229..<241: isRepG2
     - 241..<433: isRep0Long
     */
    private var probabilities: [Int] = []

    private var posDecoders: [Int] = []

    // 'Distance history table'.
    private var rep0: Int = 0
    private var rep1: Int = 0
    private var rep2: Int = 0
    private var rep3: Int = 0

    /// Is used to select exact variable from 'IsRep', 'IsRepG0', 'IsRepG1' and 'IsRepG2' arrays.
    private var state: Int = 0

    /// An array for storing output data.
    var out: [UInt8] = []
    // This array will also serve as dictionary and out window.
    private var dictStart: Int = 0
    private var dictEnd: Int = 0

    /// For proper processing of LZMA data `resetState` and `resetProperties` functions should be called at least once.
    /// If that has happened, then stateReset is true.
    private var stateReset: Bool = false

    init(_ pointerData: DataWithPointer) throws {
        self.pointerData = pointerData
        self.alignDecoder = LZMABitTreeDecoder(numBits: LZMAConstants.numAlignBits)
        // There are two types of matches so we need two decoders for them.
        self.lenDecoder = LZMALenDecoder()
        self.repLenDecoder = LZMALenDecoder()
    }

    // MARK: LZMA2 related functions.

    private func resetDictionary(_ dictSize: Int) {
        self.dictionarySize = dictSize
        self.dictStart = self.dictEnd
    }

    private func resetProperties() throws {
        var properties = pointerData.byte()
        if properties >= (9 * 5 * 5) {
            throw LZMAError.wrongProperties
        }
        /// The number of literal context bits
        self.lc = properties % 9
        properties /= 9<commit_msg>Some small improvements to LZMADecoder code<commit_after>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

struct LZMAConstants {
    static let topValue = 1 << 24
    static let numBitModelTotalBits = 11
    static let numMoveBits = 5
    static let probInitValue = ((1 << numBitModelTotalBits) / 2)
    static let numPosBitsMax = 4
    static let numStates = 12
    static let numLenToPosStates = 4
    static let numAlignBits = 4
    static let startPosModelIndex = 4
    static let endPosModelIndex = 14
    static let numFullDistances = (1 << (endPosModelIndex >> 1))
    static let matchMinLen = 2
    // LZMAConstants.numStates << LZMAConstants.numPosBitsMax = 192
}

class LZMADecoder {

    private var pointerData: DataWithPointer

    private var lc: UInt8 = 0
    private var lp: UInt8 = 0
    private var pb: UInt8 = 0
    private var dictionarySize = 0

    private var rangeDecoder = LZMARangeDecoder()
    private var posSlotDecoder  = [LZMABitTreeDecoder]()
    private var alignDecoder = LZMABitTreeDecoder(numBits: LZMAConstants.numAlignBits)
    private var lenDecoder = LZMALenDecoder()
    private var repLenDecoder = LZMALenDecoder()

    /**
     For literal decoding we need `1 << (lc + lp)` amount of tables.
     Each table contains 0x300 probabilities.
     */
    private var literalProbs = [[Int]]()

    /**
     Array with all probabilities:

     - 0..<192: isMatch
     - 193..<205: isRep
     - 205..<217: isRepG0
     - 217..<229: isRepG1
     - 229..<241: isRepG2
     - 241..<433: isRep0Long
     */
    private var probabilities = [Int]()

    private var posDecoders = [Int]()

    // 'Distance history table'.
    private var rep0 = 0
    private var rep1 = 0
    private var rep2 = 0
    private var rep3 = 0

    /// Is used to select exact variable from 'IsRep', 'IsRepG0', 'IsRepG1' and 'IsRepG2' arrays.
    private var state = 0

    /// An array for storing output data.
    var out = [UInt8]()
    // This array will also serve as dictionary and out window.
    private var dictStart = 0
    private var dictEnd = 0

    /// For proper processing of LZMA data `resetState` and `resetProperties` functions should be called at least once.
    /// If that has happened, then stateReset is true.
    private var stateReset = false

    init(_ pointerData: DataWithPointer) throws {
        self.pointerData = pointerData
    }

    // MARK: LZMA2 related functions.

    private func resetDictionary(_ dictSize: Int) {
        self.dictionarySize = dictSize
        self.dictStart = self.dictEnd
    }

    private func resetProperties() throws {
        var properties = pointerData.byte()
        if properties >= (9 * 5 * 5) {
            throw LZMAError.wrongProperties
        }
        /// The number of literal context bits
        self.lc = properties % 9
        properties /= 9
{'fraction_non_alphanumeric': 0.08603205757278377, 'fraction_numerical': 0.03173045469414459, 'mean_word_length': 2.5760233918128654, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4520594', 'n_tokens_mistral': 1970, 'n_tokens_neox': 1787, 'n_words': 643}