<commit_before>// Copyright (c) 2018 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation
import BitByteData

struct XZBlock {

    let data: Data
    let unpaddedSize: Int

    var uncompressedSize: Int {
        return data.count
    }

    init(_ blockHeaderSize: UInt8, _ byteReader: ByteReader, _ checkSize: Int) throws {
        let blockHeaderStartIndex = byteReader.offset - 1
        let realBlockHeaderSize = (blockHeaderSize.toInt() + 1) * 4

        let blockFlags = byteReader.byte()
        /**
         Bit values 00, 01, 10, 11 indicate filters number from 1 to 4,
         so we actually need to add 1 to get filters' number.
         */
        let filtersCount = blockFlags & 0x03 + 1
        guard blockFlags & 0x3C == 0
            else { throw XZError.wrongField }

        /// Should match size of compressed data.
        let compressedSize = blockFlags & 0x40 != 0 ? try byteReader.multiByteDecode() : -1

        /// Should match the size of data after decompression.
        let uncompressedSize = blockFlags & 0x80 != 0 ? try byteReader.multiByteDecode() : -1

        var filters: [(ByteReader) throws -> Data] = []
        for _ in 0..<filtersCount {
            let filterID = try byteReader.multiByteDecode()
            guard UInt64(filterID) < 0x4000000000000000
                else { throw XZError.wrongFilterID }
            // Only LZMA2 filter is supported.
            if filterID == 0x21 {
                // First, we need to check if size of LZMA2 filter's properties is equal to 1 as expected.
                let propertiesSize = try byteReader.multiByteDecode()
                guard propertiesSize == 1
                    else { throw LZMA2Error.wrongDictionarySize }
                /// Filter property for LZMA2 is a dictionary size.
                let filterPropeties = byteReader.byte()
                filters.append { try LZMA2.decompress($0, filterPropeties) }
            } else {
                throw XZError.wrongFilterID
            }
        }

        // We need to take into account 4 bytes for CRC32 so thats why "-4".
        while byteReader.offset - blockHeaderStartIndex < realBlockHeaderSize - 4 {
            let byte = byteReader.byte()<commit_msg>[XZ] Enable support for delta filter<commit_after>// Copyright (c) 2018 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation
import BitByteData

struct XZBlock {

    let data: Data
    let unpaddedSize: Int

    var uncompressedSize: Int {
        return data.count
    }

    init(_ blockHeaderSize: UInt8, _ byteReader: ByteReader, _ checkSize: Int) throws {
        let blockHeaderStartIndex = byteReader.offset - 1
        let realBlockHeaderSize = (blockHeaderSize.toInt() + 1) * 4

        let blockFlags = byteReader.byte()
        /**
         Bit values 00, 01, 10, 11 indicate filters number from 1 to 4,
         so we actually need to add 1 to get filters' number.
         */
        let filtersCount = blockFlags & 0x03 + 1
        guard blockFlags & 0x3C == 0
            else { throw XZError.wrongField }

        /// Should match size of compressed data.
        let compressedSize = blockFlags & 0x40 != 0 ? try byteReader.multiByteDecode() : -1

        /// Should match the size of data after decompression.
        let uncompressedSize = blockFlags & 0x80 != 0 ? try byteReader.multiByteDecode() : -1

        var filters: [(ByteReader) throws -> Data] = []
        for _ in 0..<filtersCount {
            let filterID = try byteReader.multiByteDecode()
            guard UInt64(filterID) < 0x4000000000000000
                else { throw XZError.wrongFilterID }
            // Only LZMA2 filter is supported.
            switch filterID {
            case 0x21:
                // First, we need to check if size of LZMA2 filter's properties is equal to 1 as expected.
                let propertiesSize = try byteReader.multiByteDecode()
                guard propertiesSize == 1
                    else { throw LZMA2Error.wrongDictionarySize }
                /// Filter property for LZMA2 is a dictionary size.
                let filterPropeties = byteReader.byte()
                filters.append { try LZMA2.decompress($0, filterPropeties) }
            case 0x03:
                // First, we need to check if size of Delta filter's properties is equal to 1 as expected.
                let propertiesSize = try byteReader.multiByteDecode()
                guard propertiesSize == 1
                    else { throw XZError.wrongField }
                let distance = (byteReader.byte() &+ 1).toInt()
                filters.append { DeltaFilter.decode($0, distance) }
            default:
                throw XZError.wrongFilterID
            }
        }

        // We need to take into account 4 bytes for CRC32 so thats why "-4".
        while byteReader.offset - blockHeaderStartIndex < realBlockHeaderSize - 4 {
            let byte = byteReader.byte()
{'fraction_non_alphanumeric': 0.076, 'fraction_numerical': 0.0302, 'mean_word_length': 1.9279859484777517, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4086180', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1268, 'n_words': 461}