<commit_before>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

/// Provides decompression function for LZMA algorithm.
public class LZMA: DecompressionAlgorithm {

    /**
     Decompresses `data` using LZMA algortihm.

     If `data` is not actually compressed with LZMA, `LZMAError` will be thrown.

     - Parameter data: Data compressed with LZMA.

     - Throws: `LZMAError` if unexpected byte (bit) sequence was encountered in `data`.
     It may indicate that either data is damaged or it might not be compressed with LZMA at all.

     - Returns: Decompressed data.
     */
    public static func decompress(data: Data) throws -> Data {
        /// Object with input data which supports convenient work with bit shifts.
        let pointerData = DataWithPointer(data: data)

        return Data(bytes: try decompress(pointerData))
    }

    static func decompress(_ pointerData: DataWithPointer) throws -> [UInt8] {
        let lzmaDecoder = try LZMADecoder(pointerData)
        try lzmaDecoder.decodeLZMA()
        return lzmaDecoder.out
    }

}<commit_msg>Use new LZMA decoder in LZMA itself<commit_after>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

/// Provides decompression function for LZMA algorithm.
public class LZMA: DecompressionAlgorithm {

    /**
     Decompresses `data` using LZMA algortihm.

     If `data` is not actually compressed with LZMA, `LZMAError` will be thrown.

     - Parameter data: Data compressed with LZMA.

     - Throws: `LZMAError` if unexpected byte (bit) sequence was encountered in `data`.
     It may indicate that either data is damaged or it might not be compressed with LZMA at all.

     - Returns: Decompressed data.
     */
    public static func decompress(data: Data) throws -> Data {
        /// Object with input data which supports convenient work with bit shifts.
        let pointerData = DataWithPointer(data: data)

        return Data(bytes: try decompress(pointerData))
    }

    static func decompress(_ pointerData: DataWithPointer) throws -> [UInt8] {
        let decoder = try LZMATempDecoder(pointerData)

        try decoder.setProperties(from: pointerData.byte())
        decoder.resetStateAndDecoders()
        decoder.dictionarySize = pointerData.uint32().toInt()

        let uncompressedSize = pointerData.uint64()
        if uncompressedSize == UInt64.max {
            decoder.uncompressedSize = -1
        } else {
            decoder.uncompressedSize = Int(truncatingIfNeeded: uncompressedSize)
        }

        try decoder.decode()
        return decoder.out
    }

}
{'fraction_non_alphanumeric': 0.07880133185349611, 'fraction_numerical': 0.006289308176100629, 'mean_word_length': 2.8739255014326646, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7424762', 'n_tokens_mistral': 776, 'n_tokens_neox': 726, 'n_words': 268}