<commit_before>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

/// Provides open function for TAR containers.
public class TarContainer: Container {

    /**
     Processes TAR container and returns an array of `TarEntry`.

     - Important: The order of entries is defined by TAR container and,
     particularly, by the creator of a given TAR container.
     It is likely that directories will be encountered earlier than files stored in those directories,
     but one SHOULD NOT rely on any particular order.

     - Parameter container: TAR container's data.

     - Throws: `TarError`, which may indicate that either container is damaged or it might not be TAR container at all.

     - Returns: Array of `TarEntry` as an array of `ContainerEntry`.
     */
    public static func open(container data: Data) throws -> [TarEntry] {
        // First, if the TAR container contains only header, it should be at least 512 bytes long.
        // So we have to check this.
        guard data.count >= 512 else { throw TarError.tooSmallFileIsPassed }

        /// Object with input data which supports convenient work with bit shifts.
        let pointerData = DataWithPointer(data: data)

        var output = [TarEntry]()

        var lastGlobalExtendedHeader: TarExtendedHeader?
        var lastLocalExtendedHeader: TarExtendedHeader?
        var longLinkName: String?
        var longName: String?

        // Container ends with two zero-filled records.
        while pointerData.data[pointerData.index..<pointerData.index + 1024] != Data(count: 1024) {
            let info = try TarEntryInfo(pointerData, lastGlobalExtendedHeader, lastLocalExtendedHeader,
                                        longName, longLinkName)

            // File data
            let dataStartIndex = info.blockStartIndex + 512
            let dataEndIndex = dataStartIndex + info.size!
            let entryData = data[dataStartIndex..<dataEndIndex]
            pointerData.index = dataEndIndex - info.size! + info.size!.roundTo512()

            let entry = TarEntry(info, entryData)

            if info.isGlobalExtendedHeader {
                lastGlobalExtendedHeader = try TarExtendedHeader(entry.data)
            } else if info.isLocalExtendedHeader {
                lastLocalExtendedHeader = try TarExtendedHeader(entry.data)
            } else if info.isLongLinkName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longLinkName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else if info.isLongName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else {
                output.append(entry)
                lastLocalExtendedHeader = nil
                longName = nil
                longLinkName = nil
            }
        }

        return output
    }

    public static func info(container data: Data) throws -> [TarEntryInfo] {
        // First, if the TAR container contains only header, it should be at least 512 bytes long.
        // So we have to check this.
        guard data.count >= 512 else { throw TarError.tooSmallFileIsPassed }
        
        /// Object with input data which supports convenient work with bit shifts.
        let pointerData = DataWithPointer(data: data)
        
        var output = [TarEntryInfo]()
        
        var lastGlobalExtendedHeader: TarExtendedHeader?
        var lastLocalExtendedHeader: TarExtendedHeader?
        var longLinkName: String?
        var longName: String?

        // TODO: First, populate infos, then get data in second loop.
        // Container ends with two zero-filled records.
        while pointerData.data[pointerData.index..<pointerData.index + 1024] != Data(count: 1024) {            
            let info = try TarEntryInfo(pointerData, lastGlobalExtendedHeader, lastLocalExtendedHeader,
                                        longName, longLinkName)
            
            if info.isGlobalExtendedHeader {
                let dataStartIndex = info.blockStartIndex + 512
                let dataEndIndex = dataStartIndex + info.size!

                lastGlobalExtendedHeader = try TarExtendedHeader(data[dataStartIndex..<dataEndIndex])
                pointerData.index = dataEndIndex - info.size! + info.size!.roundTo512()
            } else if info.isLocalExtendedHeader {
                let dataStartIndex = info.blockStartIndex + 512
                let dataEndIndex = dataStartIndex + info.size!

                lastLocalExtendedHeader = try TarExtendedHeader(data[dataStartIndex..<dataEndIndex])
                pointerData.index = dataEndIndex - info.size! + info.size!.roundTo512()
            } else if info.isLongLinkName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longLinkName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else if info.isLongName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else {
                // Skip file data.
                pointerData.index = info.blockStartIndex + 512 + info.size!.roundTo512()
                output.append(info)
                lastLocalExtendedHeader = nil
                longName = nil
                longLinkName = nil
            }
        }
        
        return output
    }

}
<commit_msg>Rework TarContainer.open to first call info() and then get data based on info entries<commit_after>// Copyright (c) 2017 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

/// Provides open function for TAR containers.
public class TarContainer: Container {

    /**
     Processes TAR container and returns an array of `TarEntry`.

     - Important: The order of entries is defined by TAR container and,
     particularly, by the creator of a given TAR container.
     It is likely that directories will be encountered earlier than files stored in those directories,
     but one SHOULD NOT rely on any particular order.

     - Parameter container: TAR container's data.

     - Throws: `TarError`, which may indicate that either container is damaged or it might not be TAR container at all.

     - Returns: Array of `TarEntry`.
     */
    public static func open(container data: Data) throws -> [TarEntry] {
        let infos = try info(container: data)

        var entries = [TarEntry]()

        for infoEntry in infos {
            let dataStartIndex = infoEntry.blockStartIndex + 512
            let dataEndIndex = dataStartIndex + infoEntry.size!
            let entryData = data[dataStartIndex..<dataEndIndex]
            entries.append(TarEntry(infoEntry, entryData))
        }

        return entries
    }

    public static func info(container data: Data) throws -> [TarEntryInfo] {
        // First, if the TAR container contains only header, it should be at least 512 bytes long.
        // So we have to check this.
        guard data.count >= 512 else { throw TarError.tooSmallFileIsPassed }
        
        /// Object with input data which supports convenient work with bit shifts.
        let pointerData = DataWithPointer(data: data)
        
        var entries = [TarEntryInfo]()
        
        var lastGlobalExtendedHeader: TarExtendedHeader?
        var lastLocalExtendedHeader: TarExtendedHeader?
        var longLinkName: String?
        var longName: String?

        // Container ends with two zero-filled records.
        while pointerData.data[pointerData.index..<pointerData.index + 1024] != Data(count: 1024) {            
            let info = try TarEntryInfo(pointerData, lastGlobalExtendedHeader, lastLocalExtendedHeader,
                                        longName, longLinkName)
            
            if info.isGlobalExtendedHeader {
                let dataStartIndex = info.blockStartIndex + 512
                let dataEndIndex = dataStartIndex + info.size!

                lastGlobalExtendedHeader = try TarExtendedHeader(data[dataStartIndex..<dataEndIndex])
                pointerData.index = dataEndIndex - info.size! + info.size!.roundTo512()
            } else if info.isLocalExtendedHeader {
                let dataStartIndex = info.blockStartIndex + 512
                let dataEndIndex = dataStartIndex + info.size!

                lastLocalExtendedHeader = try TarExtendedHeader(data[dataStartIndex..<dataEndIndex])
                pointerData.index = dataEndIndex - info.size! + info.size!.roundTo512()
            } else if info.isLongLinkName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longLinkName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else if info.isLongName {
                let dataStartIndex = info.blockStartIndex + 512
                pointerData.index = dataStartIndex

                longName = try pointerData.nullEndedAsciiString(cutoff: info.size!)
                pointerData.index = dataStartIndex + info.size!.roundTo512()
            } else {
                // Skip file data.
                pointerData.index = info.blockStartIndex + 512 + info.size!.roundTo512()
                entries.append(info)
                lastLocalExtendedHeader = nil
                longName = nil
                longLinkName = nil
            }
        }
        
        return entries
    }

}

{'fraction_non_alphanumeric': 0.07002252472823425, 'fraction_numerical': 0.012829301733424737, 'mean_word_length': 2.065746022215551, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '495079', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2435, 'n_words': 774}