<commit_before>import XCTest
@testable import SwiftDL

import Foundation

class SwiftDLTests: XCTestCase {
    func testExample() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi10.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi100.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi10.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi100.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        
        let downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
        
        let expectation = self.expectation(description: "")
        
        var progressSet: Set<String> = []
        downloader.handleProgress { bytesDownloaded, bytesExpectedToDownload in
            print("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
            progressSet.insert("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
        }
        
        downloader.handleCompletion { result in
            switch result {
            case .success:
                let data1 = try! Data(contentsOf: URL(fileURLWithPath: file1))
                XCTAssertEqual(String(bytes: data1, encoding: .utf8), "3.141592653")
                
                let data2 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                XCTAssertEqual(String(bytes: data2, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067")
            case .canceled:
                XCTFail()
            case let .failure(error):
                XCTFail("\(error)")
            }
            expectation.fulfill()
        }
        
        waitForExpectations(timeout: 20.0, handler: nil)
        
        XCTAssertTrue(progressSet.contains("11 / 112"))
        XCTAssertTrue(progressSet.contains("112 / 112"))
    }
    
    func testProgress() {
        // TODO
    }
    
    func testCompletion() {
        // TODO
    }
    
    func testCache() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi10.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi100.txt")!
        let url3 = URL(string: "http://koherent.org/pi/pi1000.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi10.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi100.txt")
        let file3 = (dir as NSString).appendingPathComponent("pi1000.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file3))
        
        for count in 0..<3 {
            let downloader: Downloader
            switch count {
            case 0, 1:
                downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
            case 2:
                downloader = Downloader(items: [(url2, file2), (url3, file3)], commonRequestHeaders: ["Accept-Encoding": "identity"])
            default:
                fatalError("Never reaches here.")
            }
            
            let expectation = self.expectation(description: "")
            
            var progressSet: Set<String> = []
            downloader.handleProgress { bytesDownloaded, bytesExpectedToDownload in
                print("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
                progressSet.insert("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
            }
            
            downloader.handleCompletion { result in
                switch result {
                case .success:
                    let data1 = try! Data(contentsOf: URL(fileURLWithPath: file1))
                    XCTAssertEqual(String(bytes: data1, encoding: .utf8), "3.141592653")
                    
                    let data2 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                    XCTAssertEqual(String(bytes: data2, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067")
                    
                    if count == 3 {
                        let data3 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                        XCTAssertEqual(String(bytes: data3, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198")
                    }
                case .canceled:
                    XCTFail()
                case let .failure(error):
                    XCTFail("\(error)")
                }
                expectation.fulfill()
            }
            
            waitForExpectations(timeout: 20.0, handler: nil)

            switch count {
            case 0:
                XCTAssertTrue(progressSet.contains("11 / 112"))
                XCTAssertTrue(progressSet.contains("112 / 112"))
            case 1:
                XCTAssertTrue(progressSet.isEmpty)
            case 2:
                XCTAssertTrue(progressSet.contains("1001 / 1001"))
            default:
                fatalError("Never reaches here.")
            }
        }
    }
    
    func testCancel() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi100000.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi1000000.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi100000.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi1000000.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        
        let downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
        
        let expectation = self.expectation(description: "")
        
        downloader.handleCompletion { result in
            switch result {
            case .success:
                XCTFail()
            case .canceled:
                break
            case let .failure(error):
                XCTFail("\(error)")
            }
            expectation.fulfill()
        }
        

        DispatchQueue.main.asyncAfter(deadline: .now() + 3) {
            downloader.cancel()
        }
        
        waitForExpectations(timeout: 20.0, handler: nil)
    }

    static var allTests : [(String, (SwiftDLTests) -> () throws -> Void)] {
        return [
            ("testExample", testExample),
            ("testProgress", testProgress),<commit_msg>Add removing files at the end of each test methods<commit_after>import XCTest
@testable import SwiftDL

import Foundation

class SwiftDLTests: XCTestCase {
    func testExample() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi10.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi100.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi10.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi100.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        
        let downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
        
        let expectation = self.expectation(description: "")
        
        var progressSet: Set<String> = []
        downloader.handleProgress { bytesDownloaded, bytesExpectedToDownload in
            print("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
            progressSet.insert("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
        }
        
        downloader.handleCompletion { result in
            switch result {
            case .success:
                let data1 = try! Data(contentsOf: URL(fileURLWithPath: file1))
                XCTAssertEqual(String(bytes: data1, encoding: .utf8), "3.141592653")
                
                let data2 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                XCTAssertEqual(String(bytes: data2, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067")
            case .canceled:
                XCTFail()
            case let .failure(error):
                XCTFail("\(error)")
            }
            expectation.fulfill()
        }
        
        waitForExpectations(timeout: 20.0, handler: nil)
        
        XCTAssertTrue(progressSet.contains("11 / 112"))
        XCTAssertTrue(progressSet.contains("112 / 112"))
        
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
    }
    
    func testProgress() {
        // TODO
    }
    
    func testCompletion() {
        // TODO
    }
    
    func testCache() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi10.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi100.txt")!
        let url3 = URL(string: "http://koherent.org/pi/pi1000.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi10.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi100.txt")
        let file3 = (dir as NSString).appendingPathComponent("pi1000.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file3))
        
        for count in 0..<3 {
            let downloader: Downloader
            switch count {
            case 0, 1:
                downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
            case 2:
                downloader = Downloader(items: [(url2, file2), (url3, file3)], commonRequestHeaders: ["Accept-Encoding": "identity"])
            default:
                fatalError("Never reaches here.")
            }
            
            let expectation = self.expectation(description: "")
            
            var progressSet: Set<String> = []
            downloader.handleProgress { bytesDownloaded, bytesExpectedToDownload in
                print("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
                progressSet.insert("\(bytesDownloaded) / \(bytesExpectedToDownload!)")
            }
            
            downloader.handleCompletion { result in
                switch result {
                case .success:
                    let data1 = try! Data(contentsOf: URL(fileURLWithPath: file1))
                    XCTAssertEqual(String(bytes: data1, encoding: .utf8), "3.141592653")
                    
                    let data2 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                    XCTAssertEqual(String(bytes: data2, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067")
                    
                    if count == 3 {
                        let data3 = try! Data(contentsOf: URL(fileURLWithPath: file2))
                        XCTAssertEqual(String(bytes: data3, encoding: .utf8), "3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198")
                    }
                case .canceled:
                    XCTFail()
                case let .failure(error):
                    XCTFail("\(error)")
                }
                expectation.fulfill()
            }
            
            waitForExpectations(timeout: 20.0, handler: nil)

            switch count {
            case 0:
                XCTAssertTrue(progressSet.contains("11 / 112"))
                XCTAssertTrue(progressSet.contains("112 / 112"))
            case 1:
                XCTAssertTrue(progressSet.isEmpty)
            case 2:
                XCTAssertTrue(progressSet.contains("1001 / 1001"))
            default:
                fatalError("Never reaches here.")
            }
        }
        
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file3))
    }
    
    func testCancel() {
        let dir: String = Bundle(for: type(of: self)).resourcePath!
        
        let url1 = URL(string: "http://koherent.org/pi/pi100000.txt")!
        let url2 = URL(string: "http://koherent.org/pi/pi1000000.txt")!
        let file1 = (dir as NSString).appendingPathComponent("pi100000.txt")
        let file2 = (dir as NSString).appendingPathComponent("pi1000000.txt")
        
        let fileManager = FileManager.default
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
        
        let downloader = Downloader(items: [(url1, file1), (url2, file2)], commonRequestHeaders: ["Accept-Encoding": "identity"])
        
        let expectation = self.expectation(description: "")
        
        downloader.handleCompletion { result in
            switch result {
            case .success:
                XCTFail()
            case .canceled:
                break
            case let .failure(error):
                XCTFail("\(error)")
            }
            expectation.fulfill()
        }
        

        DispatchQueue.main.asyncAfter(deadline: .now() + 3) {
            downloader.cancel()
        }
        
        waitForExpectations(timeout: 20.0, handler: nil)
        
        try? fileManager.removeItem(at: URL(fileURLWithPath: file1))
        try? fileManager.removeItem(at: URL(fileURLWithPath: file2))
    }

    static var allTests : [(String, (SwiftDLTests) -> () throws -> Void)] {
        return [
            ("testExample", testExample),
            ("testProgress", testProgress),
{'fraction_non_alphanumeric': 0.09756689521266913, 'fraction_numerical': 0.1678902979188156, 'mean_word_length': 2.3472786352558894, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6185157', 'n_tokens_mistral': 6488, 'n_tokens_neox': 4815, 'n_words': 800}