<commit_before>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2021 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 */

import Dispatch
import Foundation

import Basics
import PackageCollectionsModel
import TSCBasic

public protocol PackageCollectionSigner {
    /// Signs package collection using the given certificate and key.
    ///
    /// - Parameters:
    ///   - collection: The package collection to be signed
    ///   - certChainPaths: Paths to all DER-encoded certificates in the chain. The certificate used for signing
    ///                     must be the first in the array.
    ///   - certPrivateKeyPath: Path to the private key (*.pem) of the certificate
    ///   - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates
    ///   - callback: The callback to invoke when the signed collection is available.
    func sign(collection: PackageCollectionModel.V1.Collection,
              certChainPaths: [URL],
              certPrivateKeyPath: URL,
              certPolicyKey: CertificatePolicyKey,
              callback: @escaping (Result<PackageCollectionModel.V1.SignedCollection, Error>) -> Void)
}

public protocol PackageCollectionSignatureValidator {
    /// Validates a signed package collection.
    ///
    /// - Parameters:
    ///   - signedCollection: The signed package collection
    ///   - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates
    ///   - callback: The callback to invoke when the result is available.
    func validate(signedCollection: PackageCollectionModel.V1.SignedCollection,
                  certPolicyKey: CertificatePolicyKey,
                  callback: @escaping (Result<Void, Error>) -> Void)
}

// MARK: - Implementation

public struct PackageCollectionSigning: PackageCollectionSigner, PackageCollectionSignatureValidator {
    public typealias Model = PackageCollectionModel.V1

    private static let minimumRSAKeySizeInBits = 2048

    /// URL of the optional directory containing root certificates to be trusted.
    private let trustedRootCertsDir: URL?
    /// Root certificates to be trusted in additional to those found in `trustedRootCertsDir`
    private let additionalTrustedRootCerts: [Certificate]?

    /// The `DispatchQueue` to use for callbacks
    private let callbackQueue: DispatchQueue

    /// Internal cache/storage of `CertificatePolicy`s
    private let certPolicies: [CertificatePolicyKey: CertificatePolicy]

    private let encoder: JSONEncoder
    private let decoder: JSONDecoder

    private let observabilityScope: ObservabilityScope

    public init(trustedRootCertsDir: URL? = nil, additionalTrustedRootCerts: [String]? = nil, observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue) {
        self.trustedRootCertsDir = trustedRootCertsDir
        self.additionalTrustedRootCerts = additionalTrustedRootCerts.map { $0.compactMap {
            guard let data = Data(base64Encoded: $0) else {
                observabilityScope.emit(error: "The certificate \($0) is not in valid base64 encoding")
                return nil
            }
            do {
                return try Certificate(derEncoded: data)
            } catch {
                observabilityScope.emit(error: "The certificate \($0) is not in valid DER format: \(error)")
                return nil
            }
        } }

        self.callbackQueue = callbackQueue
        self.certPolicies = [:]
        self.encoder = JSONEncoder.makeWithDefaults()
        self.decoder = JSONDecoder.makeWithDefaults()
        self.observabilityScope = observabilityScope
    }

    init(certPolicy: CertificatePolicy, observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue) {
        // These should be set through the given CertificatePolicy
        self.trustedRootCertsDir = nil
        self.additionalTrustedRootCerts = nil

        self.callbackQueue = callbackQueue
        self.certPolicies = [CertificatePolicyKey.custom: certPolicy]
        self.encoder = JSONEncoder.makeWithDefaults()
        self.decoder = JSONDecoder.makeWithDefaults()
        self.observabilityScope = observabilityScope
    }

    private func getCertificatePolicy(key: CertificatePolicyKey) throws -> CertificatePolicy {
        switch key {
        case .default(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return DefaultCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                            expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .appleSwiftPackageCollection(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return AppleSwiftPackageCollectionCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                                                expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .appleDistribution(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return AppleDistributionCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                                      expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .custom:
            // Custom `CertificatePolicy` can be set using the internal initializer only
            guard let certPolicy = self.certPolicies[key] else {
                throw PackageCollectionSigningError.certPolicyNotFound
            }
            return certPolicy
        }
    }

    public func sign(collection: Model.Collection,
                     certChainPaths: [URL],
                     certPrivateKeyPath: URL,
                     certPolicyKey: CertificatePolicyKey = .default,
                     callback: @escaping (Result<Model.SignedCollection, Error>) -> Void) {
        do {
            let certChainData = try certChainPaths.map { try Data(contentsOf: $0) }
            // Check that the certificate is valid
            self.validateCertChain(certChainData, certPolicyKey: certPolicyKey) { result in
                switch result {
                case .failure(let error):
                    return callback(.failure(error))
                case .success(let certChain):
                    do {
                        let certificate = certChain.first! // !-safe because certChain cannot be empty at this point
                        let keyType = try certificate.keyType()

                        // Signature header
                        let signatureAlgorithm = Signature.Algorithm.from(keyType: keyType)
                        let header = Signature.Header(
                            algorithm: signatureAlgorithm,
                            certChain: certChainData.map { $0.base64EncodedString() }
                        )

                        // Key for signing
                        let privateKeyPEM = try Data(contentsOf: certPrivateKeyPath)

                        let privateKey: PrivateKey
                        switch keyType {
                        case .RSA:
                            privateKey = try RSAPrivateKey(pem: privateKeyPEM)
                        case .EC:
                            privateKey = try ECPrivateKey(pem: privateKeyPEM)
                        }
                        try self.validateKey(privateKey)

                        // Generate the signature
                        let signatureData = try Signature.generate(for: collection, with: header, using: privateKey, jsonEncoder: self.encoder)

                        guard let signature = String(bytes: signatureData, encoding: .utf8) else {
                            return callback(.failure(PackageCollectionSigningError.invalidSignature))
                        }

                        let collectionSignature = Model.Signature(
                            signature: signature,
                            certificate: Model.Signature.Certificate(
                                subject: Model.Signature.Certificate.Name(from: try certificate.subject()),
                                issuer: Model.Signature.Certificate.Name(from: try certificate.issuer())
                            )
                        )
                        callback(.success(Model.SignedCollection(collection: collection, signature: collectionSignature)))
                    } catch {
                        callback(.failure(error))
                    }
                }
            }<commit_msg>Allow signing from key provided as `Data` in addition to `URL` (#3831)<commit_after>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2021 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 */

import Dispatch
import Foundation

import Basics
import PackageCollectionsModel
import TSCBasic

public protocol PackageCollectionSigner {
    /// Signs package collection using the given certificate and key.
    ///
    /// - Parameters:
    ///   - collection: The package collection to be signed
    ///   - certChainPaths: Paths to all DER-encoded certificates in the chain. The certificate used for signing
    ///                     must be the first in the array.
    ///   - privateKeyPEM: Data of the private key (*.pem) of the certificate
    ///   - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates
    ///   - callback: The callback to invoke when the signed collection is available.
    func sign(collection: PackageCollectionModel.V1.Collection,
              certChainPaths: [URL],
              privateKeyPEM: Data,
              certPolicyKey: CertificatePolicyKey,
              callback: @escaping (Result<PackageCollectionModel.V1.SignedCollection, Error>) -> Void)
}

extension PackageCollectionSigner {
    /// Signs package collection using the given certificate and key.
    ///
    /// - Parameters:
    ///   - collection: The package collection to be signed
    ///   - certChainPaths: Paths to all DER-encoded certificates in the chain. The certificate used for signing
    ///                     must be the first in the array.
    ///   - certPrivateKeyPath: Path to the private key (*.pem) of the certificate
    ///   - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates
    ///   - callback: The callback to invoke when the signed collection is available.
    public func sign(collection: PackageCollectionModel.V1.Collection,
                     certChainPaths: [URL],
                     certPrivateKeyPath: URL,
                     certPolicyKey: CertificatePolicyKey = .default,
                     callback: @escaping (Result<PackageCollectionModel.V1.SignedCollection, Error>) -> Void) {
        do {
            let privateKey = try Data(contentsOf: certPrivateKeyPath)
            self.sign(collection: collection,
                      certChainPaths: certChainPaths,
                      privateKeyPEM: privateKey,
                      certPolicyKey: certPolicyKey,
                      callback: callback)
        } catch {
            callback(.failure(error))
        }
    }
}

public protocol PackageCollectionSignatureValidator {
    /// Validates a signed package collection.
    ///
    /// - Parameters:
    ///   - signedCollection: The signed package collection
    ///   - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates
    ///   - callback: The callback to invoke when the result is available.
    func validate(signedCollection: PackageCollectionModel.V1.SignedCollection,
                  certPolicyKey: CertificatePolicyKey,
                  callback: @escaping (Result<Void, Error>) -> Void)
}

// MARK: - Implementation

public struct PackageCollectionSigning: PackageCollectionSigner, PackageCollectionSignatureValidator {
    public typealias Model = PackageCollectionModel.V1

    private static let minimumRSAKeySizeInBits = 2048

    /// URL of the optional directory containing root certificates to be trusted.
    private let trustedRootCertsDir: URL?
    /// Root certificates to be trusted in additional to those found in `trustedRootCertsDir`
    private let additionalTrustedRootCerts: [Certificate]?

    /// The `DispatchQueue` to use for callbacks
    private let callbackQueue: DispatchQueue

    /// Internal cache/storage of `CertificatePolicy`s
    private let certPolicies: [CertificatePolicyKey: CertificatePolicy]

    private let encoder: JSONEncoder
    private let decoder: JSONDecoder

    private let observabilityScope: ObservabilityScope

    public init(trustedRootCertsDir: URL? = nil, additionalTrustedRootCerts: [String]? = nil, observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue) {
        self.trustedRootCertsDir = trustedRootCertsDir
        self.additionalTrustedRootCerts = additionalTrustedRootCerts.map { $0.compactMap {
            guard let data = Data(base64Encoded: $0) else {
                observabilityScope.emit(error: "The certificate \($0) is not in valid base64 encoding")
                return nil
            }
            do {
                return try Certificate(derEncoded: data)
            } catch {
                observabilityScope.emit(error: "The certificate \($0) is not in valid DER format: \(error)")
                return nil
            }
        } }

        self.callbackQueue = callbackQueue
        self.certPolicies = [:]
        self.encoder = JSONEncoder.makeWithDefaults()
        self.decoder = JSONDecoder.makeWithDefaults()
        self.observabilityScope = observabilityScope
    }

    init(certPolicy: CertificatePolicy, observabilityScope: ObservabilityScope, callbackQueue: DispatchQueue) {
        // These should be set through the given CertificatePolicy
        self.trustedRootCertsDir = nil
        self.additionalTrustedRootCerts = nil

        self.callbackQueue = callbackQueue
        self.certPolicies = [CertificatePolicyKey.custom: certPolicy]
        self.encoder = JSONEncoder.makeWithDefaults()
        self.decoder = JSONDecoder.makeWithDefaults()
        self.observabilityScope = observabilityScope
    }

    private func getCertificatePolicy(key: CertificatePolicyKey) throws -> CertificatePolicy {
        switch key {
        case .default(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return DefaultCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                            expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .appleSwiftPackageCollection(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return AppleSwiftPackageCollectionCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                                                expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .appleDistribution(let subjectUserID):
            // Create new instance each time since contents of trustedRootCertsDir might change
            return AppleDistributionCertificatePolicy(trustedRootCertsDir: self.trustedRootCertsDir, additionalTrustedRootCerts: self.additionalTrustedRootCerts,
                                                      expectedSubjectUserID: subjectUserID, observabilityScope: observabilityScope, callbackQueue: self.callbackQueue)
        case .custom:
            // Custom `CertificatePolicy` can be set using the internal initializer only
            guard let certPolicy = self.certPolicies[key] else {
                throw PackageCollectionSigningError.certPolicyNotFound
            }
            return certPolicy
        }
    }

    public func sign(collection: Model.Collection,
                     certChainPaths: [URL],
                     privateKeyPEM: Data,
                     certPolicyKey: CertificatePolicyKey = .default,
                     callback: @escaping (Result<Model.SignedCollection, Error>) -> Void) {
        do {
            let certChainData = try certChainPaths.map { try Data(contentsOf: $0) }
            // Check that the certificate is valid
            self.validateCertChain(certChainData, certPolicyKey: certPolicyKey) { result in
                switch result {
                case .failure(let error):
                    return callback(.failure(error))
                case .success(let certChain):
                    do {
                        let certificate = certChain.first! // !-safe because certChain cannot be empty at this point
                        let keyType = try certificate.keyType()

                        // Signature header
                        let signatureAlgorithm = Signature.Algorithm.from(keyType: keyType)
                        let header = Signature.Header(
                            algorithm: signatureAlgorithm,
                            certChain: certChainData.map { $0.base64EncodedString() }
                        )

                        let privateKey: PrivateKey
                        switch keyType {
                        case .RSA:
                            privateKey = try RSAPrivateKey(pem: privateKeyPEM)
                        case .EC:
                            privateKey = try ECPrivateKey(pem: privateKeyPEM)
                        }
                        try self.validateKey(privateKey)

                        // Generate the signature
                        let signatureData = try Signature.generate(for: collection, with: header, using: privateKey, jsonEncoder: self.encoder)

                        guard let signature = String(bytes: signatureData, encoding: .utf8) else {
                            return callback(.failure(PackageCollectionSigningError.invalidSignature))
                        }

                        let collectionSignature = Model.Signature(
                            signature: signature,
                            certificate: Model.Signature.Certificate(
                                subject: Model.Signature.Certificate.Name(from: try certificate.subject()),
                                issuer: Model.Signature.Certificate.Name(from: try certificate.issuer())
                            )
                        )
                        callback(.success(Model.SignedCollection(collection: collection, signature: collectionSignature)))
                    } catch {
                        callback(.failure(error))
                    }
                }
            }
{'fraction_non_alphanumeric': 0.06565784796628243, 'fraction_numerical': 0.0030467678870664703, 'mean_word_length': 2.101905811938888, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '936975', 'n_tokens_mistral': 4551, 'n_tokens_neox': 4279, 'n_words': 1321}