<commit_before>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2020 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 */

import Basics
import Dispatch
import struct Foundation.Date
import class Foundation.JSONDecoder
import struct Foundation.NSRange
import class Foundation.NSRegularExpression
import struct Foundation.URL
import PackageModel
import TSCBasic

struct GitHubPackageMetadataProvider: PackageMetadataProvider {
    public var name: String = "GitHub"

    var configuration: Configuration

    private let httpClient: HTTPClient
    private let diagnosticsEngine: DiagnosticsEngine?
    private let decoder: JSONDecoder
    private let queue: DispatchQueue

    init(configuration: Configuration = .init(), httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
        self.configuration = configuration
        self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)
        self.diagnosticsEngine = diagnosticsEngine
        self.decoder = JSONDecoder.makeWithDefaults()
        self.queue = DispatchQueue(label: "org.swift.swiftpm.GitHubPackageMetadataProvider", attributes: .concurrent)
    }

    func get(_ reference: PackageReference, callback: @escaping (Result<Model.PackageBasicMetadata, Error>) -> Void) {
        guard reference.kind == .remote else {
            return callback(.failure(Errors.invalidReferenceType(reference)))
        }
        guard let baseURL = self.apiURL(reference.path) else {
            return callback(.failure(Errors.invalidGitUrl(reference.path)))
        }

        let metadataURL = baseURL
        let tagsURL = baseURL.appendingPathComponent("tags")
        let contributorsURL = baseURL.appendingPathComponent("contributors")
        let readmeURL = baseURL.appendingPathComponent("readme")

        self.queue.async {
            let sync = DispatchGroup()
            var results = [URL: Result<HTTPClientResponse, Error>]()
            let resultsLock = Lock()

            // get the main data
            sync.enter()
            var metadataHeaders = self.makeRequestHeaders(metadataURL)
            metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
            let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
            httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
                defer { sync.leave() }
                resultsLock.withLock {
                    results[metadataURL] = result
                }
                if case .success(let response) = result {
                    let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
                    let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1

                    switch (response.statusCode, metadataHeaders.contains("Authorization"), apiRemaining) {
                    case (_, _, 0):
                        self.diagnosticsEngine?.emit(warning: "Exceeded API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
                        return callback(.failure(Errors.apiLimitsExceeded(metadataURL, apiLimit)))
                    case (401, true, _):
                        return callback(.failure(Errors.invalidAuthToken(metadataURL)))
                    case (401, false, _):
                        return callback(.failure(Errors.permissionDenied(metadataURL)))
                    case (403, _, _):
                        return callback(.failure(Errors.permissionDenied(metadataURL)))
                    case (404, _, _):
                        return callback(.failure(NotFoundError("\(baseURL)")))
                    case (200, _, _):
                        if apiRemaining < self.configuration.apiLimitWarningThreshold {
                            self.diagnosticsEngine?.emit(warning: "Approaching API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
                        }
                        // if successful, fan out multiple API calls
                        [tagsURL, contributorsURL, readmeURL].forEach { url in
                            sync.enter()
                            var headers = self.makeRequestHeaders(url)
                            headers.add(name: "Accept", value: "application/vnd.github.v3+json")
                            let options = self.makeRequestOptions(validResponseCodes: [200])
                            httpClient.get(url, headers: headers, options: options) { result in
                                defer { sync.leave() }
                                resultsLock.withLock {
                                    results[url] = result
                                }
                            }
                        }
                    default:
                        return callback(.failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)")))
                    }
                }
            }
            sync.wait()

            // process results

            do {
                // check for main request error state
                switch results[metadataURL] {
                case .none:
                    throw Errors.invalidResponse(metadataURL, "Response missing")
                case .some(.failure(let error)):
                    throw error
                case .some(.success(let metadataResponse)):
                    guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
                        throw Errors.invalidResponse(metadataURL, "Empty body")
                    }
                    let tags = try results[tagsURL]?.success?.decodeBody([Tag].self, using: self.decoder) ?? []
                    let contributors = try results[contributorsURL]?.success?.decodeBody([Contributor].self, using: self.decoder)
                    let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)

                    callback(.success(.init(
                        summary: metadata.description,
                        keywords: metadata.topics,
                        // filters out non-semantic versioned tags
                        versions: tags.compactMap { TSCUtility.Version(string: $0.name) },
                        watchersCount: metadata.watchersCount,
                        readmeURL: readme?.downloadURL,
                        authors: contributors?.map { .init(username: $0.login, url: $0.url, service: .init(name: "GitHub")) },
                        processedAt: Date()
                    )))
                }
            } catch {
                return callback(.failure(error))
            }
        }<commit_msg>fix concurrency issue with GithubMetadata provider (#3086)<commit_after>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2020 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 */

import Basics
import Dispatch
import struct Foundation.Date
import class Foundation.JSONDecoder
import struct Foundation.NSRange
import class Foundation.NSRegularExpression
import struct Foundation.URL
import PackageModel
import TSCBasic

struct GitHubPackageMetadataProvider: PackageMetadataProvider {
    public var name: String = "GitHub"

    var configuration: Configuration

    private let httpClient: HTTPClient
    private let diagnosticsEngine: DiagnosticsEngine?
    private let decoder: JSONDecoder

    init(configuration: Configuration = .init(), httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
        self.configuration = configuration
        self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)
        self.diagnosticsEngine = diagnosticsEngine
        self.decoder = JSONDecoder.makeWithDefaults()
    }

    func get(_ reference: PackageReference, callback: @escaping (Result<Model.PackageBasicMetadata, Error>) -> Void) {
        guard reference.kind == .remote else {
            return callback(.failure(Errors.invalidReferenceType(reference)))
        }
        guard let baseURL = self.apiURL(reference.path) else {
            return callback(.failure(Errors.invalidGitUrl(reference.path)))
        }

        let metadataURL = baseURL
        let tagsURL = baseURL.appendingPathComponent("tags")
        let contributorsURL = baseURL.appendingPathComponent("contributors")
        let readmeURL = baseURL.appendingPathComponent("readme")

        let sync = DispatchGroup()
        var results = [URL: Result<HTTPClientResponse, Error>]()
        let resultsLock = Lock()

        // get the main data
        sync.enter()
        var metadataHeaders = self.makeRequestHeaders(metadataURL)
        metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
        let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
        httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
            defer { sync.leave() }
            resultsLock.withLock {
                results[metadataURL] = result
            }
            if case .success(let response) = result {
                let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
                let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
                switch (response.statusCode, metadataHeaders.contains("Authorization"), apiRemaining) {
                case (_, _, 0):
                    self.diagnosticsEngine?.emit(warning: "Exceeded API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
                    resultsLock.withLock {
                        results[metadataURL] = .failure(Errors.apiLimitsExceeded(metadataURL, apiLimit))
                    }
                case (401, true, _):
                    resultsLock.withLock {
                        results[metadataURL] = .failure(Errors.invalidAuthToken(metadataURL))
                    }
                case (401, false, _):
                    resultsLock.withLock {
                        results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
                    }
                case (403, _, _):
                    resultsLock.withLock {
                        results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
                    }
                case (404, _, _):
                    resultsLock.withLock {
                        results[metadataURL] = .failure(NotFoundError("\(baseURL)"))
                    }
                case (200, _, _):
                    if apiRemaining < self.configuration.apiLimitWarningThreshold {
                        self.diagnosticsEngine?.emit(warning: "Approaching API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
                    }
                    // if successful, fan out multiple API calls
                    [tagsURL, contributorsURL, readmeURL].forEach { url in
                        sync.enter()
                        var headers = self.makeRequestHeaders(url)
                        headers.add(name: "Accept", value: "application/vnd.github.v3+json")
                        let options = self.makeRequestOptions(validResponseCodes: [200])
                        httpClient.get(url, headers: headers, options: options) { result in
                            defer { sync.leave() }
                            resultsLock.withLock {
                                results[url] = result
                            }
                        }
                    }
                default:
                    resultsLock.withLock {
                        results[metadataURL] = .failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)"))
                    }
                }
            }
        }
        sync.wait()

        // process results

        do {
            // check for main request error state
            switch results[metadataURL] {
            case .none:
                throw Errors.invalidResponse(metadataURL, "Response missing")
            case .some(.failure(let error)):
                throw error
            case .some(.success(let metadataResponse)):
                guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
                    throw Errors.invalidResponse(metadataURL, "Empty body")
                }
                let tags = try results[tagsURL]?.success?.decodeBody([Tag].self, using: self.decoder) ?? []
                let contributors = try results[contributorsURL]?.success?.decodeBody([Contributor].self, using: self.decoder)
                let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)

                callback(.success(.init(
                    summary: metadata.description,
                    keywords: metadata.topics,
                    // filters out non-semantic versioned tags
                    versions: tags.compactMap { TSCUtility.Version(string: $0.name) },
                    watchersCount: metadata.watchersCount,
                    readmeURL: readme?.downloadURL,
                    authors: contributors?.map { .init(username: $0.login, url: $0.url, service: .init(name: "GitHub")) },
                    processedAt: Date()
                )))
            }
        } catch {
            return callback(.failure(error))
        }
{'fraction_non_alphanumeric': 0.0912571349018516, 'fraction_numerical': 0.006264791869692329, 'mean_word_length': 1.9177497969130788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '420052', 'n_tokens_mistral': 3493, 'n_tokens_neox': 3198, 'n_words': 804}