<commit_before>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2018 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
*/

import TSCBasic
import TSCUtility
import SPMLLBuild
import PackageModel
import Dispatch
import Foundation
import LLBuildManifest
import SPMBuildCore

typealias Diagnostic = TSCBasic.Diagnostic

class CustomLLBuildCommand: SPMLLBuild.ExternalCommand {
    let ctx: BuildExecutionContext

    required init(_ ctx: BuildExecutionContext) {
        self.ctx = ctx
    }

    func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
        return []
    }

    func execute(
        _ command: SPMLLBuild.Command,
        _ buildSystemCommandInterface: SPMLLBuild.BuildSystemCommandInterface
    ) -> Bool {
        fatalError("subclass responsibility")
    }
}

final class TestDiscoveryCommand: CustomLLBuildCommand {

    private func write(
        tests: [IndexStore.TestCaseClass],
        forModule module: String,
        to path: AbsolutePath
    ) throws {
        let stream = try LocalFileOutputByteStream(path)

        stream <<< "import XCTest" <<< "\n"
        stream <<< "@testable import " <<< module <<< "\n"

        for klass in tests {
            stream <<< "\n"
            stream <<< "fileprivate extension " <<< klass.name <<< " {" <<< "\n"
            stream <<< indent(4) <<< "static let __allTests__\(klass.name) = [" <<< "\n"
            for method in klass.methods {
                let method = method.hasSuffix("()") ? String(method.dropLast(2)) : method
                stream <<< indent(8) <<< "(\"\(method)\", \(method))," <<< "\n"
            }
            stream <<< indent(4) <<< "]" <<< "\n"
            stream <<< "}" <<< "\n"
        }

        stream <<< """
        func __allTests_\(module)() -> [XCTestCaseEntry] {
            return [\n
        """

        for klass in tests {
            stream <<< indent(8) <<< "testCase(\(klass.name).__allTests__\(klass.name)),\n"
        }

        stream <<< """
            ]
        }
        """

        stream.flush()
    }

    private func execute(with tool: LLBuildManifest.TestDiscoveryTool) throws {
        let index = ctx.buildParameters.indexStore
        let api = try ctx.indexStoreAPI.get()
        let store = try IndexStore.open(store: index, api: api)

        // FIXME: We can speed this up by having one llbuild command per object file.
        let tests = try tool.inputs.flatMap {
            try store.listTests(inObjectFile: AbsolutePath($0.name))
        }

        let outputs = tool.outputs.compactMap{ try? AbsolutePath(validating: $0.name) }
        let testsByModule = Dictionary(grouping: tests, by: { $0.module })

        func isMainFile(_ path: AbsolutePath) -> Bool {
            return path.basename == "main.swift"
        }

        // Write one file for each test module.
        //
        // We could write everything in one file but that can easily run into type conflicts due
        // in complex packages with large number of test targets.
        for file in outputs {
            if isMainFile(file) { continue }

            // FIXME: This is relying on implementation detail of the output but passing the
            // the context all the way through is not worth it right now.
            let module = file.basenameWithoutExt

            guard let tests = testsByModule[module] else {
                // This module has no tests so just write an empty file for it.
                try localFileSystem.writeFileContents(file, bytes: "")
                continue
            }
            try write(tests: tests, forModule: module, to: file)
        }

        // Write the main file.
        let mainFile = outputs.first(where: isMainFile)!
        let stream = try LocalFileOutputByteStream(mainFile)

        stream <<< "import XCTest" <<< "\n\n"
        stream <<< "var tests = [XCTestCaseEntry]()" <<< "\n"
        for module in testsByModule.keys {
            stream <<< "tests += __allTests_\(module)()" <<< "\n"
        }
        stream <<< "\n"
        stream <<< "XCTMain(tests)" <<< "\n"

        stream.flush()
    }

    private func indent(_ spaces: Int) -> ByteStreamable {
        return Format.asRepeating(string: " ", count: spaces)
    }

    override func execute(
        _ command: SPMLLBuild.Command,
        _ buildSystemCommandInterface: SPMLLBuild.BuildSystemCommandInterface
    ) -> Bool {
        // This tool will never run without the build description.
        let buildDescription = ctx.buildDescription!
        guard let tool = buildDescription.testDiscoveryCommands[command.name] else {
            print("command \(command.name) not registered")
            return false
        }
        do {
            try execute(with: tool)
        } catch {
            // FIXME: Shouldn't use "print" here.
            print("error:", error)<commit_msg>Avoid traversing array in TestDiscoveryCommand twice (#2784)<commit_after>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2018 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
*/

import TSCBasic
import TSCUtility
import SPMLLBuild
import PackageModel
import Dispatch
import Foundation
import LLBuildManifest
import SPMBuildCore

typealias Diagnostic = TSCBasic.Diagnostic

class CustomLLBuildCommand: SPMLLBuild.ExternalCommand {
    let ctx: BuildExecutionContext

    required init(_ ctx: BuildExecutionContext) {
        self.ctx = ctx
    }

    func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
        return []
    }

    func execute(
        _ command: SPMLLBuild.Command,
        _ buildSystemCommandInterface: SPMLLBuild.BuildSystemCommandInterface
    ) -> Bool {
        fatalError("subclass responsibility")
    }
}

final class TestDiscoveryCommand: CustomLLBuildCommand {

    private func write(
        tests: [IndexStore.TestCaseClass],
        forModule module: String,
        to path: AbsolutePath
    ) throws {
        let stream = try LocalFileOutputByteStream(path)

        stream <<< "import XCTest" <<< "\n"
        stream <<< "@testable import " <<< module <<< "\n"

        for klass in tests {
            stream <<< "\n"
            stream <<< "fileprivate extension " <<< klass.name <<< " {" <<< "\n"
            stream <<< indent(4) <<< "static let __allTests__\(klass.name) = [" <<< "\n"
            for method in klass.methods {
                let method = method.hasSuffix("()") ? String(method.dropLast(2)) : method
                stream <<< indent(8) <<< "(\"\(method)\", \(method))," <<< "\n"
            }
            stream <<< indent(4) <<< "]" <<< "\n"
            stream <<< "}" <<< "\n"
        }

        stream <<< """
        func __allTests_\(module)() -> [XCTestCaseEntry] {
            return [\n
        """

        for klass in tests {
            stream <<< indent(8) <<< "testCase(\(klass.name).__allTests__\(klass.name)),\n"
        }

        stream <<< """
            ]
        }
        """

        stream.flush()
    }

    private func execute(with tool: LLBuildManifest.TestDiscoveryTool) throws {
        let index = ctx.buildParameters.indexStore
        let api = try ctx.indexStoreAPI.get()
        let store = try IndexStore.open(store: index, api: api)

        // FIXME: We can speed this up by having one llbuild command per object file.
        let tests = try tool.inputs.flatMap {
            try store.listTests(inObjectFile: AbsolutePath($0.name))
        }

        let outputs = tool.outputs.compactMap{ try? AbsolutePath(validating: $0.name) }
        let testsByModule = Dictionary(grouping: tests, by: { $0.module })

        func isMainFile(_ path: AbsolutePath) -> Bool {
            return path.basename == "main.swift"
        }

        var mainFile: AbsolutePath?
        // Write one file for each test module.
        //
        // We could write everything in one file but that can easily run into type conflicts due
        // in complex packages with large number of test targets.
        for file in outputs {
            if mainFile == nil && isMainFile(file) {
                mainFile = file
                continue 
            }

            // FIXME: This is relying on implementation detail of the output but passing the
            // the context all the way through is not worth it right now.
            let module = file.basenameWithoutExt

            guard let tests = testsByModule[module] else {
                // This module has no tests so just write an empty file for it.
                try localFileSystem.writeFileContents(file, bytes: "")
                continue
            }
            try write(tests: tests, forModule: module, to: file)
        }

        // Write the main file.
        let stream = try LocalFileOutputByteStream(mainFile!)

        stream <<< "import XCTest" <<< "\n\n"
        stream <<< "var tests = [XCTestCaseEntry]()" <<< "\n"
        for module in testsByModule.keys {
            stream <<< "tests += __allTests_\(module)()" <<< "\n"
        }
        stream <<< "\n"
        stream <<< "XCTMain(tests)" <<< "\n"

        stream.flush()
    }

    private func indent(_ spaces: Int) -> ByteStreamable {
        return Format.asRepeating(string: " ", count: spaces)
    }

    override func execute(
        _ command: SPMLLBuild.Command,
        _ buildSystemCommandInterface: SPMLLBuild.BuildSystemCommandInterface
    ) -> Bool {
        // This tool will never run without the build description.
        let buildDescription = ctx.buildDescription!
        guard let tool = buildDescription.testDiscoveryCommands[command.name] else {
            print("command \(command.name) not registered")
            return false
        }
        do {
            try execute(with: tool)
        } catch {
            // FIXME: Shouldn't use "print" here.
            print("error:", error)
{'fraction_non_alphanumeric': 0.11037247042721673, 'fraction_numerical': 0.0033238830775246847, 'mean_word_length': 2.120805369127517, 'pattern_counts': {'":': 0, '<': 207, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4638357', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2724, 'n_words': 834}