<commit_before>//
//  SwiftDocsTests.swift
//  SourceKitten
//
//  Created by <NAME> on 2015-01-03.
//  Copyright (c) 2015 SourceKitten. All rights reserved.
//

import Foundation
import SourceKittenFramework
import XCTest

func compareJSONString(withFixtureNamed name: String,
                       jsonString: CustomStringConvertible,
                       rootDirectory: String = fixturesDirectory,
                       file: StaticString = #file,
                       line: UInt = #line) {
#if os(Linux) && !swift(>=4.1)
    let jsonString = String(describing: jsonString).replacingOccurrences(of: rootDirectory, with: "")
    let actualContent = jsonString
#else
    // Strip out fixtures directory since it's dependent on the test machine's setup
    let escapedFixturesDirectory = rootDirectory.replacingOccurrences(of: "/", with: "\\/")
    let jsonString = String(describing: jsonString).replacingOccurrences(of: escapedFixturesDirectory, with: "")

    // Strip out any other absolute paths after that, since it's also dependent on the test machine's setup
    let absolutePathRegex = try! NSRegularExpression(pattern: "\"key\\.filepath\" : \"\\\\/[^\\\n]+", options: [])
    let actualContent = absolutePathRegex.stringByReplacingMatches(in: jsonString, options: [],
                                                                   range: NSRange(location: 0, length: jsonString.bridge().length),
                                                                   withTemplate: "\"key\\.filepath\" : \"\",")
#endif

    let expectedFile = File(path: versionedExpectedFilename(for: name))!

    // Use if changes are introduced by changes in SourceKitten.
    let overwrite = false
    if overwrite && actualContent != expectedFile.contents {
        _ = try? actualContent.data(using: .utf8)?.write(to: URL(fileURLWithPath: expectedFile.path!), options: [])
        return
    }

    // Use if changes are introduced by the new Swift version.
    let appendFixturesForNewSwiftVersion = false
    if appendFixturesForNewSwiftVersion && actualContent != expectedFile.contents,
        var path = expectedFile.path, let index = path.index(of: "@"),
        !path.hasSuffix("@\(buildingSwiftVersion).json") {
        path.replaceSubrange(index..<path.endIndex, with: "@\(buildingSwiftVersion).json")
        _ = try? actualContent.data(using: .utf8)?.write(to: URL(fileURLWithPath: path), options: [])
        return
    }

    func jsonValue(_ jsonString: String) -> NSObject {
        let data = jsonString.data(using: .utf8)!
        let result = try! JSONSerialization.jsonObject(with: data, options: [])
        if let dict = (result as? [String: Any])?.bridge() {
            return dict
        } else if let array = (result as? [Any])?.bridge() {
            return array
        }
        fatalError()
    }

    if jsonValue(actualContent) != jsonValue(expectedFile.contents) {
        XCTFail("output should match expected fixture", file: file, line: line)
        print(diff(original: expectedFile.contents, modified: actualContent))
    }
}

private func compareDocs(withFixtureNamed name: String, file: StaticString = #file, line: UInt = #line) {
    let swiftFilePath = fixturesDirectory + name + ".swift"
    let docs = SwiftDocs(file: File(path: swiftFilePath)!, arguments: ["-j4", swiftFilePath])!
    compareJSONString(withFixtureNamed: name, jsonString: docs, file: file, line: line)
}

private func versionedExpectedFilename(for name: String) -> String {
    #if swift(>=4.2)
        let versions = ["swift-4.2", "swift-4.1.1", "swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.1.1)
        let versions = ["swift-4.1.1", "swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.1)
        let versions = ["swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0.3)
        let versions = ["swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0.2)
        let versions = ["swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0)
        let versions = ["swift-4.0"]
    #else
        fatalError("Swift 4.0 or later is required!")
    #endif
    #if os(Linux)
        let platforms = ["Linux", ""]
    #else
        let platforms = [""]
    #endif
    for platform in platforms {
        for version in versions {
            let versionedFilename = "\(fixturesDirectory)\(platform)\(name)@\(version).json"
            if FileManager.default.fileExists(atPath: versionedFilename) {
                return versionedFilename
            }
        }
    }
    return "\(fixturesDirectory)\(name).json"
}

private func diff(original: String, modified: String) -> String {
    do {
        let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("SwiftDocsTests-diff-\(NSUUID())")
        try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)

        try original.data(using: .utf8)?.write(to: url.appendingPathComponent("original.json"))
        try modified.data(using: .utf8)?.write(to: url.appendingPathComponent("modified.json"))

        let task = Process()
        task.launchPath = "/usr/bin/env"
        task.currentDirectoryPath = url.path
        task.arguments = ["git", "diff", "original.json", "modified.json"]

        let pipe = Pipe()
        task.standardOutput = pipe
        task.standardError = pipe

        task.launch()

        let file = pipe.fileHandleForReading
        defer { file.closeFile() }

        return String(data: file.readDataToEndOfFile(), encoding: .utf8) ?? ""
    } catch {
        return "\(error)"
    }
}

private let buildingSwiftVersion: String = {
    #if swift(>=4.2)
        return "swift-4.2"
    #elseif swift(>=4.1.1)
        return "swift-4.1.1"
    #elseif swift(>=4.1)
        return "swift-4.1"
    #elseif swift(>=4.0.3)
        return "swift-4.0.3"
    #elseif swift(>=4.0.2)
        return "swift-4.0.2"
    #elseif swift(>=4.0)
        return "swift-4.0"
    #else
        fatalError("Swift 4.0 or later is required!")
    #endif
}()

class SwiftDocsTests: XCTestCase {

    func testSubscript() {
        compareDocs(withFixtureNamed: "Subscript")
    }

    func testBicycle() {
        compareDocs(withFixtureNamed: "Bicycle")
    }

    func testExtension() {
        compareDocs(withFixtureNamed: "Extension")
    }

    func testParseFullXMLDocs() {
        // swiftlint:disable:next line_length
        let xmlDocsStringPreSwift32 = "<Type file=\"file\" line=\"1\" column=\"2\"><Name>name</Name><USR>usr</USR><Declaration>declaration</Declaration><Abstract><Para>discussion</Para></Abstract><Parameters><Parameter><Name>param1</Name><Direction isExplicit=\"0\">in</Direction><Discussion><Para>param1_discussion</Para></Discussion></Parameter></Parameters><ResultDiscussion><Para>result_discussion</Para></ResultDiscussion></Type>"
        // swiftlint:disable:next line_length
        let xmlDocsStringSwift32 = "<Type file=\"file\" line=\"1\" column=\"2\"><Name>name</Name><USR>usr</USR><Declaration>declaration</Declaration><CommentParts><Abstract><Para>discussion</Para></Abstract><Parameters><Parameter><Name>param1</Name><Direction isExplicit=\"0\">in</Direction><Discussion><Para>param1_discussion</Para></Discussion></Parameter></Parameters><ResultDiscussion><Para>result_discussion</Para></ResultDiscussion></CommentParts></Type>"
        let parsedPreSwift32 = parseFullXMLDocs(xmlDocsStringPreSwift32)!
        let parsedSwift32 = parseFullXMLDocs(xmlDocsStringSwift32)!
        let expected: NSDictionary = [
            "key.doc.type": "Type",
            "key.doc.file": "file",
            "key.doc.line": 1,
            "key.doc.column": 2,
            "key.doc.name": "name",
            "key.usr": "usr",
            "key.doc.declaration": "declaration",
            "key.doc.parameters": [[
                "name": "param1",
                "discussion": [["Para": "param1_discussion"]]
            ]],
            "key.doc.result_discussion": [["Para": "result_discussion"]]
        ]
        XCTAssertEqual(toNSDictionary(parsedPreSwift32), expected)
        XCTAssertEqual(toNSDictionary(parsedSwift32), expected)
    }
}

extension SwiftDocsTests {
    static var allTests: [(String, (SwiftDocsTests) -> () throws -> Void)] {
        return [
            ("testSubscript", testSubscript),
            ("testBicycle", testBicycle),
            ("testExtension", testExtension),
            ("testParseFullXMLDocs", testParseFullXMLDocs)
        ]
    }
}
<commit_msg>Add `swift-4.1.2` to `versionedExpectedFilename(for:)`<commit_after>//
//  SwiftDocsTests.swift
//  SourceKitten
//
//  Created by <NAME> on 2015-01-03.
//  Copyright (c) 2015 SourceKitten. All rights reserved.
//

import Foundation
import SourceKittenFramework
import XCTest

func compareJSONString(withFixtureNamed name: String,
                       jsonString: CustomStringConvertible,
                       rootDirectory: String = fixturesDirectory,
                       file: StaticString = #file,
                       line: UInt = #line) {
#if os(Linux) && !swift(>=4.1)
    let jsonString = String(describing: jsonString).replacingOccurrences(of: rootDirectory, with: "")
    let actualContent = jsonString
#else
    // Strip out fixtures directory since it's dependent on the test machine's setup
    let escapedFixturesDirectory = rootDirectory.replacingOccurrences(of: "/", with: "\\/")
    let jsonString = String(describing: jsonString).replacingOccurrences(of: escapedFixturesDirectory, with: "")

    // Strip out any other absolute paths after that, since it's also dependent on the test machine's setup
    let absolutePathRegex = try! NSRegularExpression(pattern: "\"key\\.filepath\" : \"\\\\/[^\\\n]+", options: [])
    let actualContent = absolutePathRegex.stringByReplacingMatches(in: jsonString, options: [],
                                                                   range: NSRange(location: 0, length: jsonString.bridge().length),
                                                                   withTemplate: "\"key\\.filepath\" : \"\",")
#endif

    let expectedFile = File(path: versionedExpectedFilename(for: name))!

    // Use if changes are introduced by changes in SourceKitten.
    let overwrite = false
    if overwrite && actualContent != expectedFile.contents {
        _ = try? actualContent.data(using: .utf8)?.write(to: URL(fileURLWithPath: expectedFile.path!), options: [])
        return
    }

    // Use if changes are introduced by the new Swift version.
    let appendFixturesForNewSwiftVersion = false
    if appendFixturesForNewSwiftVersion && actualContent != expectedFile.contents,
        var path = expectedFile.path, let index = path.index(of: "@"),
        !path.hasSuffix("@\(buildingSwiftVersion).json") {
        path.replaceSubrange(index..<path.endIndex, with: "@\(buildingSwiftVersion).json")
        _ = try? actualContent.data(using: .utf8)?.write(to: URL(fileURLWithPath: path), options: [])
        return
    }

    func jsonValue(_ jsonString: String) -> NSObject {
        let data = jsonString.data(using: .utf8)!
        let result = try! JSONSerialization.jsonObject(with: data, options: [])
        if let dict = (result as? [String: Any])?.bridge() {
            return dict
        } else if let array = (result as? [Any])?.bridge() {
            return array
        }
        fatalError()
    }

    if jsonValue(actualContent) != jsonValue(expectedFile.contents) {
        XCTFail("output should match expected fixture", file: file, line: line)
        print(diff(original: expectedFile.contents, modified: actualContent))
    }
}

private func compareDocs(withFixtureNamed name: String, file: StaticString = #file, line: UInt = #line) {
    let swiftFilePath = fixturesDirectory + name + ".swift"
    let docs = SwiftDocs(file: File(path: swiftFilePath)!, arguments: ["-j4", swiftFilePath])!
    compareJSONString(withFixtureNamed: name, jsonString: docs, file: file, line: line)
}

private func versionedExpectedFilename(for name: String) -> String {
    #if swift(>=4.2)
        let versions = ["swift-4.2", "swift-4.1.1", "swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.1.2)
        let versions = ["swift-4.1.2", "swift-4.1.1", "swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.1.1)
        let versions = ["swift-4.1.1", "swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.1)
        let versions = ["swift-4.1", "swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0.3)
        let versions = ["swift-4.0.3", "swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0.2)
        let versions = ["swift-4.0.2", "swift-4.0"]
    #elseif swift(>=4.0)
        let versions = ["swift-4.0"]
    #else
        fatalError("Swift 4.0 or later is required!")
    #endif
    #if os(Linux)
        let platforms = ["Linux", ""]
    #else
        let platforms = [""]
    #endif
    for platform in platforms {
        for version in versions {
            let versionedFilename = "\(fixturesDirectory)\(platform)\(name)@\(version).json"
            if FileManager.default.fileExists(atPath: versionedFilename) {
                return versionedFilename
            }
        }
    }
    return "\(fixturesDirectory)\(name).json"
}

private func diff(original: String, modified: String) -> String {
    do {
        let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("SwiftDocsTests-diff-\(NSUUID())")
        try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)

        try original.data(using: .utf8)?.write(to: url.appendingPathComponent("original.json"))
        try modified.data(using: .utf8)?.write(to: url.appendingPathComponent("modified.json"))

        let task = Process()
        task.launchPath = "/usr/bin/env"
        task.currentDirectoryPath = url.path
        task.arguments = ["git", "diff", "original.json", "modified.json"]

        let pipe = Pipe()
        task.standardOutput = pipe
        task.standardError = pipe

        task.launch()

        let file = pipe.fileHandleForReading
        defer { file.closeFile() }

        return String(data: file.readDataToEndOfFile(), encoding: .utf8) ?? ""
    } catch {
        return "\(error)"
    }
}

private let buildingSwiftVersion: String = {
    #if swift(>=4.2)
        return "swift-4.2"
    #elseif swift(>=4.1.1)
        return "swift-4.1.1"
    #elseif swift(>=4.1)
        return "swift-4.1"
    #elseif swift(>=4.0.3)
        return "swift-4.0.3"
    #elseif swift(>=4.0.2)
        return "swift-4.0.2"
    #elseif swift(>=4.0)
        return "swift-4.0"
    #else
        fatalError("Swift 4.0 or later is required!")
    #endif
}()

class SwiftDocsTests: XCTestCase {

    func testSubscript() {
        compareDocs(withFixtureNamed: "Subscript")
    }

    func testBicycle() {
        compareDocs(withFixtureNamed: "Bicycle")
    }

    func testExtension() {
        compareDocs(withFixtureNamed: "Extension")
    }

    func testParseFullXMLDocs() {
        // swiftlint:disable:next line_length
        let xmlDocsStringPreSwift32 = "<Type file=\"file\" line=\"1\" column=\"2\"><Name>name</Name><USR>usr</USR><Declaration>declaration</Declaration><Abstract><Para>discussion</Para></Abstract><Parameters><Parameter><Name>param1</Name><Direction isExplicit=\"0\">in</Direction><Discussion><Para>param1_discussion</Para></Discussion></Parameter></Parameters><ResultDiscussion><Para>result_discussion</Para></ResultDiscussion></Type>"
        // swiftlint:disable:next line_length
        let xmlDocsStringSwift32 = "<Type file=\"file\" line=\"1\" column=\"2\"><Name>name</Name><USR>usr</USR><Declaration>declaration</Declaration><CommentParts><Abstract><Para>discussion</Para></Abstract><Parameters><Parameter><Name>param1</Name><Direction isExplicit=\"0\">in</Direction><Discussion><Para>param1_discussion</Para></Discussion></Parameter></Parameters><ResultDiscussion><Para>result_discussion</Para></ResultDiscussion></CommentParts></Type>"
        let parsedPreSwift32 = parseFullXMLDocs(xmlDocsStringPreSwift32)!
        let parsedSwift32 = parseFullXMLDocs(xmlDocsStringSwift32)!
        let expected: NSDictionary = [
            "key.doc.type": "Type",
            "key.doc.file": "file",
            "key.doc.line": 1,
            "key.doc.column": 2,
            "key.doc.name": "name",
            "key.usr": "usr",
            "key.doc.declaration": "declaration",
            "key.doc.parameters": [[
                "name": "param1",
                "discussion": [["Para": "param1_discussion"]]
            ]],
            "key.doc.result_discussion": [["Para": "result_discussion"]]
        ]
        XCTAssertEqual(toNSDictionary(parsedPreSwift32), expected)
        XCTAssertEqual(toNSDictionary(parsedSwift32), expected)
    }
}

extension SwiftDocsTests {
    static var allTests: [(String, (SwiftDocsTests) -> () throws -> Void)] {
        return [
            ("testSubscript", testSubscript),
            ("testBicycle", testBicycle),
            ("testExtension", testExtension),
            ("testParseFullXMLDocs", testParseFullXMLDocs)
        ]
    }
}

{'fraction_non_alphanumeric': 0.1483395262424524, 'fraction_numerical': 0.019159312587087786, 'mean_word_length': 2.8057887759611138, 'pattern_counts': {'":': 26, '<': 123, '<?xml version=': 0, '>': 158, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '8798', 'n_tokens_mistral': 5416, 'n_tokens_neox': 5086, 'n_words': 1055}