<commit_before>//
//  main.swift
//  sourcekitten
//
//  Created by <NAME> on 10/15/14.
//  Copyright (c) 2014 Realm. All rights reserved.
//

import Foundation
import XPC

// MARK: Syntax

/**
Extension on String to support subscripting with a Range<Int> to return a substring
*/
extension String {
    subscript (r: Range<Int>) -> String {
        let start = advance(startIndex, r.startIndex)
        let end = advance(startIndex, r.endIndex)
        return substringWithRange(Range(start: start, end: end))
    }
}

/**
Print syntax highlighting information as JSON to STDOUT

:param: file Path to the file to parse for syntax highlighting information
*/
func printSyntaxHighlighting(file: String) {
    // Construct a SourceKit request for getting general info about the Swift file passed as argument
    let request = xpc_dictionary_create(nil, nil, 0)
    xpc_dictionary_set_uint64(request, "key.request", sourcekitd_uid_get_from_cstr("source.request.editor.open"))
    xpc_dictionary_set_string(request, "key.name", "")
    xpc_dictionary_set_string(request, "key.sourcefile", file)

    // Initialize SourceKit XPC service
    sourcekitd_initialize()

    // Send SourceKit request and get syntaxmap XPC data
    let xpcData = xpc_dictionary_get_value(sourcekitd_send_request_sync(request), "key.syntaxmap")

    // Convert XPC data to NSData
    let data = NSData(bytes: xpc_data_get_bytes_ptr(xpcData), length: Int(xpc_data_get_length(xpcData)))

    // Convert NSData to hex string
    var hexString = "\(data)"

    // Remove first & last characters ('<' & '>')
    hexString = hexString[hexString.startIndex.successor()..<hexString.endIndex.predecessor()]

    /// Map hex type to its SourceKit string
    func stringForHexSyntaxType(hexType: String) -> String {
        let uidHex45 = NSString(format: "%02X", strtoull(hexType[0...1], nil, 16) - 0x22)
        let uidHex = "100" + hexType[4...5] + hexType[2...3] + uidHex45
        let uid = strtoull(uidHex, nil, 16) + 34
        return String(UTF8String: sourcekitd_uid_get_string_ptr(UInt64(uid)))!
    }

    println("[")

    let hexArray = hexString.componentsSeparatedByString(" ")
    let syntaxTokenCount = (hexArray.count - 5)/4
    var typeMap = [String:String]()

    for index in 0..<syntaxTokenCount {
        let typeIndex = index*4 + 4
        let hexType = hexArray[typeIndex]

        var type: String! = typeMap[hexType]
        if type == nil {
            type = stringForHexSyntaxType(hexType)
            typeMap[hexType] = type
        }
        let offsetString = hexArray[typeIndex+2]
        let offset = strtoull(offsetString[6...7] + offsetString[4...5] + offsetString[2...3] + offsetString[0...1], nil, 16)
<commit_msg>added TODO and removed useless cast<commit_after>//
//  main.swift
//  sourcekitten
//
//  Created by <NAME> on 10/15/14.
//  Copyright (c) 2014 Realm. All rights reserved.
//

import Foundation
import XPC

// MARK: Syntax

/**
Extension on String to support subscripting with a Range<Int> to return a substring
*/
extension String {
    subscript (r: Range<Int>) -> String {
        let start = advance(startIndex, r.startIndex)
        let end = advance(startIndex, r.endIndex)
        return substringWithRange(Range(start: start, end: end))
    }
}

/**
Print syntax highlighting information as JSON to STDOUT

:param: file Path to the file to parse for syntax highlighting information
*/
func printSyntaxHighlighting(file: String) {
    // TODO: Parse syntaxmap bytes instead of hex string representation

    // Construct a SourceKit request for getting general info about the Swift file passed as argument
    let request = xpc_dictionary_create(nil, nil, 0)
    xpc_dictionary_set_uint64(request, "key.request", sourcekitd_uid_get_from_cstr("source.request.editor.open"))
    xpc_dictionary_set_string(request, "key.name", "")
    xpc_dictionary_set_string(request, "key.sourcefile", file)

    // Initialize SourceKit XPC service
    sourcekitd_initialize()

    // Send SourceKit request and get syntaxmap XPC data
    let xpcData = xpc_dictionary_get_value(sourcekitd_send_request_sync(request), "key.syntaxmap")

    // Convert XPC data to NSData
    let data = NSData(bytes: xpc_data_get_bytes_ptr(xpcData), length: Int(xpc_data_get_length(xpcData)))

    // Convert NSData to hex string
    var hexString = "\(data)"

    // Remove first & last characters ('<' & '>')
    hexString = hexString[hexString.startIndex.successor()..<hexString.endIndex.predecessor()]

    /// Map hex type to its SourceKit string
    func stringForHexSyntaxType(hexType: String) -> String {
        let uidHex45 = NSString(format: "%02X", strtoull(hexType[0...1], nil, 16) - 0x22)
        let uidHex = "100" + hexType[4...5] + hexType[2...3] + uidHex45
        let uid = strtoull(uidHex, nil, 16) + 34
        return String(UTF8String: sourcekitd_uid_get_string_ptr(uid))!
    }

    println("[")

    let hexArray = hexString.componentsSeparatedByString(" ")
    let syntaxTokenCount = (hexArray.count - 5)/4
    var typeMap = [String:String]()

    for index in 0..<syntaxTokenCount {
        let typeIndex = index*4 + 4
        let hexType = hexArray[typeIndex]

        var type: String! = typeMap[hexType]
        if type == nil {
            type = stringForHexSyntaxType(hexType)
            typeMap[hexType] = type
        }
        let offsetString = hexArray[typeIndex+2]
        let offset = strtoull(offsetString[6...7] + offsetString[4...5] + offsetString[2...3] + offsetString[0...1], nil, 16)

{'fraction_non_alphanumeric': 0.11405258386219401, 'fraction_numerical': 0.019945602901178604, 'mean_word_length': 3.279286268425136, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3589328', 'n_tokens_mistral': 1805, 'n_tokens_neox': 1698, 'n_words': 462}