<commit_before>//
//  CUDACompiler.swift
//  SAD
//
//  Created by <NAME> on 9/28/16.
//
//

import CNVRTC
@_exported import struct CUDADriver.PTX
import protocol CUDADriver.CHandleCarrier
import Foundation

/// Source-level program
open class Program {

    let handle: nvrtcProgram

    public var name: String?

    /// Initialize with source program and headers
    public init(data: Data, name: String? = nil, externC: Bool = true,
                headers: [(name: String, data: Data)]) throws {
        self.name = name
        var data = data
        if externC { /// Disable mangling
            data.insert(contentsOf: "extern \"C\"{".utf8, at: 0)
            data.append(UInt8("}")!)
        }
        var headerNames: [UnsafePointer<Int8>?] = headers.map { name, _ in
            name.withCString{$0}
        }
        var headerCodes: [UnsafePointer<Int8>?] = headers.map { _, data in
            data.withUnsafeBytes{$0}
        }
        var handle: nvrtcProgram?
        try ensureSuccess(
            data.withUnsafeBytes { bytes in
                nvrtcCreateProgram(
                    &handle,                          /// Handle
                    bytes,                            /// Source
                    name?.utf8CString.map{$0} ?? nil, /// Program name
                    Int32(headerNames.count),         /// Header count
                    &headerCodes,                     /// Headers
                    &headerNames                      /// Header names
                )
            }
        )
        self.handle = handle! /// Safe
    }

    /// Initialize with source program
    public init(data: Data, name: String? = nil, externC: Bool = true) throws {
        self.name = name
        var data = data
        if externC { /// Disable mangling
            data.insert(contentsOf: "extern \"C\"{".utf8, at: 0)
            data.append(contentsOf: "}".utf8)
        }
        var handle: nvrtcProgram?
        try ensureSuccess(
            data.withUnsafeBytes { bytes in
                nvrtcCreateProgram(
                    &handle,                          /// Handle
                    bytes,                            /// Source
                    name?.utf8CString.map{$0} ?? nil, /// Program name
                    0, nil, nil                       /// No headers
                )
            }
        )
        self.handle = handle! /// Safe
    }

    public convenience init(source: String, name: String? = nil) throws {
        guard let data = source.data(using: .utf8, allowLossyConversion: true) else {
            throw CompilerError.invalidEncoding
        }
        try self.init(data: data, name: name)
    }

    public convenience init(sourceFile: String) throws {
        let url = URL(fileURLWithPath: sourceFile)
        let name = url.deletingPathExtension().lastPathComponent
        let data = try Data(contentsOf: url)
        try self.init(data: data, name: name)
    }

    deinit {
        var handle: nvrtcProgram? = self.handle
        !!nvrtcDestroyProgram(&handle)
    }

    internal func retrievePTX() throws -> PTX {
        var ptxSize = 0
        /// Get PTX size
        try ensureSuccess(nvrtcGetPTXSize(handle, &ptxSize))
        var outData = Data(capacity: ptxSize)
        outData.count = ptxSize
        /// Get PTX data
        try outData.withUnsafeMutableBytes { ptxBuf in
            try ensureSuccess(nvrtcGetPTX(handle, ptxBuf))
        }
        return PTX(data: outData, name: name)
    }

}

open class Compiler {

    open class var version: (major: Int, minor: Int) {
        var major: Int32 = 0
        var minor: Int32 = 0
        nvrtcVersion(&major, &minor)
        return (major: Int(major), minor: Int(minor))
    }

    open class func compile(_ program: Program) throws -> PTX {
        try ensureSuccess(nvrtcCompileProgram(program.handle, 0, nil))
        return try program.retrievePTX()
    }

    open class func compile(_ program: Program, options: [Option]) throws -> PTX {
        return try options.map{$0.rawArgument}.withUnsafeBufferPointer { ptr in
            ///var cStringPtr: [UnsafePointer<Int8>?] = ptr.map{$0.rawArgument.withCString{$0}}
            var cArgs: [ContiguousArray<Int8>] = ptr.map{$0.utf8CString}
            var cArgPtrs: [UnsafePointer<Int8>?] = []
            for i in cArgs.indices {
                cArgs[i].withUnsafeBufferPointer { buf in
                    cArgPtrs.append(buf.baseAddress)
                }
            }
            try ensureSuccess(
                nvrtcCompileProgram(program.handle, Int32(ptr.count), &cArgPtrs)
            )
            return try program.retrievePTX()
        }
    }

}
<commit_msg>Hold on to program data<commit_after>//
//  CUDACompiler.swift
//  SAD
//
//  Created by <NAME> on 9/28/16.
//
//

import CNVRTC
@_exported import struct CUDADriver.PTX
import protocol CUDADriver.CHandleCarrier
import Foundation

/// Source-level program
open class Program {

    let handle: nvrtcProgram

    public var name: String?
    public var data: Data
    public var headers: [(name: String, data: Data)]?

    /// Initialize with source program and headers
    public init(data: Data, name: String? = nil, externC: Bool = true,
                headers: [(name: String, data: Data)]) throws {
        self.name = name
        self.data = data
        self.headers = headers
        if externC { /// Disable mangling
            self.data.insert(contentsOf: "extern \"C\"{".utf8, at: 0)
            self.data.append(UInt8("}")!)
        }
        var headerNames: [UnsafePointer<Int8>?] = self.headers!.map { name, _ in
            name.withCString{$0}
        }
        var headerCodes: [UnsafePointer<Int8>?] = self.headers!.map { _, data in
            data.withUnsafeBytes{$0}
        }
        var handle: nvrtcProgram?
        try ensureSuccess(
            self.data.withUnsafeBytes { bytes in
                nvrtcCreateProgram(
                    &handle,                          /// Handle
                    bytes,                            /// Source
                    name?.utf8CString.map{$0} ?? nil, /// Program name
                    Int32(headerNames.count),         /// Header count
                    &headerCodes,                     /// Headers
                    &headerNames                      /// Header names
                )
            }
        )
        self.handle = handle! /// Safe
    }

    /// Initialize with source program
    public init(data: Data, name: String? = nil, externC: Bool = true) throws {
        self.name = name
        self.data = data
        if externC { /// Disable mangling
            self.data.insert(contentsOf: "extern \"C\"{".utf8, at: 0)
            self.data.append(contentsOf: "}".utf8)
        }
        var handle: nvrtcProgram?
        try ensureSuccess(
            self.data.withUnsafeBytes { bytes in
                nvrtcCreateProgram(
                    &handle,                          /// Handle
                    bytes,                            /// Source
                    name?.utf8CString.map{$0} ?? nil, /// Program name
                    0, nil, nil                       /// No headers
                )
            }
        )
        self.handle = handle! /// Safe
    }

    public convenience init(source: String, name: String? = nil) throws {
        guard let data = source.data(using: .utf8, allowLossyConversion: true) else {
            throw CompilerError.invalidEncoding
        }
        try self.init(data: data, name: name)
    }

    public convenience init(sourceFile: String) throws {
        let url = URL(fileURLWithPath: sourceFile)
        let name = url.deletingPathExtension().lastPathComponent
        let data = try Data(contentsOf: url)
        try self.init(data: data, name: name)
    }

    deinit {
        var handle: nvrtcProgram? = self.handle
        !!nvrtcDestroyProgram(&handle)
    }

    internal func retrievePTX() throws -> PTX {
        var ptxSize = 0
        /// Get PTX size
        try ensureSuccess(nvrtcGetPTXSize(handle, &ptxSize))
        var outData = Data(capacity: ptxSize)
        outData.count = ptxSize
        /// Get PTX data
        try outData.withUnsafeMutableBytes { ptxBuf in
            try ensureSuccess(nvrtcGetPTX(handle, ptxBuf))
        }
        return PTX(data: outData, name: name)
    }

}

open class Compiler {

    open class var version: (major: Int, minor: Int) {
        var major: Int32 = 0
        var minor: Int32 = 0
        nvrtcVersion(&major, &minor)
        return (major: Int(major), minor: Int(minor))
    }

    open class func compile(_ program: Program) throws -> PTX {
        try ensureSuccess(nvrtcCompileProgram(program.handle, 0, nil))
        return try program.retrievePTX()
    }

    open class func compile(_ program: Program, options: [Option]) throws -> PTX {
        return try options.map{$0.rawArgument}.withUnsafeBufferPointer { ptr in
            var cArgs: [ContiguousArray<Int8>] = ptr.map{$0.utf8CString}
            var cArgPtrs: [UnsafePointer<Int8>?] = []
            for i in cArgs.indices {
                cArgs[i].withUnsafeBufferPointer { buf in
                    cArgPtrs.append(buf.baseAddress)
                }
            }
            try ensureSuccess(
                nvrtcCompileProgram(program.handle, Int32(ptr.count), &cArgPtrs)
            )
            return try program.retrievePTX()
        }
    }

}

{'fraction_non_alphanumeric': 0.10346288255850895, 'fraction_numerical': 0.008365985386000212, 'mean_word_length': 1.585272378866685, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6617628', 'n_tokens_mistral': 2694, 'n_tokens_neox': 2580, 'n_words': 677}