<commit_before>//
//  ShellCommand.swift
//  OnBeatCore
//
//  Created by <NAME> on 11/4/16.
//
//

#if os(Linux) || os(macOS)
import Foundation

public final class ShellCommand: CustomStringConvertible {
    private let command: [String]

    #if os(Linux)
        private let process = Task()
    #else
        private let process = Process()
    #endif

    private let parentCommand: ShellCommand?

    private init(_ command: [String], parentCommand: ShellCommand?, captureOutput: Bool) {
        self.command = command
        self.parentCommand = parentCommand

        self.process.arguments = self.command
        self.process.launchPath = "/usr/bin/env"
        self.process.standardInput = Pipe()
        if let parentCommand = parentCommand {
            parentCommand.process.standardOutput = self.process.standardInput
        }
        else if captureOutput {
            self.process.standardOutput = Pipe()
        }
    }

    public convenience init(_ command: [String], captureOutput: Bool = true) {
        self.init(command, parentCommand: nil, captureOutput: captureOutput)
    }

    public convenience init(_ command: String, captureOutput: Bool = true) {
        self.init(command.components(separatedBy: " "), parentCommand: nil, captureOutput: captureOutput)
    }

    private convenience init(_ command: String, parentCommand: ShellCommand?, captureOutput: Bool = true) {
        self.init(command.components(separatedBy: " "), parentCommand: parentCommand, captureOutput: captureOutput)
    }

    @discardableResult
    public func execute() throws -> String {
        var commands = [ShellCommand]()
        var command: ShellCommand? = self
        repeat {
            commands.append(command!)
            command = command!.parentCommand
        } while command != nil

        for command in commands {
            command.process.launch()
        }

        self.process.waitUntilExit()

        guard self.process.terminationStatus == 0 else {
            throw LocalUserReportableError(source: "ShellCommand", operation: "executing command", message: "Stopping execution", reason: .user)
        }

        if let outputPipe = process.standardOutput as? Pipe {
            let outputString = String(data: outputPipe.fileHandleForReading.availableData, encoding: .utf8) ?? ""
            return outputString
        }
        else {
            return ""
        }
    }

    public func pipe(to: String) -> ShellCommand {
        return ShellCommand(to, parentCommand: self)
    }

    public var description: String {
        return self.command.joined(separator: " ")
    }
}
#endif<commit_msg>[NF] Allow asychronous shell commands<commit_after>//
//  ShellCommand.swift
//  OnBeatCore
//
//  Created by <NAME> on 11/4/16.
//
//

#if os(Linux) || os(macOS)
import Foundation

public final class ShellCommand: CustomStringConvertible {
    private let command: [String]

    #if os(Linux)
        fileprivate let process = Task()
    #else
        fileprivate let process = Process()
    #endif

    fileprivate let parentCommand: ShellCommand?

    private init(_ command: [String], parentCommand: ShellCommand?, captureOutput: Bool) {
        self.command = command
        self.parentCommand = parentCommand

        self.process.arguments = self.command
        self.process.launchPath = "/usr/bin/env"
        self.process.standardInput = Pipe()
        if let parentCommand = parentCommand {
            parentCommand.process.standardOutput = self.process.standardInput
        }
        else if captureOutput {
            self.process.standardOutput = Pipe()
        }
    }

    deinit {
        self.process.terminate()
    }

    public convenience init(_ command: [String], captureOutput: Bool = true) {
        self.init(command, parentCommand: nil, captureOutput: captureOutput)
    }

    public convenience init(_ command: String, captureOutput: Bool = true) {
        self.init(command.components(separatedBy: " "), parentCommand: nil, captureOutput: captureOutput)
    }

    private convenience init(_ command: String, parentCommand: ShellCommand?, captureOutput: Bool = true) {
        self.init(command.components(separatedBy: " "), parentCommand: parentCommand, captureOutput: captureOutput)
    }

    @discardableResult
    public func execute() throws -> String {
        self.startExecution()

        self.process.waitUntilExit()

        guard self.process.terminationStatus == 0 else {
            throw LocalUserReportableError(source: "ShellCommand", operation: "executing command", message: "Stopping execution", reason: .user)
        }

        if let outputPipe = process.standardOutput as? Pipe {
            let outputString = String(data: outputPipe.fileHandleForReading.availableData, encoding: .utf8) ?? ""
            return outputString
        }
        else {
            return ""
        }
    }

    public func executeAsync() {
        self.startExecution()
    }

    public func terminate() {
        self.process.terminate()
    }

    public func waitUntilExit() {
        self.process.waitUntilExit()
    }

    public func pipe(to: String) -> ShellCommand {
        return ShellCommand(to, parentCommand: self)
    }

    public var description: String {
        return self.command.joined(separator: " ")
    }
}

private extension ShellCommand {
    func startExecution() {
        var commands = [ShellCommand]()
        var command: ShellCommand? = self
        repeat {
            commands.append(command!)
            command = command!.parentCommand
        } while command != nil

        for command in commands {
            command.process.launch()
        }
    }
}
#endif
{'fraction_non_alphanumeric': 0.09892624537933463, 'fraction_numerical': 0.0024643548671008627, 'mean_word_length': 2.5074074074074075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4979968', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1453, 'n_words': 385}