<commit_before>//
// Released under the MIT License (MIT), http://opensource.org/licenses/MIT
//
// Copyright (c) 2015 <NAME>, NotTooBad Software (nottoobadsoftware.com)
//

import Foundation

extension FileHandle {
	/// Read what is available, as a String.
	/// - Parameter encoding: the encoding to use.
	/// - Returns: The contents as a String, or nil the end has been reached.
	public func readSome(encoding: String.Encoding) -> String? {
		let data = self.availableData

		guard data.count > 0 else { return nil }
		guard let result = String(data: data, encoding: encoding) else {
			fatalError("Could not convert binary data to text.")
		}

		return result
	}

	/// Read to the end, as a String.
	/// - Parameter encoding: the encoding to use.
	public func read(encoding: String.Encoding) -> String {
		let data = self.readDataToEndOfFile()

		guard let result = String(data: data, encoding: encoding) else {
			fatalError("Could not convert binary data to text.")
		}

		return result
	}
}

extension FileHandle {
	public func write(_ string: String, encoding: String.Encoding = .utf8) {
		#if !(os(macOS) || os(iOS) || os(tvOS) || os(watchOS))
			guard !string.isEmpty else { return }
		#endif
		guard let data = string.data(using: encoding, allowLossyConversion: false) else {
			fatalError("Could not convert text to binary data.")
		}
		self.write(data)
	}
}

#if os(iOS) || os(tvOS) || os(watchOS)
/// CommandRunning is not available on iOS, tvOS and watchOS.
public protocol CommandRunning {}
#endif

/// A stream of text. Does as much as possible lazily.
public protocol ReadableStream: class, TextOutputStreamable, CommandRunning {
	var encoding: String.Encoding {get set}
	var filehandle: FileHandle {get}

	/// All the text the stream contains so far. 
	/// If the source is a file this will read everything at once. 
	/// If the stream is empty and still open this will wait for more content or end-of-file.
	/// - Returns: more text from the stream, or nil if we have reached the end.
	func readSome() -> String?

	/// Reads everything at once.
	func read() -> String
}

extension ReadableStream {
	public func readSome() -> String? {
		return filehandle.readSome(encoding: encoding)
	}

	public func read() -> String {
		return filehandle.read(encoding: encoding)
	}

	/// Splits stream lazily into lines.
	public func lines() -> LazySequence<AnySequence<String>> {
		return AnySequence(PartialSourceLazySplitSequence({self.readSome()?.characters}, separator: "\n").map { String($0) }).lazy
	}

	/// Writes the text in this stream to the given TextOutputStream.
	public func write<Target: TextOutputStream>(to target: inout Target) {
		while let text = self.readSome() { target.write(text) }
	}

	/// Writes the text in this stream to the given WritableStream.
	public func write(to target: inout WritableStream) {
		while let text = self.readSome() { target.write(text) }
	}

	#if !(os(iOS) || os(tvOS) || os(watchOS))
	public var context: Context {
		var context = CustomContext(main)
		context.stdin = self
		return context
	}
	#endif
}

#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS)
	extension ReadableStream {
		/// `handler` will be called whenever there is new output available.
		/// Pass `nil` to remove any preexisting handlers.
		/// - Note: if the stream is read from outside of the handler, or more than once inside
		/// the handler, it may be called once when stream is closed and empty.
		public func onOutput( handler: ((ReadableStream) -> Void)? ) {
			guard let handler = handler else {
				filehandle.readabilityHandler = nil
				return
			}
			filehandle.readabilityHandler = { [unowned self] _ in
				handler(self)
			}
		}

		/// `handler` will be called whenever there is new text output available.
		/// Pass `nil` to remove any preexisting handlers.
		/// - Note: if the stream is read from outside of the handler, or more than once inside
		/// the handler, it may be called once when stream is closed and empty.
		public func onStringOutput( handler: ((String) -> Void)? ) {
			guard let handler = handler else {
				self.onOutput(handler: nil)
				return
			}
			self.onOutput { stream in
				if let output = stream.readSome() {
					handler(output)
				}
			}
		}
	}
#endif

/// An output stream, like standard output or a writeable file.
public protocol WritableStream: class, TextOutputStream {
	var encoding: String.Encoding {get set}
	var filehandle: FileHandle {get}

	/// Writes `x` to the stream.
	func write(_ x: String)

	/// Closes the stream. Must be called on non-file streams when finished writing,
	/// to prevent deadlock when reading.
	func close()
}

extension WritableStream {
	public func write(_ x: String) {<commit_msg>Mark closures to ReadableStream.onOutput and .onStringOutput as @escaping.<commit_after>//
// Released under the MIT License (MIT), http://opensource.org/licenses/MIT
//
// Copyright (c) 2015 <NAME>, NotTooBad Software (nottoobadsoftware.com)
//

import Foundation

extension FileHandle {
	/// Read what is available, as a String.
	/// - Parameter encoding: the encoding to use.
	/// - Returns: The contents as a String, or nil the end has been reached.
	public func readSome(encoding: String.Encoding) -> String? {
		let data = self.availableData

		guard data.count > 0 else { return nil }
		guard let result = String(data: data, encoding: encoding) else {
			fatalError("Could not convert binary data to text.")
		}

		return result
	}

	/// Read to the end, as a String.
	/// - Parameter encoding: the encoding to use.
	public func read(encoding: String.Encoding) -> String {
		let data = self.readDataToEndOfFile()

		guard let result = String(data: data, encoding: encoding) else {
			fatalError("Could not convert binary data to text.")
		}

		return result
	}
}

extension FileHandle {
	public func write(_ string: String, encoding: String.Encoding = .utf8) {
		#if !(os(macOS) || os(iOS) || os(tvOS) || os(watchOS))
			guard !string.isEmpty else { return }
		#endif
		guard let data = string.data(using: encoding, allowLossyConversion: false) else {
			fatalError("Could not convert text to binary data.")
		}
		self.write(data)
	}
}

#if os(iOS) || os(tvOS) || os(watchOS)
/// CommandRunning is not available on iOS, tvOS and watchOS.
public protocol CommandRunning {}
#endif

/// A stream of text. Does as much as possible lazily.
public protocol ReadableStream: class, TextOutputStreamable, CommandRunning {
	var encoding: String.Encoding {get set}
	var filehandle: FileHandle {get}

	/// All the text the stream contains so far. 
	/// If the source is a file this will read everything at once. 
	/// If the stream is empty and still open this will wait for more content or end-of-file.
	/// - Returns: more text from the stream, or nil if we have reached the end.
	func readSome() -> String?

	/// Reads everything at once.
	func read() -> String
}

extension ReadableStream {
	public func readSome() -> String? {
		return filehandle.readSome(encoding: encoding)
	}

	public func read() -> String {
		return filehandle.read(encoding: encoding)
	}

	/// Splits stream lazily into lines.
	public func lines() -> LazySequence<AnySequence<String>> {
		return AnySequence(PartialSourceLazySplitSequence({self.readSome()?.characters}, separator: "\n").map { String($0) }).lazy
	}

	/// Writes the text in this stream to the given TextOutputStream.
	public func write<Target: TextOutputStream>(to target: inout Target) {
		while let text = self.readSome() { target.write(text) }
	}

	/// Writes the text in this stream to the given WritableStream.
	public func write(to target: inout WritableStream) {
		while let text = self.readSome() { target.write(text) }
	}

	#if !(os(iOS) || os(tvOS) || os(watchOS))
	public var context: Context {
		var context = CustomContext(main)
		context.stdin = self
		return context
	}
	#endif
}

#if os(macOS) || os(iOS) || os(tvOS) || os(watchOS)
	extension ReadableStream {
		/// `handler` will be called whenever there is new output available.
		/// - Note: if the stream is read from outside of the handler, or more than once inside
		/// the handler, it may be called once when stream is closed and empty.
		public func onOutput(_ handler: @escaping (ReadableStream) -> Void) {
			filehandle.readabilityHandler = { [unowned self] _ in
				handler(self)
			}
		}

		/// `handler` will be called whenever there is new text output available.
		/// - Note: if the stream is read from outside of the handler, or more than once inside
		/// the handler, it may be called once when stream is closed and empty.
		public func onStringOutput(_ handler: @escaping (String) -> Void) {
			self.onOutput { stream in
				if let output = stream.readSome() {
					handler(output)
				}
			}
		}
	}
#endif

/// An output stream, like standard output or a writeable file.
public protocol WritableStream: class, TextOutputStream {
	var encoding: String.Encoding {get set}
	var filehandle: FileHandle {get}

	/// Writes `x` to the stream.
	func write(_ x: String)

	/// Closes the stream. Must be called on non-file streams when finished writing,
	/// to prevent deadlock when reading.
	func close()
}

extension WritableStream {
	public func write(_ x: String) {
{'fraction_non_alphanumeric': 0.11044548524125912, 'fraction_numerical': 0.0015248883563881931, 'mean_word_length': 3.4703018500486853, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3089856', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2682, 'n_words': 1065}