<commit_before>import Foundation

#if os(Linux)
  import Glibc

  typealias NSRegularExpression = RegularExpression
#else
  import Darwin
#endif

public struct FileCheckOptions: OptionSet {
  public let rawValue: UInt64

  /// Convert from a value of `RawValue`, succeeding unconditionally.
  public init(rawValue: UInt64) {
    self.rawValue = rawValue
  }

  /// Do not treat all horizontal whitespace as equivalent.
  public static let strictWhitespace = FileCheckOptions(rawValue: 1 << 0)
  /// Add an implicit negative check with this pattern to every positive
  /// check. This can be used to ensure that no instances of this pattern
  /// occur which are not matched by a positive pattern.
  public static let implicitCheckNot = FileCheckOptions(rawValue: 1 << 1)
  /// Allow the input file to be empty. This is useful when making checks that
  /// some error message does not occur, for example.
  public static let allowEmptyInput = FileCheckOptions(rawValue: 1 << 2)
  /// Require all positive matches to cover an entire input line.  Allows
  /// leading and trailing whitespace if `.strictWhitespace` is not also
  /// passed.
  public static let matchFullLines = FileCheckOptions(rawValue: 1 << 3)
}

/// `FileCheckFD` represents the standard output streams `FileCheck` is capable
/// of overriding to gather output.
public enum FileCheckFD {
  /// Standard output.
  case stdout
  /// Standard error.
  case stderr
  /// A custom output stream.
  case custom(fileno: Int32, ptr: UnsafeMutablePointer<FILE>)

  /// Retrieve the file descriptor for this output stream.
  var fileno : Int32 {
    switch self {
    case .stdout:
      return STDOUT_FILENO
    case .stderr:
      return STDERR_FILENO
    case let .custom(fileno: fd, ptr: _):
      return fd
    }
  }

  /// Retrieve the FILE pointer for this stream.
  var filePtr : UnsafeMutablePointer<FILE>! {
    switch self {
    case .stdout:
      #if os(Linux)
        return Glibc.stdout
      #else
        return Darwin.stdout
      #endif
    case .stderr:
      #if os(Linux)
        return Glibc.stderr
      #else
        return Darwin.stderr
      #endif
    case let .custom(fileno: _, ptr: ptr):
      return ptr
    }
  }
}

public func fileCheckOutput(of FD : FileCheckFD = .stdout, withPrefixes prefixes : [String] = ["CHECK"], against file : String = #file, options: FileCheckOptions = [], block : () -> ()) -> Bool {
  guard let validPrefixes = validateCheckPrefixes(prefixes) else {
    print("Supplied check-prefix is invalid! Prefixes must be unique and ",
          "start with a letter and contain only alphanumeric characters, ",
          "hyphens and underscores")
    return false
  }
  guard let PrefixRE = try? NSRegularExpression(pattern: validPrefixes.joined(separator: "|"), options: []) else {
    print("Unable to combine check-prefix strings into a prefix regular ",
          "expression! This is likely a bug in FileCheck's verification of ",
          "the check-prefix strings. Regular expression parsing failed.")
    return false
  }

  let input = overrideFDAndCollectOutput(file: FD, of: block)
  if (input.isEmpty && !options.contains(.allowEmptyInput)) {
    print("FileCheck error: input from file descriptor \(FD) is empty.\n")
    return false
  }

  guard let contents = try? String(contentsOfFile: file, encoding: .utf8) else {
    return false
  }
  let buf = contents.cString(using: .utf8)?.withUnsafeBufferPointer { buffer in
    return readCheckStrings(in: buffer, withPrefixes: validPrefixes, options: options, PrefixRE)
  }
  guard let checkStrings = buf else {
    return false
  }
  return check(input: input, against: checkStrings)
}

private func overrideFDAndCollectOutput(file : FileCheckFD, of block : () -> ()) -> String {
  fflush(stdout)
  let oldFd = dup(file.fileno)

  let template = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("output.XXXXXX")
  return template.withUnsafeFileSystemRepresentation { buffer in
    guard let buffer = buffer else {
      return ""
    }

   	let newFd = mkstemp(UnsafeMutablePointer(mutating: buffer))
    guard newFd != -1 else {
      return ""
    }

    dup2(newFd, file.fileno)

    block()

    close(newFd)
    fflush(file.filePtr)


    dup2(oldFd, file.fileno)
    close(oldFd)

    let url = URL(fileURLWithFileSystemRepresentation: buffer, isDirectory: false, relativeTo: nil)
    guard let s = try? String(contentsOf: url) else {
      return ""
    }
    return s
  }
}

func validateCheckPrefixes(_ prefixes : [String]) -> [String]? {
  let validator = try! NSRegularExpression(pattern: "^[a-zA-Z0-9_-]*$", options: [])

  for prefix in prefixes {
    // Reject empty prefixes.
    if prefix.isEmpty {
      return nil
    }

    let range = NSRange(
      location: 0,<commit_msg>Specify an encoding<commit_after>import Foundation

#if os(Linux)
  import Glibc

  typealias NSRegularExpression = RegularExpression
#else
  import Darwin
#endif

public struct FileCheckOptions: OptionSet {
  public let rawValue: UInt64

  /// Convert from a value of `RawValue`, succeeding unconditionally.
  public init(rawValue: UInt64) {
    self.rawValue = rawValue
  }

  /// Do not treat all horizontal whitespace as equivalent.
  public static let strictWhitespace = FileCheckOptions(rawValue: 1 << 0)
  /// Add an implicit negative check with this pattern to every positive
  /// check. This can be used to ensure that no instances of this pattern
  /// occur which are not matched by a positive pattern.
  public static let implicitCheckNot = FileCheckOptions(rawValue: 1 << 1)
  /// Allow the input file to be empty. This is useful when making checks that
  /// some error message does not occur, for example.
  public static let allowEmptyInput = FileCheckOptions(rawValue: 1 << 2)
  /// Require all positive matches to cover an entire input line.  Allows
  /// leading and trailing whitespace if `.strictWhitespace` is not also
  /// passed.
  public static let matchFullLines = FileCheckOptions(rawValue: 1 << 3)
}

/// `FileCheckFD` represents the standard output streams `FileCheck` is capable
/// of overriding to gather output.
public enum FileCheckFD {
  /// Standard output.
  case stdout
  /// Standard error.
  case stderr
  /// A custom output stream.
  case custom(fileno: Int32, ptr: UnsafeMutablePointer<FILE>)

  /// Retrieve the file descriptor for this output stream.
  var fileno : Int32 {
    switch self {
    case .stdout:
      return STDOUT_FILENO
    case .stderr:
      return STDERR_FILENO
    case let .custom(fileno: fd, ptr: _):
      return fd
    }
  }

  /// Retrieve the FILE pointer for this stream.
  var filePtr : UnsafeMutablePointer<FILE>! {
    switch self {
    case .stdout:
      #if os(Linux)
        return Glibc.stdout
      #else
        return Darwin.stdout
      #endif
    case .stderr:
      #if os(Linux)
        return Glibc.stderr
      #else
        return Darwin.stderr
      #endif
    case let .custom(fileno: _, ptr: ptr):
      return ptr
    }
  }
}

public func fileCheckOutput(of FD : FileCheckFD = .stdout, withPrefixes prefixes : [String] = ["CHECK"], against file : String = #file, options: FileCheckOptions = [], block : () -> ()) -> Bool {
  guard let validPrefixes = validateCheckPrefixes(prefixes) else {
    print("Supplied check-prefix is invalid! Prefixes must be unique and ",
          "start with a letter and contain only alphanumeric characters, ",
          "hyphens and underscores")
    return false
  }
  guard let PrefixRE = try? NSRegularExpression(pattern: validPrefixes.joined(separator: "|"), options: []) else {
    print("Unable to combine check-prefix strings into a prefix regular ",
          "expression! This is likely a bug in FileCheck's verification of ",
          "the check-prefix strings. Regular expression parsing failed.")
    return false
  }

  let input = overrideFDAndCollectOutput(file: FD, of: block)
  if (input.isEmpty && !options.contains(.allowEmptyInput)) {
    print("FileCheck error: input from file descriptor \(FD) is empty.\n")
    return false
  }

  guard let contents = try? String(contentsOfFile: file, encoding: .utf8) else {
    return false
  }
  let buf = contents.cString(using: .utf8)?.withUnsafeBufferPointer { buffer in
    return readCheckStrings(in: buffer, withPrefixes: validPrefixes, options: options, PrefixRE)
  }
  guard let checkStrings = buf else {
    return false
  }
  return check(input: input, against: checkStrings)
}

private func overrideFDAndCollectOutput(file : FileCheckFD, of block : () -> ()) -> String {
  fflush(file.filePtr)
  let oldFd = dup(file.fileno)

  let template = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("output.XXXXXX")
  return template.withUnsafeFileSystemRepresentation { buffer in
    guard let buffer = buffer else {
      return ""
    }

   	let newFd = mkstemp(UnsafeMutablePointer(mutating: buffer))
    guard newFd != -1 else {
      return ""
    }

    dup2(newFd, file.fileno)

    block()

    close(newFd)
    fflush(file.filePtr)


    dup2(oldFd, file.fileno)
    close(oldFd)

    let url = URL(fileURLWithFileSystemRepresentation: buffer, isDirectory: false, relativeTo: nil)
    guard let s = try? String(contentsOf: url, encoding: .utf8) else {
      return ""
    }
    return s
  }
}

func validateCheckPrefixes(_ prefixes : [String]) -> [String]? {
  let validator = try! NSRegularExpression(pattern: "^[a-zA-Z0-9_-]*$", options: [])

  for prefix in prefixes {
    // Reject empty prefixes.
    if prefix.isEmpty {
      return nil
    }

    let range = NSRange(
      location: 0,
{'fraction_non_alphanumeric': 0.0960116630219723, 'fraction_numerical': 0.005102572112881391, 'mean_word_length': 3.1414402759810263, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4715257', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2766, 'n_words': 949}