<commit_before>//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

import SwiftShims

/// Convert the given numeric value to a hexadecimal string.
// FIXME(integers): support a more general BinaryInteger protocol
public func asHex<T : FixedWidthInteger>(_ x: T) -> String {
    return "0x\(String(x, radix: 16))"
}

/// Convert the given sequence of numeric values to a string representing
/// their hexadecimal values.
// FIXME(integers): support a more general BinaryInteger protocol
public func asHex<S : Sequence>(_ x: S) -> String
    where
    S.Element : FixedWidthInteger {
        return "[ \(x.lazy.map { asHex($0) }.joined(separator: ", ")) ]"
}

/// Compute the prefix sum of `seq`.
public func scan<
    S : Sequence, U
    >(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] {
    var result: [U] = []
    result.reserveCapacity(seq.underestimatedCount)
    var runningResult = initial
    for element in seq {
        runningResult = combine(runningResult, element)
        result.append(runningResult)
    }
    return result
}

public func gather<C : Collection, IndicesSequence : Sequence>(
    _ collection: C, _ indices: IndicesSequence
    ) -> [C.Element]
    where IndicesSequence.Element == C.Index {
        return Array(indices.map { collection[$0] })
}

public func scatter<T>(_ a: [T], _ idx: [Int]) -> [T] {
    var result = a
    for i in 0..<a.count {
        result[idx[i]] = a[i]
    }
    return result
}

public func withArrayOfCStrings<R>(
    _ args: [String], _ body: ([UnsafeMutablePointer<CChar>?]) -> R
    ) -> R {
    let argsCounts = Array(args.map { $0.utf8.count + 1 })
    let argsOffsets = [ 0 ] + scan(argsCounts, 0, +)
    let argsBufferSize = argsOffsets.last!
    
    var argsBuffer: [UInt8] = []
    argsBuffer.reserveCapacity(argsBufferSize)
    for arg in args {
        argsBuffer.append(contentsOf: arg.utf8)
        argsBuffer.append(0)
    }
    
    return argsBuffer.withUnsafeMutableBufferPointer {
        (argsBuffer) in
        let ptr = UnsafeMutableRawPointer(argsBuffer.baseAddress!).bindMemory(
            to: CChar.self, capacity: argsBuffer.count)
        var cStrings: [UnsafeMutablePointer<CChar>?] = argsOffsets.map { ptr + $0 }
        cStrings[cStrings.count - 1] = nil
        return body(cStrings)
    }
}<commit_msg>Fixed indentation to 2 spaces in SwiftPrivate.swift<commit_after>//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

import SwiftShims

/// Convert the given numeric value to a hexadecimal string.
// FIXME(integers): support a more general BinaryInteger protocol
public func asHex<T : FixedWidthInteger>(_ x: T) -> String {
  return "0x\(String(x, radix: 16))"
}

/// Convert the given sequence of numeric values to a string representing
/// their hexadecimal values.
// FIXME(integers): support a more general BinaryInteger protocol
public func asHex<S : Sequence>(_ x: S) -> String
  where
  S.Element : FixedWidthInteger {
    return "[ \(x.lazy.map { asHex($0) }.joined(separator: ", ")) ]"
}

/// Compute the prefix sum of `seq`.
public func scan<
  S : Sequence, U
  >(_ seq: S, _ initial: U, _ combine: (U, S.Element) -> U) -> [U] {
  var result: [U] = []
  result.reserveCapacity(seq.underestimatedCount)
  var runningResult = initial
  for element in seq {
    runningResult = combine(runningResult, element)
    result.append(runningResult)
  }
  return result
}

public func gather<C : Collection, IndicesSequence : Sequence>(
  _ collection: C, _ indices: IndicesSequence
  ) -> [C.Element]
  where IndicesSequence.Element == C.Index {
    return Array(indices.map { collection[$0] })
}

public func scatter<T>(_ a: [T], _ idx: [Int]) -> [T] {
  var result = a
  for i in 0..<a.count {
    result[idx[i]] = a[i]
  }
  return result
}

public func withArrayOfCStrings<R>(
  _ args: [String], _ body: ([UnsafeMutablePointer<CChar>?]) -> R
  ) -> R {
  let argsCounts = Array(args.map { $0.utf8.count + 1 })
  let argsOffsets = [ 0 ] + scan(argsCounts, 0, +)
  let argsBufferSize = argsOffsets.last!
  
  var argsBuffer: [UInt8] = []
  argsBuffer.reserveCapacity(argsBufferSize)
  for arg in args {
    argsBuffer.append(contentsOf: arg.utf8)
    argsBuffer.append(0)
  }
  
  return argsBuffer.withUnsafeMutableBufferPointer {
    (argsBuffer) in
    let ptr = UnsafeMutableRawPointer(argsBuffer.baseAddress!).bindMemory(
      to: CChar.self, capacity: argsBuffer.count)
    var cStrings: [UnsafeMutablePointer<CChar>?] = argsOffsets.map { ptr + $0 }
    cStrings[cStrings.count - 1] = nil
    return body(cStrings)
  }
}
{'fraction_non_alphanumeric': 0.17579408543263966, 'fraction_numerical': 0.009675063891931361, 'mean_word_length': 2.6093544137022397, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 35, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3623261', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1633, 'n_words': 484}