<commit_before>// Copyright (c) 2020 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

infix operator >>>

@inline(__always)
private func >>> (num: UInt32, count: Int) -> UInt32 {
    // This implementation assumes without checking that `count` is in the 1...31 range.
    return (num >> UInt32(truncatingIfNeeded: count)) | (num << UInt32(truncatingIfNeeded: 32 - count))
}

struct Sha256 {

    private static let k: [UInt32] =
        [0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98,
         0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
         0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8,
         0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
         0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819,
         0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
         0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
         0xc67178f2]

    static func hash(data: Data) -> [UInt8] {
        var h0 = 0x6a09e667 as UInt32
        var h1 = 0xbb67ae85 as UInt32
        var h2 = 0x3c6ef372 as UInt32
        var h3 = 0xa54ff53a as UInt32
        var h4 = 0x510e527f as UInt32
        var h5 = 0x9b05688c as UInt32
        var h6 = 0x1f83d9ab as UInt32
        var h7 = 0x5be0cd19 as UInt32

        // Padding
        #if compiler(>=5.0)
            var bytes = data.withUnsafeBytes { $0.map { $0 } }
        #else
            var bytes = data.withUnsafeBytes {
                [UInt8](UnsafeBufferPointer(start: $0, count: data.count / MemoryLayout<UInt8>.size))
            }
        #endif

        let originalLength = bytes.count
        var newLength = originalLength * 8 + 1
        while newLength % 512 != 448 {
            newLength += 1
        }
        newLength /= 8

        bytes.append(0x80)
        for _ in 0..<(newLength - originalLength - 1) {
            bytes.append(0x00)
        }

        // Length<commit_msg>Remove Swift-4.2-support code-path from Sha256.swift<commit_after>// Copyright (c) 2020 <NAME>
// Licensed under MIT License
//
// See LICENSE for license information

import Foundation

infix operator >>>

@inline(__always)
private func >>> (num: UInt32, count: Int) -> UInt32 {
    // This implementation assumes without checking that `count` is in the 1...31 range.
    return (num >> UInt32(truncatingIfNeeded: count)) | (num << UInt32(truncatingIfNeeded: 32 - count))
}

struct Sha256 {

    private static let k: [UInt32] =
        [0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98,
         0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
         0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8,
         0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
         0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819,
         0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
         0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
         0xc67178f2]

    static func hash(data: Data) -> [UInt8] {
        var h0 = 0x6a09e667 as UInt32
        var h1 = 0xbb67ae85 as UInt32
        var h2 = 0x3c6ef372 as UInt32
        var h3 = 0xa54ff53a as UInt32
        var h4 = 0x510e527f as UInt32
        var h5 = 0x9b05688c as UInt32
        var h6 = 0x1f83d9ab as UInt32
        var h7 = 0x5be0cd19 as UInt32

        // Padding
        var bytes = data.withUnsafeBytes { $0.map { $0 } }

        let originalLength = bytes.count
        var newLength = originalLength * 8 + 1
        while newLength % 512 != 448 {
            newLength += 1
        }
        newLength /= 8

        bytes.append(0x80)
        for _ in 0..<(newLength - originalLength - 1) {
            bytes.append(0x00)
        }

        // Length
{'fraction_non_alphanumeric': 0.09039802113784574, 'fraction_numerical': 0.22891837193613673, 'mean_word_length': 2.759932375316991, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4375946', 'n_tokens_mistral': 2475, 'n_tokens_neox': 1931, 'n_words': 399}