<commit_before>//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#if DEPLOYMENT_RUNTIME_SWIFT

#if os(OSX) || os(iOS)
import Darwin
#elseif os(Linux)
import Glibc
#endif
    
import CoreFoundation

internal func __NSDataInvokeDeallocatorUnmap(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void {
    munmap(mem, length)
}

internal func __NSDataInvokeDeallocatorFree(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void {
    free(mem)
}

#else

@_exported import Foundation // Clang module

@_silgen_name("__NSDataInvokeDeallocatorVM")
internal func __NSDataInvokeDeallocatorVM(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("__NSDataInvokeDeallocatorUnmap")
internal func __NSDataInvokeDeallocatorUnmap(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("__NSDataInvokeDeallocatorFree")
internal func __NSDataInvokeDeallocatorFree(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("_NSWriteDataToFile_Swift")
internal func _NSWriteDataToFile_Swift(url: NSURL, data: NSData, options: UInt, error: NSErrorPointer) -> Bool

#endif

public final class _DataStorage {
    public enum Backing {
        // A mirror of the Objective-C implementation that is suitable to inline in Swift
        case swift
        
        // these two storage points for immutable and mutable data are reserved for references that are returned by "known"
        // cases from Foundation in which implement the backing of struct Data, these have signed up for the concept that
        // the backing bytes/mutableBytes pointer does not change per call (unless mutated) as well as the length is ok
        // to be cached, this means that as long as the backing reference is retained no further objc_msgSends need to be
        // dynamically dispatched out to the reference.
        case immutable(NSData) // This will most often (perhaps always) be NSConcreteData
        case mutable(NSMutableData) // This will often (perhaps always) be NSConcreteMutableData
        
        // These are reserved for foreign sources where neither Swift nor Foundation are fully certain whom they belong
        // to from an object inheritance standpoint, this means that all bets are off and the values of bytes, mutableBytes,
        // and length cannot be cached. This also means that all methods are expected to dynamically dispatch out to the
        // backing reference.
        case customReference(NSData) // tracks data references that are only known to be immutable
        case customMutableReference(NSMutableData) // tracks data references that are known to be mutable
    }
    
    public static let maxSize = Int.max >> 1
    public static let vmOpsThreshold = NSPageSize() * 4
    
    public static func allocate(_ size: Int, _ clear: Bool) -> UnsafeMutableRawPointer? {
        if clear {
            return calloc(1, size)
        } else {
            return malloc(size)
        }
    }
    
    
    public static func move(_ dest_: UnsafeMutableRawPointer, _ source_: UnsafeRawPointer?, _ num_: Int) {
        var dest = dest_
        var source = source_
        var num = num_
        if _DataStorage.vmOpsThreshold <= num && ((unsafeBitCast(source, to: Int.self) | unsafeBitCast(dest, to: Int.self)) & (NSPageSize() - 1)) == 0 {
            let pages = NSRoundDownToMultipleOfPageSize(num)
            NSCopyMemoryPages(source!, dest, pages)
            source = source!.advanced(by: pages)
            dest = dest.advanced(by: pages)
            num -= pages
        }
        if num > 0 {
            memmove(dest, source!, num)
        }
    }
    
    public static func shouldAllocateCleared(_ size: Int) -> Bool {
        return (size > (128 * 1024))
    }
    
    public var _bytes: UnsafeMutableRawPointer?
    public var _length: Int
    public var _capacity: Int
    public var _needToZero: Bool
    public var _deallocator: ((UnsafeMutableRawPointer, Int) -> Void)? = nil
    public var _backing: Backing = .swift
    
    public var bytes: UnsafeRawPointer? {<commit_msg>remove redundant optional nil assignment in stdlib source<commit_after>//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#if DEPLOYMENT_RUNTIME_SWIFT

#if os(OSX) || os(iOS)
import Darwin
#elseif os(Linux)
import Glibc
#endif
    
import CoreFoundation

internal func __NSDataInvokeDeallocatorUnmap(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void {
    munmap(mem, length)
}

internal func __NSDataInvokeDeallocatorFree(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void {
    free(mem)
}

#else

@_exported import Foundation // Clang module

@_silgen_name("__NSDataInvokeDeallocatorVM")
internal func __NSDataInvokeDeallocatorVM(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("__NSDataInvokeDeallocatorUnmap")
internal func __NSDataInvokeDeallocatorUnmap(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("__NSDataInvokeDeallocatorFree")
internal func __NSDataInvokeDeallocatorFree(_ mem: UnsafeMutableRawPointer, _ length: Int) -> Void

@_silgen_name("_NSWriteDataToFile_Swift")
internal func _NSWriteDataToFile_Swift(url: NSURL, data: NSData, options: UInt, error: NSErrorPointer) -> Bool

#endif

public final class _DataStorage {
    public enum Backing {
        // A mirror of the Objective-C implementation that is suitable to inline in Swift
        case swift
        
        // these two storage points for immutable and mutable data are reserved for references that are returned by "known"
        // cases from Foundation in which implement the backing of struct Data, these have signed up for the concept that
        // the backing bytes/mutableBytes pointer does not change per call (unless mutated) as well as the length is ok
        // to be cached, this means that as long as the backing reference is retained no further objc_msgSends need to be
        // dynamically dispatched out to the reference.
        case immutable(NSData) // This will most often (perhaps always) be NSConcreteData
        case mutable(NSMutableData) // This will often (perhaps always) be NSConcreteMutableData
        
        // These are reserved for foreign sources where neither Swift nor Foundation are fully certain whom they belong
        // to from an object inheritance standpoint, this means that all bets are off and the values of bytes, mutableBytes,
        // and length cannot be cached. This also means that all methods are expected to dynamically dispatch out to the
        // backing reference.
        case customReference(NSData) // tracks data references that are only known to be immutable
        case customMutableReference(NSMutableData) // tracks data references that are known to be mutable
    }
    
    public static let maxSize = Int.max >> 1
    public static let vmOpsThreshold = NSPageSize() * 4
    
    public static func allocate(_ size: Int, _ clear: Bool) -> UnsafeMutableRawPointer? {
        if clear {
            return calloc(1, size)
        } else {
            return malloc(size)
        }
    }
    
    
    public static func move(_ dest_: UnsafeMutableRawPointer, _ source_: UnsafeRawPointer?, _ num_: Int) {
        var dest = dest_
        var source = source_
        var num = num_
        if _DataStorage.vmOpsThreshold <= num && ((unsafeBitCast(source, to: Int.self) | unsafeBitCast(dest, to: Int.self)) & (NSPageSize() - 1)) == 0 {
            let pages = NSRoundDownToMultipleOfPageSize(num)
            NSCopyMemoryPages(source!, dest, pages)
            source = source!.advanced(by: pages)
            dest = dest.advanced(by: pages)
            num -= pages
        }
        if num > 0 {
            memmove(dest, source!, num)
        }
    }
    
    public static func shouldAllocateCleared(_ size: Int) -> Bool {
        return (size > (128 * 1024))
    }
    
    public var _bytes: UnsafeMutableRawPointer?
    public var _length: Int
    public var _capacity: Int
    public var _needToZero: Bool
    public var _deallocator: ((UnsafeMutableRawPointer, Int) -> Void)?
    public var _backing: Backing = .swift
    
    public var bytes: UnsafeRawPointer? {
{'fraction_non_alphanumeric': 0.1059536934950386, 'fraction_numerical': 0.005071664829106946, 'mean_word_length': 2.7085036794766966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3419301', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2372, 'n_words': 908}