<commit_before>import libc

/**
    A replacement for Foundation's NSFileManager using
    implementation from Swift's core libraries.
*/
class FileManager {
    enum Error: ErrorProtocol {
        case CouldNotOpenFile
        case Unreadable
    }

    static func readBytesFromFile(_ path: String) throws -> [UInt8] {
        let fd = open(path, O_RDONLY)

        if fd < 0 {
            throw Error.CouldNotOpenFile
        }
        defer {
            close(fd)
        }

        var info = stat()
        let ret = withUnsafeMutablePointer(&info) { infoPointer -> Bool in
            if fstat(fd, infoPointer) < 0 {
                return false
            }
            return true
        }

        if !ret {
            print("Read bytes 1")
            throw Error.Unreadable
        }

        let length = Int(info.st_size)

        let rawData = malloc(length)
        var remaining = Int(info.st_size)
        var total = 0
        while remaining > 0 {
            let advanced = rawData.advanced(by: total)

            let amt = read(fd, advanced, remaining)
            if amt < 0 {
                break
            }
            remaining -= amt
            total += amt
        }

        if remaining != 0 {
            print("Read bytes 2")
            throw Error.Unreadable
        }

        //thanks @Danappelxx
        let data = UnsafeMutablePointer<UInt8>(rawData)
        let buffer = UnsafeMutableBufferPointer<UInt8>(start: data, count: length)
        return Array(buffer)
    }

    static func fileAtPath(_ path: String) -> (exists: Bool, isDirectory: Bool) {
        var isDirectory = false
        var s = stat()
        if lstat(path, &s) >= 0 {
            if (s.st_mode & S_IFMT) == S_IFLNK {
                if stat(path, &s) >= 0 {
                    isDirectory = (s.st_mode & S_IFMT) == S_IFDIR
                } else {
                    return (false, isDirectory)
                }
            } else {
                isDirectory = (s.st_mode & S_IFMT) == S_IFDIR
            }

            // don't chase the link for this magic case -- we might be /Net/foo
            // which is a symlink to /private/Net/foo which is not yet mounted...
            if (s.st_mode & S_IFMT) == S_IFLNK {<commit_msg>testing nsdata load over manual<commit_after>import Foundation
import libc

/**
    A replacement for Foundation's NSFileManager using
    implementation from Swift's core libraries.
*/
class FileManager {
    enum Error: ErrorProtocol {
        case CouldNotOpenFile
        case Unreadable
    }

    static func readBytesFromFile(_ path: String) throws -> [UInt8] {
        let data = NSData(contentsOfFile: path)
        return data?.byteArray ?? []
//        let fd = open(path, O_RDONLY)
//
//        if fd < 0 {
//            throw Error.CouldNotOpenFile
//        }
//        defer {
//            close(fd)
//        }
//
//        var info = stat()
//        let ret = withUnsafeMutablePointer(&info) { infoPointer -> Bool in
//            if fstat(fd, infoPointer) < 0 {
//                return false
//            }
//            return true
//        }
//
//        if !ret {
//            print("Read bytes 1")
//            throw Error.Unreadable
//        }
//
//        let length = Int(info.st_size)
//
//        let rawData = malloc(length)
//        var remaining = Int(info.st_size)
//        var total = 0
//        while remaining > 0 {
//            let advanced = rawData.advanced(by: total)
//
//            let amt = read(fd, advanced, remaining)
//            if amt < 0 {
//                break
//            }
//            remaining -= amt
//            total += amt
//        }
//
//        if remaining != 0 {
//            print("Read bytes 2")
//            throw Error.Unreadable
//        }
//
//        //thanks @Danappelxx
//        let data = UnsafeMutablePointer<UInt8>(rawData)
//        let buffer = UnsafeMutableBufferPointer<UInt8>(start: data, count: length)
//        return Array(buffer)
    }

    static func fileAtPath(_ path: String) -> (exists: Bool, isDirectory: Bool) {
        var isDirectory = false
        var s = stat()
        if lstat(path, &s) >= 0 {
            if (s.st_mode & S_IFMT) == S_IFLNK {
                if stat(path, &s) >= 0 {
                    isDirectory = (s.st_mode & S_IFMT) == S_IFDIR
                } else {
                    return (false, isDirectory)
                }
            } else {
                isDirectory = (s.st_mode & S_IFMT) == S_IFDIR
            }

            // don't chase the link for this magic case -- we might be /Net/foo
            // which is a symlink to /private/Net/foo which is not yet mounted...
            if (s.st_mode & S_IFMT) == S_IFLNK {
{'fraction_non_alphanumeric': 0.10704463719060715, 'fraction_numerical': 0.0055003173259995765, 'mean_word_length': 1.3746860873932698, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5269858', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1318, 'n_words': 380}