<commit_before>import Foundation
import HTTP

/// Servers files from the supplied public directory
/// on not found errors.
public final class FileMiddleware: Middleware {
    
    private var publicDir: String
    private let loader = DataFile()

    public init(publicDir: String) {
        // Remove last "/" from the publicDir if present, so we can directly append uri path from the request.
        self.publicDir = publicDir.finished(with: "/")
    }
    
    public func respond(to request: Request, chainingTo next: Responder) throws -> Response {
        do {
            return try next.respond(to: request)
        } catch let error as AbortError where error.status == .notFound {
            // Check in file system
            var path = request.uri.path
            guard !path.contains("../") else { throw HTTP.Status.forbidden }
            if path.hasPrefix("/") {
                path = String(path.characters.dropFirst())
            }
            let filePath = publicDir + path

            guard
                let attributes = try? Foundation.FileManager.default.attributesOfItem(atPath: filePath),
                let modifiedAt = attributes[.modificationDate] as? Date,
                let fileSize = attributes[.size] as? NSNumber
            else {
                throw Abort.notFound
            }

            var headers: [HeaderKey: String] = [:]
            
            // Generate ETag value, "HEX value of last modified date" + "-" + "file size"
            let fileETag = "\(modifiedAt.timeIntervalSince1970)-\(fileSize.intValue)"
            headers["ETag"] = fileETag

            // Check if file has been cached already and return NotModified response if the etags match
            if fileETag == request.headers["If-None-Match"] {
                return Response(status: .notModified, headers: headers, body: .data([]))
            }

            // Set Content-Type header based on the media type
            // Only set Content-Type if file not modified and returned above.
            if
                let fileExtension = filePath.components(separatedBy: ".").last,
                let type = Request.mediaTypes[fileExtension]
            {
                headers["Content-Type"] = type
            }

            // File exists and was not cached, returning content of file.
            if let fileBody = try? loader.read(at:filePath) {
                return Response(status: .ok, headers: headers, body: .data(fileBody))
            } else {
                print("unable to load path")
                throw Abort.notFound
            }
        }
    }
}
<commit_msg>Use chunked transfer for serving static files<commit_after>import Foundation
import HTTP
import libc

/// Servers files from the supplied public directory
/// on not found errors.
public final class FileMiddleware: Middleware {

    private var publicDir: String
    private let loader = DataFile()

    public init(publicDir: String) {
        // Remove last "/" from the publicDir if present, so we can directly append uri path from the request.
        self.publicDir = publicDir.finished(with: "/")
    }

    public func respond(to request: Request, chainingTo next: Responder) throws -> Response {
        do {
            return try next.respond(to: request)
        } catch let error as AbortError where error.status == .notFound {
            // Check in file system
            var path = request.uri.path
            guard !path.contains("../") else { throw HTTP.Status.forbidden }
            if path.hasPrefix("/") {
                path = String(path.characters.dropFirst())
            }
            let filePath = publicDir + path

            guard
                let attributes = try? Foundation.FileManager.default.attributesOfItem(atPath: filePath),
                let modifiedAt = attributes[.modificationDate] as? Date,
                let fileSize = attributes[.size] as? NSNumber
            else {
                throw Abort.notFound
            }

            var headers: [HeaderKey: String] = [:]

            // Generate ETag value, "HEX value of last modified date" + "-" + "file size"
            let fileETag = "\(modifiedAt.timeIntervalSince1970)-\(fileSize.intValue)"
            headers["ETag"] = fileETag

            // Check if file has been cached already and return NotModified response if the etags match
            if fileETag == request.headers["If-None-Match"] {
                return Response(status: .notModified, headers: headers, body: .data([]))
            }

            // Set Content-Type header based on the media type
            // Only set Content-Type if file not modified and returned above.
            if
                let fileExtension = filePath.components(separatedBy: ".").last,
                let type = Request.mediaTypes[fileExtension]
            {
                headers["Content-Type"] = type
            }

            // Try to open the file for reading.
            // This is the last chance to report a Not Found error to the client.
            guard let file = fopen(filePath, "r") else {
                throw Abort.notFound
            }

            return Response(status: .ok, headers: headers, chunked: { stream in
                let chunkSize = 32768
 
                defer {
                    fclose(file)
                }

                guard let buffer = malloc(chunkSize) else {
                    try? stream.close()
                    return
                }

                defer {
                    free(buffer)
                }

                do {
                    var count: size_t = 0

                    repeat {
                        count = fread(buffer, 1, chunkSize, file)
                        if count > 0 {
                            let chunk = Array(UnsafeRawBufferPointer(start: buffer, count: count))
                            try stream.write(chunk)
                        }
                    } while count == chunkSize
                }
                catch {
                    // In chunked mode, the server has no way to indicate errors inside the body,
                    // so closing the stream after the first write error is effectively
                    // the only option to keep the server running.
                }

                try? stream.close()
            })
        }
    }
}

{'fraction_non_alphanumeric': 0.08380414312617702, 'fraction_numerical': 0.0025109855618330196, 'mean_word_length': 1.5360127337843215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3615780', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1478, 'n_words': 520}