<commit_before>import Foundation

// inspired by: http://jordansmith.io/performant-date-parsing/

class SQLiteDateParser {
    
    private struct ParserComponents {
        var year: Int32 = 0
        var month: Int32 = 0
        var day: Int32 = 0
        var hour: Int32 = 0
        var minute: Int32 = 0
        var second: Int32 = 0
        var nanosecond = ContiguousArray<CChar>.init(repeating: 0, count: 10) // 9 digits, and trailing \0
    }
    
    func components(from dateString: String) -> DatabaseDateComponents? {
        return dateString.withCString { cString in
            components(cString: cString, length: strlen(cString))
        }
    }
    
    func components(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        assert(strlen(cString) == length)
        
        guard length >= 5 else { return nil }
        
        if cString.advanced(by: 4).pointee == 45 /* '-' */ {
            return datetimeComponents(cString: cString, length: length)
        }
        
        if cString.advanced(by: 2).pointee == 58 /* ':' */ {
            return timeComponents(cString: cString, length: length)
        }
        
        return nil
    }
    
    // - YYYY-MM-DD
    // - YYYY-MM-DD HH:MM
    // - YYYY-MM-DD HH:MM:SS
    // - YYYY-MM-DD HH:MM:SS.SSS
    // - YYYY-MM-DDTHH:MM
    // - YYYY-MM-DDTHH:MM:SS
    // - YYYY-MM-DDTHH:MM:SS.SSS
    private func datetimeComponents(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        var parserComponents = ParserComponents()
        
        // TODO: Get rid of this pyramid when SE-0210 has shipped
        let parseCount = withUnsafeMutablePointer(to: &parserComponents.year) { yearP in
            withUnsafeMutablePointer(to: &parserComponents.month) { monthP in
                withUnsafeMutablePointer(to: &parserComponents.day) { dayP in
                    withUnsafeMutablePointer(to: &parserComponents.hour) { hourP in
                        withUnsafeMutablePointer(to: &parserComponents.minute) { minuteP in
                            withUnsafeMutablePointer(to: &parserComponents.second) { secondP in
                                parserComponents.nanosecond.withUnsafeMutableBufferPointer { nanosecondBuffer in
                                    withVaList([yearP, monthP, dayP, hourP, minuteP, secondP, nanosecondBuffer.baseAddress!]) { pointer in
                                        vsscanf(cString, "%d-%d-%d%*c%d:%d:%d.%10s", pointer)
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        
        guard parseCount >= 3 else { return nil }
        
        var components = DateComponents()
        components.year = Int(parserComponents.year)
        components.month = Int(parserComponents.month)
        components.day = Int(parserComponents.day)
        
        guard parseCount >= 5 else { return DatabaseDateComponents(components, format: .YMD) }
        
        components.hour = Int(parserComponents.hour)
        components.minute = Int(parserComponents.minute)
        
        guard parseCount >= 6 else { return DatabaseDateComponents(components, format: .YMD_HM) }
        
        components.second = Int(parserComponents.second)
        
        guard parseCount >= 7 else { return DatabaseDateComponents(components, format: .YMD_HMS) }
        
        components.nanosecond = nanosecondsInt(for: parserComponents.nanosecond)
        
        return DatabaseDateComponents(components, format: .YMD_HMSS)
    }
    
    // - HH:MM
    // - HH:MM:SS
    // - HH:MM:SS.SSS
    private func timeComponents(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        var parserComponents = ParserComponents()
        
        // TODO: Get rid of this pyramid when SE-0210 has shipped
        let parseCount = withUnsafeMutablePointer(to: &parserComponents.hour) { hourP in
            withUnsafeMutablePointer(to: &parserComponents.minute) { minuteP in
                withUnsafeMutablePointer(to: &parserComponents.second) { secondP in
                    parserComponents.nanosecond.withUnsafeMutableBufferPointer { nanosecondBuffer in
                        withVaList([hourP, minuteP, secondP, nanosecondBuffer.baseAddress!]) { pointer in
                            vsscanf(cString, "%d:%d:%d.%10s", pointer)
                        }
                    }
                }
            }
        }
        
        guard parseCount >= 2 else { return nil }
        
        var components = DateComponents()
        components.hour = Int(parserComponents.hour)
        components.minute = Int(parserComponents.minute)
        
        guard parseCount >= 3 else { return DatabaseDateComponents(components, format: .HM) }
        
        components.second = Int(parserComponents.second)
        
        guard parseCount >= 4 else { return DatabaseDateComponents(components, format: .HMS) }
        
        guard let nanoseconds = nanosecondsInt(for: parserComponents.nanosecond) else { return nil }
        components.nanosecond = nanoseconds
        
        return DatabaseDateComponents(components, format: .HMSS)
    }
    
    private func nanosecondsInt(for nanosecond: ContiguousArray<CChar>) -> Int? {
        // truncate after the third digit
        var result = 0
        for char in nanosecond.prefix(3) {
            let digit = Int(char) - 48 /* '0' */
            guard (0...9).contains(digit) else { return nil }
            result = result * 10 + digit
        }
        return 1_000_000 * result
    }
}<commit_msg>SQLiteDateParser: fix nanosecond parsing<commit_after>import Foundation

// inspired by: http://jordansmith.io/performant-date-parsing/

class SQLiteDateParser {
    
    private struct ParserComponents {
        var year: Int32 = 0
        var month: Int32 = 0
        var day: Int32 = 0
        var hour: Int32 = 0
        var minute: Int32 = 0
        var second: Int32 = 0
        var nanosecond = ContiguousArray<CChar>.init(repeating: 0, count: 10) // 9 digits, and trailing \0
    }
    
    func components(from dateString: String) -> DatabaseDateComponents? {
        return dateString.withCString { cString in
            components(cString: cString, length: strlen(cString))
        }
    }
    
    func components(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        assert(strlen(cString) == length)
        
        guard length >= 5 else { return nil }
        
        if cString.advanced(by: 4).pointee == 45 /* '-' */ {
            return datetimeComponents(cString: cString, length: length)
        }
        
        if cString.advanced(by: 2).pointee == 58 /* ':' */ {
            return timeComponents(cString: cString, length: length)
        }
        
        return nil
    }
    
    // - YYYY-MM-DD
    // - YYYY-MM-DD HH:MM
    // - YYYY-MM-DD HH:MM:SS
    // - YYYY-MM-DD HH:MM:SS.SSS
    // - YYYY-MM-DDTHH:MM
    // - YYYY-MM-DDTHH:MM:SS
    // - YYYY-MM-DDTHH:MM:SS.SSS
    private func datetimeComponents(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        var parserComponents = ParserComponents()
        
        // TODO: Get rid of this pyramid when SE-0210 has shipped
        let parseCount = withUnsafeMutablePointer(to: &parserComponents.year) { yearP in
            withUnsafeMutablePointer(to: &parserComponents.month) { monthP in
                withUnsafeMutablePointer(to: &parserComponents.day) { dayP in
                    withUnsafeMutablePointer(to: &parserComponents.hour) { hourP in
                        withUnsafeMutablePointer(to: &parserComponents.minute) { minuteP in
                            withUnsafeMutablePointer(to: &parserComponents.second) { secondP in
                                parserComponents.nanosecond.withUnsafeMutableBufferPointer { nanosecondBuffer in
                                    withVaList([yearP, monthP, dayP, hourP, minuteP, secondP, nanosecondBuffer.baseAddress!]) { pointer in
                                        vsscanf(cString, "%d-%d-%d%*c%d:%d:%d.%10s", pointer)
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        
        guard parseCount >= 3 else { return nil }
        
        var components = DateComponents()
        components.year = Int(parserComponents.year)
        components.month = Int(parserComponents.month)
        components.day = Int(parserComponents.day)
        
        guard parseCount >= 5 else { return DatabaseDateComponents(components, format: .YMD) }
        
        components.hour = Int(parserComponents.hour)
        components.minute = Int(parserComponents.minute)
        
        guard parseCount >= 6 else { return DatabaseDateComponents(components, format: .YMD_HM) }
        
        components.second = Int(parserComponents.second)
        
        guard parseCount >= 7 else { return DatabaseDateComponents(components, format: .YMD_HMS) }
        
        components.nanosecond = nanosecondsInt(for: parserComponents.nanosecond)
        
        return DatabaseDateComponents(components, format: .YMD_HMSS)
    }
    
    // - HH:MM
    // - HH:MM:SS
    // - HH:MM:SS.SSS
    private func timeComponents(cString: UnsafePointer<CChar>, length: Int) -> DatabaseDateComponents? {
        var parserComponents = ParserComponents()
        
        // TODO: Get rid of this pyramid when SE-0210 has shipped
        let parseCount = withUnsafeMutablePointer(to: &parserComponents.hour) { hourP in
            withUnsafeMutablePointer(to: &parserComponents.minute) { minuteP in
                withUnsafeMutablePointer(to: &parserComponents.second) { secondP in
                    parserComponents.nanosecond.withUnsafeMutableBufferPointer { nanosecondBuffer in
                        withVaList([hourP, minuteP, secondP, nanosecondBuffer.baseAddress!]) { pointer in
                            vsscanf(cString, "%d:%d:%d.%10s", pointer)
                        }
                    }
                }
            }
        }
        
        guard parseCount >= 2 else { return nil }
        
        var components = DateComponents()
        components.hour = Int(parserComponents.hour)
        components.minute = Int(parserComponents.minute)
        
        guard parseCount >= 3 else { return DatabaseDateComponents(components, format: .HM) }
        
        components.second = Int(parserComponents.second)
        
        guard parseCount >= 4 else { return DatabaseDateComponents(components, format: .HMS) }
        
        guard let nanoseconds = nanosecondsInt(for: parserComponents.nanosecond) else { return nil }
        components.nanosecond = nanoseconds
        
        return DatabaseDateComponents(components, format: .HMSS)
    }
    
    private func nanosecondsInt(for nanosecond: ContiguousArray<CChar>) -> Int? {
        // truncate after the third digit
        var result = 0
        let multipliers = [100_000_000, 10_000_000, 1_000_000, 100_000, 10_000, 1_000, 100, 10, 1]
        for (char, multiplier) in zip(nanosecond.prefix(3), multipliers) {
            if char == 0 { return result }
            let digit = Int(char) - 48 /* '0' */
            guard (0...9).contains(digit) else { return nil }
            result += multiplier * digit
        }
        return result
    }
}
{'fraction_non_alphanumeric': 0.09313003551936239, 'fraction_numerical': 0.014467642727193971, 'mean_word_length': 1.8773678963110667, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '249363', 'n_tokens_mistral': 3330, 'n_tokens_neox': 3073, 'n_words': 707}