<commit_before>//
//  JsonSerializer.swift
//  JsonSerializer
//
//  Created by <NAME> on 2014/09/11.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//  License: The MIT License
//

#if os(Linux)
    import Glibc
#else
    import Darwin
#endif

internal final class JsonDeserializer: Parser {
    internal  typealias ByteSequence = [UInt8]
    internal  typealias Char = UInt8
    
    // MARK: Public Readable
    
    internal private(set) var lineNumber = 1
    internal private(set) var columnNumber = 1
    
    // MARK: Source
    
    private let source: [UInt8]
    
    // MARK: State
    
    private var cur: Int
    private let end: Int
    
    // MARK: Accessors
    
    private var currentChar: Char {
        return source[cur]
    }
    
    private var nextChar: Char {
        return source[cur.successor()]
    }
    
    private var currentSymbol: Character {
        return Character(UnicodeScalar(currentChar))
    }
    
    // MARK: Initializer
    
    internal required convenience init<ByteSequence: CollectionType where ByteSequence.Generator.Element == UInt8>(_ sequence: ByteSequence) {
        self.init(Array(sequence))
    }
    
    internal required init(_ source: ByteSequence) {
        self.source = source
        self.cur = source.startIndex
        self.end = source.endIndex
    }
    
    // MARK: Serialize
    
    internal func deserialize() throws -> Json {
        let json = try deserializeNextValue()
        skipWhitespaces()
        
        guard cur == end else {
            throw ExtraTokenError("extra tokens found", self)
        }
        
        return json
    }
    
    private func deserializeNextValue() throws -> Json {
        skipWhitespaces()
        guard cur != end else {
            throw InsufficientTokenError("unexpected end of tokens", self)
        }
        
        switch currentChar {
        case Char(ascii: "n"):
            return try parseSymbol("null", Json.NullValue)
        case Char(ascii: "t"):
            return try parseSymbol("true", Json.BooleanValue(true))
        case Char(ascii: "f"):
            return try parseSymbol("false", Json.BooleanValue(false))
        case Char(ascii: "-"), Char(ascii: "0") ... Char(ascii: "9"):
            return try parseNumber()
        case Char(ascii: "\""):
            return try parseString()
        case Char(ascii: "{"):
            return try parseObject()
        case Char(ascii: "["):
            return try parseArray()
        case let c:
            throw UnexpectedTokenError("unexpected token: \(c)", self)
        }
    }
    
    private func parseSymbol(target: StaticString, @autoclosure _ iftrue:  () -> Json) throws -> Json {
        guard expect(target) else {
            throw UnexpectedTokenError("expected \"\(target)\" but \(currentSymbol)", self)
        }
        
        return iftrue()
    }
    
    private func parseString() throws -> Json {
        assert(currentChar == Char(ascii: "\""), "points a double quote")
        advance()
        
        var buffer = [CChar]()

        while cur != end && currentChar != Char(ascii: "\"") {
            switch currentChar {
            case Char(ascii: "\\"):
                advance()
                
                guard cur != end else {
                    throw InvalidStringError("unexpected end of a string literal", self)
                }
                
                guard let escapedChar = parseEscapedChar() else {
                    throw InvalidStringError("invalid escape sequence", self)
                }
                
                String(escapedChar).utf8.forEach {
                    buffer.append(CChar(bitPattern: $0))
                }
            default:
                buffer.append(CChar(bitPattern: currentChar))
            }
            
            advance()
        }
        
        guard expect("\"") else {
            throw InvalidStringError("missing double quote", self)
        }
        
        buffer.append(0) // trailing nul
        
        guard let string = String.fromCString(buffer) else {
            throw InvalidStringError("Unable to parse CString", self)
        }
        
        return .StringValue(string)
    }
    
    private func parseEscapedChar() -> UnicodeScalar? {
        let character = UnicodeScalar(currentChar)
        
        // 'u' indicates unicode
        guard character == "u" else {
            return unescapeMapping[character] ?? character
        }
        
        guard let surrogateValue = parseEscapedUnicodeSurrogate() else { return nil }
<commit_msg>Explain double \u#### sequences<commit_after>//
//  JsonSerializer.swift
//  JsonSerializer
//
//  Created by <NAME> on 2014/09/11.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//  License: The MIT License
//

#if os(Linux)
    import Glibc
#else
    import Darwin
#endif

internal final class JsonDeserializer: Parser {
    internal  typealias ByteSequence = [UInt8]
    internal  typealias Char = UInt8
    
    // MARK: Public Readable
    
    internal private(set) var lineNumber = 1
    internal private(set) var columnNumber = 1
    
    // MARK: Source
    
    private let source: [UInt8]
    
    // MARK: State
    
    private var cur: Int
    private let end: Int
    
    // MARK: Accessors
    
    private var currentChar: Char {
        return source[cur]
    }
    
    private var nextChar: Char {
        return source[cur.successor()]
    }
    
    private var currentSymbol: Character {
        return Character(UnicodeScalar(currentChar))
    }
    
    // MARK: Initializer
    
    internal required convenience init<ByteSequence: CollectionType where ByteSequence.Generator.Element == UInt8>(_ sequence: ByteSequence) {
        self.init(Array(sequence))
    }
    
    internal required init(_ source: ByteSequence) {
        self.source = source
        self.cur = source.startIndex
        self.end = source.endIndex
    }
    
    // MARK: Serialize
    
    internal func deserialize() throws -> Json {
        let json = try deserializeNextValue()
        skipWhitespaces()
        
        guard cur == end else {
            throw ExtraTokenError("extra tokens found", self)
        }
        
        return json
    }
    
    private func deserializeNextValue() throws -> Json {
        skipWhitespaces()
        guard cur != end else {
            throw InsufficientTokenError("unexpected end of tokens", self)
        }
        
        switch currentChar {
        case Char(ascii: "n"):
            return try parseSymbol("null", Json.NullValue)
        case Char(ascii: "t"):
            return try parseSymbol("true", Json.BooleanValue(true))
        case Char(ascii: "f"):
            return try parseSymbol("false", Json.BooleanValue(false))
        case Char(ascii: "-"), Char(ascii: "0") ... Char(ascii: "9"):
            return try parseNumber()
        case Char(ascii: "\""):
            return try parseString()
        case Char(ascii: "{"):
            return try parseObject()
        case Char(ascii: "["):
            return try parseArray()
        case let c:
            throw UnexpectedTokenError("unexpected token: \(c)", self)
        }
    }
    
    private func parseSymbol(target: StaticString, @autoclosure _ iftrue:  () -> Json) throws -> Json {
        guard expect(target) else {
            throw UnexpectedTokenError("expected \"\(target)\" but \(currentSymbol)", self)
        }
        
        return iftrue()
    }
    
    private func parseString() throws -> Json {
        assert(currentChar == Char(ascii: "\""), "points a double quote")
        advance()
        
        var buffer = [CChar]()

        while cur != end && currentChar != Char(ascii: "\"") {
            switch currentChar {
            case Char(ascii: "\\"):
                advance()
                
                guard cur != end else {
                    throw InvalidStringError("unexpected end of a string literal", self)
                }
                
                guard let escapedChar = parseEscapedChar() else {
                    throw InvalidStringError("invalid escape sequence", self)
                }
                
                String(escapedChar).utf8.forEach {
                    buffer.append(CChar(bitPattern: $0))
                }
            default:
                buffer.append(CChar(bitPattern: currentChar))
            }
            
            advance()
        }
        
        guard expect("\"") else {
            throw InvalidStringError("missing double quote", self)
        }
        
        buffer.append(0) // trailing nul
        
        guard let string = String.fromCString(buffer) else {
            throw InvalidStringError("Unable to parse CString", self)
        }
        
        return .StringValue(string)
    }
    
    private func parseEscapedChar() -> UnicodeScalar? {
        let character = UnicodeScalar(currentChar)
        
        // 'u' indicates unicode
        guard character == "u" else {
            return unescapeMapping[character] ?? character
        }
        
        guard let surrogateValue = parseEscapedUnicodeSurrogate() else { return nil }

        // two consecutive \u#### sequences represent 32 bit unicode characters
{'fraction_non_alphanumeric': 0.09894145603802117, 'fraction_numerical': 0.005184705119896306, 'mean_word_length': 1.734494979326639, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4190756', 'n_tokens_mistral': 2460, 'n_tokens_neox': 2365, 'n_words': 670}