<commit_before>//
//  NTriples.swift
//  PageDatabase
//
//  Created by <NAME> on 6/4/16.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import Foundation
import SPARQLSyntax

extension String {
    static let unicodeScalarsNeedingNTriplesIRIEscaping : Set<UnicodeScalar> = {
        var charactersNeedingEscaping = Set<UnicodeScalar>()
        for i in 0x00...0x20 {
            charactersNeedingEscaping.insert(UnicodeScalar(i)!)
        }
        charactersNeedingEscaping.insert(UnicodeScalar(0x3c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x3e))
        charactersNeedingEscaping.insert(UnicodeScalar(0x22))
        charactersNeedingEscaping.insert(UnicodeScalar(0x5c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x5e))
        charactersNeedingEscaping.insert(UnicodeScalar(0x60))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7b))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7d))
        return charactersNeedingEscaping
    }()
    
    static let unicodeScalarsNeedingNTriplesLiteralEscaping = Set<UnicodeScalar>([
        UnicodeScalar(0x22),
        UnicodeScalar(0x5c),
        UnicodeScalar(0x0a),
        UnicodeScalar(0x0d)
        ])
    
    var ntriplesStringEscaped: String {
        let needsEscaping = self.unicodeScalars.contains { String.unicodeScalarsNeedingNTriplesLiteralEscaping.contains($0) || $0.value >= 0x80 }
        if !needsEscaping {
            return self
        } else {
            var escaped = ""
            for c in self {
                switch c {
                case _ where c.unicodeScalars.first != nil && c.unicodeScalars.first!.value >= 0x80:
                    for s in c.unicodeScalars {
                        let value = s.value
                        if value <= 0xFFFF {
                            escaped += String(format: "\\u%04X", value)
                        } else {
                            escaped += String(format: "\\U%08X", value)
                        }
                    }
                case Character(UnicodeScalar(0x22)):
                    escaped += "\\\""
                case Character(UnicodeScalar(0x5c)):
                    escaped += "\\\\"
                case Character(UnicodeScalar(0x0a)):
                    escaped += "\\n"
                case Character(UnicodeScalar(0x0d)):
                    escaped += "\\r"
                default:
                    escaped.append(c)
                }
            }
            return escaped
        }
    }
    
    var ntriplesIRIEscaped: String {
        let needsEscaping = self.unicodeScalars.contains {
            $0.value <= 0x20 || String.unicodeScalarsNeedingNTriplesIRIEscaping.contains($0) || $0.value >= 0x80
        }
        if !needsEscaping {
            return self
        } else {
            var escaped = ""
            for c in self {
                switch c {
                case _ where String.unicodeScalarsNeedingNTriplesIRIEscaping.contains(c.unicodeScalars.first!),
                     _ where c.unicodeScalars.first != nil && c.unicodeScalars.first!.value >= 0x80:
                    for s in c.unicodeScalars {
                        let value = s.value
                        if value <= 0xFFFF {
                            escaped += String(format: "\\u%04X", value)
                        } else {
                            escaped += String(format: "\\U%08X", value)
                        }
                    }
                default:
                    escaped.append(c)
                }
            }
            return escaped
        }
    }
}

extension Term {
    func ntriplesData() -> Data? {
        switch self.type {
        case .iri:
            return "<\(self.value.ntriplesIRIEscaped)>".data(using: .utf8)
        case .blank:
            return "_:\(self.value)".data(using: .utf8)
        case .language(let l):
            return "\"\(value.ntriplesStringEscaped)\"@\(l)".data(using: .utf8)
        case .datatype(.string):
            return "\"\(value.ntriplesStringEscaped)\"".data(using: .utf8)
        case .datatype(let dt):
            return "\"\(value.ntriplesStringEscaped)\"^^<\(dt.value)>".data(using: .utf8)
        }<commit_msg>Fixed N-Triples escaping of combining characters on ascii characters.<commit_after>//
//  NTriples.swift
//  PageDatabase
//
//  Created by <NAME> on 6/4/16.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import Foundation
import SPARQLSyntax

extension String {
    static let unicodeScalarsNeedingNTriplesIRIEscaping : Set<UnicodeScalar> = {
        var charactersNeedingEscaping = Set<UnicodeScalar>()
        for i in 0x00...0x20 {
            charactersNeedingEscaping.insert(UnicodeScalar(i)!)
        }
        charactersNeedingEscaping.insert(UnicodeScalar(0x3c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x3e))
        charactersNeedingEscaping.insert(UnicodeScalar(0x22))
        charactersNeedingEscaping.insert(UnicodeScalar(0x5c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x5e))
        charactersNeedingEscaping.insert(UnicodeScalar(0x60))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7b))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7c))
        charactersNeedingEscaping.insert(UnicodeScalar(0x7d))
        return charactersNeedingEscaping
    }()
    
    static let unicodeScalarsNeedingNTriplesLiteralEscaping = Set<UnicodeScalar>([
        UnicodeScalar(0x22),
        UnicodeScalar(0x5c),
        UnicodeScalar(0x0a),
        UnicodeScalar(0x0d)
        ])
    
    var ntriplesStringEscaped: String {
        let needsEscaping = self.unicodeScalars.contains { String.unicodeScalarsNeedingNTriplesLiteralEscaping.contains($0) || $0.value >= 0x80 }
        if !needsEscaping {
            return self
        } else {
            var escaped = ""
            for c in self {
                switch c {
                case _ where c.unicodeScalars.first != nil && c.unicodeScalars.first!.value >= 0x80,
                     _ where c.unicodeScalars.count > 1:
                    for s in c.unicodeScalars {
                        let value = s.value
                        if value <= 0xFFFF {
                            escaped += String(format: "\\u%04X", value)
                        } else {
                            escaped += String(format: "\\U%08X", value)
                        }
                    }
                case Character(UnicodeScalar(0x22)):
                    escaped += "\\\""
                case Character(UnicodeScalar(0x5c)):
                    escaped += "\\\\"
                case Character(UnicodeScalar(0x0a)):
                    escaped += "\\n"
                case Character(UnicodeScalar(0x0d)):
                    escaped += "\\r"
                default:
                    escaped.append(c)
                }
            }
            return escaped
        }
    }
    
    var ntriplesIRIEscaped: String {
        let needsEscaping = self.unicodeScalars.contains {
            $0.value <= 0x20 || String.unicodeScalarsNeedingNTriplesIRIEscaping.contains($0) || $0.value >= 0x80
        }
        if !needsEscaping {
            return self
        } else {
            var escaped = ""
            for c in self {
                switch c {
                case _ where String.unicodeScalarsNeedingNTriplesIRIEscaping.contains(c.unicodeScalars.first!),
                     _ where c.unicodeScalars.first != nil && c.unicodeScalars.first!.value >= 0x80,
                     _ where c.unicodeScalars.count > 1:
                    for s in c.unicodeScalars {
                        let value = s.value
                        if value <= 0xFFFF {
                            escaped += String(format: "\\u%04X", value)
                        } else {
                            escaped += String(format: "\\U%08X", value)
                        }
                    }
                default:
                    escaped.append(c)
                }
            }
            return escaped
        }
    }
}

extension Term {
    func ntriplesData() -> Data? {
        switch self.type {
        case .iri:
            return "<\(self.value.ntriplesIRIEscaped)>".data(using: .utf8)
        case .blank:
            return "_:\(self.value)".data(using: .utf8)
        case .language(let l):
            return "\"\(value.ntriplesStringEscaped)\"@\(l)".data(using: .utf8)
        case .datatype(.string):
            return "\"\(value.ntriplesStringEscaped)\"".data(using: .utf8)
        case .datatype(let dt):
            return "\"\(value.ntriplesStringEscaped)\"^^<\(dt.value)>".data(using: .utf8)
        }
{'fraction_non_alphanumeric': 0.10441950996107167, 'fraction_numerical': 0.020151133501259445, 'mean_word_length': 1.5698734922035893, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3375160', 'n_tokens_mistral': 2577, 'n_tokens_neox': 2411, 'n_words': 426}