<commit_before>// XGAF file format parser for Swift.
// (c) 2016 <NAME>. MIT license. See LICENSE for more information.

import Foundation

let wordCharacters = CharacterSet.whitespacesAndNewlines.union(CharacterSet(charactersIn: "/;:()[]=")).inverted

extension Scanner {
    
    func skipping(_ characters: CharacterSet?, closure: () throws->()) throws {
        let previous = charactersToBeSkipped
        defer { charactersToBeSkipped = previous }
        charactersToBeSkipped = characters
        try closure()
    }
    
    func skipComments() throws {
        while true {
            if skipString(";") {
                let previousCharactersToBeSkipped = charactersToBeSkipped
                charactersToBeSkipped = nil
                defer { charactersToBeSkipped = previousCharactersToBeSkipped }
                
                // If at "\n" already, do nothing
                guard !skipString("\n") else { continue }

                guard skipUpTo("\n") else { return }
                skipString("\n")
            } else if skipString("/*") {
                guard skipUpTo("*/") else {
                    throw ParseError(kind: .unterminatedComment, scanner: self)
                }
                skipString("*/")
            } else {
                return
            }
        }
    }
    
    func scanWord() -> String? {
        //return scanUpToCharacters(from: CharacterSet.whitespacesAndNewlines)
        return scanCharacters(from: wordCharacters)
    }
    
    func peekUtf16CodeUnit() -> UTF16.CodeUnit? {
        let originalScanLocation = scanLocation
        defer { scanLocation = originalScanLocation }
        
        let originalCharactersToBeSkipped = charactersToBeSkipped
        defer { charactersToBeSkipped = originalCharactersToBeSkipped }
        
        if let characters = charactersToBeSkipped {
            charactersToBeSkipped = nil
            let _ = scanCharacters(from: characters)
        }<commit_msg>Fix newlines handling in comments<commit_after>// XGAF file format parser for Swift.
// (c) 2016 <NAME>. MIT license. See LICENSE for more information.

import Foundation

let wordCharacters = CharacterSet.whitespacesAndNewlines.union(CharacterSet(charactersIn: "/;:()[]=")).inverted

extension Scanner {
    
    func skipping(_ characters: CharacterSet?, closure: () throws->()) throws {
        let previous = charactersToBeSkipped
        defer { charactersToBeSkipped = previous }
        charactersToBeSkipped = characters
        try closure()
    }
    
    func skipComments() throws {
        while true {
            if skipString(";") {
                let previousCharactersToBeSkipped = charactersToBeSkipped
                charactersToBeSkipped = nil
                defer { charactersToBeSkipped = previousCharactersToBeSkipped }
                
                // If at "\n" already, do nothing
                guard !skipString("\n") else { continue }
                guard !skipString("\r\n") else { continue }

                guard skipUpToCharacters(from: CharacterSet.newlines) else {
                    return
                }
                if !skipString("\n") {
                    skipString("\r\n")
                }
            } else if skipString("/*") {
                guard skipUpTo("*/") else {
                    throw ParseError(kind: .unterminatedComment, scanner: self)
                }
                skipString("*/")
            } else {
                return
            }
        }
    }
    
    func scanWord() -> String? {
        //return scanUpToCharacters(from: CharacterSet.whitespacesAndNewlines)
        return scanCharacters(from: wordCharacters)
    }
    
    func peekUtf16CodeUnit() -> UTF16.CodeUnit? {
        let originalScanLocation = scanLocation
        defer { scanLocation = originalScanLocation }
        
        let originalCharactersToBeSkipped = charactersToBeSkipped
        defer { charactersToBeSkipped = originalCharactersToBeSkipped }
        
        if let characters = charactersToBeSkipped {
            charactersToBeSkipped = nil
            let _ = scanCharacters(from: characters)
        }
{'fraction_non_alphanumeric': 0.08545979778526722, 'fraction_numerical': 0.0038517091959557053, 'mean_word_length': 1.8854166666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4862790', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1052, 'n_words': 249}