<commit_before>//
//  IncompatibleCharacter.swift
//
//  CotEditor
//  https://coteditor.com
//
//  Created by 1024jp on 2016-05-28.
//
//  ---------------------------------------------------------------------------
//
//  © 2004-2007 nakamuxu
//  © 2014-2020 1024jp
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//  https://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

import Foundation

final class IncompatibleCharacter: NSObject {  // -> inherit NSObject for NSArrayController
    
    @objc let character: String
    @objc let convertedCharacter: String?
    @objc let location: Int
    @objc let lineNumber: Int
    
    
    required init(character: Character, convertedCharacter: String?, location: Int, lineNumber: Int) {
        
        self.character = String(character)
        self.convertedCharacter = convertedCharacter
        self.location = location
        self.lineNumber = lineNumber
    }
    
    
    var range: NSRange {
        
        return NSRange(location: self.location, length: 1)
    }
    
    
    override var debugDescription: String {
        
        return "<\(self): \(self.character) -\(self.location)>"
    }
    
}



// MARK: -

extension String {
    
    /// list-up characters cannot be converted to the passed-in encoding
    func scanIncompatibleCharacters(for encoding: String.Encoding) -> [IncompatibleCharacter] {
        
        guard !self.canBeConverted(to: encoding) else { return [] }
        
        guard
            let data = self.data(using: encoding, allowLossyConversion: true),  // lossy conversion must always success
            let convertedString = String(data: data, encoding: encoding)
            else { assertionFailure(); return [] }
        
        return convertedString.difference(from: self).removals
            .map { (change) in
                guard case let .remove(offset, character, _) = change else { preconditionFailure() }
                
                let converted: String? = String(character)
                    .data(using: encoding, allowLossyConversion: true)
                    .flatMap { String(data: $0, encoding: encoding) }
                let location = self.index(self.startIndex, offsetBy: offset).utf16Offset(in: self)
                
                return IncompatibleCharacter(character: character,
                                             convertedCharacter: converted,
                                             location: location,
                                             lineNumber: self.lineNumber(at: location))
            }
    }
    
}<commit_msg>Switch algorithm for large incompatibles scan (#1218)<commit_after>//
//  IncompatibleCharacter.swift
//
//  CotEditor
//  https://coteditor.com
//
//  Created by 1024jp on 2016-05-28.
//
//  ---------------------------------------------------------------------------
//
//  © 2004-2007 nakamuxu
//  © 2014-2020 1024jp
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//  https://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

import Foundation

final class IncompatibleCharacter: NSObject {  // -> inherit NSObject for NSArrayController
    
    @objc let character: String
    @objc let convertedCharacter: String?
    @objc let location: Int
    @objc let lineNumber: Int
    
    
    required init(character: Character, convertedCharacter: String?, location: Int, lineNumber: Int) {
        
        self.character = String(character)
        self.convertedCharacter = convertedCharacter
        self.location = location
        self.lineNumber = lineNumber
    }
    
    
    var range: NSRange {
        
        NSRange(location: self.location, length: self.character.utf16.count)
    }
    
    
    override var debugDescription: String {
        
        "<\(self): \(self.character) -\(self.location)>"
    }
    
}



// MARK: -

extension String {
    
    /// list-up characters cannot be converted to the passed-in encoding
    func scanIncompatibleCharacters(for encoding: String.Encoding) -> [IncompatibleCharacter] {
        
        guard !self.canBeConverted(to: encoding) else { return [] }
        
        guard
            let data = self.data(using: encoding, allowLossyConversion: true),  // lossy conversion must always success
            let convertedString = String(data: data, encoding: encoding)
            else { assertionFailure(); return [] }
        
        if self.length == convertedString.length, self.length > 10_000 {
            return self.quickIncompatibleFind(with: convertedString)
        }
        
        return convertedString.difference(from: self).removals
            .map { (change) in
                guard case let .remove(offset, character, _) = change else { preconditionFailure() }
                
                let converted: String? = String(character)
                    .data(using: encoding, allowLossyConversion: true)
                    .flatMap { String(data: $0, encoding: encoding) }
                let location = self.index(self.startIndex, offsetBy: offset).utf16Offset(in: self)
                
                return IncompatibleCharacter(character: character,
                                             convertedCharacter: converted,
                                             location: location,
                                             lineNumber: self.lineNumber(at: location))
            }
    }
    
    
    
    // MARK: Private Methods
    
    private func quickIncompatibleFind(with convertedString: String) -> [IncompatibleCharacter] {
        
        zip(self, convertedString).enumerated()
            .filter { $1.0 != $1.1 }
            .map { (offset, characters) in
                let location = self.index(self.startIndex, offsetBy: offset).utf16Offset(in: self)
                
                return IncompatibleCharacter(character: characters.0,
                                             convertedCharacter: String(characters.1),
                                             location: location,
                                             lineNumber: self.lineNumber(at: location))
            }
    }
    
}
{'fraction_non_alphanumeric': 0.11022727272727273, 'fraction_numerical': 0.013920454545454545, 'mean_word_length': 1.7808056872037914, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6192637', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1689, 'n_words': 524}