<commit_before>//
//  NetworkUtilities.swift
//  piwigo
//
//  Created by <NAME> on 29/12/2020.
//  Copyright © 2020 Piwigo.org. All rights reserved.
//

@objc
class NetworkUtilities: NSObject {
    
    @objc
    class func utf8mb4String(from string: String?) -> String? {
        // Return empty string is nothing provided
        guard let strToConvert = string else {
            return ""
        }
        // Convert string to UTF-8 encoding
        let serverEncoding = String.Encoding.init(rawValue: Model.sharedInstance()?.stringEncoding ?? String.Encoding.utf8.rawValue )
        if let strData = strToConvert.data(using: serverEncoding, allowLossyConversion: true) {
            return String(data: strData, encoding: .utf8)
        }
        return ""
    }

    // Piwigo supports the 3-byte UTF-8, not the standard UTF-8 (4 bytes)
    // See https://github.com/Piwigo/Piwigo-Mobile/issues/429, https://github.com/Piwigo/Piwigo/issues/750
    @objc
    class func utf8mb3String(from string: String?) -> String? {
        // Return empty string is nothing provided
        guard let strToFilter = string else {
            return ""
        }

        // Replace characters encoded on 4 bytes
        var utf8mb3String = ""
        for char in strToFilter {
            if char.utf8.count > 2 {
                // 4-byte char => Not handled by Piwigo Server
                utf8mb3String.append("\u{FFFD}")  // Use the Unicode replacement character
            } else {
                // Up to 3-byte char
                utf8mb3String.append(char)
            }
        }
        return utf8mb3String
    }
}<commit_msg>Fixes conversion to 3-byte UTF-8<commit_after>//
//  NetworkUtilities.swift
//  piwigo
//
//  Created by <NAME> on 29/12/2020.
//  Copyright © 2020 Piwigo.org. All rights reserved.
//

@objc
class NetworkUtilities: NSObject {
    
    @objc
    class func utf8mb4String(from string: String?) -> String? {
        // Return empty string is nothing provided
        guard let strToConvert = string else {
            return ""
        }
        // Convert string to UTF-8 encoding
        let serverEncoding = String.Encoding.init(rawValue: Model.sharedInstance()?.stringEncoding ?? String.Encoding.utf8.rawValue )
        if let strData = strToConvert.data(using: serverEncoding, allowLossyConversion: true) {
            return String(data: strData, encoding: .utf8)
        }
        return ""
    }

    // Piwigo supports the 3-byte UTF-8, not the standard UTF-8 (4 bytes)
    // See https://github.com/Piwigo/Piwigo-Mobile/issues/429, https://github.com/Piwigo/Piwigo/issues/750
    @objc
    class func utf8mb3String(from string: String?) -> String? {
        // Return empty string is nothing provided
        guard let strToFilter = string else {
            return ""
        }

        // Replace characters encoded on 4 bytes
        var utf8mb3String = ""
        for char in strToFilter {
            if char.utf8.count > 3 {
                // 4-byte char => Not handled by Piwigo Server
                utf8mb3String.append("\u{FFFD}")  // Use the Unicode replacement character
            } else {
                // Up to 3-byte char
                utf8mb3String.append(char)
            }
        }
        return utf8mb3String
    }
}
{'fraction_non_alphanumeric': 0.10039549741405537, 'fraction_numerical': 0.026163675083662914, 'mean_word_length': 1.9071618037135278, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4538439', 'n_tokens_mistral': 994, 'n_tokens_neox': 932, 'n_words': 264}