<commit_before>// swiftlint:disable function_body_length type_body_length file_length

import BartyCrouchTranslator
import Foundation
import HandySwift
import MungoHealer

// swiftlint:disable cyclomatic_complexity

// NOTE:
// This file was not refactored as port of the work/big-refactoring branch for version 4.0 to prevent unexpected behavior changes.
// A rewrite after writing extensive tests for the expected behavior could improve readebility, extensibility and performance.

public class StringsFileUpdater {
    // MARK: - Sub Types
    typealias TranslationEntry = (key: String, value: String, comment: String?, line: Int)
    typealias LocaleInfo = (language: String, region: String?)
    typealias DuplicateEntry = (String, [TranslationEntry])

    // MARK: - Stored Type Properties
    public static let defaultIgnoreKeys: [String] = ["#bartycrouch-ignore!", "#bc-ignore!", "#i!"]

    // MARK: - Stored Instance Properties
    let path: String
    var oldContentString: String = ""

    // MARK: - Initializers
    public init?(path: String) {
        self.path = path
        do {
            self.oldContentString = try String(contentsOfFile: path)
        } catch {
            return nil
        }
    }

    // MARK: - Methods
    // Updates the keys of this instances strings file with those of the given strings file.
    public func incrementallyUpdateKeys(
        withStringsFileAtPath otherStringFilePath: String,
        addNewValuesAsEmpty: Bool,
        ignoreBaseKeysAndComment ignores: [String] = defaultIgnoreKeys,
        override: Bool = false,
        updateCommentWithBase: Bool = true,
        keepExistingKeys: Bool = false,
        overrideComments: Bool = false,
        keepWhitespaceSurroundings: Bool = false,
        ignoreEmptyStrings: Bool = false
    ) {
        do {
            let newContentString = try String(contentsOfFile: otherStringFilePath)

            let oldTranslations = findTranslations(inString: oldContentString)
            var newTranslations = findTranslations(inString: newContentString)

            if let lastOldTranslation = oldTranslations.last {
                newTranslations = newTranslations.map { ($0.key, $0.value, $0.comment, $0.line + lastOldTranslation.line + 1) }
            }

            let updatedTranslations: [TranslationEntry] = {
                var translations: [TranslationEntry] = []

                if keepExistingKeys {
                    translations += oldTranslations.filter { oldKey, _, _, _ in
                        return newTranslations.filter { newKey, _, _, _ in oldKey == newKey }.isEmpty
                    }
                }

                for newTranslation in newTranslations {
                    // skip keys marked for ignore
                    guard !newTranslation.value.containsAny(of: ignores) else { continue }
                    if ignoreEmptyStrings && newTranslation.value.isBlank { continue }

                    // Skip keys that have been marked for ignore in comment
                    if let newComment = newTranslation.comment, newComment.containsAny(of: ignores) { continue }

                    let oldTranslation = oldTranslations.first { oldKey, _, _, _ in oldKey == newTranslation.key }

                    // get value from default comment structure if possible
                    let oldBaseValue: String? = {
                        guard let oldComment = oldTranslation?.comment, let foundMatch = defaultCommentStructureMatches(inString: oldComment) else {
                            return nil
                        }

                        return (oldComment as NSString).substring(with: foundMatch.range(at: 1))
                    }()

                    let updatedComment: String? = {
                        // add new comment if none existed before
                        guard let oldComment = oldTranslation?.comment else { return newTranslation.comment }

                        // keep old comment if no new comment exists
                        guard let newComment = newTranslation.comment else { return oldComment }

                        // override with comment in force update mode
                        if override || overrideComments { return newComment }

                        // update if implicit requirements fullfilled
                        if updateCommentWithBase && defaultCommentStructureMatches(inString: oldComment) != nil { return newComment }

                        return oldComment
                    }()

                    let updatedValue: String = {
                        // get new translation value corrected by false % placeholders
                        var newTranslationValue = newTranslation.value
                        // swiftlint:disable:next force_try
                        let regex = try! Regex("%\\d\\$((?![@dcixoufegap]|[l]{1,2}[duixo]|[h][duixo]|L[dfega]|\\.\\d))", options: [])
                        newTranslationValue = regex.replacingMatches(in: newTranslationValue, with: "%$1")

                        guard let oldValue = oldTranslation?.value else {
                            // add new key with empty value
                            guard !addNewValuesAsEmpty else { return "" }

                            // add new key with Base value
                            return newTranslationValue
                        }

                        if override { return newTranslationValue } // override with new value in force update mode

                        if let oldBaseValue = oldBaseValue, oldBaseValue == oldValue { return newTranslationValue } // update base value

                        // keep existing translation
                        return oldValue
                    }()

                    // don't change order of existing translations if no specific order specified
                    let updatedLine: Int = oldTranslation?.line ?? newTranslation.line

                    translations.append((newTranslation.key, updatedValue, updatedComment, updatedLine))
                }
<commit_msg>Remove magic of gathering of new translation via comment<commit_after>// swiftlint:disable function_body_length type_body_length file_length

import BartyCrouchTranslator
import Foundation
import HandySwift
import MungoHealer

// swiftlint:disable cyclomatic_complexity

// NOTE:
// This file was not refactored as port of the work/big-refactoring branch for version 4.0 to prevent unexpected behavior changes.
// A rewrite after writing extensive tests for the expected behavior could improve readebility, extensibility and performance.

public class StringsFileUpdater {
    // MARK: - Sub Types
    typealias TranslationEntry = (key: String, value: String, comment: String?, line: Int)
    typealias LocaleInfo = (language: String, region: String?)
    typealias DuplicateEntry = (String, [TranslationEntry])

    // MARK: - Stored Type Properties
    public static let defaultIgnoreKeys: [String] = ["#bartycrouch-ignore!", "#bc-ignore!", "#i!"]

    // MARK: - Stored Instance Properties
    let path: String
    var oldContentString: String = ""

    // MARK: - Initializers
    public init?(path: String) {
        self.path = path
        do {
            self.oldContentString = try String(contentsOfFile: path)
        } catch {
            return nil
        }
    }

    // MARK: - Methods
    // Updates the keys of this instances strings file with those of the given strings file.
    public func incrementallyUpdateKeys(
        withStringsFileAtPath otherStringFilePath: String,
        addNewValuesAsEmpty: Bool,
        ignoreBaseKeysAndComment ignores: [String] = defaultIgnoreKeys,
        override: Bool = false,
        updateCommentWithBase: Bool = true,
        keepExistingKeys: Bool = false,
        overrideComments: Bool = false,
        keepWhitespaceSurroundings: Bool = false,
        ignoreEmptyStrings: Bool = false
    ) {
        do {
            let newContentString = try String(contentsOfFile: otherStringFilePath)

            let oldTranslations = findTranslations(inString: oldContentString)
            var newTranslations = findTranslations(inString: newContentString)

            if let lastOldTranslation = oldTranslations.last {
                newTranslations = newTranslations.map { ($0.key, $0.value, $0.comment, $0.line + lastOldTranslation.line + 1) }
            }

            let updatedTranslations: [TranslationEntry] = {
                var translations: [TranslationEntry] = []

                if keepExistingKeys {
                    translations += oldTranslations.filter { oldKey, _, _, _ in
                        return newTranslations.filter { newKey, _, _, _ in oldKey == newKey }.isEmpty
                    }
                }

                for newTranslation in newTranslations {
                    // skip keys marked for ignore
                    guard !newTranslation.value.containsAny(of: ignores) else { continue }
                    if ignoreEmptyStrings && newTranslation.value.isBlank { continue }

                    // Skip keys that have been marked for ignore in comment
                    if let newComment = newTranslation.comment, newComment.containsAny(of: ignores) { continue }

                    let oldTranslation = oldTranslations.first { oldKey, _, _, _ in oldKey == newTranslation.key }

                    let updatedComment: String? = {
                        // add new comment if none existed before
                        guard let oldComment = oldTranslation?.comment else { return newTranslation.comment }

                        // keep old comment if no new comment exists
                        guard let newComment = newTranslation.comment else { return oldComment }

                        // override with comment in force update mode
                        if override || overrideComments { return newComment }

                        // update if implicit requirements fullfilled
                        if updateCommentWithBase && defaultCommentStructureMatches(inString: oldComment) != nil { return newComment }

                        return oldComment
                    }()

                    let updatedValue: String = {
                        // get new translation value corrected by false % placeholders
                        var newTranslationValue = newTranslation.value
                        // swiftlint:disable:next force_try
                        let regex = try! Regex("%\\d\\$((?![@dcixoufegap]|[l]{1,2}[duixo]|[h][duixo]|L[dfega]|\\.\\d))", options: [])
                        newTranslationValue = regex.replacingMatches(in: newTranslationValue, with: "%$1")

                        guard let oldValue = oldTranslation?.value else {
                            // add new key with empty value
                            guard !addNewValuesAsEmpty else { return "" }

                            // add new key with Base value
                            return newTranslationValue
                        }

                        if override { return newTranslationValue } // override with new value in force update mode

                        // keep existing translation
                        return oldValue
                    }()

                    // don't change order of existing translations if no specific order specified
                    let updatedLine: Int = oldTranslation?.line ?? newTranslation.line

                    translations.append((newTranslation.key, updatedValue, updatedComment, updatedLine))
                }

{'fraction_non_alphanumeric': 0.07210708117443869, 'fraction_numerical': 0.0018134715025906736, 'mean_word_length': 1.8405690458670592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2650877', 'n_tokens_mistral': 2749, 'n_tokens_neox': 2556, 'n_words': 831}