<commit_before>import Foundation
import Core
import PathIndexable

public class Localization {
    private let localizations: [String: Node]
    private let defaultDialect: String
    
    public convenience init(localizationDirectory: String) throws {
        // Finish path with "/"
        let localizationDirectory = localizationDirectory.finished(with: "/")
        
        // List the files in the directory
        let contents = try FileManager.contentsOfDirectory(localizationDirectory)
        
        // Read the files into nodes mapped to their appropriate language
        var localizations = [String: Node]()
        for path in contents where path.hasSuffix(".json") {
            // Get the name
            guard let nameRaw = path.components(separatedBy: "/").last?.characters.split(separator: ".").first else {
                continue
            }
            let name = String(nameRaw)
            
            // Read the
            let data = try DataFile().load(path: path)
            localizations[name] = try JSON(bytes: data).makeNode()
        }
        
        self.init(localizations: localizations)
    }

    public init(localizations: [String: Node]? = nil, defaultLocale: String? = nil) {
        self.localizations = localizations ?? [:]
        self.defaultDialect = defaultLocale ?? localizations?.keys.first ?? "default"
    }

    public subscript(_ languageCode: String, _ paths: PathIndex...) -> String {
        return self[languageCode, paths]
    }

    public subscript(_ languageCode: String, _ paths: [PathIndex]) -> String {
        return localizations[languageCode]?[paths]?.string  // Index by language
            ?? localizations[defaultDialect]?[paths]?.string // Index the default language
            ?? paths.map { "\($0)" }.joined(separator: ".") // Return the literal path indexed if no translation<commit_msg>lowercase localization<commit_after>import Foundation
import Core
import PathIndexable

public class Localization {
    private let localizations: [String: Node]
    private let defaultDialect: String
    
    public convenience init(localizationDirectory: String) throws {
        // Finish path with "/"
        let localizationDirectory = localizationDirectory.finished(with: "/")
        
        // List the files in the directory
        let contents = try FileManager.contentsOfDirectory(localizationDirectory)
        
        // Read the files into nodes mapped to their appropriate language
        var localizations = [String: Node]()
        for path in contents where path.hasSuffix(".json") {
            // Get the name
            guard let nameRaw = path.components(separatedBy: "/").last?.characters.split(separator: ".").first else {
                continue
            }
            let name = String(nameRaw)
            
            // Read the
            let data = try DataFile().load(path: path)
            localizations[name.lowercased()] = try JSON(bytes: data).makeNode()
        }
        
        self.init(localizations: localizations)
    }

    public init(localizations: [String: Node]? = nil, defaultLocale: String? = nil) {
        self.localizations = localizations ?? [:]
        self.defaultDialect = defaultLocale ?? localizations?.keys.first ?? "default"
    }

    public subscript(_ languageCode: String, _ paths: PathIndex...) -> String {
        return self[languageCode, paths]
    }

    public subscript(_ languageCode: String, _ paths: [PathIndex]) -> String {
        return localizations[languageCode.lowercased()]?[paths]?.string  // Index by language
            ?? localizations[defaultDialect]?[paths]?.string // Index the default language
            ?? paths.map { "\($0)" }.joined(separator: ".") // Return the literal path indexed if no translation
{'fraction_non_alphanumeric': 0.10409511228533685, 'fraction_numerical': 0.0005284015852047556, 'mean_word_length': 2.473394495412844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6950988', 'n_tokens_mistral': 952, 'n_tokens_neox': 928, 'n_words': 276}