<commit_before>//
//  ExpressionsProcessor.swift
//  Swifternalization
//
//  Created by <NAME> on 26/07/15.
//  Copyright (c) 2015 <NAME>. All rights reserved.
//

import Foundation

/**
Swifternalization contains some built-in country-related shared expressions.
Developer can create its own expressions in expressions.json file for
base and preferred languages.

The class is responsible for proper loading the built-in shared ones and
those loaded from project's files. It handles overriding of built-in and loads
those from Base and preferred language version.

It always load base expressions, then looking for language specific.
If in Base are some expressions that overrides built-ins, that's fine.
The class adds developer's custom expressions and adds only those of built-in
that are not contained in the Base.

The same is for preferred languages but also here what is important is that
at first preferred language file expressions are loaded, then if something
different is defined in Base it will be also loaded and then the built-in
expression that differs.
*/
class SharedExpressionsProcessor {
    
    /**
    Method takes expression for both Base and preferred language localizations
    and also internally loads built-in expressions, combine them and returns 
    expressions for Base and prefered language.
    
    :param: preferedLanguage A user device's language.
    :param: preferedLanguageExpressions Expressions from expressions.json
    :param: baseLanguageExpressions Expressions from base section of 
            expression.json.
    
    :returns: array of shared expressions for Base and preferred language.
    */
    class func processSharedExpression(preferedLanguage: CountryCode, preferedLanguageExpressions: [SharedExpression], baseLanguageExpressions: [SharedExpression]) -> [SharedExpression] {
        /*
        Get unique base expressions that are not presented in prefered language
        expressions. Those from base will be used in a case when programmer 
        will ask for string localization and when there is no such expression
        in prefered language section defined. 

        It means two things:
        1. Programmer make this expression shared through prefered language
        and this is good as base expression.
        2. He forgot to define such expression for prefered language.
        */
        var uniqueBaseExpressions = baseLanguageExpressions
        if preferedLanguageExpressions.count > 0 {
            uniqueBaseExpressions = baseLanguageExpressions.filter({
                let pref = $0
                return preferedLanguageExpressions.filter({$0.identifier == pref.identifier}).count == 0
            })
        }
        
        // Expressions from json files.
        var loadedExpressions = uniqueBaseExpressions + preferedLanguageExpressions

        
        // Load prefered language nad base built-in expressions. Get unique.
        let prefBuiltInExpressions = loadBuiltInExpressions(preferedLanguage)
        let baseBuiltInExpressions = SharedBaseExpression.allExpressions()
        let uniqueBaseBuiltInExpressions = baseBuiltInExpressions.filter({
            let pref = $0
            return prefBuiltInExpressions.filter({$0.identifier == pref.identifier}).count == 0
        })
        
        // Unique built-in expressions made of base + prefered language.
        let builtInExpressions = uniqueBaseBuiltInExpressions + prefBuiltInExpressions
        
        /*
        To get it done we must get only unique built-in expressions that are not 
        in loaded expressions.
        */
        let uniqueBuiltInExpressions = builtInExpressions.filter({
            let builtIn = $0
            return loadedExpressions.filter({$0.identifier == builtIn.identifier}).count == 0
        })
        
        return loadedExpressions + uniqueBuiltInExpressions
    }
    
    /**
    Method loads built-in framework's built-in expressions for specific language.
    
    :param: language A preferred user's language.
    :returns: Shared expressions for specific language. If there is no
    expression for passed language empty array is returned.
    */
    private class func loadBuiltInExpressions(language: CountryCode) -> [SharedExpression] {
        switch language {
        case "pl": return SharedPolishExpression.allExpressions()
        default: return []
        }
    }
}<commit_msg>Create operator for getting unique values of two arrays<commit_after>//
//  ExpressionsProcessor.swift
//  Swifternalization
//
//  Created by <NAME> on 26/07/15.
//  Copyright (c) 2015 <NAME>. All rights reserved.
//

import Foundation

/**
Swifternalization contains some built-in country-related shared expressions.
Developer can create its own expressions in expressions.json file for
base and preferred languages.

The class is responsible for proper loading the built-in shared ones and
those loaded from project's files. It handles overriding of built-in and loads
those from Base and preferred language version.

It always load base expressions, then looking for language specific.
If in Base are some expressions that overrides built-ins, that's fine.
The class adds developer's custom expressions and adds only those of built-in
that are not contained in the Base.

The same is for preferred languages but also here what is important is that
at first preferred language file expressions are loaded, then if something
different is defined in Base it will be also loaded and then the built-in
expression that differs.
*/
class SharedExpressionsProcessor {
    
    /**
    Method takes expression for both Base and preferred language localizations
    and also internally loads built-in expressions, combine them and returns 
    expressions for Base and prefered language.
    
    :param: preferedLanguage A user device's language.
    :param: preferedLanguageExpressions Expressions from expressions.json
    :param: baseLanguageExpressions Expressions from base section of 
            expression.json.
    :returns: array of shared expressions for Base and preferred language.
    */
    class func processSharedExpression(preferedLanguage: CountryCode, preferedLanguageExpressions: [SharedExpression], baseLanguageExpressions: [SharedExpression]) -> [SharedExpression] {
        /*
        Get unique base expressions that are not presented in prefered language
        expressions. Those from base will be used in a case when programmer 
        will ask for string localization and when there is no such expression
        in prefered language section defined. 

        It means two things:
        1. Programmer make this expression shared through prefered language
        and this is good as base expression.
        2. He forgot to define such expression for prefered language.
        */
        let uniqueBaseExpressions = baseLanguageExpressions <! preferedLanguageExpressions
        
        // Expressions from json files.
        var loadedExpressions = uniqueBaseExpressions + preferedLanguageExpressions
        
        // Load prefered language nad base built-in expressions. Get unique.
        let prefBuiltInExpressions = loadBuiltInExpressions(preferedLanguage)
        let baseBuiltInExpressions = SharedBaseExpression.allExpressions()
        let uniqueBaseBuiltInExpressions = baseBuiltInExpressions <! prefBuiltInExpressions
        
        // Unique built-in expressions made of base + prefered language.
        let builtInExpressions = uniqueBaseBuiltInExpressions + prefBuiltInExpressions
        
        /*
        To get it done we must get only unique built-in expressions that are not 
        in loaded expressions.
        */
        return loadedExpressions + (builtInExpressions <! loadedExpressions)
    }
    
    /**
    Method loads built-in framework's built-in expressions for specific language.
    
    :param: language A preferred user's language.
    :returns: Shared expressions for specific language. If there is no
    expression for passed language empty array is returned.
    */
    private class func loadBuiltInExpressions(language: CountryCode) -> [SharedExpression] {
        switch language {
        case "pl": return SharedPolishExpression.allExpressions()
        default: return []
        }
    }
}

infix operator <! {}
func <! (lhs: [SharedExpression], rhs: [SharedExpression]) -> [SharedExpression] {
    /*
    "Get Unique" operator. It helps in getting unique shared expressions from two arrays.
    Content of `lhs` array will be checked in terms on uniqueness. The operator does
    check is there is any shared expression in `lhs` that is presented in `rhs`.
    If element from `lhs` is not in `rhs` then this element is correct and is returned
    in new array which is a result of this operation.
    */
    var result = lhs
    if rhs.count > 0 {
        result = lhs.filter({
            let tmp = $0
            return rhs.filter({$0.identifier == tmp.identifier}).count == 0
        })
    }
    return result
}
{'fraction_non_alphanumeric': 0.0552964690206529, 'fraction_numerical': 0.004219409282700422, 'mean_word_length': 3.013814616755793, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6640644', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2055, 'n_words': 922}