<commit_before>//
//  StringExtension.swift
//  Client
//
//  Created by <NAME> on 11/9/15.
//  Copyright © 2015 Cliqz. All rights reserved.
//

import Foundation

extension String {
    
    var boolValue: Bool {
        return NSString(string: self).boolValue
    }
    
    
    func trim() -> String {
        let newString = self.stringByTrimmingCharactersInSet(NSCharacterSet.whitespaceCharacterSet())
        return newString
    }
    
    static func generateRandomString(length: Int, alphabet: String = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") -> String {
        var randomString = ""
        
        let rangeLength = UInt32 (alphabet.characters.count)
        
        for _ in 0 ..< length {
            let randomIndex = Int(arc4random_uniform(rangeLength))
            randomString.append(alphabet[alphabet.startIndex.advancedBy(randomIndex)])
        }
        
        return String(randomString)
    }
    
    func replace(string:String, replacement:String) -> String {
        return self.stringByReplacingOccurrencesOfString(string, withString: replacement, options: NSStringCompareOptions.LiteralSearch, range: nil)
    }
    
    func removeWhitespaces() -> String {
        return self.replace(" ", replacement: "")
    }
    
    func escapeURL() -> String {
        if self.contains("%") { // String already escaped
            return self
        } else {
            let allowedCharacterSet = NSMutableCharacterSet()
            allowedCharacterSet.formUnionWithCharacterSet(NSCharacterSet.URLQueryAllowedCharacterSet())
            allowedCharacterSet.removeCharactersInString("=")
            
            return self.stringByAddingPercentEncodingWithAllowedCharacters(allowedCharacterSet)!
        }
    }
	
	// Cliqz added extra URL encoding methods because stringByAddingPercentEncodingWithAllowedCharacters doesn't encodes &
	func encodeURL() -> String {
		return CFURLCreateStringByAddingPercentEscapes(nil,
			self as CFStringRef,
			nil,
			String("!*'\"();:@&=+$,/?%#[]% ") as CFStringRef, CFStringConvertNSStringEncodingToEncoding(NSUTF8StringEncoding)) as String
	}

}
<commit_msg>Included new line in trim method for String<commit_after>//
//  StringExtension.swift
//  Client
//
//  Created by <NAME> on 11/9/15.
//  Copyright © 2015 Cliqz. All rights reserved.
//

import Foundation

extension String {
    
    var boolValue: Bool {
        return NSString(string: self).boolValue
    }
    
    
    func trim() -> String {
        let newString = self.stringByTrimmingCharactersInSet(NSCharacterSet.whitespaceAndNewlineCharacterSet())
        return newString
    }
    
    static func generateRandomString(length: Int, alphabet: String = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") -> String {
        var randomString = ""
        
        let rangeLength = UInt32 (alphabet.characters.count)
        
        for _ in 0 ..< length {
            let randomIndex = Int(arc4random_uniform(rangeLength))
            randomString.append(alphabet[alphabet.startIndex.advancedBy(randomIndex)])
        }
        
        return String(randomString)
    }
    
    func replace(string:String, replacement:String) -> String {
        return self.stringByReplacingOccurrencesOfString(string, withString: replacement, options: NSStringCompareOptions.LiteralSearch, range: nil)
    }
    
    func removeWhitespaces() -> String {
        return self.replace(" ", replacement: "")
    }
    
    func escapeURL() -> String {
        if self.contains("%") { // String already escaped
            return self
        } else {
            let allowedCharacterSet = NSMutableCharacterSet()
            allowedCharacterSet.formUnionWithCharacterSet(NSCharacterSet.URLQueryAllowedCharacterSet())
            allowedCharacterSet.removeCharactersInString("=")
            
            return self.stringByAddingPercentEncodingWithAllowedCharacters(allowedCharacterSet)!
        }
    }
	
	// Cliqz added extra URL encoding methods because stringByAddingPercentEncodingWithAllowedCharacters doesn't encodes &
	func encodeURL() -> String {
		return CFURLCreateStringByAddingPercentEscapes(nil,
			self as CFStringRef,
			nil,
			String("!*'\"();:@&=+$,/?%#[]% ") as CFStringRef, CFStringConvertNSStringEncodingToEncoding(NSUTF8StringEncoding)) as String
	}

}

{'fraction_non_alphanumeric': 0.0940959409594096, 'fraction_numerical': 0.01107011070110701, 'mean_word_length': 2.907207207207207, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5213706', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1179, 'n_words': 243}