<commit_before>//
//  String+IVGFoundation.swift
//  IVGFoundation
//
//  Created by <NAME> on 2/10/17.
//  Copyright © 2017 <NAME>. All rights reserved.
//

import Foundation

public extension String {

    public func stringByAppendingPathComponent(_ value: String) -> String {
        return (self as NSString).appendingPathComponent(value)
    }

    public var attributedStringFromHTML: NSAttributedString? {
        guard let data = self.data(using: .utf16, allowLossyConversion: false) else { return nil }
        guard let result = try? NSAttributedString(data: data, options: [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html], documentAttributes: nil) else { return nil }
        return result
    }

    public var stringFromHTML: String? {
        return attributedStringFromHTML?.string
    }

    public func substring(withNSRange nsRange: NSRange) -> String? {
        guard let range = Range(nsRange) else { return nil }

        let start = utf16.index(utf16.startIndex, offsetBy: range.lowerBound)
        let end = utf16.index(utf16.startIndex, offsetBy: range.upperBound)
        return String(describing: utf16[start..<end])
    }

}

public extension String {

    public func isValid(withRegEx pattern: String) -> Bool {
        guard let regEx = try? NSRegularExpression(pattern: pattern, options: []) else { return false }

        let range = NSRange(location: 0, length: self.count)
        return !regEx.matches(in: self, options: [], range: range).isEmpty
    }

    public static let emailRegEx = "^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$"

    public var isValidEmail: Bool { return isValid(withRegEx: String.emailRegEx) }
<commit_msg>Fix substring for swift 4<commit_after>//
//  String+IVGFoundation.swift
//  IVGFoundation
//
//  Created by <NAME> on 2/10/17.
//  Copyright © 2017 <NAME>. All rights reserved.
//

import Foundation

public extension String {

    public func stringByAppendingPathComponent(_ value: String) -> String {
        return (self as NSString).appendingPathComponent(value)
    }

    public var attributedStringFromHTML: NSAttributedString? {
        guard let data = self.data(using: .utf16, allowLossyConversion: false) else { return nil }
        guard let result = try? NSAttributedString(data: data, options: [NSAttributedString.DocumentReadingOptionKey.documentType: NSAttributedString.DocumentType.html], documentAttributes: nil) else { return nil }
        return result
    }

    public var stringFromHTML: String? {
        return attributedStringFromHTML?.string
    }

    public func substring(withNSRange nsRange: NSRange) -> String? {
        guard let range = Range(nsRange) else { return nil }

        let start = index(startIndex, offsetBy: range.lowerBound)
        let end = index(startIndex, offsetBy: range.upperBound)
        return String(describing: self[start..<end])
    }

}

public extension String {

    public func isValid(withRegEx pattern: String) -> Bool {
        guard let regEx = try? NSRegularExpression(pattern: pattern, options: []) else { return false }

        let range = NSRange(location: 0, length: self.count)
        return !regEx.matches(in: self, options: [], range: range).isEmpty
    }

    public static let emailRegEx = "^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$"

    public var isValidEmail: Bool { return isValid(withRegEx: String.emailRegEx) }

{'fraction_non_alphanumeric': 0.11265113535830139, 'fraction_numerical': 0.01032143910350929, 'mean_word_length': 3.3431498079385404, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3432895', 'n_tokens_mistral': 1031, 'n_tokens_neox': 964, 'n_words': 274}