<commit_before>//
//  String+Punycode.swift
//  Punycode
//
//  Created by <NAME> on 2020-03-16.
//

import Foundation

public extension String {

	/// The IDNA-encoded representation of a Unicode domain.
	///
	/// This will properly split domains on periods; e.g.,
	/// "www.bücher.ch" becomes "www.xn--bcher-kva.ch".
	var idnaEncoded: String? {
		guard let mapped = try? self.mapUTS46() else { return nil }

		let nonASCII = CharacterSet(charactersIn: UnicodeScalar(0)...UnicodeScalar(127)).inverted
		var result = ""

		let s = Scanner(string: mapped.precomposedStringWithCanonicalMapping)
		let dotAt = CharacterSet(charactersIn: ".@")

		while !s.isAtEnd {
			if let input = s.shimScanUpToCharacters(from: dotAt) {
				if !input.isValidLabel { return nil }

				if input.rangeOfCharacter(from: nonASCII) != nil {
					result.append("xn--")

					if let encoded = input.punycodeEncoded {
						result.append(encoded)
					}
				} else {
					result.append(input)
				}
			}

			if let input = s.shimScanCharacters(from: dotAt) {
				result.append(input)
			}
		}

		return result
	}

	/// The Unicode representation of an IDNA-encoded domain.
	///
	/// This will properly split domains on periods; e.g.,
	/// "www.xn--bcher-kva.ch" becomes "www.bücher.ch".
	var idnaDecoded: String? {
		var result = ""
		let s = Scanner(string: self)
		let dotAt = CharacterSet(charactersIn: ".@")

		while !s.isAtEnd {
			if let input = s.shimScanUpToCharacters(from: dotAt) {
				if input.lowercased().hasPrefix("xn--") {
					let start = input.index(input.startIndex, offsetBy: 4)
					guard let substr = input[start...].punycodeDecoded else { return nil }
					guard substr.isValidLabel else { return nil }
					result.append(substr)
				} else {
					result.append(input)
				}
			}

			if let input = s.shimScanCharacters(from: dotAt) {
				result.append(input)
			}
		}

		return result
	}

	/// The IDNA- and percent-encoded representation of a URL string.
	var encodedURLString: String? {
		let urlParts = self.urlParts
		var pathAndQuery = urlParts.pathAndQuery

		var allowedCharacters = CharacterSet.urlPathAllowed
		allowedCharacters.insert(charactersIn: "%")
		allowedCharacters.insert(charactersIn: "?")
		pathAndQuery = pathAndQuery.addingPercentEncoding(withAllowedCharacters: allowedCharacters) ?? ""

		var result = "\(urlParts.scheme)\(urlParts.delim)"

		if let username = urlParts.username?.addingPercentEncoding(withAllowedCharacters: .urlUserAllowed) {
			if let password = urlParts.password?.addingPercentEncoding(withAllowedCharacters: .urlPasswordAllowed) {
				result.append("\(username):\(password)@")
			} else {
				result.append("\(username)@")
			}
		}

		guard let host = urlParts.host.idnaEncoded else { return nil }

		result.append("\(host)\(pathAndQuery)")

		if var fragment = urlParts.fragment {
			var fragmentAlloweCharacters = CharacterSet.urlFragmentAllowed
			fragmentAlloweCharacters.insert(charactersIn: "%")
			fragment = fragment.addingPercentEncoding(withAllowedCharacters: fragmentAlloweCharacters) ?? ""

			result.append("#\(fragment)")
		}

		return result
	}

	/// The Unicode representation of an IDNA- and percent-encoded URL string.
	var decodedURLString: String? {
		let urlParts = self.urlParts
		var usernamePassword = ""

		if let username = urlParts.username?.removingPercentEncoding {
			if let password = urlParts.password?.removingPercentEncoding {
				usernamePassword = "\(username):\(password)@"
			} else {
				usernamePassword = "\(username)@"
			}
		}

		guard let host = urlParts.host.idnaDecoded else { return nil }

		var result = "\(urlParts.scheme)\(urlParts.delim)\(usernamePassword)\(host)\(urlParts.pathAndQuery.removingPercentEncoding ?? "")"

		if let fragment = urlParts.fragment?.removingPercentEncoding {
			result.append("#\(fragment)")
		}

		return result
	}

}

public extension URL {

	/// Initializes a URL with a Unicode URL string.
	///
	/// If `unicodeString` can be successfully encoded, equivalent to
	///
	/// ```
	/// URL(string: unicodeString.encodedURLString!)
	/// ```
	///
	/// - Parameter unicodeString: The unicode URL string with which to create a URL.
	init?(unicodeString: String) {
		guard let encodedString = unicodeString.encodedURLString else { return nil }
		self.init(string: encodedString)
	}

	/// The IDNA- and percent-decoded representation of the URL.
	///
	/// Equivalent to
	///
	///	```
	/// self.absoluteString.decodedURLString
	/// ```<commit_msg>Try the native init() first; only try encoding if that fails<commit_after>//
//  String+Punycode.swift
//  Punycode
//
//  Created by <NAME> on 2020-03-16.
//

import Foundation

public extension String {

	/// The IDNA-encoded representation of a Unicode domain.
	///
	/// This will properly split domains on periods; e.g.,
	/// "www.bücher.ch" becomes "www.xn--bcher-kva.ch".
	var idnaEncoded: String? {
		guard let mapped = try? self.mapUTS46() else { return nil }

		let nonASCII = CharacterSet(charactersIn: UnicodeScalar(0)...UnicodeScalar(127)).inverted
		var result = ""

		let s = Scanner(string: mapped.precomposedStringWithCanonicalMapping)
		let dotAt = CharacterSet(charactersIn: ".@")

		while !s.isAtEnd {
			if let input = s.shimScanUpToCharacters(from: dotAt) {
				if !input.isValidLabel { return nil }

				if input.rangeOfCharacter(from: nonASCII) != nil {
					result.append("xn--")

					if let encoded = input.punycodeEncoded {
						result.append(encoded)
					}
				} else {
					result.append(input)
				}
			}

			if let input = s.shimScanCharacters(from: dotAt) {
				result.append(input)
			}
		}

		return result
	}

	/// The Unicode representation of an IDNA-encoded domain.
	///
	/// This will properly split domains on periods; e.g.,
	/// "www.xn--bcher-kva.ch" becomes "www.bücher.ch".
	var idnaDecoded: String? {
		var result = ""
		let s = Scanner(string: self)
		let dotAt = CharacterSet(charactersIn: ".@")

		while !s.isAtEnd {
			if let input = s.shimScanUpToCharacters(from: dotAt) {
				if input.lowercased().hasPrefix("xn--") {
					let start = input.index(input.startIndex, offsetBy: 4)
					guard let substr = input[start...].punycodeDecoded else { return nil }
					guard substr.isValidLabel else { return nil }
					result.append(substr)
				} else {
					result.append(input)
				}
			}

			if let input = s.shimScanCharacters(from: dotAt) {
				result.append(input)
			}
		}

		return result
	}

	/// The IDNA- and percent-encoded representation of a URL string.
	var encodedURLString: String? {
		let urlParts = self.urlParts
		var pathAndQuery = urlParts.pathAndQuery

		var allowedCharacters = CharacterSet.urlPathAllowed
		allowedCharacters.insert(charactersIn: "%")
		allowedCharacters.insert(charactersIn: "?")
		pathAndQuery = pathAndQuery.addingPercentEncoding(withAllowedCharacters: allowedCharacters) ?? ""

		var result = "\(urlParts.scheme)\(urlParts.delim)"

		if let username = urlParts.username?.addingPercentEncoding(withAllowedCharacters: .urlUserAllowed) {
			if let password = urlParts.password?.addingPercentEncoding(withAllowedCharacters: .urlPasswordAllowed) {
				result.append("\(username):\(password)@")
			} else {
				result.append("\(username)@")
			}
		}

		guard let host = urlParts.host.idnaEncoded else { return nil }

		result.append("\(host)\(pathAndQuery)")

		if var fragment = urlParts.fragment {
			var fragmentAlloweCharacters = CharacterSet.urlFragmentAllowed
			fragmentAlloweCharacters.insert(charactersIn: "%")
			fragment = fragment.addingPercentEncoding(withAllowedCharacters: fragmentAlloweCharacters) ?? ""

			result.append("#\(fragment)")
		}

		return result
	}

	/// The Unicode representation of an IDNA- and percent-encoded URL string.
	var decodedURLString: String? {
		let urlParts = self.urlParts
		var usernamePassword = ""

		if let username = urlParts.username?.removingPercentEncoding {
			if let password = urlParts.password?.removingPercentEncoding {
				usernamePassword = "\(username):\(password)@"
			} else {
				usernamePassword = "\(username)@"
			}
		}

		guard let host = urlParts.host.idnaDecoded else { return nil }

		var result = "\(urlParts.scheme)\(urlParts.delim)\(usernamePassword)\(host)\(urlParts.pathAndQuery.removingPercentEncoding ?? "")"

		if let fragment = urlParts.fragment?.removingPercentEncoding {
			result.append("#\(fragment)")
		}

		return result
	}

}

public extension URL {

	/// Initializes a URL with a Unicode URL string.
	///
	/// If `unicodeString` can be successfully encoded, equivalent to
	///
	/// ```
	/// URL(string: unicodeString.encodedURLString!)
	/// ```
	///
	/// - Parameter unicodeString: The unicode URL string with which to create a URL.
	init?(unicodeString: String) {
		if let url = URL(string: unicodeString) {
			self = url
		} else {
			guard let encodedString = unicodeString.encodedURLString else { return nil }
			self.init(string: encodedString)
		}
	}

	/// The IDNA- and percent-decoded representation of the URL.
	///
	/// Equivalent to
	///
	///	```
	/// self.absoluteString.decodedURLString
	/// ```
{'fraction_non_alphanumeric': 0.1296747067935384, 'fraction_numerical': 0.0033193184332816995, 'mean_word_length': 3.404970760233918, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '345029', 'n_tokens_mistral': 3236, 'n_tokens_neox': 2817, 'n_words': 658}