<commit_before>//
//  Symbols.swift
//  Covfefe
//
//  Created by <NAME> on 07.08.17.
//  Copyright (c) 2017 <NAME>
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in all
//  copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//  SOFTWARE.

import Foundation

/// A non-terminal symbol, which cannot occurr in a word recognized by a parser
public struct NonTerminal: Codable {
	
	/// Name of the non-terminal
	public let name: String
	
	/// Creates a new non-terminal symbol with a given name
	///
	/// - Parameter name: Name of the non-terminal symbol
	public init(name: String) {
		self.name = name
	}
}

extension NonTerminal: CustomStringConvertible {
	public var description: String {
		return name
	}
}

extension NonTerminal: ExpressibleByStringLiteral {
	public typealias StringLiteralType = String
	
	public init(stringLiteral value: String) {
		self.init(name: value)
	}
}

extension NonTerminal: Hashable {
	public var hashValue: Int {
		return name.hashValue
	}
	
	public static func ==(lhs: NonTerminal, rhs: NonTerminal) -> Bool {
		return lhs.name == rhs.name
	}
}

/// A terminal symbol which can occur in a string recognized by a parser and which cannot be
/// replaced by any production
public struct Terminal: Codable {
	
	/// Value of the terminal
	public let value: String
	
	/// Indicates whether the value of the non-terminal is a regular expression
	public let isRegularExpression: Bool
	
	/// Creates a new terminal value which can occurr in a string
	///
	/// **Note**: The value of the terminal string may not overlap partially with any other non-terminal
	/// contained in a grammar. For regular terminals, it it may be deriable to add word boundary markers: `\b`.
	///
	/// - Parameters:
	///   - value: Value of the terminal which can occurr in a string
	///   - isRegularExpression: Indicates whether the value is a regular expression
	/// - Throws: An error if the value is an invalid regular expression
	public init(value: String, isRegularExpression: Bool = false) throws {
		self.value = value
		self.isRegularExpression = isRegularExpression
		
		if isRegularExpression {
			_ = try NSRegularExpression(pattern: value, options: [])
		}
	}
}

extension Terminal: ExpressibleByStringLiteral {
	public typealias StringLiteralType = String
	
	public init(stringLiteral value: String) {
		try! self.init(value: value)
	}
}

extension Terminal: Hashable {
	public var hashValue: Int {
		return value.hashValue
	}
	
	public static func ==(lhs: Terminal, rhs: Terminal) -> Bool {
		return lhs.value == rhs.value
	}
}

extension Terminal: CustomStringConvertible {
	public var description: String {
		return value
	}
}

/// A symbol which can either be a terminal or a non-terminal character
///
/// - terminal: A terminal character
/// - nonTerminal: A non-terminal character
public enum Symbol: Codable {
	/// A terminal symbol
	case terminal(Terminal)
	
	/// A non-terminal symbol<commit_msg>Improves parsing performance by memoizing string hashes<commit_after>//
//  Symbols.swift
//  Covfefe
//
//  Created by <NAME> on 07.08.17.
//  Copyright (c) 2017 <NAME>
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in all
//  copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//  SOFTWARE.

import Foundation

/// A non-terminal symbol, which cannot occurr in a word recognized by a parser
public struct NonTerminal: Codable, Hashable {
	
	public static func ==(lhs: NonTerminal, rhs: NonTerminal) -> Bool {
		return lhs.name == rhs.name
	}
	
	/// Name of the non-terminal
	public let name: String
	
	public let hashValue: Int
	
	/// Creates a new non-terminal symbol with a given name
	///
	/// - Parameter name: Name of the non-terminal symbol
	public init(name: String) {
		self.name = name
		self.hashValue = name.hashValue
	}
}

extension NonTerminal: CustomStringConvertible {
	public var description: String {
		return name
	}
}

extension NonTerminal: ExpressibleByStringLiteral {
	public typealias StringLiteralType = String
	
	public init(stringLiteral value: String) {
		self.init(name: value)
	}
}

/// A terminal symbol which can occur in a string recognized by a parser and which cannot be
/// replaced by any production
public struct Terminal: Codable, Hashable {
	
	/// Value of the terminal
	public let value: String
	
	/// Indicates whether the value of the non-terminal is a regular expression
	public let isRegularExpression: Bool
	
	public let hashValue: Int
	
	/// Creates a new terminal value which can occurr in a string
	///
	/// **Note**: The value of the terminal string may not overlap partially with any other non-terminal
	/// contained in a grammar. For regular terminals, it it may be deriable to add word boundary markers: `\b`.
	///
	/// - Parameters:
	///   - value: Value of the terminal which can occurr in a string
	///   - isRegularExpression: Indicates whether the value is a regular expression
	/// - Throws: An error if the value is an invalid regular expression
	public init(value: String, isRegularExpression: Bool = false) throws {
		self.value = value
		self.isRegularExpression = isRegularExpression
		self.hashValue = value.hashValue
		
		if isRegularExpression {
			_ = try NSRegularExpression(pattern: value, options: [])
		}
	}
	
	public static func ==(lhs: Terminal, rhs: Terminal) -> Bool {
		return lhs.value == rhs.value
	}
}

extension Terminal: ExpressibleByStringLiteral {
	public typealias StringLiteralType = String
	
	public init(stringLiteral value: String) {
		try! self.init(value: value)
	}
}

extension Terminal: CustomStringConvertible {
	public var description: String {
		return value
	}
}

/// A symbol which can either be a terminal or a non-terminal character
///
/// - terminal: A terminal character
/// - nonTerminal: A non-terminal character
public enum Symbol: Codable {
	/// A terminal symbol
	case terminal(Terminal)
	
	/// A non-terminal symbol
{'fraction_non_alphanumeric': 0.0835368204600951, 'fraction_numerical': 0.0025703637064644646, 'mean_word_length': 3.459598853868195, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4712465', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2078, 'n_words': 965}