<commit_before>﻿#region License information
/*
    ZScript Game Scripting Programming Language
    Copyright (C) 2015  <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/
#endregion
using System;
using System.Linq;
using ZScript.Runtime.Execution;

namespace ZScript.Elements
{
    /// <summary>
    /// Defines a single token for a virtual machine
    /// </summary>
    public class Token : IEquatable<Token>
    {
        /// <summary>
        /// The type of this token
        /// </summary>
        public readonly TokenType Type;

        /// <summary>
        /// The token itself. When the Type corresponds to a VM instruction, this value is not used
        /// </summary>
        public readonly Object TokenObject;

        /// <summary>
        /// An instruction that can be executed by a VM, in case this token has a token type of
        /// TokenType.Instruction or TokenType.Operator, otherwise this value should be ignored
        /// </summary>
        public readonly VmInstruction Instruction;

        /// <summary>
        /// Gets or sets a value specifying the reachability of this token.
        /// This is a meta-data type property used by IntermediaryTokenLists to mark the reachability of instruction tokens
        /// </summary>
        public bool Reachable { get; set; }

        /// <summary>
        /// Initializes a new instance of the Token class
        /// </summary>
        /// <param name="type">A type of token to create</param>
        /// <param name="tokenObject">The underlying token object to create</param>
        public Token(TokenType type, Object tokenObject)
            : this(type, tokenObject, VmInstruction.Noop)
        {

        }

        /// <summary>
        /// Initializes a new instance of the Token class
        /// </summary>
        /// <param name="type">A type of token to create</param>
        /// <param name="tokenObject">The underlying token object to create</param>
        /// <param name="instruction">An instruction this token represents</param>
        public Token(TokenType type, Object tokenObject, VmInstruction instruction)
        {
            Type = type;
            TokenObject = tokenObject;
            Instruction = instruction;
            Reachable = true; // By deafult, tokens always start marked as reachable
        }

        /// <summary>
        /// Returns a string representation of this token object
        /// </summary>
        /// <returns>A string representation of this token object</returns>
        public override string ToString()
        {
            return "{ Token type:" + Type + ", value:'" + TokenObject + "' (" + (TokenObject != null ? TokenObject.GetType().ToString() : "") + "), instruction:" + Instruction + " }";
        }

        #region Equality members

        public bool Equals(Token other)
        {
            if (ReferenceEquals(null, other)) return false;
            if (ReferenceEquals(this, other)) return true;
            if (other.GetType() != GetType()) return false;

            if (TokenObject.GetType() != other.TokenObject.GetType()) return false;

            // Array of objects checking
            if (TokenObject is object[])
            {
                if (!((object[])TokenObject).SequenceEqual((object[])other.TokenObject))
                    return false;
            }
            else if (!Equals(TokenObject, other.TokenObject))
            {
                return false;
            }

            return Type == other.Type && Instruction == other.Instruction;
        }

        public override bool Equals(object obj)
        {
            if (ReferenceEquals(null, obj)) return false;
            if (ReferenceEquals(this, obj)) return true;
            if (obj.GetType() != GetType()) return false;<commit_msg>Fixing null reference in Token.Equals method<commit_after>﻿#region License information
/*
    ZScript Game Scripting Programming Language
    Copyright (C) 2015  <NAME>

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/
#endregion
using System;
using System.Linq;
using ZScript.Runtime.Execution;

namespace ZScript.Elements
{
    /// <summary>
    /// Defines a single token for a virtual machine
    /// </summary>
    public class Token : IEquatable<Token>
    {
        /// <summary>
        /// The type of this token
        /// </summary>
        public readonly TokenType Type;

        /// <summary>
        /// The token itself. When the Type corresponds to a VM instruction, this value is not used
        /// </summary>
        public readonly Object TokenObject;

        /// <summary>
        /// An instruction that can be executed by a VM, in case this token has a token type of
        /// TokenType.Instruction or TokenType.Operator, otherwise this value should be ignored
        /// </summary>
        public readonly VmInstruction Instruction;

        /// <summary>
        /// Gets or sets a value specifying the reachability of this token.
        /// This is a meta-data type property used by IntermediaryTokenLists to mark the reachability of instruction tokens
        /// </summary>
        public bool Reachable { get; set; }

        /// <summary>
        /// Initializes a new instance of the Token class
        /// </summary>
        /// <param name="type">A type of token to create</param>
        /// <param name="tokenObject">The underlying token object to create</param>
        public Token(TokenType type, Object tokenObject)
            : this(type, tokenObject, VmInstruction.Noop)
        {

        }

        /// <summary>
        /// Initializes a new instance of the Token class
        /// </summary>
        /// <param name="type">A type of token to create</param>
        /// <param name="tokenObject">The underlying token object to create</param>
        /// <param name="instruction">An instruction this token represents</param>
        public Token(TokenType type, Object tokenObject, VmInstruction instruction)
        {
            Type = type;
            TokenObject = tokenObject;
            Instruction = instruction;
            Reachable = true; // By deafult, tokens always start marked as reachable
        }

        /// <summary>
        /// Returns a string representation of this token object
        /// </summary>
        /// <returns>A string representation of this token object</returns>
        public override string ToString()
        {
            return "{ Token type:" + Type + ", value:'" + TokenObject + "' (" + (TokenObject != null ? TokenObject.GetType().ToString() : "") + "), instruction:" + Instruction + " }";
        }

        #region Equality members

        public bool Equals(Token other)
        {
            if (ReferenceEquals(null, other)) return false;
            if (ReferenceEquals(this, other)) return true;
            if (other.GetType() != GetType()) return false;

            if ((TokenObject != null) && (other.TokenObject != null) && (TokenObject.GetType() != other.TokenObject.GetType())) return false;

            // Array of objects checking
            if (TokenObject is object[])
            {
                if (!((object[])TokenObject).SequenceEqual((object[])other.TokenObject))
                    return false;
            }
            else if (!Equals(TokenObject, other.TokenObject))
            {
                return false;
            }

            return Type == other.Type && Instruction == other.Instruction;
        }

        public override bool Equals(object obj)
        {
            if (ReferenceEquals(null, obj)) return false;
            if (ReferenceEquals(this, obj)) return true;
            if (obj.GetType() != GetType()) return false;
{'fraction_non_alphanumeric': 0.09604582002423175, 'fraction_numerical': 0.003744905826632889, 'mean_word_length': 2.120274914089347, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '206951', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2135, 'n_words': 884}