<commit_before>﻿/*

 Copyright (c) 2006 <NAME>.

 The use and distribution terms for this software are contained in the file named License.txt, 
 which can be found in the root of the Phalanger distribution. By using this software 
 in any fashion, you are agreeing to be bound by the terms of this license.
 
 You must not remove this notice from this software.

*/

using System;
using System.IO;
using System.Text;
using System.Reflection;
using System.Diagnostics;
using System.Collections;
using System.Collections.Generic;

namespace PHP.Core.Parsers
{
	public sealed class Tokenizer : Lexer
	{
		[Flags]
		public enum Features
		{
			V5Keywords = 1,
			ContextKeywords = 2,
			Linq = 4,
			SkipWhitespace = 8,

			AllowAspTags = 16,
			AllowShortTags = 32,
			TypeKeywords = 64,

			Default = V5Keywords | AllowAspTags | AllowShortTags | Linq | ContextKeywords | TypeKeywords
		}

		private Features features;

		public Tokenizer(TextReader/*!*/ reader)
			: this(reader, Features.Default)
		{
		}

		public Tokenizer(TextReader/*!*/ reader, Features features)
			: base(reader)
		{
			if (reader == null)
				throw new ArgumentNullException("reader");

			AllowAspTags = (features & Features.AllowAspTags) != 0;
			AllowShortTags = (features & Features.AllowShortTags) != 0;

			this.features = features;
		}

		public TokenCategory TokenCategory { get { return tokenCategory; } }
		private TokenCategory tokenCategory;

		public Tokens RealToken { get { return realToken; } }
		private Tokens realToken;

		public string TokenText { get { return tokenText; } }
		private string tokenText;

		#region Compressed State

		public struct CompressedState : IEquatable<CompressedState>
		{
			internal string HereDocLabel { get { return hereDocLabel; } }
			private readonly string hereDocLabel;

			internal LexicalStates CurrentState { get { return currentState; } }
			private readonly LexicalStates currentState;

			private readonly LexicalStates[]/*!*/ stateStack;

			public CompressedState(Tokenizer tokenizer)
			{
				this.hereDocLabel = tokenizer.hereDocLabel;
				this.currentState = tokenizer.CurrentLexicalState;
				this.stateStack = tokenizer.StateStack.ToArray();
			}

			public override int GetHashCode()
			{
				unchecked
				{
					int result = (hereDocLabel != null) ? hereDocLabel.GetHashCode() : 0x2312347;
					for (int i = 0; i < stateStack.Length; i++)
						result ^= (int)stateStack[i] << i;
					return result ^ ((int)currentState << 7);
				}
			}

			public bool Equals(CompressedState other)
			{
				if (hereDocLabel != other.hereDocLabel) return false;
				if (stateStack.Length != other.stateStack.Length) return false;

				for (int i = 0; i < stateStack.Length; i++)
				{
					if (stateStack[i] != other.stateStack[i])
						return false;
				}

				return true;
			}

			public Stack<LexicalStates> GetStateStack()
			{
				return new Stack<LexicalStates>(stateStack);
			}
		}

		public CompressedState GetCompressedState()
		{
			return new CompressedState(this);
		}

		public void RestoreCompressedState(CompressedState state)
		{
			hereDocLabel = state.HereDocLabel;
			StateStack = state.GetStateStack();
			CurrentLexicalState = state.CurrentState;
		}

		#endregion

		public static bool IsCharToken(Tokens token)
		{
			return (int)token > 0 && (int)token < (int)Toks.ERROR;
		}

		public new Tokens GetNextToken()
		{
			for (; ; )
			{
				inString = false;
				isCode = false;

				Tokens token = realToken = base.GetNextToken();
				tokenText = base.GetTokenString();

				switch (token)
				{
					case Tokens.EOF:
					case Tokens.ERROR:
						tokenCategory = TokenCategory.Unknown;
						return token;

					case Tokens.ErrorInvalidIdentifier:
					case Tokens.ErrorNotSupported:
						tokenCategory = TokenCategory.Unknown;
						return Tokens.ERROR;

					#region Token Postprocessing

					// following two cases have to determine the token type
					// (T_LNUMBER/T_L64NUMBER or T_DNUMBER) depending on the actual value
					case Tokens.ParseDecimalNumber:
						tokenCategory = TokenCategory.Number;
						return GetHexIntegerTokenType(0);
					case Tokens.ParseHexadecimalNumber:
						tokenCategory = TokenCategory.Number; 
						return GetHexIntegerTokenType(2);
                    case Tokens.ParseBinaryNumber:
                        tokenCategory = TokenCategory.Number;<commit_msg>Tokenizer gets TokenText lazily<commit_after>﻿/*

 Copyright (c) 2006 <NAME>.

 The use and distribution terms for this software are contained in the file named License.txt, 
 which can be found in the root of the Phalanger distribution. By using this software 
 in any fashion, you are agreeing to be bound by the terms of this license.
 
 You must not remove this notice from this software.

*/

using System;
using System.IO;
using System.Text;
using System.Reflection;
using System.Diagnostics;
using System.Collections;
using System.Collections.Generic;

namespace PHP.Core.Parsers
{
	public sealed class Tokenizer : Lexer
	{
		[Flags]
		public enum Features
		{
			V5Keywords = 1,
			ContextKeywords = 2,
			Linq = 4,
			SkipWhitespace = 8,

			AllowAspTags = 16,
			AllowShortTags = 32,
			TypeKeywords = 64,

			Default = V5Keywords | AllowAspTags | AllowShortTags | Linq | ContextKeywords | TypeKeywords
		}

		private Features features;

		public Tokenizer(TextReader/*!*/ reader)
			: this(reader, Features.Default)
		{
		}

		public Tokenizer(TextReader/*!*/ reader, Features features)
			: base(reader)
		{
			if (reader == null)
				throw new ArgumentNullException("reader");

			AllowAspTags = (features & Features.AllowAspTags) != 0;
			AllowShortTags = (features & Features.AllowShortTags) != 0;

			this.features = features;
		}

		public TokenCategory TokenCategory { get { return tokenCategory; } }
		private TokenCategory tokenCategory;

		public Tokens RealToken { get { return realToken; } }
		private Tokens realToken;

        public string TokenText { get { return _tokenText ?? (_tokenText = base.GetTokenString()); } }
		[System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)]
        private string _tokenText;

		#region Compressed State

		public struct CompressedState : IEquatable<CompressedState>
		{
			internal string HereDocLabel { get { return hereDocLabel; } }
			private readonly string hereDocLabel;

			internal LexicalStates CurrentState { get { return currentState; } }
			private readonly LexicalStates currentState;

			private readonly LexicalStates[]/*!*/ stateStack;

			public CompressedState(Tokenizer tokenizer)
			{
				this.hereDocLabel = tokenizer.hereDocLabel;
				this.currentState = tokenizer.CurrentLexicalState;
				this.stateStack = tokenizer.StateStack.ToArray();
			}

			public override int GetHashCode()
			{
				unchecked
				{
					int result = (hereDocLabel != null) ? hereDocLabel.GetHashCode() : 0x2312347;
					for (int i = 0; i < stateStack.Length; i++)
						result ^= (int)stateStack[i] << i;
					return result ^ ((int)currentState << 7);
				}
			}

			public bool Equals(CompressedState other)
			{
				if (hereDocLabel != other.hereDocLabel) return false;
				if (stateStack.Length != other.stateStack.Length) return false;

				for (int i = 0; i < stateStack.Length; i++)
				{
					if (stateStack[i] != other.stateStack[i])
						return false;
				}

				return true;
			}

			public Stack<LexicalStates> GetStateStack()
			{
				return new Stack<LexicalStates>(stateStack);
			}
		}

		public CompressedState GetCompressedState()
		{
			return new CompressedState(this);
		}

		public void RestoreCompressedState(CompressedState state)
		{
			hereDocLabel = state.HereDocLabel;
			StateStack = state.GetStateStack();
			CurrentLexicalState = state.CurrentState;
		}

		#endregion

		public static bool IsCharToken(Tokens token)
		{
			return (int)token > 0 && (int)token < (int)Toks.ERROR;
		}

		public new Tokens GetNextToken()
		{
			for (; ; )
			{
				inString = false;
				isCode = false;

				Tokens token = realToken = base.GetNextToken();
				_tokenText = null;

				switch (token)
				{
					case Tokens.EOF:
					case Tokens.ERROR:
						tokenCategory = TokenCategory.Unknown;
						return token;

					case Tokens.ErrorInvalidIdentifier:
					case Tokens.ErrorNotSupported:
						tokenCategory = TokenCategory.Unknown;
						return Tokens.ERROR;

					#region Token Postprocessing

					// following two cases have to determine the token type
					// (T_LNUMBER/T_L64NUMBER or T_DNUMBER) depending on the actual value
					case Tokens.ParseDecimalNumber:
						tokenCategory = TokenCategory.Number;
						return GetHexIntegerTokenType(0);
					case Tokens.ParseHexadecimalNumber:
						tokenCategory = TokenCategory.Number; 
						return GetHexIntegerTokenType(2);
                    case Tokens.ParseBinaryNumber:
                        tokenCategory = TokenCategory.Number;
{'fraction_non_alphanumeric': 0.08871331828442439, 'fraction_numerical': 0.007674943566591422, 'mean_word_length': 3.314021421616358, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5877667', 'n_tokens_mistral': 3208, 'n_tokens_neox': 2537, 'n_words': 719}