<commit_before>#region License
/*---------------------------------------------------------------------------------*\

	Distributed under the terms of an MIT-style license:

	The MIT License

	Copyright (c) 2006-2010 <NAME>

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in
	all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	THE SOFTWARE.

\*---------------------------------------------------------------------------------*/
#endregion License

using System;
using System.IO;

using JsonFx.Common;
using JsonFx.Json;
using JsonFx.Serialization;

namespace JsonFx.EcmaScript
{
	/// <summary>
	/// Represents an ECMAScript identifier for serialization.
	/// </summary>
	public class EcmaScriptIdentifier : ITextFormattable<CommonTokenType>
	{
		#region Fields

		private readonly string identifier;

		#endregion Fields

		#region Init

		/// <summary>
		/// Ctor
		/// </summary>
		public EcmaScriptIdentifier()
			: this(null)
		{
		}

		/// <summary>
		/// Ctor
		/// </summary>
		/// <remarks></remarks>
		public EcmaScriptIdentifier(string ident)
		{
			this.identifier =
				String.IsNullOrEmpty(ident) ?
				String.Empty :
				EcmaScriptIdentifier.EnsureValidIdentifier(ident, true);
		}

		#endregion Init

		#region Properties

		/// <summary>
		/// Gets the ECMAScript identifier represented by this instance
		/// </summary>
		public string Identifier
		{
			get { return this.identifier; }
		}

		#endregion Properties

		#region Methods

		/// <summary>
		/// Ensures is a valid EcmaScript variable expression.
		/// </summary>
		/// <param name="varExpr">the variable expression</param>
		/// <returns>varExpr</returns>
		public static string EnsureValidIdentifier(string varExpr, bool nested)
		{
			return EcmaScriptIdentifier.EnsureValidIdentifier(varExpr, nested, true);
		}

		/// <summary>
		/// Ensures is a valid EcmaScript variable expression.
		/// </summary>
		/// <param name="varExpr">the variable expression</param>
		/// <returns>varExpr</returns>
		public static string EnsureValidIdentifier(string varExpr, bool nested, bool throwOnEmpty)
		{
			if (String.IsNullOrEmpty(varExpr))
			{
				if (throwOnEmpty)
				{
					throw new ArgumentException("Variable expression is empty.");
				}
				return String.Empty;
			}

			varExpr = varExpr.Replace(" ", "");

			if (!EcmaScriptIdentifier.IsValidIdentifier(varExpr, nested))
			{
				throw new ArgumentException("Variable expression \""+varExpr+"\" is not supported.");
			}

			return varExpr;
		}

		/// <summary>
		/// Verifies is a valid EcmaScript variable expression.
		/// </summary>
		/// <param name="varExpr">the variable expression</param>
		/// <returns>varExpr</returns>
		/// <remarks>
		/// http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
		/// 
		/// IdentifierName =
		///		IdentifierStart | IdentifierName IdentifierPart
		/// IdentifierStart =
		///		Letter | '$' | '_'
		/// IdentifierPart =
		///		IdentifierStart | Digit
		/// </remarks>
		public static bool IsValidIdentifier(string varExpr, bool nested)
		{
			if (String.IsNullOrEmpty(varExpr))
			{
				return false;
			}

			if (nested)
			{
				string[] parts = varExpr.Split('.');
				foreach (string part in parts)
				{
					if (!EcmaScriptIdentifier.IsValidIdentifier(part, false))
					{
						return false;
					}
				}
				return true;
			}

			if (EcmaScriptIdentifier.IsReservedWord(varExpr))
			{
				return false;
			}

			bool indentPart = false;
			foreach (char ch in varExpr)
			{
				if (indentPart && Char.IsDigit(ch))
				{
					// digits are only allowed after first char
					continue;
				}

				// can be start or part
				if (Char.IsLetter(ch) || ch == '_' || ch == '$')
				{
					indentPart = true;
					continue;
				}

				return false;
			}

			return true;
		}

		private static bool IsReservedWord(string varExpr)
		{
			// TODO: investigate doing this like Rhino does (switch on length check first letter or two)
			switch (varExpr)
			{
				// literals
				case "null":
				case "false":
				case "true":

				// ES5 Keywords
				case "break":
				case "case":
				case "catch":
				case "continue":
				case "debugger":
				case "default":
				case "delete":
				case "do":
				case "else":
				case "finally":
				case "for":
				case "function":
				case "if":
				case "in":
				case "instanceof":
				case "new":
				case "return":
				case "switch":
				case "this":<commit_msg>- cleaning up ident checks<commit_after>#region License
/*---------------------------------------------------------------------------------*\

	Distributed under the terms of an MIT-style license:

	The MIT License

	Copyright (c) 2006-2010 <NAME>

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in
	all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	THE SOFTWARE.

\*---------------------------------------------------------------------------------*/
#endregion License

using System;
using System.IO;

using JsonFx.Common;
using JsonFx.Json;
using JsonFx.Serialization;

namespace JsonFx.EcmaScript
{
	/// <summary>
	/// Represents an ECMAScript identifier for serialization.
	/// </summary>
	public class EcmaScriptIdentifier : ITextFormattable<CommonTokenType>
	{
		#region Fields

		private readonly string identifier;

		#endregion Fields

		#region Init

		/// <summary>
		/// Ctor
		/// </summary>
		public EcmaScriptIdentifier()
			: this(null)
		{
		}

		/// <summary>
		/// Ctor
		/// </summary>
		/// <remarks></remarks>
		public EcmaScriptIdentifier(string ident)
		{
			this.identifier =
				String.IsNullOrEmpty(ident) ?
				String.Empty :
				EcmaScriptIdentifier.VerifyIdentifier(ident, true);
		}

		#endregion Init

		#region Properties

		/// <summary>
		/// Gets the ECMAScript identifier represented by this instance
		/// </summary>
		public string Identifier
		{
			get { return this.identifier; }
		}

		#endregion Properties

		#region Methods

		/// <summary>
		/// Verifies is a valid EcmaScript identifier
		/// </summary>
		/// <param name="ident">the identifier</param>
		/// <returns>identifier</returns>
		public static string VerifyIdentifier(string ident, bool nested)
		{
			return EcmaScriptIdentifier.VerifyIdentifier(ident, nested, true);
		}

		/// <summary>
		/// Verifies is a valid EcmaScript identifier
		/// </summary>
		/// <param name="ident">the identifier</param>
		/// <returns>identifier</returns>
		public static string VerifyIdentifier(string ident, bool nested, bool throwOnEmpty)
		{
			if (String.IsNullOrEmpty(ident))
			{
				if (throwOnEmpty)
				{
					throw new ArgumentException("Identifier is empty.");
				}
				return String.Empty;
			}

			ident = ident.Replace(" ", "");

			if (!EcmaScriptIdentifier.IsValidIdentifier(ident, nested))
			{
				throw new ArgumentException("Identifier \""+ident+"\" is not supported.");
			}

			return ident;
		}

		/// <summary>
		/// Verifies is a valid EcmaScript variable expression
		/// </summary>
		/// <param name="ident">the identifier</param>
		/// <returns>identifier</returns>
		/// <remarks>
		/// http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
		/// 
		/// IdentifierName =
		///		IdentifierStart | IdentifierName IdentifierPart
		/// IdentifierStart =
		///		Letter | '$' | '_'
		/// IdentifierPart =
		///		IdentifierStart | Digit
		/// </remarks>
		public static bool IsValidIdentifier(string ident, bool nested)
		{
			if (String.IsNullOrEmpty(ident))
			{
				return false;
			}

			if (nested)
			{
				string[] parts = ident.Split('.');
				foreach (string part in parts)
				{
					if (!EcmaScriptIdentifier.IsValidIdentifier(part, false))
					{
						return false;
					}
				}
				return true;
			}

			if (EcmaScriptIdentifier.IsReservedWord(ident))
			{
				return false;
			}

			bool indentPart = false;
			for (int i=0, length=ident.Length; i<length; i++)
			{
				char ch = ident[i];
				if (indentPart && ((ch >= '0') && (ch <= '9')))
				{
					// digits are only allowed after first char
					continue;
				}

				// can be start or part
				if (((ch >= 'a') && (ch <= 'z')) ||
					((ch >= 'A') && (ch <= 'Z')) ||
					(ch == '_') || (ch == '$'))
				{
					indentPart = true;
					continue;
				}

				return false;
			}

			return true;
		}

		private static bool IsReservedWord(string ident)
		{
			// TODO: investigate doing this like Rhino does (switch on length check first letter or two)
			switch (ident)
			{
				// literals
				case "null":
				case "false":
				case "true":

				// ES5 Keywords
				case "break":
				case "case":
				case "catch":
				case "continue":
				case "debugger":
				case "default":
				case "delete":
				case "do":
				case "else":
				case "finally":
				case "for":
				case "function":
				case "if":
				case "in":
				case "instanceof":
				case "new":
				case "return":
				case "switch":
				case "this":
{'fraction_non_alphanumeric': 0.14458622630887596, 'fraction_numerical': 0.002533308313004316, 'mean_word_length': 2.593728927848955, 'pattern_counts': {'":': 44, '<': 71, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2393033', 'n_tokens_mistral': 3759, 'n_tokens_neox': 3049, 'n_words': 998}