<commit_before>﻿using System;

namespace IronAHK.Scripting
{
    partial class Parser
    {
        enum Token { Unknown, Assign, Command, Label, Hotkey, Flow, Expression, Directive }

        Token GetToken(string code)
        {
            code = code.TrimStart(Spaces);

            if (code.Length == 0)
                return Token.Unknown;

            if (IsFlowOperator(code))
                return Token.Flow;
            else if (IsLabel(code))
                return Token.Label;
            else if (IsHotkeyLabel(code) || IsHotstringLabel(code))
                return Token.Hotkey;
            else if (IsAssignment(code))
                return Token.Assign;
            else if (IsDirective(code))
                return Token.Directive;
            else if (IsCommand(code))
                return Token.Command;
            else
                return Token.Expression;
        }

        bool IsSpace(char sym)
        {
            return Array.IndexOf<char>(Spaces, sym) != -1;
        }

        bool IsSpace(string code)
        {
            foreach (char sym in code)
                if (!IsSpace(sym))
                    return false;
            return true;
        }

        bool IsFlowOperator(string code)
        {
            char[] delimiters = new char[Spaces.Length + 1];
            delimiters[0] = Multicast;
            Spaces.CopyTo(delimiters, 1);
            string word = code.Split(delimiters, 2)[0].ToLowerInvariant();

            switch (word)
            {
                case FlowBreak:
                case FlowContinue:
                case FlowElse:
                case FlowGosub:
                case FlowGoto:
                case FlowIf:
                case FlowLoop:
                case FlowReturn:
                case FlowWhile:
                case FunctionLocal:
                case FunctionGlobal:
                case FunctionStatic:
                    return true;
            }

            return false;
        }

        bool IsLabel(string code)
        {
            for (int i = 0; i < code.Length; i++)
            {
                char sym = code[i];

                if (IsIdentifier(sym))
                    continue;

                switch (sym)
                {
                    case HotkeyBound:
                        if (i == 0)
                            return false;
                        else if (i == code.Length - 1)
                            return true;
                        else
                        {
                            string sub = StripCommentSingle(code.Substring(i));
                            return sub.Length == 0 || IsSpace(sub);
                        }
                        break;

                    case ParenOpen:
                    case ParenClose:
                        break;

                    default:
                        return false;
                }
            }

            return false;
        }<commit_msg>Removed unreachable code.<commit_after>﻿using System;

namespace IronAHK.Scripting
{
    partial class Parser
    {
        enum Token { Unknown, Assign, Command, Label, Hotkey, Flow, Expression, Directive }

        Token GetToken(string code)
        {
            code = code.TrimStart(Spaces);

            if (code.Length == 0)
                return Token.Unknown;

            if (IsFlowOperator(code))
                return Token.Flow;
            else if (IsLabel(code))
                return Token.Label;
            else if (IsHotkeyLabel(code) || IsHotstringLabel(code))
                return Token.Hotkey;
            else if (IsAssignment(code))
                return Token.Assign;
            else if (IsDirective(code))
                return Token.Directive;
            else if (IsCommand(code))
                return Token.Command;
            else
                return Token.Expression;
        }

        bool IsSpace(char sym)
        {
            return Array.IndexOf<char>(Spaces, sym) != -1;
        }

        bool IsSpace(string code)
        {
            foreach (char sym in code)
                if (!IsSpace(sym))
                    return false;
            return true;
        }

        bool IsFlowOperator(string code)
        {
            char[] delimiters = new char[Spaces.Length + 1];
            delimiters[0] = Multicast;
            Spaces.CopyTo(delimiters, 1);
            string word = code.Split(delimiters, 2)[0].ToLowerInvariant();

            switch (word)
            {
                case FlowBreak:
                case FlowContinue:
                case FlowElse:
                case FlowGosub:
                case FlowGoto:
                case FlowIf:
                case FlowLoop:
                case FlowReturn:
                case FlowWhile:
                case FunctionLocal:
                case FunctionGlobal:
                case FunctionStatic:
                    return true;
            }

            return false;
        }

        bool IsLabel(string code)
        {
            for (int i = 0; i < code.Length; i++)
            {
                char sym = code[i];

                if (IsIdentifier(sym))
                    continue;

                switch (sym)
                {
                    case HotkeyBound:
                        if (i == 0)
                            return false;
                        else if (i == code.Length - 1)
                            return true;
                        else
                        {
                            string sub = StripCommentSingle(code.Substring(i));
                            return sub.Length == 0 || IsSpace(sub);
                        }

                    case ParenOpen:
                    case ParenClose:
                        break;

                    default:
                        return false;
                }
            }

            return false;
        }
{'fraction_non_alphanumeric': 0.07312552653748947, 'fraction_numerical': 0.0037068239258635214, 'mean_word_length': 0.9443170651817884, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4488495', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1344, 'n_words': 357}