<commit_before>//#define JIT

using System;
using System.Collections.Generic;
using System.Linq;
using NiL.JS.Core;
using NiL.JS.BaseLibrary;
using NiL.JS.Expressions;
using System.Text;

namespace NiL.JS.Statements
{
#if !(PORTABLE || NETCORE)
    [Serializable]
#endif
    public enum SuppressScopeIsolationMode
    {
        Auto,
        Suppress,
        DoNotSuppress
    }

#if !(PORTABLE || NETCORE)
    [Serializable]
#endif
    public sealed class CodeBlock : CodeNode
    {
        private sealed class SuspendData
        {
            public int LineIndex;
            public Context Context;
        }

        internal static readonly VariableDescriptor[] emptyVariables = new VariableDescriptor[0];

        private string code;
#if (NET40 || !NETSTANDARD1_3 && !NET40) && JIT
        internal Func<Context, JSObject> compiledVersion;
#endif
#if DEBUG
        internal HashSet<string> directives;
#endif
        internal VariableDescriptor[] _variables;
        internal CodeNode[] _lines;
        internal bool _strict;
        internal bool built;
        internal SuppressScopeIsolationMode _suppressScopeIsolation;

        public VariableDescriptor[] Variables { get { return _variables; } }
        public CodeNode[] Body { get { return _lines; } }
        public bool Strict { get { return _strict; } }
        public string Code
        {
            get
            {
                var res = ToString();
                return res.Substring(1, res.Length - 2);
            }
        }

        public override int Length
        {
            get
            {
                return base.Length < 0 ? -base.Length : base.Length;
            }
            internal set
            {
                base.Length = value;
            }
        }

        public CodeBlock(CodeNode[] body)
        {
            if (body == null)
                throw new ArgumentNullException("body");

            code = "";
            _lines = body;
            _variables = null;
            _strict = false;
        }

        internal static CodeNode Parse(ParseInfo state, ref int index)
        {
            int position = index;
            bool sroot = position == 0 && state.AllowDirectives;
            if (!sroot)
            {
                if (state.Code[position] != '{')
                    throw new ArgumentException("code (" + position + ")");
                position++;
            }

            Tools.SkipSpaces(state.Code, ref position);

            var body = new List<CodeNode>();
#if DEBUG
            HashSet<string> directives = null;
#endif

            var oldFunctionScopeLevel = state.FunctionScopeLevel;
            state.LexicalScopeLevel++;
            if (state.AllowDirectives)
                state.FunctionScopeLevel = state.LexicalScopeLevel;

            var oldVariablesCount = state.Variables.Count;
            VariableDescriptor[] variables = null;
            state.LabelsCount = 0;

            var allowDirectives = state.AllowDirectives;
            using (state.WithCodeContext())
            {
                state.CodeContext &= ~CodeContext.AllowDirectives;

                try
                {
                    if (allowDirectives)
                    {
                        int start = position;
                        do
                        {
                            var s = position;
                            if (position >= state.Code.Length)
                                break;
                            if (Parser.ValidateValue(state.Code, ref position))
                            {
                                while (position < state.Code.Length && Tools.IsWhiteSpace(state.Code[position]))
                                    position++;
                                if (position < state.Code.Length && (Parser.IsOperator(state.Code[position])
                                    || Parser.Validate(state.Code, "instanceof", position)
                                    || Parser.Validate(state.Code, "in", position)))
                                {
                                    position = s;
                                    break;
                                }
                                var t = s;
                                if (Parser.ValidateString(state.Code, ref t, true))
                                {
                                    var str = state.Code.Substring(s + 1, t - s - 2);
                                    if (str == "use strict")
                                    {
                                        state.CodeContext |= CodeContext.Strict;
                                    }
#if DEBUG
                                    if (directives == null)
                                        directives = new HashSet<string>();

                                    directives.Add(str);
#endif
                                }
                                else
                                {
                                    position = s;
                                    break;
                                }
                            }
                            else if (state.Code[position] == ';')
                            {
#if DEBUG
                                if (directives == null)
                                    break;
#endif

                                do position++; while (position < state.Code.Length && Tools.IsWhiteSpace(state.Code[position]));
                            }
                            else
                                break;
                        }
                        while (true);
                        position = start;
                    }
<commit_msg>Last one fix for tests<commit_after>//#define JIT

using System;
using System.Collections.Generic;
using System.Linq;
using NiL.JS.Core;
using NiL.JS.BaseLibrary;
using NiL.JS.Expressions;
using System.Text;

namespace NiL.JS.Statements
{
#if !(PORTABLE || NETCORE)
    [Serializable]
#endif
    public enum SuppressScopeIsolationMode
    {
        Auto,
        Suppress,
        DoNotSuppress
    }

#if !(PORTABLE || NETCORE)
    [Serializable]
#endif
    public sealed class CodeBlock : CodeNode
    {
        private sealed class SuspendData
        {
            public int LineIndex;
            public Context Context;
        }

        internal static readonly VariableDescriptor[] emptyVariables = new VariableDescriptor[0];

        private string code;
#if (NET40 || !NETSTANDARD1_3 && !NET40) && JIT
        internal Func<Context, JSObject> compiledVersion;
#endif
#if DEBUG
        internal HashSet<string> directives;
#endif
        internal VariableDescriptor[] _variables;
        internal CodeNode[] _lines;
        internal bool _strict;
        internal bool built;
        internal SuppressScopeIsolationMode _suppressScopeIsolation;

        public VariableDescriptor[] Variables { get { return _variables; } }
        public CodeNode[] Body { get { return _lines; } }
        public bool Strict { get { return _strict; } }
        public string Code
        {
            get
            {
                var res = ToString();
                return res.Substring(1, res.Length - 2);
            }
        }

        public override int Length
        {
            get
            {
                return base.Length < 0 ? -base.Length : base.Length;
            }
            internal set
            {
                base.Length = value;
            }
        }

        public CodeBlock(CodeNode[] body)
        {
            if (body == null)
                throw new ArgumentNullException("body");

            code = "";
            _lines = body;
            _variables = null;
            _strict = false;
        }

        internal static CodeNode Parse(ParseInfo state, ref int index)
        {
            int position = index;
            bool sroot = position == 0 && state.AllowDirectives;
            if (!sroot)
            {
                if (state.Code[position] != '{')
                    throw new ArgumentException("code (" + position + ")");
                position++;
            }

            Tools.SkipSpaces(state.Code, ref position);

            var body = new List<CodeNode>();
#if DEBUG
            HashSet<string> directives = null;
#endif

            var oldFunctionScopeLevel = state.FunctionScopeLevel;
            state.LexicalScopeLevel++;
            if (state.AllowDirectives)
                state.FunctionScopeLevel = state.LexicalScopeLevel;

            var oldVariablesCount = state.Variables.Count;
            VariableDescriptor[] variables = null;
            state.LabelsCount = 0;

            var allowDirectives = state.AllowDirectives;
            using (state.WithCodeContext())
            {
                state.CodeContext &= ~CodeContext.AllowDirectives;

                try
                {
                    if (allowDirectives)
                    {
                        int start = position;
                        var hasDirectives = false;
                        do
                        {
                            var s = position;
                            if (position >= state.Code.Length)
                                break;

                            if (Parser.ValidateValue(state.Code, ref position))
                            {
                                while (position < state.Code.Length && Tools.IsWhiteSpace(state.Code[position]))
                                    position++;
                                if (position < state.Code.Length && (Parser.IsOperator(state.Code[position])
                                    || Parser.Validate(state.Code, "instanceof", position)
                                    || Parser.Validate(state.Code, "in", position)))
                                {
                                    position = s;
                                    break;
                                }

                                var t = s;
                                if (Parser.ValidateString(state.Code, ref t, true))
                                {
                                    var str = state.Code.Substring(s + 1, t - s - 2);
                                    if (str == "use strict")
                                    {
                                        state.CodeContext |= CodeContext.Strict;
                                    }

                                    hasDirectives = true;
#if DEBUG
                                    if (directives == null)
                                        directives = new HashSet<string>();

                                    directives.Add(str);
#endif
                                }
                                else
                                {
                                    position = s;
                                    break;
                                }
                            }
                            else if (state.Code[position] == ';')
                            {
                                if (!hasDirectives)
                                    break;

                                do position++; while (position < state.Code.Length && Tools.IsWhiteSpace(state.Code[position]));
                            }
                            else
                                break;
                        }
                        while (true);
                        position = start;
                    }

{'fraction_non_alphanumeric': 0.07779705117085862, 'fraction_numerical': 0.0024284475281873374, 'mean_word_length': 0.9805908622466506, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 47, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5299426', 'n_tokens_mistral': 2624, 'n_tokens_neox': 2464, 'n_words': 611}