<commit_before>﻿using System;
using System.Collections.Generic;
using JSONx.AST;
using JSONx.Lexers;

namespace JSONx.Parsers
{
    public partial class Parser
    {
        private readonly List<Token> _tokens;
        private readonly int _tokensCount;
        protected int _index;


        public Parser(List<Token> tokens)
        {
            if(tokens == null)
                throw new ArgumentNullException(nameof(tokens));
            if (tokens.Count == 0)
                throw new ArgumentException("List of token can't be empty.", nameof(tokens));

            _tokens = tokens;
            _tokensCount = tokens.Count;
            _index = 0;
        }

        public JSONxNode Parse()
        {
            var value = Ensure(ParseValue, "Expected <value> got {0}", _tokens[_index]);
            Ensure(TokenType.EOF, "Expected <EOF> got {0}", _tokens[_index]);
            return value;
        }

        protected JSONxNode ParseValue()
        {
            return Attempt(ParseTrue) ??
                   Attempt(ParseFalse) ??
                   Attempt(ParseNull) ??
                   Attempt(ParseString) ??
                   Attempt(ParseNumber) ??
                   Attempt(ParseList) ??
                   Attempt(ParseObject);
        }

        protected JSONxNode ParseObject()
        {
            if (!Check(TokenType.LeftCurlyBracket)) return null;
            var props = Attempt(ParsePropertyList);
            Ensure(TokenType.RightCurlyBracket, "Expected '}}' got {0}", _tokens[_index].Type);
            return new ObjectNode(props);
        }<commit_msg>little hack<commit_after>﻿using System;
using System.Collections.Generic;
using JSONx.AST;
using JSONx.Lexers;

namespace JSONx.Parsers
{
    public partial class Parser
    {
        private readonly List<Token> _tokens;
        private readonly int _tokensCount;
        protected int _index;


        public Parser(List<Token> tokens)
        {
            if(tokens == null)
                throw new ArgumentNullException(nameof(tokens));
            if (tokens.Count == 0)
                throw new ArgumentException("List of token can't be empty.", nameof(tokens));

            _tokens = tokens;
            _tokensCount = tokens.Count;
            _index = 0;
        }

        public JSONxNode Parse()
        {
            var value = Ensure(ParseValue, "Expected <value> got {0}", _tokens[_index]);
            Ensure(TokenType.EOF, "Expected <EOF> got {0}", _tokens[_index]);
            return value;
        }

        protected JSONxNode ParseValue()
        {
            return Attempt(ParseTrue) ??
                   Attempt(ParseFalse) ??
                   Attempt(ParseNull) ??
                   Attempt(ParseString) ??
                   Attempt(ParseNumber) ??
                   Attempt(ParseList) ??
                   (JSONxNode)Attempt(ParseObject); // HACK
        }

        protected ObjectNode ParseObject()
        {
            if (!Check(TokenType.LeftCurlyBracket)) return null;
            var props = Attempt(ParsePropertyList);
            Ensure(TokenType.RightCurlyBracket, "Expected '}}' got {0}", _tokens[_index].Type);
            return new ObjectNode(props);
        }
{'fraction_non_alphanumeric': 0.09819763828464885, 'fraction_numerical': 0.003107520198881293, 'mean_word_length': 1.8040069686411149, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7623194', 'n_tokens_mistral': 850, 'n_tokens_neox': 803, 'n_words': 200}