<commit_before>require 'muskox/types'

module Muskox

  class ParserError < StandardError
  end

  class UnexpectedProperty < ParserError
    attr_reader :unexpected_property, :allowed_properties

    def initialize unexpected_property, allowed_properties
      super "Unexpected property: [#{unexpected_property}] at root. Allowed properties: [#{allowed_properties.join(", ")}]"
      @unexpected_property = unexpected_property
      @allowed_properties = allowed_properties
    end
  end

  class Parser
    attr_reader :schema

    def initialize schema
      @schema = schema
    end

    def parse input
      schema_stack = [schema]
      stack = [[:root, nil]]
      lexer = Pure::Lexer.new input, :quirks_mode => true
      lexer.lex do |type, value|
#        puts "token #{type}: #{value}"
#        puts "stack #{stack.inspect}"
#        puts "schema stack #{schema_stack.last["type"]}"
        case type
        when :property
          unless expected_property? schema_stack, value
            raise UnexpectedProperty.new value,
                                         (schema_stack.last["properties"]&& schema_stack.last["properties"].keys||[])<commit_msg>replace ugly andand with fetch<commit_after>require 'muskox/types'

module Muskox

  class ParserError < StandardError
  end

  class UnexpectedProperty < ParserError
    attr_reader :unexpected_property, :allowed_properties

    def initialize unexpected_property, allowed_properties
      super "Unexpected property: [#{unexpected_property}] at root. Allowed properties: [#{allowed_properties.join(", ")}]"
      @unexpected_property = unexpected_property
      @allowed_properties = allowed_properties
    end
  end

  class Parser
    attr_reader :schema

    def initialize schema
      @schema = schema
    end

    def parse input
      schema_stack = [schema]
      stack = [[:root, nil]]
      lexer = Pure::Lexer.new input, :quirks_mode => true
      lexer.lex do |type, value|
#        puts "token #{type}: #{value}"
#        puts "stack #{stack.inspect}"
#        puts "schema stack #{schema_stack.last["type"]}"
        case type
        when :property
          unless expected_property? schema_stack, value
            raise UnexpectedProperty.new value,
                                         schema_stack.last.fetch("properties", {}).keys
{'fraction_non_alphanumeric': 0.09411259131929522, 'fraction_numerical': 0.0, 'mean_word_length': 2.847933884297521, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73241', 'n_tokens_mistral': 630, 'n_tokens_neox': 615, 'n_words': 171}