<commit_before>require 'rexml/document'


module KotlinGrammar
  class Generator < Jekyll::Generator
    def generate(site)
      file_path = "#{File.dirname(__FILE__)}/../#{site.config['grammar_file_path']}"

      unless File.file?(file_path)
        return nil
      end

      data = []
      file_contents = File.read(file_path)
      doc = REXML::Document.new file_contents
      grammar_page = site.pages.detect {|page| page.name.downcase.split('.')[0] == 'grammar'}

      # set
      doc.elements.each('/tokens/set') do |node|
        set = {
            'file-name' => node.attributes['file-name'],
            'content' => []
        }

        node.elements.each('./*') do |node|
          case node.name
            # comment
            when 'doc'
              set['content'].push({
                'type' => 'comment',
                'content' => node.text
              })

            # item<commit_msg>Kotlin grammar generator improvements<commit_after>require 'rexml/document'


module KotlinGrammar
  class Generator < Jekyll::Generator
    def generate(site)
      file_path = "#{File.dirname(__FILE__)}/../#{site.config['grammar_file_path']}"

      unless File.file?(file_path)
        return nil
      end

      data = []
      file_contents = File.read(file_path)
      doc = REXML::Document.new file_contents
      grammar_page = site.pages.detect {|page| page.name.downcase.split('.')[0] == 'grammar'}

      unless grammar_page
        return nil
      end

      # set
      doc.elements.each('/tokens/set') do |node|
        set = {
            'file-name' => node.attributes['file-name'],
            'content' => []
        }

        node.elements.each('./*') do |node|
          case node.name
            # comment
            when 'doc'
              set['content'].push({
                'type' => 'comment',
                'content' => node.text
              })

            # item
{'fraction_non_alphanumeric': 0.14241164241164242, 'fraction_numerical': 0.0010395010395010396, 'mean_word_length': 1.814327485380117, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2998056', 'n_tokens_mistral': 578, 'n_tokens_neox': 562, 'n_words': 105}