<commit_before>module Khaleesi
  class Generator

    def initialize(src_dir, dest_dir, line_numbers, css_class, time_pattern, date_pattern, diff_plus)
      @src_dir = src_dir
      @dest_dir = dest_dir
      $line_numbers = line_numbers.eql?('true')
      $css_class = css_class
      @time_pattern = time_pattern
      @date_pattern = date_pattern
      @diff_plus = diff_plus.eql?('true')

      # puts "src_dir : #{@src_dir}"
      # puts "dest_dir : #{@dest_dir}"
      # puts "line_numbers : #{$line_numbers}"
      # puts "css_class : #{$css_class}"
      # puts "time_pattern : #{@time_pattern}"
      # puts "date_pattern : #{@date_pattern}"
      # puts "diff_plus : #{@diff_plus}"
    end

    def generate
      @decrt_regexp = produce_variable_regex('decorator')
      @title_regexp = produce_variable_regex('title')
      @var_regexp = /(\p{Word}+):(\p{Word}+)/
      @doc_regexp = /‡{6,}/

      @page_dir = "#{@src_dir}/_pages/"
      @variable_stack = Array.new
      start_time = Time.now

      @page_stack = Array.new

      Dir.glob("#{@page_dir}/**/*") do |page_file|
        next unless File.readable? page_file
        next unless is_valid_file page_file

        @page_stack.clear
        @page_stack.push File.expand_path(page_file)
        single_start_time = Time.now

        if @diff_plus
          file_status = nil
          base_name = File.basename(page_file)
          Dir.chdir(File.expand_path('..', page_file)) do
            file_status = %x[git status -s #{base_name} 2>&1]
          end
          file_status = file_status.to_s.strip

          # only haven't commit pages available, Git will return nothing if page committed.
          next if file_status.empty?

          # a correct message from Git should included the file name, may occur errors
          # in command running such as Git didn't install if not include.
          unless file_status.include? base_name
            puts file_status
            next
          end
        end

        extract_page_structure(page_file)

        variables = @variable_stack.pop
        # page can't stand without decorator
        next unless variables and variables[@decrt_regexp, 3]

        # isn't legal page if title missing
        next unless variables[@title_regexp, 3]

        content = is_html_file(page_file) ? parse_html_file(page_file, '') : parse_markdown_file(page_file)

        page_path = File.expand_path(@dest_dir + get_link(page_file, variables))
        page_dir_path = File.dirname(page_path)
        unless File.directory?(page_dir_path)
          FileUtils.mkdir_p(page_dir_path)
        end

        bytes = IO.write(page_path, content)
        puts "Done (#{Generator.humanize(Time.now - single_start_time)}) => '#{page_path}' bytes[#{bytes}]."
      end

      puts "\nGenerator time elapsed : #{Generator.humanize(Time.now - start_time)}."
    end

    def parse_markdown_file(file_path)
      content = extract_page_structure(file_path)
      content = parse_decorator_file(handle_markdown(content))
      @variable_stack.pop
      content
    end

    def parse_decorator_file(bore_content)
      variables = @variable_stack.last
      decorator = variables ? variables[@decrt_regexp, 3] : nil
      decorator ? parse_html_file("#{@src_dir}/_decorators/#{decorator.strip}.html", bore_content) : bore_content
    end

    def parse_html_file(file_path, bore_content)
      content = extract_page_structure(file_path)

      content = parse_html_content(content.to_s, bore_content)
      content = parse_decorator_file(content) # recurse parse

      @variable_stack.pop
      content
    end

    def parse_html_content(html_content, bore_content)
      parsed_text = handle_html_content(html_content, '')


      # http://www.ruby-doc.org/core-2.1.0/Regexp.html#class-Regexp-label-Repetition use '.+?' to disable greedy match.
      regexp = /(#foreach\p{Blank}?\(\$(\p{Graph}+)\p{Blank}?:\p{Blank}?\$(\p{Graph}+)([^\)]*)\)(.+?)#end)/m
      while (foreach_snippet = parsed_text.match(regexp))
        foreach_snippet = handle_foreach_snippet(foreach_snippet)

        # because the Regexp cannot skip a unhandled foreach snippet,
        # so we claim every snippet must successful, and if not, we shall use blank as replacement.
        parsed_text.sub!(regexp, foreach_snippet.to_s)
      end


      regexp = /(#if\p{Blank}?chain:(prev|next)\(\$(\p{Graph}+)\)(.+?)#end)/m
      while (chain_snippet = parsed_text.match(regexp))
        chain_snippet = handle_chain_snippet(chain_snippet)
        parsed_text.sub!(regexp, chain_snippet.to_s)
      end


      parsed_text.sub!(/\$\{decorator:content}/, bore_content)

      parsed_text
    end

    def handle_html_content(html_content, added_scope)
      page_file = @page_stack.last
      parsed_text = ''
      sub_script = ''

      html_content.each_char do |char|
        is_valid = sub_script.start_with?('${')
        case char
          when '$'
            sub_script.clear << char

          when '{'
            is_valid = sub_script.eql? '$'
            (is_valid ? sub_script : parsed_text) << char

          when ':'
            is_valid = is_valid && sub_script.length > 3
            (is_valid ? sub_script : parsed_text) << char

          when '}'
            is_valid = is_valid && sub_script.length > 4
            if is_valid
              sub_script << char

              form_scope = sub_script[@var_regexp, 1]
              form_value = sub_script[@var_regexp, 2]

              case form_scope
                when 'variable', added_scope

                  case form_value
                    when 'createtime'<commit_msg>an invalid variable expression will output as plain text.<commit_after>module Khaleesi
  class Generator

    def initialize(src_dir, dest_dir, line_numbers, css_class, time_pattern, date_pattern, diff_plus)
      @src_dir = src_dir
      @dest_dir = dest_dir
      $line_numbers = line_numbers.eql?('true')
      $css_class = css_class
      @time_pattern = time_pattern
      @date_pattern = date_pattern
      @diff_plus = diff_plus.eql?('true')

      # puts "src_dir : #{@src_dir}"
      # puts "dest_dir : #{@dest_dir}"
      # puts "line_numbers : #{$line_numbers}"
      # puts "css_class : #{$css_class}"
      # puts "time_pattern : #{@time_pattern}"
      # puts "date_pattern : #{@date_pattern}"
      # puts "diff_plus : #{@diff_plus}"
    end

    def generate
      @decrt_regexp = produce_variable_regex('decorator')
      @title_regexp = produce_variable_regex('title')
      @var_regexp = /(\p{Word}+):(\p{Word}+)/
      @doc_regexp = /‡{6,}/

      @page_dir = "#{@src_dir}/_pages/"
      @variable_stack = Array.new
      start_time = Time.now

      @page_stack = Array.new

      Dir.glob("#{@page_dir}/**/*") do |page_file|
        next unless File.readable? page_file
        next unless is_valid_file page_file

        @page_stack.clear
        @page_stack.push File.expand_path(page_file)
        single_start_time = Time.now

        if @diff_plus
          file_status = nil
          base_name = File.basename(page_file)
          Dir.chdir(File.expand_path('..', page_file)) do
            file_status = %x[git status -s #{base_name} 2>&1]
          end
          file_status = file_status.to_s.strip

          # only haven't commit pages available, Git will return nothing if page committed.
          next if file_status.empty?

          # a correct message from Git should included the file name, may occur errors
          # in command running such as Git didn't install if not include.
          unless file_status.include? base_name
            puts file_status
            next
          end
        end

        extract_page_structure(page_file)

        variables = @variable_stack.pop
        # page can't stand without decorator
        next unless variables and variables[@decrt_regexp, 3]

        # isn't legal page if title missing
        next unless variables[@title_regexp, 3]

        content = is_html_file(page_file) ? parse_html_file(page_file, '') : parse_markdown_file(page_file)

        page_path = File.expand_path(@dest_dir + get_link(page_file, variables))
        page_dir_path = File.dirname(page_path)
        unless File.directory?(page_dir_path)
          FileUtils.mkdir_p(page_dir_path)
        end

        bytes = IO.write(page_path, content)
        puts "Done (#{Generator.humanize(Time.now - single_start_time)}) => '#{page_path}' bytes[#{bytes}]."
      end

      puts "\nGenerator time elapsed : #{Generator.humanize(Time.now - start_time)}."
    end

    def parse_markdown_file(file_path)
      content = extract_page_structure(file_path)
      content = parse_decorator_file(handle_markdown(content))
      @variable_stack.pop
      content
    end

    def parse_decorator_file(bore_content)
      variables = @variable_stack.last
      decorator = variables ? variables[@decrt_regexp, 3] : nil
      decorator ? parse_html_file("#{@src_dir}/_decorators/#{decorator.strip}.html", bore_content) : bore_content
    end

    def parse_html_file(file_path, bore_content)
      content = extract_page_structure(file_path)

      content = parse_html_content(content.to_s, bore_content)
      content = parse_decorator_file(content) # recurse parse

      @variable_stack.pop
      content
    end

    def parse_html_content(html_content, bore_content)
      parsed_text = handle_html_content(html_content, '')


      # http://www.ruby-doc.org/core-2.1.0/Regexp.html#class-Regexp-label-Repetition use '.+?' to disable greedy match.
      regexp = /(#foreach\p{Blank}?\(\$(\p{Graph}+)\p{Blank}?:\p{Blank}?\$(\p{Graph}+)([^\)]*)\)(.+?)#end)/m
      while (foreach_snippet = parsed_text.match(regexp))
        foreach_snippet = handle_foreach_snippet(foreach_snippet)

        # because the Regexp cannot skip a unhandled foreach snippet,
        # so we claim every snippet must successful, and if not, we shall use blank as replacement.
        parsed_text.sub!(regexp, foreach_snippet.to_s)
      end


      regexp = /(#if\p{Blank}?chain:(prev|next)\(\$(\p{Graph}+)\)(.+?)#end)/m
      while (chain_snippet = parsed_text.match(regexp))
        chain_snippet = handle_chain_snippet(chain_snippet)
        parsed_text.sub!(regexp, chain_snippet.to_s)
      end


      parsed_text.sub!(/\$\{decorator:content}/, bore_content)

      parsed_text
    end

    def handle_html_content(html_content, added_scope)
      page_file = @page_stack.last
      parsed_text = ''
      sub_script = ''

      html_content.each_char do |char|
        is_valid = sub_script.start_with?('${')
        case char
          when '$'
            parsed_text << sub_script unless sub_script.empty?
            sub_script.clear << char

          when '{'
            is_valid = sub_script.eql? '$'
            (is_valid ? sub_script : parsed_text) << char

          when ':'
            is_valid = is_valid && sub_script.length > 3
            (is_valid ? sub_script : parsed_text) << char

          when '}'
            is_valid = is_valid && sub_script.length > 4
            if is_valid
              sub_script << char

              form_scope = sub_script[@var_regexp, 1]
              form_value = sub_script[@var_regexp, 2]

              case form_scope
                when 'variable', added_scope

                  case form_value
                    when 'createtime'
{'fraction_non_alphanumeric': 0.10935163191695259, 'fraction_numerical': 0.0022873229524060876, 'mean_word_length': 2.5547217010631647, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611743', 'n_tokens_mistral': 3629, 'n_tokens_neox': 3489, 'n_words': 727}