<commit_before>require_relative "rp_tags"
require_relative "rp_parser"

module RpLogs

  class RpLogGenerator < Jekyll::Generator
    safe true
    priority :normal

    @@parsers = {}

    def RpLogGenerator.add(parser) 
      @@parsers[parser::FORMAT_STR] = parser
    end

    def initialize(config)
      config['rp_convert'] ||= true
    end

    def generate(site)
      return unless site.config['rp_convert']
      @site = site

      # Directory of RPs
      index = site.pages.detect { |page| page.data['rp_index'] }
      index.data['rps'] = {'canon' => [], 'noncanon' => []}

      # Convert all of the posts to be pretty
      # Also build up our hash of tags
      site.pages.select { |p| p.data['layout'] == 'rp' }
        .each { |page|
          # puts page.inspect
          page.data['rp_tags'] = page.data['rp_tags'].split(',').map { |t| Tag.new t }.sort
          
          convertRp page

          key = if page.data['canon'] then 'canon' else 'noncanon' end
          index.data['rps'][key].push page
        }

      index.data['rps']['canon'].sort_by! { |p| p.data['start_date'] }.reverse!
      index.data['rps']['noncanon'].sort_by! { |p| p.data['start_date'] }.reverse!
    end

    def convertRp(page)
      options = get_options page

      compiled_lines = []
      page.content.each_line { |raw_line| 
        page.data['format'].each { |format| 
          log_line = @@parsers[format].parse_line(raw_line, options)
          if log_line then
            compiled_lines << log_line 
            break
          end
        }
      }

      merge_lines! compiled_lines
      stats = extract_stats compiled_lines
<commit_msg>Add canon/noncanon auto-tags<commit_after>require_relative "rp_tags"
require_relative "rp_parser"

module RpLogs

  class RpLogGenerator < Jekyll::Generator
    safe true
    priority :normal

    @@parsers = {}

    def RpLogGenerator.add(parser) 
      @@parsers[parser::FORMAT_STR] = parser
    end

    def initialize(config)
      config['rp_convert'] ||= true
    end

    def generate(site)
      return unless site.config['rp_convert']
      @site = site

      # Directory of RPs
      index = site.pages.detect { |page| page.data['rp_index'] }
      index.data['rps'] = {'canon' => [], 'noncanon' => []}

      # Convert all of the posts to be pretty
      # Also build up our hash of tags
      site.pages.select { |p| p.data['layout'] == 'rp' }
        .each { |page|
          # puts page.inspect
          page.data['rp_tags'] = page.data['rp_tags'].split(',').map { |t| Tag.new t }

          convertRp page

          key = if page.data['canon'] then 'canon' else 'noncanon' end
          # Add key for canon/noncanon
          index.data['rps'][key].push page
          # Add tag for canon/noncanon
          page.data['rp_tags'] << (Tag.new key)
          page.data['rp_tags'].sort!
        }

      index.data['rps']['canon'].sort_by! { |p| p.data['start_date'] }.reverse!
      index.data['rps']['noncanon'].sort_by! { |p| p.data['start_date'] }.reverse!
    end

    def convertRp(page)
      options = get_options page

      compiled_lines = []
      page.content.each_line { |raw_line| 
        page.data['format'].each { |format| 
          log_line = @@parsers[format].parse_line(raw_line, options)
          if log_line then
            compiled_lines << log_line 
            break
          end
        }
      }

      merge_lines! compiled_lines
      stats = extract_stats compiled_lines

{'fraction_non_alphanumeric': 0.13459879206212252, 'fraction_numerical': 0.0, 'mean_word_length': 2.2904446546830655, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7121583', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1099, 'n_words': 245}