<commit_before>require_relative "rp_parser"
require_relative "rp_logline"
require_relative "rp_page"
require_relative "rp_arcs"
require_relative "rp_tags"

module Jekyll
  module RpLogs
    # Consider renaming since it is more of a converter in practice
    # It can't actually be a subclass of Jekyll::Converter because those work
    # very differently. Converters convert single files, while a generator can
    # work with the entire site.
    class RpLogGenerator < Jekyll::Generator
      safe true
      priority :normal

      @parsers = {}

      class << self
        attr_reader :parsers, :rp_key

        def add(parser)
          @parsers[parser::FORMAT_STR] = parser
        end

        ##
        # Extract global settings from the config file.
        # The rp directory and collection name is pulled out; it must be the
        # first collection defined.
        def extract_settings(config)
          @rp_key = config["collections"].keys[0].freeze
        end
      end

      def initialize(config)
        # Should actually probably complain if things are undefined or missing
        config["rp_convert"] = true unless config.key? "rp_convert"

        RpLogGenerator.extract_settings(config)
        LogLine.extract_settings(config)
        Page.extract_settings(config)

        Jekyll.logger.info "Loaded jekyll-rp_logs #{RpLogs::VERSION}"
      end

      def generate(site)
        return unless site.config["rp_convert"]

        # There doesn't seem to be a better way to add this to all pages than
        # by modifying the configuration file, which is added onto the `site`
        # liquid variable.
        site.config["rp_logs_version"] = RpLogs::VERSION

        Jekyll.logger.info("RpLogGenerator#generate called")

        main_index, arc_index = extract_indexes(site)

        disable_liquid_rendering(site)
        # Pull out all the pages that are error-free
        rp_pages = extract_valid_rps(site)

        convert_all_pages(site, main_index, arc_index, rp_pages)
      end

      private

      ##
      # Convenience method for accessing the collection key name
      def rp_key
        self.class.rp_key
      end

      ##
      #
      def extract_indexes(site)
        # Directory of RPs
        main_index = site.pages.find { |page| page.data["rp_index"] }
        Jekyll.logger.abort_with "Main index page missing" if main_index.nil?
        main_index.data["rps"] = { "canon" => [], "noncanon" => [] }

        # Arc-style directory
        arc_index = site.pages.find { |page| page.data["rp_arcs"] }
        Jekyll.logger.abort_with "Arc index page missing" if arc_index.nil?

        site.data["menu_pages"] = [main_index, arc_index]
      end

      ##
      # Redefine the #render_with_liquid? method for every RP Document. This
      # speeds up the rendering process a little, and also avoids Liquid
      # throwing a fit if someone typed {{ in the log.
      def disable_liquid_rendering(site)
        site.collections[rp_key].docs.each do |doc|
          # https://github.com/jekyll/jekyll/blob/6e8fd8cb50eab4dab527eaaa0b23d08593b9972b/lib/jekyll/document.rb#L150
          def doc.render_with_liquid?
            false
          end
        end
      end

      ##
      # Returns a list of RpLogs::Page objects that are error-free.
      def extract_valid_rps(site)
        site.collections[rp_key].docs.map { |p| RpLogs::Page.new(p) }
          .reject do |p|
            message = p.errors?(self.class.parsers)
            skip_page(site, p, message) if message
            message
          end
      end

      def convert_all_pages(site, main_index, arc_index, rp_pages)
        arcs = Hash.new { |hash, key| hash[key] = Arc.new(key) }
        no_arc_rps = []<commit_msg>Helper function for finding index pages<commit_after>require_relative "rp_parser"
require_relative "rp_logline"
require_relative "rp_page"
require_relative "rp_arcs"
require_relative "rp_tags"

module Jekyll
  module RpLogs
    # Consider renaming since it is more of a converter in practice
    # It can't actually be a subclass of Jekyll::Converter because those work
    # very differently. Converters convert single files, while a generator can
    # work with the entire site.
    class RpLogGenerator < Jekyll::Generator
      safe true
      priority :normal

      @parsers = {}

      class << self
        attr_reader :parsers, :rp_key

        def add(parser)
          @parsers[parser::FORMAT_STR] = parser
        end

        ##
        # Extract global settings from the config file.
        # The rp directory and collection name is pulled out; it must be the
        # first collection defined.
        def extract_settings(config)
          @rp_key = config["collections"].keys[0].freeze
        end
      end

      def initialize(config)
        # Should actually probably complain if things are undefined or missing
        config["rp_convert"] = true unless config.key? "rp_convert"

        RpLogGenerator.extract_settings(config)
        LogLine.extract_settings(config)
        Page.extract_settings(config)

        Jekyll.logger.info "Loaded jekyll-rp_logs #{RpLogs::VERSION}"
      end

      def generate(site)
        return unless site.config["rp_convert"]

        # There doesn't seem to be a better way to add this to all pages than
        # by modifying the configuration file, which is added onto the `site`
        # liquid variable.
        site.config["rp_logs_version"] = RpLogs::VERSION

        Jekyll.logger.info("RpLogGenerator#generate called")

        main_index, arc_index = extract_indexes(site)

        disable_liquid_rendering(site)
        # Pull out all the pages that are error-free
        rp_pages = extract_valid_rps(site)

        convert_all_pages(site, main_index, arc_index, rp_pages)
      end

      private

      ##
      # Convenience method for accessing the collection key name
      def rp_key
        self.class.rp_key
      end

      ##
      #
      def extract_indexes(site)
        # Directory of RPs
        main_index = find_index(site, "rp_index")
        Jekyll.logger.abort_with "Main index page missing" if main_index.nil?
        main_index.data["rps"] = { "canon" => [], "noncanon" => [] }

        # Arc-style directory
        arc_index = find_index(site, "rp_arcs")
        Jekyll.logger.abort_with "Arc index page missing" if arc_index.nil?

        site.data["menu_pages"] = [main_index, arc_index]
      end

      def find_index(site, key)
        site.pages.find { |page| page.data[key] }
      end

      ##
      # Redefine the #render_with_liquid? method for every RP Document. This
      # speeds up the rendering process a little, and also avoids Liquid
      # throwing a fit if someone typed {{ in the log.
      def disable_liquid_rendering(site)
        site.collections[rp_key].docs.each do |doc|
          # https://github.com/jekyll/jekyll/blob/6e8fd8cb50eab4dab527eaaa0b23d08593b9972b/lib/jekyll/document.rb#L150
          def doc.render_with_liquid?
            false
          end
        end
      end

      ##
      # Returns a list of RpLogs::Page objects that are error-free.
      def extract_valid_rps(site)
        site.collections[rp_key].docs.map { |p| RpLogs::Page.new(p) }
          .reject do |p|
            message = p.errors?(self.class.parsers)
            skip_page(site, p, message) if message
            message
          end
      end

      def convert_all_pages(site, main_index, arc_index, rp_pages)
        arcs = Hash.new { |hash, key| hash[key] = Arc.new(key) }
        no_arc_rps = []
{'fraction_non_alphanumeric': 0.08330027766759222, 'fraction_numerical': 0.006611133148221605, 'mean_word_length': 2.4334997730367682, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673786', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2227, 'n_words': 626}