<commit_before>module Middleman
  module Robots
    class Extension < ::Middleman::Extension
      option :rules, [], 'List of rules about sitemap.xml'
      option :sitemap, false, 'URI of sitemap.xml'

      def initialize(app, options_hash = {}, &block)
        super

        data = rules(options.rules) + sitemap(options.sitemap)
        data.gsub!(/\n+$/, "\n")

        build_dir = app.build_dir
        app.after_build do
          File.open(File.join(build_dir, "robots.txt"), "w") do |file|
            file.puts(data)
          end
          logger.info '== middleman-robots: robots.txt created =='
        end
      end

      def rules(rules)
        return '' if rules.empty?

        data = []
        rules.each do |rule|
          row = []

          if (rule["user-agent"] || rule[:user_agent])
            user_agent = rule[:user_agent] || rule["user-agent"]
            row << "User-Agent: #{user_agent}"
          end

          if (rule[:disallow])
            rule[:disallow].each do |path|
              path = "/" + path unless /^\// =~ path
              row << "Disallow: #{path}"
            end
          end

          if (rule[:allow])
            rule[:allow].each do |path|
              path = "/" + path unless /^\// =~ path
              row << "Allow: #{path}"
            end
          end

          data << row.join("\n") + "\n\n" if row.length > 0
        end

        data.join('')
      end

      def sitemap(path)
        path ? "Sitemap: #{path}" : ""
      end
    end
  end
end
<commit_msg>Refactor with Rubocop<commit_after>module Middleman
  module Robots
    # Robots Extension Class
    #
    # Create robots.txt when `$ middleman build`
    class Extension < ::Middleman::Extension
      option :rules, [], 'List of rules about sitemap.xml'
      option :sitemap, false, 'URI of sitemap.xml'

      def initialize(app, options_hash = {}, &block)
        super
        build_dir = app.build_dir

        data = rules(options.rules) + sitemap(options.sitemap)
        data.gsub!(/\n+$/, "\n")

        app.after_build do
          File.open(File.join(build_dir, 'robots.txt'), 'w') do |file|
            file.puts(data)
          end
          logger.info '== middleman-robots: robots.txt created =='
        end
      end

      def rules(rules)
        return '' if rules.empty?
        data = []
        rules.each do |rule|
          row = []
          row << user_agent(rule)
          row << disallow(rule)
          row << allow(rule)
          row.compact!
          data << row.join("\n") + "\n\n" if row.length > 0
        end
        data.join('')
      end

      def user_agent(rule)
        return unless rule.key?('user-agent') || rule.key?(:user_agent)
        user_agent = rule[:user_agent] || rule['user-agent']
        "User-Agent: #{user_agent}"
      end

      def disallow(rule)
        return unless rule.key?(:disallow)
        lines = []
        rule[:disallow].each do |path|
          path = File.join('/', path) unless /^\// =~ path
          lines << "Disallow: #{path}"
        end
        lines
      end

      def allow(rule)
        return unless rule.key?(:allow)
        lines = []
        rule[:allow].each do |path|
          path = File.join('/' + path) unless /^\// =~ path
          lines << "Allow: #{path}"
        end
        lines
      end

      def sitemap(path)
        path ? "Sitemap: #{path}" : ''
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.14144065325167687, 'fraction_numerical': 0.0005832604257801109, 'mean_word_length': 1.636433512682552, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '343896', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1038, 'n_words': 257}