<commit_before>class Middleman::Extensions::ResourceHelpers < ::Middleman::Extension
  def initialize(app, options_hash={})
    super
    require 'rest_client'
  end

  helpers do
    def include_javascripts(javascripts)
      include_assets(:javascript_include_tag, javascripts)
    end

    def include_stylesheets(stylesheets)
      include_assets(:stylesheet_link_tag, stylesheets)
    end

    def resource_file(resource)
      resource.path.split('/').last.gsub(resource.ext, '')
    end

    def resource_dir(resource)
      resource.url.split('/').last
    end

    def resources_for(dir, ext: 'html', exclude_indexes: false, sort_by: nil, ignore: nil)
      filtered = sitemap.resources
        .reject  {|r| r.url == dir}                      # Exclude main directory index
        .reject  {|r| r.data.hidden}                     # reject hidden (Front matter)
        .select  {|r| r.url.start_with?(dir)}            # Select files in the given dir

      collect_resources(filtered,
        { ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore}
      )
    end

    def resource_tree(dir, ext: 'html', exclude_indexes: false, sort_by: nil, ignore: nil)
      res = resources_for(dir, ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore)
      res
        .select { |r| r.children.any? }
        .map { |r| parse_resource(r, { ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore }) }
        .reject { |r| r.parent != '/' }  # Remove non-root directories from the root
    end

    def local_data(path)
      result = sitemap.find_resource_by_path(relative_dir(current_page.path, path).to_s)
      raise "#{path} not found" unless result

      case result.ext
      when '.yaml', '.yml'
        result = YAML.load(result.render)
      when '.json'
        result = Oj.load(result.render)
      end

      result
    end

    # Use in layouts, in templates either Frontmatter or this method can be used
    def append_class(block_name, appended_classes='')
      current_page_classes = current_page.data[block_name] || ''
      existing_classes_a = current_page_classes.split(' ')
      appended_classes_a = appended_classes.split(' ')
      classes = existing_classes_a.concat(appended_classes_a).reverse.join(' ')
      content_for(block_name, classes)
    end

    def get(url, options = {})
      begin
        result = RestClient.get(url, options)
      rescue => e
        raise "GET - #{e.message}: '#{url}'"
      end

      Oj.load(result)
    end

    def post(url, params = {}, headers = {})
      begin
        result = RestClient.post(url, params, headers)
      rescue => e
        raise "POST - #{e.message}: '#{url}'"
      end

      Oj.load(result)
    end

    private

    def collect_resources(resources, options)
      resources
        .select  {|r| r.ext == ".#{options[:ext]}"}      # Select files only HTML files
        .sort_by {|r| r.url }                            # Sort by url (default)
        .sort_by {|r| r.data[options[:sort_by]] || -1}   # Sort by `sort_by` param
        .reject  {|r|                                    # Exclude all directory indexes
          options[:exclude_indexes] ? r.directory_index? : false
        }
        .reject  {|r| ignore ? r.url.match(options[:ignore]) : false }  # Ignore URLs matching pattern (if provided)
        .reject  {|r| r.path.end_with? ("-standalone#{r.ext}")}         # Ignore proxied '-standalone' mode pages
    end

    def parse_resource(r, options)
      data = {}
      data[:title] = r.data.title || resource_dir(r)
      data[:url] = r.url
      data[:children] = collect_resources(r.children, options).map { |c| parse_resource(c, options) } if r.children.any?
      data[:parent] = r.parent.url
      data
    end<commit_msg>Exclude partials from resource list<commit_after>class Middleman::Extensions::ResourceHelpers < ::Middleman::Extension
  def initialize(app, options_hash={})
    super
    require 'rest_client'
  end

  helpers do
    def include_javascripts(javascripts)
      include_assets(:javascript_include_tag, javascripts)
    end

    def include_stylesheets(stylesheets)
      include_assets(:stylesheet_link_tag, stylesheets)
    end

    def resource_file(resource)
      resource.path.split('/').last.gsub(resource.ext, '')
    end

    def resource_dir(resource)
      resource.url.split('/').last
    end

    def resources_for(dir, ext: 'html', exclude_indexes: false, sort_by: nil, ignore: nil)
      filtered = sitemap.resources
        .reject  {|r| r.url == dir}                      # Exclude main directory index
        .reject  {|r| r.data.hidden}                     # reject hidden (Front matter)
        .select  {|r| r.url.start_with?(dir)}            # Select files in the given dir

      collect_resources(filtered,
        { ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore}
      )
    end

    def resource_tree(dir, ext: 'html', exclude_indexes: false, sort_by: nil, ignore: nil)
      res = resources_for(dir, ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore)
      res
        .select { |r| r.children.any? }
        .map { |r| parse_resource(r, { ext: ext, exclude_indexes: exclude_indexes, sort_by: sort_by, ignore: ignore }) }
        .reject { |r| r.parent != '/' }  # Remove non-root directories from the root
    end

    def local_data(path)
      result = sitemap.find_resource_by_path(relative_dir(current_page.path, path).to_s)
      raise "#{path} not found" unless result

      case result.ext
      when '.yaml', '.yml'
        result = YAML.load(result.render)
      when '.json'
        result = Oj.load(result.render)
      end

      result
    end

    # Use in layouts, in templates either Frontmatter or this method can be used
    def append_class(block_name, appended_classes='')
      current_page_classes = current_page.data[block_name] || ''
      existing_classes_a = current_page_classes.split(' ')
      appended_classes_a = appended_classes.split(' ')
      classes = existing_classes_a.concat(appended_classes_a).reverse.join(' ')
      content_for(block_name, classes)
    end

    def get(url, options = {})
      begin
        result = RestClient.get(url, options)
      rescue => e
        raise "GET - #{e.message}: '#{url}'"
      end

      Oj.load(result)
    end

    def post(url, params = {}, headers = {})
      begin
        result = RestClient.post(url, params, headers)
      rescue => e
        raise "POST - #{e.message}: '#{url}'"
      end

      Oj.load(result)
    end

    private

    def collect_resources(resources, options)
      resources
        .select  {|r| r.ext == ".#{options[:ext]}"}      # Select files only HTML files
        .sort_by {|r| r.url }                            # Sort by url (default)
        .sort_by {|r| r.data[options[:sort_by]] || -1}   # Sort by `sort_by` param
        .reject  {|r|                                    # Exclude all directory indexes
          options[:exclude_indexes] ? r.directory_index? : false
        }
        .reject  {|r| ignore ? r.url.match(options[:ignore]) : false }  # Ignore URLs matching pattern (if provided)
        .reject  {|r| r.path.end_with? ("-standalone#{r.ext}")}         # Ignore proxied '-standalone' mode pages
        .reject  {|r| r.path.start_with? ("partials/")}                 # Ignore partials
    end

    def parse_resource(r, options)
      data = {}
      data[:title] = r.data.title || resource_dir(r)
      data[:url] = r.url
      data[:children] = collect_resources(r.children, options).map { |c| parse_resource(c, options) } if r.children.any?
      data[:parent] = r.parent.url
      data
    end
{'fraction_non_alphanumeric': 0.12105535437144335, 'fraction_numerical': 0.0002586652871184687, 'mean_word_length': 2.4630541871921183, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4218820', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2348, 'n_words': 583}