<commit_before>module Morion
  class FoldersAndFilesFinder
    EXTENSION_FILTER_REGEX = /.*(\.jpg|\.jpeg|\.png|\.svg)/

    def initialize
      @rails_root = Rails.root.to_s
    end

    def folder_paths
      app_assets_paths = Rails.application.assets.paths.select {|path| path =~ /#{@rails_root}/}
      app_assets_paths.map do |path|
        pathname = Pathname.new(path)
        pathname.exist? ? pathname.cleanpath : nil
      end.compact
    end

    def file_paths
      Find.find(*folder_paths).select do |path|
        path =~ EXTENSION_FILTER_REGEX
      end
    end

    def files_by_folders
      files = file_paths.map do |path|
        File.new(path, folder_paths, @rails_root)
      end

      @folders = files.group_by {|file| file.folder_path }
    end
  end
end<commit_msg>adds filter to ignore blacklisted path<commit_after>module Morion
  class FoldersAndFilesFinder
    EXTENSION_FILTER_REGEX = /.*(\.jpg|\.jpeg|\.png|\.svg)/

    def initialize
      @rails_root = Rails.root.to_s
    end

    def folder_paths
      app_assets_paths = Rails.application.assets.paths.select {|path| path =~ /#{@rails_root}/}
      app_assets_paths.map do |path|
        pathname = Pathname.new(path)
        pathname.exist? ? pathname.cleanpath : nil
      end.compact
    end

    def file_paths
      Find.find(*folder_paths).select do |path|
        path =~ EXTENSION_FILTER_REGEX && permitted_path?(path)
      end
    end

    def files_by_folders
      files = file_paths.map do |path|
        File.new(path, folder_paths, @rails_root)
      end

      @folders = files.group_by {|file| file.folder_path }
    end

    private

    def permitted_path?(path)
      if Morion::Config.blacklist.any?
        !blacklisted?(path)
      else
        true
      end
    end

    def blacklisted?(path)
      include_path?(path, Morion::Config.blacklist)
    end

    def include_path?(path, list)
      list.any? do |filter|
        path.match(/#{filter}/)
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10402010050251256, 'fraction_numerical': 0.0, 'mean_word_length': 2.4268502581755595, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4026665', 'n_tokens_mistral': 684, 'n_tokens_neox': 662, 'n_words': 137}