<commit_before>module Hammer
  class ContentProxy
    @@variables = { }

    def initialize(source_dir = nil, filename = nil)
      @@variables[:current_parser_filename] = filename if filename
      @@variables[:current_parser_source_dir] = source_dir if source_dir
    end

    def contentful
      @contentful ||= Hammer::ContentfulHelper.new(Settings.contentful)
      @contentful.content_proxy ||= self
      @contentful
    end

    def markdown(text)
      Hammer::MarkdownParser.new.parse(text) if text
    end

    # hack to return "registered variables"
    def method_missing(method_name, *arguments, &block)
      @@variables[method_name] || super
    end

    def fill_exception(exeption)
      exeption.source_path = current_parser_source_dir + '/' + current_parser_filename
      exeption.line = find_file_in_slim_backtrace
      exeption.input_directory = current_parser_source_dir
      exeption
    end

    def find_file_in_slim_backtrace
      line = Thread.current.backtrace.find{ |x| x =~ /TEMPLATE/ }
      match = line.match(/:(\d+)/)
      match ? match[1].to_i : nil
    end

    class << self
      def register_variable(name, value)
        @@variables[name] = value
      end

      def unregister_variable(name)
        @@variables.delete(name)
      end

      def add_paths(paths)
        @@variables['autogenerated_content_files'] ||= []
        @@variables['autogenerated_content_files'].concat(paths)
      end

      def find_file(filename)
        return nil unless @@variables['autogenerated_content_files']
        @@variables['autogenerated_content_files'].select do |path|
          path.match(filename)
        end.max_by(&:length)
      end
    end
  end
end<commit_msg>implement = path(file) helper for slim<commit_after>module Hammer
  class ContentProxy
    include Hammer::FindingFiles
    @@variables = { }

    def initialize(source_dir = nil, filename = nil)
      @@variables[:current_parser_filename] = filename if filename
      @@variables[:current_parser_source_dir] = source_dir if source_dir
    end

    def contentful
      @contentful ||= Hammer::ContentfulHelper.new(Settings.contentful)
      @contentful.content_proxy ||= self
      @contentful
    end

    def markdown(text)
      Hammer::MarkdownParser.new.parse(text) if text
    end

    def path(file)
      find_file(file)
    end

    # hack to return "registered variables"
    def method_missing(method_name, *arguments, &block)
      @@variables[method_name] || super
    end

    def fill_exception(exeption)
      exeption.source_path = current_parser_source_dir + '/' + current_parser_filename
      exeption.line = find_file_in_slim_backtrace
      exeption.input_directory = current_parser_source_dir
      exeption
    end

    def find_file_in_slim_backtrace
      line = Thread.current.backtrace.find{ |x| x =~ /TEMPLATE/ }
      match = line.match(/:(\d+)/)
      match ? match[1].to_i : nil
    end

    class << self
      def register_variable(name, value)
        @@variables[name] = value
      end

      def unregister_variable(name)
        @@variables.delete(name)
      end

      def add_paths(paths)
        @@variables['autogenerated_content_files'] ||= []
        @@variables['autogenerated_content_files'].concat(paths)
      end

      def find_file(filename)
        return nil unless @@variables['autogenerated_content_files']
        @@variables['autogenerated_content_files'].select do |path|
          path.match(filename)
        end.max_by(&:length)
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09824462061155152, 'fraction_numerical': 0.0005662514156285391, 'mean_word_length': 2.992090395480226, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4720998', 'n_tokens_mistral': 1107, 'n_tokens_neox': 1040, 'n_words': 211}