<commit_before>require 'digest'

module Nydp
  PLUGINS = []
  COMMENT_RX = /^.*##> /

  module PluginHelper
    def base_path ; "" ; end # override this to provide common prefix for plugin filenames

    def file_readers filenames, bp=base_path
      filenames.map { |f| Nydp::FileReader.new(f.gsub(bp, ""), f) }
    end

    def relative_path name
      # File.join File.expand_path(File.dirname(__FILE__)), name
      File.join File.expand_path(File.dirname(caller[0].split(/:\d+:/)[0])), name
    end
  end

  def self.nydp_from_backtrace str
    file, original_line, meth = str.split(/:/)
    line = original_line.to_i - 1
    filepath = File.expand_path file

    if filepath.start_with? base_gen_path
      code = File.read filepath
      lines = code.split /\n/

      while line > 0 && !(lines[line] =~ COMMENT_RX)
        line = line - 1
      end

      if (line >= 0 && (lines[line] =~ COMMENT_RX))
        comment = lines[line].sub(COMMENT_RX, '')
        return [(filepath.sub(base_gen_path + '/', '') + ":" + original_line), comment].join("\n")
      else
        return str
      end
    end

    str
  end

  def self.enhance_backtrace bt
    bt.map { |s| nydp_from_backtrace s }
  end
<commit_msg>errors: improve source-mapping a little bit more<commit_after>require 'digest'

module Nydp
  PLUGINS = []
  COMMENT_RX = /^.*##> /

  module PluginHelper
    def base_path ; "" ; end # override this to provide common prefix for plugin filenames

    def file_readers filenames, bp=base_path
      filenames.map { |f| Nydp::FileReader.new(f.gsub(bp, ""), f) }
    end

    def relative_path name
      # File.join File.expand_path(File.dirname(__FILE__)), name
      File.join File.expand_path(File.dirname(caller[0].split(/:\d+:/)[0])), name
    end
  end

  def self.nydp_from_backtrace str
    file, original_line, meth = str.split(/:/)
    line = original_line.to_i - 2 # -1 to convert from 1-based index to zero-based index ; -1 to start looking backwards from previous line
    filepath = File.expand_path file

    if filepath.start_with? base_gen_path
      code = File.read filepath
      lines = code.split /\n/

      while line > 0 && !(lines[line] =~ COMMENT_RX)
        line = line - 1
      end

      if (line >= 0 && (lines[line] =~ COMMENT_RX))
        comment = lines[line].sub(COMMENT_RX, '')
        return [(filepath.sub(base_gen_path + '/', '') + ":" + original_line), comment].join("\n")
      else
        return str
      end
    end

    str
  end

  def self.enhance_backtrace bt
    bt.map { |s| nydp_from_backtrace s }
  end

{'fraction_non_alphanumeric': 0.12767613857532115, 'fraction_numerical': 0.0058388478007006615, 'mean_word_length': 2.645390070921986, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1156880', 'n_tokens_mistral': 908, 'n_tokens_neox': 870, 'n_words': 217}