<commit_before>class StarScope::Datum

  def self.build(file, args)
    args[:file] = file

    if args[:line_no]
      args[:line] = File.readlines(file)[args[:line_no]-1].chomp
    end

    if args[:scope]
      args[:scope] = args[:scope].map {|x| x.to_sym}
    end

    args
  end

  def self.score_match(dat, fqn)
    score = 0

    i = -1
    fqn[0...-1].reverse.each do |test|
      if test.to_sym == dat[:scope][i]
        score += 5
      elsif Regexp.new(test, Regexp::IGNORECASE).match(dat[:scope][i])
        score += 2
      end
      i -= 1
    end

    score - dat[:scope].count - i + 1
  end

  def self.location(dat)
    "#{dat[:file]}:#{dat[:line_no]}"
  end

  def self.to_s(key, dat)
    str = ""
    str << "#{dat[:scope].join " "} " unless dat[:scope].empty?
    str << "#{key} -- #{location dat}"
    str << " (#{dat[:line].strip})"
  end

  def self.ctag_line(key, dat)
    "#{key}\t#{dat[:file]}\t/^#{dat[:line]}$/;"
  end
end<commit_msg>Don't rely on having a scope<commit_after>class StarScope::Datum

  def self.build(file, args)
    args[:file] = file

    if args[:line_no]
      args[:line] = File.readlines(file)[args[:line_no]-1].chomp
    end

    if args[:scope]
      if args[:scope].empty?
        args.delete(:scope)
      else
        args[:scope] = args[:scope].map {|x| x.to_sym}
      end
    end

    args
  end

  def self.score_match(dat, fqn)
    return 0 if not dat[:scope]

    score = 0

    i = -1
    fqn[0...-1].reverse.each do |test|
      if test.to_sym == dat[:scope][i]
        score += 5
      elsif Regexp.new(test, Regexp::IGNORECASE).match(dat[:scope][i])
        score += 2
      end
      i -= 1
    end

    score - dat[:scope].count - i + 1
  end

  def self.location(dat)
    "#{dat[:file]}:#{dat[:line_no]}"
  end

  def self.to_s(key, dat)
    str = ""
    str << "#{dat[:scope].join " "} " if dat[:scope]
    str << "#{key} -- #{location dat}"
    str << " (#{dat[:line].strip})"
  end

  def self.ctag_line(key, dat)
    "#{key}\t#{dat[:file]}\t/^#{dat[:line]}$/;"
  end
end
{'fraction_non_alphanumeric': 0.19530102790014683, 'fraction_numerical': 0.00930004894762604, 'mean_word_length': 2.361842105263158, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1513043', 'n_tokens_mistral': 821, 'n_tokens_neox': 787, 'n_words': 164}