<commit_before>require 'rdoc'
require 'comment_extractor/extractor'

class CommentExtractor::Extractor::Ruby < CommentExtractor::Extractor
  include CommentExtractor::CodeObject::Comment::Type

  filename /(?:Rakefile|Gemfile|\.rb|\.gemspec|Guardfile)$/
  filetype 'ruby'
  shebang /.*ruby$/

  class Options < Hash
    def tab_width
      2
    end
  end

  def scan
    tokens = RDoc::RubyLex.tokenize(content, Options.new)

    tokens.each do |token|
      case token
      when RDoc::RubyToken::TkRD_COMMENT # =begin ... =end
        token.value.split("\n").each_with_index do |comment, index|
          line_no = token.line_no + 1 + index
          add_comment(line_no, comment, type: BLOCK_COMMENT)
        end
      when RDoc::RubyToken::TkCOMMENT # # ...
        comment = token.value.sub(/^\s*#\s?/, '')
        add_comment(token.line_no, comment, type: ONE_LINER_COMMENT)
      end
    end
  end

  private

  def add_comment(line, comment, **metadata)
    comment_object = build_comment(line, comment, **metadata)
    code_objects << comment_object
  end

  def line_is_comment_of_begin_keyword?(token)
    File.readlines(self.file.path)[token.line_no] == /^=begin/
  end
end<commit_msg>Remove unused line<commit_after>require 'rdoc'
require 'comment_extractor/extractor'

class CommentExtractor::Extractor::Ruby < CommentExtractor::Extractor
  include CommentExtractor::CodeObject::Comment::Type

  filename /(?:Rakefile|Gemfile|\.rb|\.gemspec|Guardfile)$/
  filetype 'ruby'
  shebang /.*ruby$/

  class Options < Hash
    def tab_width
      2
    end
  end

  def scan
    tokens = RDoc::RubyLex.tokenize(content, Options.new)

    tokens.each do |token|
      case token
      when RDoc::RubyToken::TkRD_COMMENT # =begin ... =end
        token.value.split("\n").each_with_index do |comment, index|
          line_no = token.line_no + 1 + index
          add_comment(line_no, comment, type: BLOCK_COMMENT)
        end
      when RDoc::RubyToken::TkCOMMENT # # ...
        comment = token.value.sub(/^\s*#\s?/, '')
        add_comment(token.line_no, comment, type: ONE_LINER_COMMENT)
      end
    end
  end

  private

  def add_comment(line, comment, **metadata)
    comment_object = build_comment(line, comment, **metadata)
    code_objects << comment_object
  end
end
{'fraction_non_alphanumeric': 0.12215411558669002, 'fraction_numerical': 0.0017513134851138354, 'mean_word_length': 3.2314814814814814, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4612268', 'n_tokens_mistral': 783, 'n_tokens_neox': 742, 'n_words': 154}