<commit_before>module Deadlink
  class Scanner
    def initialize(target_dir)

      if target_dir.nil?
        @repo_root = repo_root(".")
        @target_dir = @repo_root
      else
        validate(target_dir)
        @target_dir = target_dir
        @repo_root = repo_root(@target_dir)
      end
    end

    def md_files
      if File.directory?(@target_dir)
        Dir.glob(File.join(@target_dir, '/**/*.{md,markdown}'))
      else
        files = []
        files.push(@target_dir)
      end
    end

    def paths(files)
      paths = []
      files.each do |file|
        File.open(file) do |f|
          f.each_with_index do |line, index|
            line.scan /\[[^\]]*\]\(([^)]+)\)/ do |link|
              paths.push Path.new(f.path, link[0], index + 1, @repo_root)
            end
          end
        end
      end
      Paths.new(paths)
    end

    private

    def repo_root(target_dir)
      dir = target_dir
      until dir.empty? do
        return dir if git_repo?(dir)
        dir = prev_dir(dir)
      end
      dir
    end

    def git_repo?(dir)
      Dir.exist?(File.join(dir, ".git"))
    end

    def prev_dir(dir)
      return "" if dir == "/"
      File.expand_path("../", dir)
    end

    def validate(target_dir)
      unless FileTest.exist?(target_dir)
        raise ArgumentError, "wrong path"
      end
    end

  end
end<commit_msg>more cool message<commit_after>module Deadlink
  class Scanner
    def initialize(target_dir)

      if target_dir.nil?
        @repo_root = repo_root(".")
        @target_dir = @repo_root
      else
        validate(target_dir)
        @target_dir = target_dir
        @repo_root = repo_root(@target_dir)
      end
    end

    def md_files
      if File.directory?(@target_dir)
        Dir.glob(File.join(@target_dir, '/**/*.{md,markdown}'))
      else
        files = []
        files.push(@target_dir)
      end
    end

    def paths(files)
      paths = []
      files.each do |file|
        File.open(file) do |f|
          f.each_with_index do |line, index|
            line.scan /\[[^\]]*\]\(([^)]+)\)/ do |link|
              paths.push Path.new(f.path, link[0], index + 1, @repo_root)
            end
          end
        end
      end
      Paths.new(paths)
    end

    private

    def repo_root(target_dir)
      dir = target_dir
      until dir.empty? do
        return dir if git_repo?(dir)
        dir = prev_dir(dir)
      end
      dir
    end

    def git_repo?(dir)
      Dir.exist?(File.join(dir, ".git"))
    end

    def prev_dir(dir)
      return "" if dir == "/"
      File.expand_path("../", dir)
    end

    def validate(target_dir)
      unless FileTest.exist?(target_dir)
        raise ArgumentError, target_dir + ": No such file or directory"
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.12103746397694524, 'fraction_numerical': 0.001440922190201729, 'mean_word_length': 1.76318407960199, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3792357', 'n_tokens_mistral': 927, 'n_tokens_neox': 907, 'n_words': 207}