<commit_before>namespace :dep do
  desc "list dependency gems order by less referenced"
  task :list do
    require "set"
    deps = Set.new
    context = false
    current_parent  = false
    versions = {
      "bundler" => "1.7.4" # bundler version does not appear in Gemfile.lock
    }
    lock_file = ENV["GEMFILE_LOCK"] || "Gemfile.production.lock"
    File.open(lock_file).each_line do |line|
      # GEM
      #   remote: https://rubygems.org/
      #   specs:
      #     foo_gem (0.1.2)
      #       foo_dep1 (>= 1.0.0)
      #       foo_dep2
      #     bar_gem (1.2.1)
      #       bar_dep1
      #       ...
      #   ...
      if line == "  specs:\n"
        context = true
        next
      end
      gem_name = line[/[^ \n]+/, 0] # "    foo_gem (0.1.2)\n" => "foo_gem"
      if line.match(/^ {4}[^ ]/)    # "    foo_gem (0.1.2)"
        current_parent = gem_name
        versions[current_parent] = line[/\((.*?)\)/, 1] # foobar (1.2.3) => 1.2.3<commit_msg>Ensure Gemfile.produciton.lock is up to date before processing<commit_after>namespace :dep do
  desc "list dependency gems order by less referenced"
  task :list do
    require "set"
    deps = Set.new
    context = false
    current_parent  = false
    versions = {
      "bundler" => "1.7.4" # bundler version does not appear in Gemfile.lock
    }
    lock_file = "Gemfile.production.lock"
    unless ENV["SKIP_BUNDLE_INSTALL"]
      system("bundle install --gemfile Gemfile.production", out: STDERR) # ensure lock file is up to date
    end
    File.open(lock_file).each_line do |line|
      # GEM
      #   remote: https://rubygems.org/
      #   specs:
      #     foo_gem (0.1.2)
      #       foo_dep1 (>= 1.0.0)
      #       foo_dep2
      #     bar_gem (1.2.1)
      #       bar_dep1
      #       ...
      #   ...
      if line == "  specs:\n"
        context = true
        next
      end
      gem_name = line[/[^ \n]+/, 0] # "    foo_gem (0.1.2)\n" => "foo_gem"
      if line.match(/^ {4}[^ ]/)    # "    foo_gem (0.1.2)"
        current_parent = gem_name
        versions[current_parent] = line[/\((.*?)\)/, 1] # foobar (1.2.3) => 1.2.3
{'fraction_non_alphanumeric': 0.15051740357478832, 'fraction_numerical': 0.028222013170272814, 'mean_word_length': 1.755181347150259, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306109', 'n_tokens_mistral': 794, 'n_tokens_neox': 775, 'n_words': 175}