<commit_before>class Bun
  class Archive
    class IndexBot < Thor
    
      desc "build", "Build file index for archive"
      option 'archive', :aliases=>'-a', :type=>'string',  :desc=>'Archive location'
      option 'quiet',   :aliases=>'-q', :type=>'boolean', :desc=>'Run quietly'
      def build
        # TODO the following two lines are a common pattern; refactor
        directory = options[:archive] || Archive.location
        archive = Archive.new(directory)
        archive.build_and_save_index(:verbose=>!options[:quiet])
      end
    
      desc "clear", "Clear file index for archive"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      def clear
        # TODO the following two lines are a common pattern; refactor
        directory = options[:archive] || Archive.location
        archive = Archive.new(directory)
        archive.clear_index
      end

      desc "ls_original", "List the original index file for the archive"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      def ls_original
        archive = Archive.new(options[:archive])
        # TODO Use Array.justify_rows
        archive.original_index.each do |spec|
          puts "#{spec[:tape]}  #{spec[:date].strftime('%Y/%d/%m')}  #{spec[:file]}"
        end
      end
      
      desc "set_dates", "Set file modification dates for archived files, based on original index"
      option 'archive', :aliases=>'-a', :type=>'string',  :desc=>'Archive location'
      option 'dryrun',  :aliases=>'-d', :type=>'boolean', :desc=>"Perform a dry run. Do not actually set dates"
      def set_dates
        archive = Archive.new(options[:archive])
        shell = Bun::Shell.new(options)
        archive.original_index.each do |spec|
          file = archive.expanded_tape_path(spec[:tape])
          timestamp = spec[:date]
          puts "About to set timestamp: #{file} #{timestamp.strftime('%Y/%m/%d %H:%M:%S')}" unless options[:quiet]
          shell.set_timestamp(file, timestamp)
        end
      end
      
      VALID_MESSAGES = %w{missing name old new old_file new_file}
      
      # TODO Create check method: Check that an index file entry exists for each tape
      # file, check frozen file dates and content vs. index, check 
      # text archive file contents vs. index
      desc "check_original", "Check contents of the original index"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      option "include", :aliases=>'-i', :type=>'string', :desc=>"Include only certain messages. Options include #{VALID_MESSAGES.join(',')}"<commit_msg>Fix bun archive index set_dates<commit_after>class Bun
  class Archive
    class IndexBot < Thor
    
      desc "build", "Build file index for archive"
      option 'archive', :aliases=>'-a', :type=>'string',  :desc=>'Archive location'
      option 'quiet',   :aliases=>'-q', :type=>'boolean', :desc=>'Run quietly'
      def build
        # TODO the following two lines are a common pattern; refactor
        directory = options[:archive] || Archive.location
        archive = Archive.new(directory)
        archive.build_and_save_index(:verbose=>!options[:quiet])
      end
    
      desc "clear", "Clear file index for archive"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      def clear
        # TODO the following two lines are a common pattern; refactor
        directory = options[:archive] || Archive.location
        archive = Archive.new(directory)
        archive.clear_index
      end

      desc "ls_original", "List the original index file for the archive"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      def ls_original
        archive = Archive.new(options[:archive])
        # TODO Use Array.justify_rows
        archive.original_index.each do |spec|
          puts "#{spec[:tape]}  #{spec[:date].strftime('%Y/%d/%m')}  #{spec[:file]}"
        end
      end
      
      desc "set_dates", "Set file modification dates for archived files, based on original index"
      option 'archive', :aliases=>'-a', :type=>'string',  :desc=>'Archive location'
      option 'dryrun',  :aliases=>'-d', :type=>'boolean', :desc=>"Perform a dry run. Do not actually set dates"
      def set_dates
        archive = Archive.new(options[:archive])
        shell = Bun::Shell.new(options)
        archive.each do |tape|
          descr = archive.descriptor(tape)
          timestamp = descr[:updated]
          if timestamp
            puts "About to set timestamp: #{tape} #{timestamp.strftime('%Y/%m/%d %H:%M:%S')}" unless options[:quiet]
            shell.set_timestamp(archive.expanded_tape_path(tape), timestamp)
          else
            puts "No updated time available for #{tape}" unless options[:quiet]
          end
        end
      end
      
      VALID_MESSAGES = %w{missing name old new old_file new_file}
      
      # TODO Create check method: Check that an index file entry exists for each tape
      # file, check frozen file dates and content vs. index, check 
      # text archive file contents vs. index
      desc "check_original", "Check contents of the original index"
      option 'archive', :aliases=>'-a', :type=>'string', :desc=>'Archive location'
      option "include", :aliases=>'-i', :type=>'string', :desc=>"Include only certain messages. Options include #{VALID_MESSAGES.join(',')}"
{'fraction_non_alphanumeric': 0.1337134449144749, 'fraction_numerical': 0.0, 'mean_word_length': 2.9462989840348333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2615858', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1526, 'n_words': 476}