<commit_before>require 'aws/s3'
require 'yaml'

module AP
  class Replayer

    attr_accessor :done

    def initialize(crawler)
      @crawler = crawler
      @s3_config = YAML.load_file("#{@crawler.dir}/config/s3.yml")
      @done = false
    end

    def init
      if ['production', 'internal'].include?(@crawler.env)
        @crawler.logger.err "Can't run replays in production environment"
        exit
      end

      connect
      bucket = AWS::S3::Bucket.find(@s3_config['bucket'])
      @crawler.params[:replaydate] = bucket.objects(:prefix => "#{@s3_config['dir']}/").map{|o| o.key.split('/')[1, 1].first}.uniq.sort.last.split('.').first unless @crawler.params[:replaydate]

      local_gzip = "#{@crawler.datadir}/#{@crawler.params[:replaydate]}.tar.gz"
      unless File.exist?(local_gzip)
        puts "Downloading replay from #{@crawler.params[:replaydate]}..."
        s3_object = bucket.objects(:prefix => "#{@s3_config['dir']}/#{@crawler.params[:replaydate]}.tar.gz").first
        File.open(local_gzip, 'w') {|f| f.write(s3_object.value)}
        system "tar -zxvf #{local_gzip} -C #{@crawler.datadir}/"
      end

      @timekeys = Dir.glob("#{@crawler.datadir}/#{@crawler.params[:replaydate]}/*").map{|d| d.split('/').last}.uniq.sort
      @timekey_idx = 0
    end

    def replay
      timekey = @timekeys[@timekey_idx]
      @crawler.logger.log "Started replaying #{timekey}"
<commit_msg>Make crawling work without an s3.yml<commit_after>require 'aws/s3'
require 'yaml'

module AP
  class Replayer

    attr_accessor :done

    def initialize(crawler)
      @crawler = crawler
      @done = false
    end

    def init
      if ['production', 'internal'].include?(@crawler.env)
        @crawler.logger.err "Can't run replays in production environment"
        exit
      end

      @s3_config = YAML.load_file("#{@crawler.dir}/config/s3.yml")
      connect
      bucket = AWS::S3::Bucket.find(@s3_config['bucket'])
      @crawler.params[:replaydate] = bucket.objects(:prefix => "#{@s3_config['dir']}/").map{|o| o.key.split('/')[1, 1].first}.uniq.sort.last.split('.').first unless @crawler.params[:replaydate]

      local_gzip = "#{@crawler.datadir}/#{@crawler.params[:replaydate]}.tar.gz"
      unless File.exist?(local_gzip)
        puts "Downloading replay from #{@crawler.params[:replaydate]}..."
        s3_object = bucket.objects(:prefix => "#{@s3_config['dir']}/#{@crawler.params[:replaydate]}.tar.gz").first
        File.open(local_gzip, 'w') {|f| f.write(s3_object.value)}
        system "tar -zxvf #{local_gzip} -C #{@crawler.datadir}/"
      end

      @timekeys = Dir.glob("#{@crawler.datadir}/#{@crawler.params[:replaydate]}/*").map{|d| d.split('/').last}.uniq.sort
      @timekey_idx = 0
    end

    def replay
      timekey = @timekeys[@timekey_idx]
      @crawler.logger.log "Started replaying #{timekey}"

{'fraction_non_alphanumeric': 0.18299964875307342, 'fraction_numerical': 0.008781173164734809, 'mean_word_length': 3.3216995447647952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6560837', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1034, 'n_words': 165}