<commit_before># Visit AWS-S3, and pull the logs for a date range. 
#
require 'aws-sdk'
require 'date'
require 'models'

class Snarfer
  def initialize(access_key, secret)
    @s3 = AWS::S3.new(
      access_key_id: access_key, 
      secret_access_key: secret,
      region: 'eu-west-1'
    )
    
    Download.raise_on_save_failure = true
    UserAgent.raise_on_save_failure = true
    IpAddress.raise_on_save_failure = true

    @bucket_name = 'verbose-ireland'
    @downloads_regex = /.* verbose-ireland \[(.*)\] (\d+.\d+.\d+.\d+) .* (\d+) REST.GET.OBJECT \d+_TheVerbosePodcast_-_Episode(\d+).mp3 .* - (\d+) (\d+) (\d+) (\d+) "(.*)" "(.*)".*/
  end

  # Retrieve S3 logs from the range of dates provided.
  # If only one date, then just retrieve for that date.
  def snarf(start_date, end_date = nil)
    start_date = condition(start_date)
    end_date = condition(end_date)

    if (end_date.nil? or start_date.eql?(end_date)) 
      download_logs(start_date)
    else
      if (start_date > end_date)
        throw ArgumentError.new("Start date is after end date") 
      else
        throw ArgumentError.new("Not implemented for ranges > 1 day :P")
      end        
    end
  end

  protected

  def download_logs(date)
    throw ArgumentError.new("Can only get logs in the past") if (date.eql?(Time.now.utc.to_date))
    activity = Activity.new({ start: Time.now.utc})
    begin
      activity.processed = download_objects(@s3.buckets[@bucket_name], 'logs/' + date.strftime("%Y-%m-%d"))
      activity.end = Time.now.utc
    rescue StandardError => e
      activity.exception = e.to_s
    end
    activity.save
  end

  # Grab all of the objects in the logs bucket for the
  # particular date. If the date is today, then that's 
  # an issue, because the logging may not be finished
  #
  def download_objects(bucket, prefix)
    log_count = 0
    bucket.objects.with_prefix(prefix).each do |log|
      log.read.each_line do |entry|
        unless (m = @downloads_regex.match(entry)).nil?
          store(
            m[3],
            DateTime.strptime(m[1], "%d/%b/%Y:%H:%M:%S %z"), 
            m[2], m[4], m[7], m[10], 
            (m[9].eql?('-')) ? nil : m[9]
          )
        end
      end
      log_count += 1
    end
    log_count
  end

  def store(reqid, date, ip, episode, spent, agent, refer)
    # Only make a new record if there isn't one with
    # this request id.
    unless Download.first(arid: reqid)
      Download.create(
        arid: reqid,
        at: date, 
        episode: episode.to_i, 
        spent: spent.to_f,
        user_agent: user_agent_with_description(agent),
        ip_address: iprecord_for(ip),
        referrer: refer
      )
    end
  end

  def user_agent_with_description(desc)
    user_agent = UserAgent.first(description: desc) or UserAgent.new({ description: agent })
    user_agent.save and user_agent
  end

  def ip_record_for(ip)
    iprecord = IpAddress.get(ip) or IpAddress.new({ ip: ip })
    iprecord.visits += 1
    iprecord.save and iprecord
  end

  def condition(date)
    (date.nil? or date.is_a?(Date)) ? date : Date.parse(date)
  end
end<commit_msg>Updated the regex to pull out the Amazon request id for dup detection. Was using and/or instead of &&/|| in evaluations. Couple of variable issues hidden by mock/stub usage Processed counter was counting duplicates.<commit_after># Visit AWS-S3, and pull the logs for a date range. 
#
require 'aws-sdk'
require 'date'
require 'models'

class Snarfer
  def initialize(access_key, secret)
    @s3 = AWS::S3.new(
      access_key_id: access_key, 
      secret_access_key: secret,
      region: 'eu-west-1'
    )
    
    Download.raise_on_save_failure = true
    UserAgent.raise_on_save_failure = true
    IpAddress.raise_on_save_failure = true

    @bucket_name = 'verbose-ireland'
    @downloads_regex = /.* verbose-ireland \[(.*)\] (\d+.\d+.\d+.\d+) .* ([0-9A-F]+) REST.GET.OBJECT \d+_TheVerbosePodcast_-_Episode(\d+).mp3 .* \d+ - (\d+) (\d+) (\d+) (\d+) "(.*)" "(.*)" */
  end

  # Retrieve S3 logs from the range of dates provided.
  # If only one date, then just retrieve for that date.
  def snarf(start_date, end_date = nil)
    start_date = condition(start_date)
    end_date = condition(end_date)

    if (end_date.nil? or start_date.eql?(end_date)) 
      download_logs(start_date)
    else
      if (start_date > end_date)
        throw ArgumentError.new("Start date is after end date") 
      else
        throw ArgumentError.new("Not implemented for ranges > 1 day :P")
      end        
    end
  end

  protected

  def download_logs(date)
    throw ArgumentError.new("Can only get logs in the past") if (date.eql?(Time.now.utc.to_date))
    activity = Activity.new({ start: Time.now.utc})
    begin
      activity.processed = download_objects(@s3.buckets[@bucket_name], 'logs/' + date.strftime("%Y-%m-%d"))
      activity.end = Time.now.utc
    rescue StandardError => e
      activity.exception = e.to_s
      raise e
    end
    activity.save
  end

  # Grab all of the objects in the logs bucket for the
  # particular date. If the date is today, then that's 
  # an issue, because the logging may not be finished
  #
  def download_objects(bucket, prefix)
    log_count = 0
    bucket.objects.with_prefix(prefix).each do |log|
      log.read.each_line do |entry|
        unless (m = @downloads_regex.match(entry)).nil?
          store(
            m[3],
            DateTime.strptime(m[1], "%d/%b/%Y:%H:%M:%S %z"), 
            m[2], m[4], m[7], m[10], 
            (m[9].eql?('-')) ? nil : m[9]
          ) && log_count += 1
        end
      end
    end
    log_count
  end

  def store(reqid, date, ip, episode, spent, agent, refer)
    # Only make a new record if there isn't one with
    # this request id.
    unless Download.first(arid: reqid)
      Download.create(
        arid: reqid,
        at: date, 
        episode: episode.to_i, 
        spent: spent.to_f,
        user_agent: user_agent_with_description(agent),
        ip_address: ip_record_for(ip),
        referrer: refer
      )
    else
      nil
    end
  end

  def user_agent_with_description(desc)
    user_agent = UserAgent.first(description: desc) || UserAgent.new({ description: desc })
    user_agent.save && user_agent
  end

  def ip_record_for(ip)
    iprecord = IpAddress.get(ip) || IpAddress.new({ ip: ip })
    iprecord.visits += 1
    iprecord.save && iprecord
  end

  def condition(date)
    (date.nil? or date.is_a?(Date)) ? date : Date.parse(date)
  end
end
{'fraction_non_alphanumeric': 0.12419106317411402, 'fraction_numerical': 0.0064714946070878274, 'mean_word_length': 2.610122358175751, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1909620', 'n_tokens_mistral': 2252, 'n_tokens_neox': 2185, 'n_words': 582}