<commit_before>require 'nokogiri'
require 'typhoeus'
require 'date'

module Agents
  class WebsiteAgent < Agent
    cannot_receive_events!

    description <<-MD
      The WebsiteAgent scrapes a website, XML document, or JSON feed and creates Events based on the results.

      Specify a `url` and select a `mode` for when to create Events based on the scraped data, either `all` or `on_change`.

      The `type` value can be `xml`, `html`, or `json`.

      To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.

      When parsing HTML or XML, these sub-hashes specify how to extract with a `css` CSS selector and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:

          'extract': {
            'url': { 'css': "#comic img", 'attr': "src" },
            'title': { 'css': "#comic img", 'attr': "title" },
            'body_text': { 'css': "div.main", 'text': true }
          }

      When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:

          'extract': {
            'title': { 'path': "results.data[*].title" },
            'description': { 'path': "results.data[*].description" }
          }

      Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.

      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.

      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
    MD

    event_description do
      "Events will have the fields you specified.  Your options look like:\n\n    #{Utils.pretty_print options['extract']}"
    end

    default_schedule "every_12h"

    UNIQUENESS_LOOK_BACK = 30

    def working?
      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
    end

    def default_options
      {
          'expected_update_period_in_days' => "2",
          'url' => "http://xkcd.com",
          'type' => "html",
          'mode' => :on_change,
          'extract' => {
            'url' => {'css' => "#comic img", 'attr' => "src"},
            'title' => {'css' => "#comic img", 'attr' => "title"}
          }
      }
    end

    def validate_options
      errors.add(:base, "url and expected_update_period_in_days are required") unless options['expected_update_period_in_days'].present? && options['url'].present?
      if !options['extract'].present? && extraction_type != "json"
        errors.add(:base, "extract is required for all types except json")
      end
    end

    def check
      hydra = Typhoeus::Hydra.new
      log "Fetching #{options['url']}"
      request_opts = {:followlocation => true}
      if !options['basic_auth'].blank?
        request_opts[:userpwd] = options['basic_auth']
      end
      request = Typhoeus::Request.new(options['url'], request_opts)
      request.on_failure do |response|
        error "Failed: #{response.inspect}"
      end
      request.on_success do |response|
        doc = parse(response.body)

        if extract_full_json?
          result = doc
          if store_payload? result
            log "Storing new result for '#{name}': #{result.inspect}"
            create_event :payload => result
          end
        else
          output = {}<commit_msg>blank? -> present?<commit_after>require 'nokogiri'
require 'typhoeus'
require 'date'

module Agents
  class WebsiteAgent < Agent
    cannot_receive_events!

    description <<-MD
      The WebsiteAgent scrapes a website, XML document, or JSON feed and creates Events based on the results.

      Specify a `url` and select a `mode` for when to create Events based on the scraped data, either `all` or `on_change`.

      The `type` value can be `xml`, `html`, or `json`.

      To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.

      When parsing HTML or XML, these sub-hashes specify how to extract with a `css` CSS selector and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:

          'extract': {
            'url': { 'css': "#comic img", 'attr': "src" },
            'title': { 'css': "#comic img", 'attr': "title" },
            'body_text': { 'css': "div.main", 'text': true }
          }

      When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:

          'extract': {
            'title': { 'path': "results.data[*].title" },
            'description': { 'path': "results.data[*].description" }
          }

      Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.

      Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `username:password`.

      Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
    MD

    event_description do
      "Events will have the fields you specified.  Your options look like:\n\n    #{Utils.pretty_print options['extract']}"
    end

    default_schedule "every_12h"

    UNIQUENESS_LOOK_BACK = 30

    def working?
      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
    end

    def default_options
      {
          'expected_update_period_in_days' => "2",
          'url' => "http://xkcd.com",
          'type' => "html",
          'mode' => :on_change,
          'extract' => {
            'url' => {'css' => "#comic img", 'attr' => "src"},
            'title' => {'css' => "#comic img", 'attr' => "title"}
          }
      }
    end

    def validate_options
      errors.add(:base, "url and expected_update_period_in_days are required") unless options['expected_update_period_in_days'].present? && options['url'].present?
      if !options['extract'].present? && extraction_type != "json"
        errors.add(:base, "extract is required for all types except json")
      end
    end

    def check
      hydra = Typhoeus::Hydra.new
      log "Fetching #{options['url']}"
      request_opts = {:followlocation => true}
      if options['basic_auth'].present?
        request_opts[:userpwd] = options['basic_auth']
      end
      request = Typhoeus::Request.new(options['url'], request_opts)
      request.on_failure do |response|
        error "Failed: #{response.inspect}"
      end
      request.on_success do |response|
        doc = parse(response.body)

        if extract_full_json?
          result = doc
          if store_payload? result
            log "Storing new result for '#{name}': #{result.inspect}"
            create_event :payload => result
          end
        else
          output = {}
{'fraction_non_alphanumeric': 0.11579378068739771, 'fraction_numerical': 0.001363884342607747, 'mean_word_length': 2.7185598377281948, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7300367', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2059, 'n_words': 717}