<commit_before># encoding: utf-8

module Pluto

class FeedFetcherCondGetWithCache

  include LogUtils::Logging
  include Models   # for easy convenience access for Activity etc.

  def initialize
    @worker  = Fetcher::Worker.new
  end

  def debug=(value)  @debug = value;   end
  def debug?()       @debug || false;  end


  def fetch( feed_rec )
    #############
    # try smart http update; will update db records

    feed_url = feed_rec.feed_url
    feed_key = feed_rec.key

    ### todo/fix: normalize/unifiy feed_url
    ##  - same in fetcher - use shared utitlity method or similar

    @worker.use_cache = true
    @worker.cache[ feed_url ] = {
      'etag'          => feed_rec.http_etag,
      'last-modified' => feed_rec.http_last_modified
    }

    begin
      response = @worker.get( feed_url )
    rescue SocketError => e
      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
      ###  will result in SocketError -- getaddrinfo: Name or service not known
      logger.error "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )

      ### todo/fix: update feed rec in db
      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
      return nil
    end

    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false

    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
      logger.info "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
      logger.info "no change; request returns not modified (304); skipping parsing feed"
      return nil   # no updates available; nothing to do
    end

    feed_fetched = Time.now

    if response.code != '200'   # note Net::HTTP response.code is a string in ruby

      logger.error "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"

      feed_attribs = {
        http_code:          response.code.to_i,
        http_server:        response.header[ 'server' ],
        http_etag:          nil,<commit_msg>Handle SystemCallError in feedfetcher gracefully (#20)<commit_after># encoding: utf-8

module Pluto

class FeedFetcherCondGetWithCache

  include LogUtils::Logging
  include Models   # for easy convenience access for Activity etc.

  def initialize
    @worker  = Fetcher::Worker.new
  end

  def debug=(value)  @debug = value;   end
  def debug?()       @debug || false;  end


  def fetch( feed_rec )
    #############
    # try smart http update; will update db records

    feed_url = feed_rec.feed_url
    feed_key = feed_rec.key

    ### todo/fix: normalize/unifiy feed_url
    ##  - same in fetcher - use shared utitlity method or similar

    @worker.use_cache = true
    @worker.cache[ feed_url ] = {
      'etag'          => feed_rec.http_etag,
      'last-modified' => feed_rec.http_last_modified
    }

    begin
      response = @worker.get( feed_url )
    rescue SocketError, SystemCallError => e
      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
      ###  will result in SocketError -- getaddrinfo: Name or service not known
      logger.error "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )

      ### todo/fix: update feed rec in db
      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
      return nil
    end

    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false

    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
      logger.info "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
      logger.info "no change; request returns not modified (304); skipping parsing feed"
      return nil   # no updates available; nothing to do
    end

    feed_fetched = Time.now

    if response.code != '200'   # note Net::HTTP response.code is a string in ruby

      logger.error "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"

      feed_attribs = {
        http_code:          response.code.to_i,
        http_server:        response.header[ 'server' ],
        http_etag:          nil,
{'fraction_non_alphanumeric': 0.11222171740118128, 'fraction_numerical': 0.004997728305315766, 'mean_word_length': 2.7408666100254884, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6986348', 'n_tokens_mistral': 1389, 'n_tokens_neox': 1323, 'n_words': 415}