<commit_before>require 'feedzirra'
require 'rest_client'
require 'nokogiri'
require 'uri'
require 'feed_autodiscovery'
require 'feed_caching'
require 'feed_parser'

##
# This class can fetch feeds and parse them. It also takes care of caching, sending HTTP headers
# that indicate the server to send only new entries.

class FeedClient

  ##
  # Fetch a feed, parse it and save the entries in the database. This is a class method.
  #
  # Receives as argument the id of the feed to fetch. It must already be saved in the database and its
  # fetch_url field must have value.
  #
  # Optionally receives as argument a boolean that tells the method whether to perform feed autodiscovery. It
  # defaults to false.
  #
  # The method tries to use the last received etag with the if-none-match header to indicate the server to send only new
  # entries.
  #
  # If no etag was received last time the feed was fetched, it tries to use the last received last-modified header with the
  # if-modified-since request header to indicate the server to send only new entries.
  #
  # If the last time the feed was fetched no etag and no last-modified headers were in the response, this method fetches
  # the full feed without sending caching headers.<commit_msg>Correct an Rdoc comment to explain the way we actually handle caching headers in FeedClient.<commit_after>require 'feedzirra'
require 'rest_client'
require 'nokogiri'
require 'uri'
require 'feed_autodiscovery'
require 'feed_caching'
require 'feed_parser'

##
# This class can fetch feeds and parse them. It also takes care of caching, sending HTTP headers
# that indicate the server to send only new entries.

class FeedClient

  ##
  # Fetch a feed, parse it and save the entries in the database. This is a class method.
  #
  # Receives as argument the id of the feed to fetch. It must already be saved in the database and its
  # fetch_url field must have value.
  #
  # Optionally receives as argument a boolean that tells the method whether to perform feed autodiscovery. It
  # defaults to false.
  #
  # The method tries to use the last received etag with the if-none-match header and the last received
  # last-modified with the if-modified-since header to tell the server to send the feed only if it has new entries.
  #
  # If the last time the feed was fetched no etag and no last-modified headers were in the response, this method fetches
  # the full feed without sending caching headers.
{'fraction_non_alphanumeric': 0.04730831973898858, 'fraction_numerical': 0.0, 'mean_word_length': 3.847826086956522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5185724', 'n_tokens_mistral': 668, 'n_tokens_neox': 654, 'n_words': 367}