<commit_before>class FeedWorker
  include Sidekiq::Worker

  sidekiq_options :retry => false

  def up_to_date?(feed, feed_record)
    (not feed_record.etag.nil?) and     \
    (feed.etag == feed_record.etag) and \
    not feed_record.newsitems.empty?
  end

  def create(url, category_name)
    feed_url = Feedbag.find(url).first
    unless feed_url.nil? then
      objCategory = Category.where(name: category_name).first_or_create
      objFeed = objCategory.feeds.build(feed_params(feed_url))
      objFeed.save && fetch(objFeed.id) && objFeed.reload
      $redis.set("create-#{jid}", {
        id: objFeed.id,
        title: objFeed.meta[:title],
        path: "/#{objFeed.to_param}",
        favicon: objFeed.favicon,
        category: { id: objCategory.id, name: objCategory.name }
      }.to_json, :ex => 60)
    else
      # Handle the case where url has no feeds
      $redis.set("create-#{jid}", {
        error: "Can't find any feed, are you sure the url is valid?"
      }.to_json, :ex => 60)
    end
  end

  def fetch(feed_id)
    feed_record = Feed.find(feed_id)

    feed = Feedzirra::Feed.fetch_and_parse(feed_record.feed_url)
    return if feed.is_a? Fixnum # Got HTTP response code instead of Feed object ^^;
    return if up_to_date?(feed, feed_record)

    # TODO: Use config to select which field should be sanitized
    #feed.sanitize_entries!

    feed_record.update(title: feed.title, etag: feed.etag, url: feed.url)
    feed.entries.each do | entry |
      item = Newsitem.where({
        permalink: entry.url,
      }).first
      if item.nil? and not Itemhash.has? entry.url
        item = feed_record.newsitems.build(newsitem_params(entry))
        item.save
      end
    end
  end

  def perform(action, *params)
    send(action, *params)
  end

  private
  def newsitem_params(entry)
    ActionController::Parameters.new({
      title: entry.title,<commit_msg>Refactor FeedWorker, add mark_as_read...<commit_after>class FeedWorker
  include Sidekiq::Worker

  sidekiq_options :retry => false

  def up_to_date?(feed, feed_record)
    (not feed_record.etag.nil?) and     \
    (feed.etag == feed_record.etag) and \
    not feed_record.newsitems.empty?
  end

  def create(url, category_name)
    feed_url = Feedbag.find(url).first
    unless feed_url.nil? then
      objCategory = Category.where(name: category_name).first_or_create
      objFeed = objCategory.feeds.build(feed_params(feed_url))
      objFeed.save && fetch(objFeed.id) && objFeed.reload
      $redis.set("create-#{jid}", {
        view: "create",
        id: objFeed.id
      }.to_json, :ex => 60)
    else
      # Handle the case where url has no feeds
      $redis.set("create-#{jid}", {
        error: "Can't find any feed, are you sure the url is valid?"
      }.to_json, :ex => 60)
    end
  end

  def fetch(feed_id)
    feed_record = Feed.find(feed_id)

    feed = Feedzirra::Feed.fetch_and_parse(feed_record.feed_url)
    return if feed.is_a? Fixnum # Got HTTP response code instead of Feed object ^^;
    return if up_to_date?(feed, feed_record)

    # TODO: Use config to select which field should be sanitized
    #feed.sanitize_entries!

    feed_record.update(title: feed.title, etag: feed.etag, url: feed.url)
    feed.entries.each do | entry |
      item = Newsitem.where({
        permalink: entry.url,
      }).first
      if item.nil? and not Itemhash.has? entry.url
        item = feed_record.newsitems.build(newsitem_params(entry))
        item.save
      end
    end
  end

  def mark_as_read(feed_id)
    feed_record = Feed.find(feed_id)
    feed_record.mark_all_as_read
    $redis.set("markAsRead-#{jid}", {
      view: "mark_feed_as_read",
      id: feed_id
    }.to_json, :ex => 60)
  rescue
    $redis.set("markAsRead-#{jid}", {
      error: "Feed can not marked as read."
    }.to_json, :ex => 60)
  end

  def perform(action, *params)
    send(action, *params)
  end

  private
  def newsitem_params(entry)
    ActionController::Parameters.new({
      title: entry.title,
{'fraction_non_alphanumeric': 0.11633725293970477, 'fraction_numerical': 0.003002251688766575, 'mean_word_length': 2.892891918208374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '742529', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1361, 'n_words': 319}