<commit_before>require 'redis_helper'

class Activity
  include Mongoid::Document
  include Mongoid::Timestamps
  include RedisHelper

  field :activity_type, :type => String

  field :user1, :as => :actor1, :type => BSON::ObjectId
  field :username1, :type => String
  field :thumb1, :type => String

  field :user2, :as => :actor2, :type => BSON::ObjectId
  field :username2, :type => String

  field :subject, :type => BSON::ObjectId
  field :subject_title, :type => String

  embedded_in :activity_feed_chunk

  validates_presence_of :actor1, :username1

  # push status to a specific feed
  def push(id, location="feed")
    $redis.zadd key(location, id), timestamp, encoded
  end

  # push to followers (assumes an array of follower ids)
  def push_to_followers(user)
    user.followers.each do |follower|
      push(follower.id) unless user.blocked?(follower)
    end

    push(user.id)

    if user
    end

    #push onto the superfeed
    $redis.zadd "FIREHOSEFEED", timestamp, encoded
  end

  def self.decode(json)
    Activity.new(ActiveSupport::JSON.decode(json))
  end

  def as_json(options={})
    if self.thumb1.nil?
      user = User.find_by_username(self.username1)
      if user && user.thumb_url
        self.thumb1 = user.thumb_url
      end
    end

    if !self.updated_at
      self.updated_at = self.created_at
    end

    self.attributes.merge(:id => self[:_id])
  end

end<commit_msg>cap the firehosefeed at 300 elements<commit_after>require 'redis_helper'

class Activity
  include Mongoid::Document
  include Mongoid::Timestamps
  include RedisHelper

  field :activity_type, :type => String

  field :user1, :as => :actor1, :type => BSON::ObjectId
  field :username1, :type => String
  field :thumb1, :type => String

  field :user2, :as => :actor2, :type => BSON::ObjectId
  field :username2, :type => String

  field :subject, :type => BSON::ObjectId
  field :subject_title, :type => String

  embedded_in :activity_feed_chunk

  validates_presence_of :actor1, :username1

  # push status to a specific feed
  def push(id, location="feed")
    $redis.zadd key(location, id), timestamp, encoded
  end

  # push to followers (assumes an array of follower ids)
  def push_to_followers(user)
    user.followers.each do |follower|
      push(follower.id) unless user.blocked?(follower)
    end

    push(user.id)

    if user
    end

    #push onto the superfeed
    $redis.zadd "FIREHOSEFEED", timestamp, encoded
    trim_feed("FIREHOSEFEED")
  end

  def self.decode(json)
    Activity.new(ActiveSupport::JSON.decode(json))
  end

  def as_json(options={})
    if self.thumb1.nil?
      user = User.find_by_username(self.username1)
      if user && user.thumb_url
        self.thumb1 = user.thumb_url
      end
    end

    if !self.updated_at
      self.updated_at = self.created_at
    end

    self.attributes.merge(:id => self[:_id])
  end


  def trim_feed(key, max_feed=300)
    if ($redis.zcard key) >= max_feed
      n = max_feed - 100
      if (r = $redis.zrevrange key, n, n, :with_scores => true)
        $redis.zremrangebyscore key, "-inf", "(#{r.last}"
      end
    end
  end

end
{'fraction_non_alphanumeric': 0.11160857051487048, 'fraction_numerical': 0.01055324592260953, 'mean_word_length': 3.326417704011065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7210834', 'n_tokens_mistral': 1103, 'n_tokens_neox': 1054, 'n_words': 276}