<commit_before># Defines periodic jobs run by DelayedJobs.
#
# Scheduling is done by rufus-scheduler
#
# You should only use "cron" type jobs, not "every". Cron jobs have deterministic
# times to run which help across daemon restarts, and distributing periodic
# jobs across multiple job servers.
#
# Periodic jobs default to low priority. You can override this in the arguments
# passed to Delayed::Periodic.cron

if ActionController::Base.session_store == ActiveRecord::SessionStore
  expire_after = (Setting.from_config("session_store") || {})[:expire_after]
  expire_after ||= 1.day

  Delayed::Periodic.cron 'ActiveRecord::SessionStore::Session.delete_all', '*/5 * * * *' do
    ActiveRecord::SessionStore::Session.delete_all(['updated_at < ?', expire_after.ago])
  end
end

persistence_token_expire_after = (Setting.from_config("session_store") || {})[:expire_remember_me_after]
persistence_token_expire_after ||= 1.month
Delayed::Periodic.cron 'SessionPersistenceToken.delete_all', '35 11 * * *' do
  SessionPersistenceToken.delete_all(['updated_at < ?', persistence_token_expire_after.ago])
end

Delayed::Periodic.cron 'ExternalFeedAggregator.process', '*/30 * * * *' do
  ExternalFeedAggregator.process
end

Delayed::Periodic.cron 'SummaryMessageConsolidator.process', '*/15 * * * *' do
  SummaryMessageConsolidator.process
end

Delayed::Periodic.cron 'Attachment.process_scribd_conversion_statuses', '*/5 * * * *' do
  Attachment.process_scribd_conversion_statuses
end

Delayed::Periodic.cron 'Twitter processing', '*/15 * * * *' do
  TwitterSearcher.process
  TwitterUserPoller.process
end

Delayed::Periodic.cron 'Reporting::CountsReport.process', '0 11 * * *' do
  Reporting::CountsReport.process
end

Delayed::Periodic.cron 'StreamItem.destroy_stream_items', '45 11 * * *' do
  # we pass false for the touch_users argument, on the assumption that these
  # stream items that we delete aren't visible on the user's dashboard anymore
  # anyway, so there's no need to invalidate all the caches.
  StreamItem.destroy_stream_items(4.weeks.ago, false)
end

if Mailman.config.poll_interval == 0 && Mailman.config.ignore_stdin == true
  Delayed::Periodic.cron 'IncomingMessageProcessor.process', '*/1 * * * *' do
    IncomingMessageProcessor.process
  end
end

if PageView.page_view_method == :cache
  # periodically pull new page views off the cache and insert them into the db
  Delayed::Periodic.cron 'PageView.process_cache_queue', '*/1 * * * *' do
    PageView.process_cache_queue
  end
end

Delayed::Periodic.cron 'ErrorReport.destroy_error_reports', '35 */1 * * *' do
  cutoff = Setting.get('error_reports_retain_for', 3.months.to_s).to_i
  if cutoff > 0
    ErrorReport.destroy_error_reports(cutoff.ago)
  end
end

if Delayed::Stats.enabled?
  Delayed::Periodic.cron 'Delayed::Stats.cleanup', '0 11 * * *' do
    Delayed::Stats.cleanup
  end
end

Delayed::Periodic.cron 'Alert.process', '30 11 * * *', :priority => Delayed::LOW_PRIORITY do
  Alert.process<commit_msg>run periodic jobs for all shards fixes #7722<commit_after># Defines periodic jobs run by DelayedJobs.
#
# Scheduling is done by rufus-scheduler
#
# You should only use "cron" type jobs, not "every". Cron jobs have deterministic
# times to run which help across daemon restarts, and distributing periodic
# jobs across multiple job servers.
#
# Periodic jobs default to low priority. You can override this in the arguments
# passed to Delayed::Periodic.cron

if ActionController::Base.session_store == ActiveRecord::SessionStore
  expire_after = (Setting.from_config("session_store") || {})[:expire_after]
  expire_after ||= 1.day

  Delayed::Periodic.cron 'ActiveRecord::SessionStore::Session.delete_all', '*/5 * * * *' do
    Shard.with_each_shard do
      ActiveRecord::SessionStore::Session.delete_all(['updated_at < ?', expire_after.ago])
    end
  end
end

persistence_token_expire_after = (Setting.from_config("session_store") || {})[:expire_remember_me_after]
persistence_token_expire_after ||= 1.month
Delayed::Periodic.cron 'SessionPersistenceToken.delete_all', '35 11 * * *' do
  Shard.with_each_shard do
    SessionPersistenceToken.delete_all(['updated_at < ?', persistence_token_expire_after.ago])
  end
end

Delayed::Periodic.cron 'ExternalFeedAggregator.process', '*/30 * * * *' do
  Shard.with_each_shard do
    ExternalFeedAggregator.process
  end
end

Delayed::Periodic.cron 'SummaryMessageConsolidator.process', '*/15 * * * *' do
  Shard.with_each_shard do
    SummaryMessageConsolidator.process
  end
end

Delayed::Periodic.cron 'Attachment.process_scribd_conversion_statuses', '*/5 * * * *' do
  Shard.with_each_shard do
    Attachment.process_scribd_conversion_statuses
  end
end

Delayed::Periodic.cron 'Twitter processing', '*/15 * * * *' do
  Shard.with_each_shard do
    TwitterSearcher.process
    TwitterUserPoller.process
  end
end

Delayed::Periodic.cron 'Reporting::CountsReport.process', '0 11 * * *' do
  Reporting::CountsReport.process
end

Delayed::Periodic.cron 'StreamItem.destroy_stream_items', '45 11 * * *' do
  Shard.with_each_shard do
    # we pass false for the touch_users argument, on the assumption that these
    # stream items that we delete aren't visible on the user's dashboard anymore
    # anyway, so there's no need to invalidate all the caches.
    StreamItem.destroy_stream_items(4.weeks.ago, false)
  end
end

if Mailman.config.poll_interval == 0 && Mailman.config.ignore_stdin == true
  Delayed::Periodic.cron 'IncomingMessageProcessor.process', '*/1 * * * *' do
    IncomingMessageProcessor.process
  end
end

if PageView.page_view_method == :cache
  # periodically pull new page views off the cache and insert them into the db
  Delayed::Periodic.cron 'PageView.process_cache_queue', '*/1 * * * *' do
    Shard.with_each_shard do
      PageView.process_cache_queue
    end
  end
end

Delayed::Periodic.cron 'ErrorReport.destroy_error_reports', '35 */1 * * *' do
  cutoff = Setting.get('error_reports_retain_for', 3.months.to_s).to_i
  if cutoff > 0
    Shard.with_each_shard do
      ErrorReport.destroy_error_reports(cutoff.ago)
    end
  end
end

if Delayed::Stats.enabled?
  Delayed::Periodic.cron 'Delayed::Stats.cleanup', '0 11 * * *' do
    Delayed::Stats.cleanup
  end
end

Delayed::Periodic.cron 'Alert.process', '30 11 * * *', :priority => Delayed::LOW_PRIORITY do
  Shard.with_each_shard do
    Alert.process
  end
{'fraction_non_alphanumeric': 0.11264728986645718, 'fraction_numerical': 0.01225451688923802, 'mean_word_length': 5.017013232514178, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5412739', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2084, 'n_words': 494}