<commit_before>require "#{Rails.root}/lib/data_cycle/batch_update_logging"
require "#{Rails.root}/lib/importers/assigned_article_importer"
require "#{Rails.root}/lib/articles_courses_cleaner"
require "#{Rails.root}/lib/importers/rating_importer"
require "#{Rails.root}/lib/article_status_manager"
require "#{Rails.root}/lib/importers/view_importer"
require "#{Rails.root}/lib/importers/upload_importer"
require "#{Rails.root}/lib/data_cycle/cache_updater"

# Executes all the steps of 'update_constantly' data import task
class DailyUpdate
  include BatchUpdateLogging

  def initialize
    setup_logger
    return if updates_paused?
    return if daily_update_running?
    wait_until_constant_update_finishes

    begin
      create_pid_file
      run_update
    ensure
      delete_pid_file
    end
  end

  private

  def run_update
    log_start_of_update
    update_article_data
    update_article_views unless ENV['no_views'] == 'true'
    update_commons_uploads
    CacheUpdater.update_all_caches
    log_end_of_update 'Daily update finished.'
  end

  ###############
  # Data import #
  ###############

  def update_article_data
    log_message 'Finding articles that match assignment titles'
    AssignedArticleImporter.import_articles_for_assignments

    log_message 'Rebuilding ArticlesCourses for all current students'
    ArticlesCoursesCleaner.rebuild_articles_courses

    log_message 'Updating ratings for all articles'
    RatingImporter.update_all_ratings

    log_message 'Updating article namespace and deleted status'
    ArticleStatusManager.update_article_status
  end

  def update_article_views
    log_message 'Updating article views'
    ViewImporter.update_all_views(true)
  end

  def update_commons_uploads
    log_message 'Identifying deleted Commons uploads'
    UploadImporter.find_deleted_files(CommonsUpload.where(deleted: false))

    log_message 'Updating Commons uploads'
    UploadImporter.import_all_uploads(User.all)

    log_message 'Updating Commons uploads usage counts'
    UploadImporter.update_usage_count_by_course(Course.all)

    log_message 'Getting thumbnail urls for Commons uploads'
    thumbless_uploads = CommonsUpload.where(thumburl: nil, deleted: false)
    UploadImporter.import_urls_in_batches(thumbless_uploads)
  end
<commit_msg>Limit upload fetching to current student users<commit_after>require "#{Rails.root}/lib/data_cycle/batch_update_logging"
require "#{Rails.root}/lib/importers/assigned_article_importer"
require "#{Rails.root}/lib/articles_courses_cleaner"
require "#{Rails.root}/lib/importers/rating_importer"
require "#{Rails.root}/lib/article_status_manager"
require "#{Rails.root}/lib/importers/view_importer"
require "#{Rails.root}/lib/importers/upload_importer"
require "#{Rails.root}/lib/data_cycle/cache_updater"

# Executes all the steps of 'update_constantly' data import task
class DailyUpdate
  include BatchUpdateLogging

  def initialize
    setup_logger
    return if updates_paused?
    return if daily_update_running?
    wait_until_constant_update_finishes

    begin
      create_pid_file
      run_update
    ensure
      delete_pid_file
    end
  end

  private

  def run_update
    log_start_of_update
    update_article_data
    update_article_views unless ENV['no_views'] == 'true'
    update_commons_uploads
    CacheUpdater.update_all_caches
    log_end_of_update 'Daily update finished.'
  end

  ###############
  # Data import #
  ###############

  def update_article_data
    log_message 'Finding articles that match assignment titles'
    AssignedArticleImporter.import_articles_for_assignments

    log_message 'Rebuilding ArticlesCourses for all current students'
    ArticlesCoursesCleaner.rebuild_articles_courses

    log_message 'Updating ratings for all articles'
    RatingImporter.update_all_ratings

    log_message 'Updating article namespace and deleted status'
    ArticleStatusManager.update_article_status
  end

  def update_article_views
    log_message 'Updating article views'
    ViewImporter.update_all_views(true)
  end

  def update_commons_uploads
    log_message 'Identifying deleted Commons uploads'
    UploadImporter.find_deleted_files(CommonsUpload.where(deleted: false))

    log_message 'Updating Commons uploads for current students'
    UploadImporter.import_all_uploads(User.current.role('student'))

    log_message 'Updating Commons uploads usage counts'
    UploadImporter.update_usage_count_by_course(Course.all)

    log_message 'Getting thumbnail urls for Commons uploads'
    thumbless_uploads = CommonsUpload.where(thumburl: nil, deleted: false)
    UploadImporter.import_urls_in_batches(thumbless_uploads)
  end

{'fraction_non_alphanumeric': 0.07486057486057486, 'fraction_numerical': 0.0, 'mean_word_length': 4.985879332477535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7564215', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1392, 'n_words': 291}