<commit_before>class BlogJob < ProgressJob::Base 
  #Parameters are id of blog and total number of posts for blog
  def initialize(id, number_of_posts)
    super progress_max: number_of_posts
    @id = id
    @number_of_posts = number_of_posts
  end

  def perform 
    #initialize blog variables
    blog = Blog.find(@id)
    search_query = blog.name
    show_reblogs = blog.reblogs
    post_types = blog.post_types

    # Create new client to make requests to Tumblr API
    myClient = Tumblr::Client.new
    
    
    update_stage ("Searching " + 0.to_s + "/" + @number_of_posts.to_s + " blog posts...")

    #final post array
    posts = Array.new

    post_types.each do |post_type|
      #temp variables for loop
      i = 0
      temp_posts = Array.new
        loop do
          
          temp_posts = (myClient.posts(search_query, :type => post_type, "reblog_info" => !show_reblogs, "offset" => i ) )["posts"]
          
          #Increment by 50 bc need to increase offset for request. only 20 posts returned at a time.
          i = i + 20
          update_progress(step: temp_posts.length)
          update_stage (posts.length.to_s + "/" + @number_of_posts.to_s + " blog posts processed")
          
          #add the fetched posts to the final post array, break if no posts were received
          break if temp_posts.blank?
          posts += temp_posts
        end
    end
    #Ignore reblogs if user doesn't want reblogs
    if !@show_reblogs
      posts = posts.select {|x| x["reblogged_from_id"] == nil}
    end

    #Don't include posts that don't have note_count (raises error when sorting), or a note_count of 0
    #Sort by descending note count, and return first 30
    blog.update(posts: posts.delete_if {|x| x["note_count"].nil? || x["note_count"] == 0 }.sort_by {|x| x["note_count"] }.reverse![0,30])

    update_progress_max(100)
  end

  def max_run_time
    600
  end
<commit_msg>Fixed error in comments, and replaced variable i with more descriptive variable, offset<commit_after>class BlogJob < ProgressJob::Base 
  #Parameters are id of blog and total number of posts for blog
  def initialize(id, number_of_posts)
    super progress_max: number_of_posts
    @id = id
    @number_of_posts = number_of_posts
  end

  def perform 
    #initialize blog variables
    blog = Blog.find(@id)
    search_query = blog.name
    show_reblogs = blog.reblogs
    post_types = blog.post_types

    # Create new client to make requests to Tumblr API
    myClient = Tumblr::Client.new
    
    
    update_stage ("Searching " + 0.to_s + "/" + @number_of_posts.to_s + " blog posts...")

    #final post array
    posts = Array.new

    post_types.each do |post_type|
      #temp variables for loop
      offset = 0
      temp_posts = Array.new
        loop do
          
          temp_posts = (myClient.posts(search_query, :type => post_type, "reblog_info" => !show_reblogs, "offset" => offset ) )["posts"]
          
          #Need to increment offset by 20 for next request.
          offset = offset + 20
          update_progress(step: temp_posts.length)
          update_stage (posts.length.to_s + "/" + @number_of_posts.to_s + " blog posts processed")
          
          #add the fetched posts to the final post array, break if no posts were received
          break if temp_posts.blank?
          posts += temp_posts
        end
    end
    #Ignore reblogs if user doesn't want reblogs
    if !@show_reblogs
      posts = posts.select {|x| x["reblogged_from_id"] == nil}
    end

    #Don't include posts that don't have note_count (raises error when sorting), or a note_count of 0
    #Sort by descending note count, and return first 30
    blog.update(posts: posts.delete_if {|x| x["note_count"].nil? || x["note_count"] == 0 }.sort_by {|x| x["note_count"] }.reverse![0,30])

    update_progress_max(100)
  end

  def max_run_time
    600
  end

{'fraction_non_alphanumeric': 0.09212903225806451, 'fraction_numerical': 0.01032258064516129, 'mean_word_length': 2.6292134831460676, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1028888', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1186, 'n_words': 382}