<commit_before>class Populate < Thor

  desc "posts", "Generate posts in a topic"
  long_desc <<-LONGDESC
    Create a topic with any number of posts, or add posts to an existing topic.

    Examples:

    Create a new topic with 100 posts from batman and spiderman:

    > $ thor populate:posts -n 100 -u batman spiderman -t "So many posts"

    Add 10 posts to topic with id 123:

    > $ thor populate:posts -n 10 -u batman spiderman -i 123

  LONGDESC
  method_option :num_posts, aliases: '-n',  type: :numeric, required: true, desc: "Number of posts to make"
  method_option :users,     aliases: '-u',  required: true, type: :array, desc: "Usernames of users who will make the posts"
  method_option :title,     aliases: '-t',  desc: "The title of the topic, if making a new topic"
  method_option :topic_id,  aliases: '-i', type: :numeric, desc: "The id of the topic where the posts will be added"
  def posts
    require './config/environment'

    users = User.find_all_by_username_lower(options[:users].map(&:downcase))
    if users.length != options[:users].length
      not_found = options[:users].map(&:downcase) - users.map(&:username_lower)
      puts "No user found for these usernames: #{not_found.join(', ')}"
      exit 1
    end

    RateLimiter.disable

    topic = nil
    start_post = 1
    if options[:topic_id]
      topic = Topic.find(options[:topic_id])
      puts "Adding more posts to '#{topic.title}'"
    else
      topic_title = options[:title] || hipster_words.sample(6).join(' ')

      puts "Making a new topic: '#{topic_title}'"

      post_creator = PostCreator.new(users[0], title: topic_title, raw: hipster_words.sample(10).join(' '))
      first_post = post_creator.create
      if post_creator.errors.present?
        puts "ERROR creating the topic!"
        puts post_creator.errors.full_messages
        exit 1
      end
      topic = first_post.topic
      start_post = 2
    end

    puts "Making #{options[:num_posts]} posts"

    (start_post..options[:num_posts]).each do |num|
      print '.'
      raw = rand(4) == 0 ? (rand(2) == 0 ? image_posts.sample : wikipedia_posts.sample ) : hipster_words.sample(20).join(' ')
      post_creator = PostCreator.new(users[num % (users.length)], topic_id: topic.id, raw: raw)
      post_creator.create
    end

    puts ''

    RateLimiter.enable

    puts "Done. Topic id = #{topic.id}"
  ensure
    RateLimiter.enable
  end

  private

    def hipster_words
      @hipster_words ||= "retro put a bird on it wolf vegan gluten-free swag trust fund master cleanse four loko synth gentrify literally lomo bitters Keytar try-hard semiotics gastropub marfa YOLO bicycle rights street art authentic DIY fashion axe Chia letterpress twee mlkshk Typewriter umami Etsy keffiyeh direct trade Distillery Odd Future narwhal Pitchfork roof party pork belly Austin McSweeney's cliche Dreamcatcher +1 drinking vinegar Neutra Vice pickled Brooklyn Williamsburg hella small batch ennui squid Schlitz Kale chips food truck butcher vinyl ethnic fixie shabby chic iPhone <NAME> Deep v hoodie forage Retro fanny pack wayfarers messenger bag pug you probably haven't heard of them Art disrupt High Life hashtag chambray readymade selfies sartorial PBR leggings flexitarian chillwave Cosby sweater Marfa typewriter freegan chia biodiesel 8-bit occupy Tonx sustainable cray PBR&B Yr skateboard Tumblr 3 moon fingerstache Intelligentsia plaid Thundercats XOXO craft beer mustache Shoreditch Next level Godard actually mumblecore keytar stumptown irony Disrupt tofu scenester lo-fi single-origin coffee kogi beard yr tattooed viral 90's aesthetic pop-up mixtape Pinterest Asymmetrical kitsch farm-to-table photo booth cardigan Squid Helvetica Direct kale salvia American Apparel artisan tousled".split(' ') +
                          (["\n\n"] * 20)<commit_msg>In populate:posts tool, catch post validation errors and retry<commit_after>class Populate < Thor

  MAX_ERRORS = 5

  desc "posts", "Generate posts in a topic"
  long_desc <<-LONGDESC
    Create a topic with any number of posts, or add posts to an existing topic.

    Examples:

    Create a new topic with 100 posts from batman and spiderman:

    > $ thor populate:posts -n 100 -u batman spiderman -t "So many posts"

    Add 10 posts to topic with id 123:

    > $ thor populate:posts -n 10 -u batman spiderman -i 123

  LONGDESC
  method_option :num_posts, aliases: '-n',  type: :numeric, required: true, desc: "Number of posts to make"
  method_option :users,     aliases: '-u',  required: true, type: :array, desc: "Usernames of users who will make the posts"
  method_option :title,     aliases: '-t',  desc: "The title of the topic, if making a new topic"
  method_option :topic_id,  aliases: '-i', type: :numeric, desc: "The id of the topic where the posts will be added"
  def posts
    require './config/environment'

    users = User.find_all_by_username_lower(options[:users].map(&:downcase))
    if users.length != options[:users].length
      not_found = options[:users].map(&:downcase) - users.map(&:username_lower)
      puts "No user found for these usernames: #{not_found.join(', ')}"
      exit 1
    end

    RateLimiter.disable

    topic = nil
    start_post = 1
    if options[:topic_id]
      topic = Topic.find(options[:topic_id])
      puts "Adding more posts to '#{topic.title}'"
    else
      topic_title = options[:title] || hipster_words.sample(6).join(' ')

      puts "Making a new topic: '#{topic_title}'"

      post_creator = PostCreator.new(users[0], title: topic_title, raw: hipster_words.sample(10).join(' '))
      first_post = post_creator.create
      if post_creator.errors.present?
        puts "ERROR creating the topic!"
        puts post_creator.errors.full_messages
        exit 1
      end
      topic = first_post.topic
      start_post = 2
    end

    puts "Making #{options[:num_posts]} posts"

    num_errors = 0

    (start_post..options[:num_posts]).each do |num|
      print '.'
      raw = rand(4) == 0 ? (rand(2) == 0 ? image_posts.sample : wikipedia_posts.sample ) : hipster_words.sample(20).join(' ')
      post_creator = PostCreator.new(users[num % (users.length)], topic_id: topic.id, raw: raw)
      post_creator.create
      if post_creator.errors.present?
        # It's probably a "Body is too similar to what you recently posted" error.
        # Try one more time using more random words.
        post_creator = PostCreator.new(users[num % (users.length)], topic_id: topic.id, raw: hipster_words.sample(40).join(' '))
        post_creator.create
        if post_creator.errors.present?
          # Still failing! Show the error.
          puts '', "--------------------------"
          puts "ERROR creating a post!"
          puts post_creator.errors.full_messages
          puts "--------------------------"

          # Stop looping after MAX_ERRORS errors
          num_errors += 1
          if num_errors > MAX_ERRORS
            puts "Giving up. Too many errors."
            exit 1
          end
        end
      end
    end

    puts ''

    puts "Done. Topic id = #{topic.id}"
  ensure
    RateLimiter.enable
  end

  private

    def hipster_words
      @hipster_words ||= "retro put a bird on it wolf vegan gluten-free swag trust fund master cleanse four loko synth gentrify literally lomo bitters Keytar try-hard semiotics gastropub marfa YOLO bicycle rights street art authentic DIY fashion axe Chia letterpress twee mlkshk Typewriter umami Etsy keffiyeh direct trade Distillery Odd Future narwhal Pitchfork roof party pork belly Austin McSweeney's cliche Dreamcatcher +1 drinking vinegar Neutra Vice pickled Brooklyn Williamsburg hella small batch ennui squid Schlitz Kale chips food truck butcher vinyl ethnic fixie shabby chic iPhone Terry Richardson Deep v hoodie forage Retro fanny pack wayfarers messenger bag pug you probably haven't heard of them Art disrupt High Life hashtag chambray readymade selfies sartorial PBR leggings flexitarian chillwave Cosby sweater Marfa typewriter freegan chia biodiesel 8-bit occupy Tonx sustainable cray PBR&B Yr skateboard Tumblr 3 moon fingerstache Intelligentsia plaid Thundercats XOXO craft beer mustache Shoreditch Next level Godard actually mumblecore keytar stumptown irony Disrupt tofu scenester lo-fi single-origin coffee kogi beard yr tattooed viral 90's aesthetic pop-up mixtape Pinterest Asymmetrical kitsch farm-to-table photo booth cardigan Squid Helvetica Direct kale salvia American Apparel artisan tousled".split(' ') +
                          (["\n\n"] * 20)
{'fraction_non_alphanumeric': 0.088287020752726, 'fraction_numerical': 0.009379763160980185, 'mean_word_length': 3.241670810542019, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1169407', 'n_tokens_mistral': 2675, 'n_tokens_neox': 2501, 'n_words': 931}