<commit_before>class Booru
  # Get post by id
  # Only new API
  # http://danbooru.donmai.us/posts/$id.json
  # === Returns
  # Hash:: Data of found post
  def post(id)
    only_new_api
    do_request("posts/#{id}.json")
  end

  # Get posts by tags
  # http://danbooru.donmai.us/posts.json?tags=touhou&page=1&limit=100
  # https://yande.re/post.json?tags=touhou&page=1&limit=100
  # === Returns
  # Array:: Hashes of found posts
  def posts_by_tags(tags, page = 1, limit = options[:limits][:per_page])
    tags = clean_tags(tags)
    posts_url = self.class::OLD_API ? 'post.json' : 'posts.json'
    do_request(posts_url, tags: tags, page: page, limit: limit)
  end

  # Get posts count by tag
  # http://danbooru.donmai.us/counts/posts.json?tags=touhou
  # https://yande.re/post/index.xml?tags=touhou&limit=1
  # === Returns
  # Integer:: Number of posts
  def posts_count_by_tag(tags)
    tags = clean_tags(tags)
    if self.class::OLD_API
      do_request('post/index.xml', { tags: tags, limit: 1 }, :get, nil, :xml).root['count'].to_i
    else
      do_request('counts/posts.json', tags: tags)['counts']['posts']
    end
  end

  def download_by_tags(tags)
    tags = clean_tags(tags)
    tags_dir = sanitize_filename(tags)
    puts "Tags are #{tags}."
    FileUtils.mkdir_p tags_dir

    bbs_path = File.join(options[:storage] || tags_dir, 'files.bbs')
    bbs = File.new(bbs_path, 'a+')
    old_bbs = bbs.read

    count = posts_count_by_tag(tags)
    if count.zero?
      puts 'No posts, nothing to do.'
    else
      pages = (count.to_f / options[:limits][:per_page]).ceil
      pages = options[:limits][:pages] if options[:limits][:pages] && options[:limits][:pages] < pages
      num = 1
      1.upto(pages) do |page|
        puts "Page #{page}/#{pages}:"
        posts_by_tags(tags, page, options[:limits][:per_page]).each do |post_data|
          download_post(post_data, tags_dir, num, count, bbs, old_bbs)
          num += 1
        end
      end
    end
  end

  private

  def download_post(post_data, target, num, count, bbs, old_bbs)
    exit if options[:limits][:posts] && options[:limits][:posts] < num

    # Prepare post data
    if post_data['file_url']
      url = get_url(post_data['file_url'])
      md5 = post_data['md5']
    else
      url, md5 = get_data_from_html(post_data['id'])
      return nil if url.nil?
    end
    filename = get_filename(post_data, md5, url)
    tag_string = self.class::OLD_API ? post_data['tags'] : post_data['tag_string']

    path =
      if options[:storage]
        File.join(options[:storage], filename)
      else
        File.join(target, filename)
      end
    if File.exist?(path) && md5 == Digest::MD5.hexdigest(File.read(path))
      puts "File exists - #{path} (#{num}/#{count})"
    else
      puts "Saving #{path}... (#{num}/#{count})"
      download_with_tool(url, path)
      puts 'saved!'
    end
    FileUtils.ln_sf(File.join('..', path), File.join(target, filename)) if options[:storage]<commit_msg>Disable retry with HTML as it doesn't work anymore.<commit_after>class Booru
  # Get post by id
  # Only new API
  # http://danbooru.donmai.us/posts/$id.json
  # === Returns
  # Hash:: Data of found post
  def post(id)
    only_new_api
    do_request("posts/#{id}.json")
  end

  # Get posts by tags
  # http://danbooru.donmai.us/posts.json?tags=touhou&page=1&limit=100
  # https://yande.re/post.json?tags=touhou&page=1&limit=100
  # === Returns
  # Array:: Hashes of found posts
  def posts_by_tags(tags, page = 1, limit = options[:limits][:per_page])
    tags = clean_tags(tags)
    posts_url = self.class::OLD_API ? 'post.json' : 'posts.json'
    do_request(posts_url, tags: tags, page: page, limit: limit)
  end

  # Get posts count by tag
  # http://danbooru.donmai.us/counts/posts.json?tags=touhou
  # https://yande.re/post/index.xml?tags=touhou&limit=1
  # === Returns
  # Integer:: Number of posts
  def posts_count_by_tag(tags)
    tags = clean_tags(tags)
    if self.class::OLD_API
      do_request('post/index.xml', { tags: tags, limit: 1 }, :get, nil, :xml).root['count'].to_i
    else
      do_request('counts/posts.json', tags: tags)['counts']['posts']
    end
  end

  def download_by_tags(tags)
    tags = clean_tags(tags)
    tags_dir = sanitize_filename(tags)
    puts "Tags are #{tags}."
    FileUtils.mkdir_p tags_dir

    bbs_path = File.join(options[:storage] || tags_dir, 'files.bbs')
    bbs = File.new(bbs_path, 'a+')
    old_bbs = bbs.read

    count = posts_count_by_tag(tags)
    if count.zero?
      puts 'No posts, nothing to do.'
    else
      pages = (count.to_f / options[:limits][:per_page]).ceil
      pages = options[:limits][:pages] if options[:limits][:pages] && options[:limits][:pages] < pages
      num = 1
      1.upto(pages) do |page|
        puts "Page #{page}/#{pages}:"
        posts_by_tags(tags, page, options[:limits][:per_page]).each do |post_data|
          download_post(post_data, tags_dir, num, count, bbs, old_bbs)
          num += 1
        end
      end
    end
  end

  private

  def download_post(post_data, target, num, count, bbs, old_bbs)
    exit if options[:limits][:posts] && options[:limits][:posts] < num

    # Prepare post data
    if post_data['file_url']
      url = get_url(post_data['file_url'])
      md5 = post_data['md5']
    else
      # url, md5 = get_data_from_html(post_data['id'])
      # return if url.nil?
      puts "File url is unknown for #{post_data['id']}."
      return
    end
    filename = get_filename(post_data, md5, url)
    tag_string = self.class::OLD_API ? post_data['tags'] : post_data['tag_string']

    path =
      if options[:storage]
        File.join(options[:storage], filename)
      else
        File.join(target, filename)
      end
    if File.exist?(path) && md5 == Digest::MD5.hexdigest(File.read(path))
      puts "File exists - #{path} (#{num}/#{count})"
    else
      puts "Saving #{path}... (#{num}/#{count})"
      download_with_tool(url, path)
      puts 'saved!'
    end
    FileUtils.ln_sf(File.join('..', path), File.join(target, filename)) if options[:storage]
{'fraction_non_alphanumeric': 0.15275486638073243, 'fraction_numerical': 0.006598482349059716, 'mean_word_length': 3.1527397260273973, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848667', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2177, 'n_words': 462}