<commit_before>require 'wordpress'

namespace :wordpress do
  desc "Reset the blog relevant tables for a clean import"
  task :reset_blog do
    Rake::Task["environment"].invoke

    %w(posts post_translations taggings tags).each do |table_name|
      p "Truncating #{table_name} ..."
      ActiveRecord::Base.connection.execute "DELETE FROM #{table_name}"
    end

  end

  desc "import blog data from a WordPressImport XML dump"
  task :import_blog, :file_name, :blog_slug do |task, params|
    Rake::Task["environment"].invoke
    p "Loading XML from #{params[:file_name]} (using blog #{params[:blog_slug]}) ..."
    dump = WordPressImport::Dump.new(params[:file_name])

    p "Importing #{dump.authors.count} authors ..."
    dump.authors.each(&:to_rails)
    
    # by default, import all; unless $ONLY_PUBLISHED = "true"
    only_published = ENV['ONLY_PUBLISHED'] == 'true' ? true : false
    p "Importing #{dump.posts(only_published).count} posts ..."
    
    if only_published
      p "(only published posts)" 
    else
      p "(export ONLY_PUBLISHED=true to import only published posts)"
    end

    dump.posts(only_published).each{|p| p.to_rails(params[:blog_slug]) }
  end

  desc "reset blog tables and then import blog data from a WordPressImport XML dump"
  task :reset_and_import_blog, :file_name, :blog_slug do |task, params|
    Rake::Task["environment"].invoke
    Rake::Task["wordpress:reset_blog"].invoke
    Rake::Task["wordpress:import_blog"].invoke(params[:file_name], params[:blog_slug])
  end


  desc "download images in posts to public folder"
  task :download_post_images, :host_match do |task, params|
    raise "Error: you must specify a host to match for this download (i.e. rake wordpress:download_post_images['mywebsite']" if params[:host_match].blank?

    Rake::Task["environment"].invoke
    
    # scrape images
    @posts = ::Post.all
    @posts.each do |post|
      doc = Nokogiri::HTML(post.body)
      doc.css("img").each do |img|
        # find remote file path
        remote_file = img.attributes["src"].text
        # load uri
        begin
          remote_uri = URI(remote_file)
        rescue => error
          puts "Error parsing URL #{remote_file}: #{error.message}"
        end

        # only download if the image is a LFA-hosted image
        if remote_uri && remote_uri.host.match(params[:host_match]) != nil
          # find a local path for it
          local_file = File.expand_path(File.join(Rails.public_path,remote_uri.path))
          # only download if not already there or if it's zero bytes
          unless File.size?(local_file)
            # create local folders if necessary
            dirname = File.dirname(local_file)
            unless File.directory?(dirname)
              FileUtils.mkdir_p(dirname)
            end
            # save remote file to local
            begin
              remote_file_io = open(remote_file)
              File.open(local_file,'wb'){ |f| f.write(remote_file_io.read) }
              puts "Saved file: #{File.basename(local_file)}"
            rescue OpenURI::HTTPError => error
              puts "Error saving file #{remote_file}: #{error.message}"
            end
          end
        end
      end
    end
    puts "Finished downloding images from #{@posts.count} posts"

  end

  # desc "Reset the cms relevant tables for a clean import"
  # task :reset_pages do<commit_msg>Minor filename tweak<commit_after>require 'wordpress'

namespace :wordpress do
  desc "Reset the blog relevant tables for a clean import"
  task :reset_blog do
    Rake::Task["environment"].invoke

    %w(posts post_translations taggings tags).each do |table_name|
      p "Truncating #{table_name} ..."
      ActiveRecord::Base.connection.execute "DELETE FROM #{table_name}"
    end

  end

  desc "import blog data from a WordPressImport XML dump"
  task :import_blog, :file_name, :blog_slug do |task, params|
    Rake::Task["environment"].invoke
    p "Loading XML from #{params[:file_name]} (using blog #{params[:blog_slug]}) ..."
    dump = WordPressImport::Dump.new(params[:file_name])

    p "Importing #{dump.authors.count} authors ..."
    dump.authors.each(&:to_rails)
    
    # by default, import all; unless $ONLY_PUBLISHED = "true"
    only_published = ENV['ONLY_PUBLISHED'] == 'true' ? true : false
    p "Importing #{dump.posts(only_published).count} posts ..."
    
    if only_published
      p "(only published posts)" 
    else
      p "(export ONLY_PUBLISHED=true to import only published posts)"
    end

    dump.posts(only_published).each{|p| p.to_rails(params[:blog_slug]) }
  end

  desc "reset blog tables and then import blog data from a WordPressImport XML dump"
  task :reset_and_import_blog, :file_name, :blog_slug do |task, params|
    Rake::Task["environment"].invoke
    Rake::Task["wordpress:reset_blog"].invoke
    Rake::Task["wordpress:import_blog"].invoke(params[:file_name], params[:blog_slug])
  end


  desc "download images in posts to public folder"
  task :download_post_images, :host_match do |task, params|
    raise "Error: you must specify a host to match for this download (i.e. rake wordpress:download_post_images['mywebsite']" if params[:host_match].blank?

    Rake::Task["environment"].invoke
    
    # scrape images
    @posts = ::Post.all
    @posts.each do |post|
      doc = Nokogiri::HTML(post.body)
      doc.css("img").each do |img|
        # find remote file path
        remote_file = img.attributes["src"].text
        # load uri
        begin
          remote_uri = URI(remote_file)
        rescue => error
          puts "Error parsing URL #{remote_file}: #{error.message}"
        end

        # only download if the image is a LFA-hosted image
        if remote_uri && remote_uri.host.match(params[:host_match]) != nil
          # find a local path for it
          local_file = File.expand_path(File.join(Rails.public_path,remote_uri.path))
          # only download if not already there or if it's zero bytes
          unless File.size?(local_file)
            # create local folders if necessary
            dirname = File.dirname(local_file)
            unless File.directory?(dirname)
              FileUtils.mkdir_p(dirname)
            end
            # save remote file to local
            begin
              remote_file_io = open(remote_file)
              File.open(local_file,'wb'){ |f| f.write(remote_file_io.read) }
              puts "Saved file: #{local_file}"
            rescue OpenURI::HTTPError => error
              puts "Error saving file #{remote_file}: #{error.message}"
            end
          end
        end
      end
    end
    puts "Finished downloding images from #{@posts.count} posts"

  end

  # desc "Reset the cms relevant tables for a clean import"
  # task :reset_pages do
{'fraction_non_alphanumeric': 0.10853514471352628, 'fraction_numerical': 0.0, 'mean_word_length': 2.822234762979684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '738425', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1973, 'n_words': 573}