<commit_before>
namespace :db do

  desc "generate random articles (ARTICLES_NR, 10 by default) with tags and comments"
  task :generate_random_articles => :environment do
    
    articles_nr = ((ENV['ARTICLES_NR'] =~ /^\d+$/) ? ENV['ARTICLES_NR'].to_i : 10)
    
    (1..articles_nr).to_a.each do |i|
      article = Factory.build(:article)
      3.times { Factory.create(:comment, :article => article) }
      puts "#{i}. #{article.title}"
    end
    
  end
    
end
<commit_msg>add task for exporting articles (and task for quick export test)<commit_after>
namespace :db do

  desc "generate random articles (ARTICLES_NR, 10 by default) with tags and comments"
  task :generate_random_articles => :environment do
    
    articles_nr = ((ENV['ARTICLES_NR'] =~ /^\d+$/) ? ENV['ARTICLES_NR'].to_i : 10)
    
    (1..articles_nr).to_a.each do |i|
      article = Factory.build(:article)
      3.times { Factory.create(:comment, :article => article) }
      puts "#{i}. #{article.title}"
    end
    
  end

  desc "export articles into JSON file (EXPORT_FILE or 'exported_articles_TIMESTAMP.json')"
  task :export_articles => :environment do
  
    ENV['EXPORT_FILE'] ||= "exported_articles_#{Time.now.strftime('%d-%m-%Y_%H-%M')}.json"

    File.open(ENV['EXPORT_FILE'], 'w') do |f|
      f.puts "[\n" + Article.all.map { |article| article.attributes.to_json }.join(",\n") + "\n]"
    end

    puts "Done, #{Article.count} articles exported into '#{ENV['EXPORT_FILE']}'."    
  end

  # PS: import is meant to be done by throwaway scripts, this is just for validaning exported articles
  # This is just a quick check, to properly test, load into database or write tests
  desc "test exported articles from EXPORT_FILE"
  task :test_export do
  
    raise Exception, "EXPORT_FILE env. variable is not set!" unless ENV['EXPORT_FILE']

    begin
      articles = JSON::parse(File.read(ENV['EXPORT_FILE']))

      puts "Exported #{articles.size} articles, it looks OK."
    rescue => e
      puts "Export file is invalid:\n#{e}"
    end
  end
    
end

{'fraction_non_alphanumeric': 0.13853658536585367, 'fraction_numerical': 0.005853658536585366, 'mean_word_length': 3.3178947368421055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5865166', 'n_tokens_mistral': 697, 'n_tokens_neox': 666, 'n_words': 195}