<commit_before>require 'config/environment'
namespace :mms do
  namespace :cache do
    cummulative_categories_cleanup_description = "Deletes cummulative information"    
    desc cummulative_categories_cleanup_description
    task :cummulative_categories_cleanup do |t|
      [CumulativeMediaCategoryAssociation, CachedCategoryCount].each { |model| model.connection.execute("TRUNCATE TABLE #{model.table_name}") }
      options = {:group =>'category_id', :order => 'category_id'}
      options.merge!({:joins => :medium, :conditions => {'media.application_filter_id' => ApplicationFilter.application_filter.id}}) if !ApplicationFilter.application_filter.nil?
      MediaCategoryAssociation.find(:all, options).collect(&:category).each do |category|
        medium_ids = ApplicationFilter.application_filter.nil? ? MediaCategoryAssociation.find(:all, :conditions => {:category_id => category.id}).collect(&:medium_id) : MediaCategoryAssociation.find(:all, :joins => :medium, :conditions => {:category_id => category.id, 'media.application_filter_id' => ApplicationFilter.application_filter.id}).collect(&:medium_id)
        ([category] + category.ancestors).each{ |c| medium_ids.each { |medium_id| CumulativeMediaCategoryAssociation.create(:category => c, :medium_id => medium_id) if CumulativeMediaCategoryAssociation.find(:first, :conditions => {:category_id => c.id, :medium_id => medium_id}).nil? } }
      end
    end
    
    desc "Deletes view cache"
    task :view_cleanup do |t|
      ['categories', 'documents', 'media_objects', 'places'].each{ |folder| `rm -rf #{File.join('public', folder)}` }
    end
  end
end<commit_msg>added deletion of fragment caching to rake cleanup<commit_after>require 'config/environment'
namespace :mms do
  namespace :cache do
    cummulative_categories_cleanup_description = "Deletes cummulative information"    
    desc cummulative_categories_cleanup_description
    task :cummulative_categories_cleanup do |t|
      [CumulativeMediaCategoryAssociation, CachedCategoryCount].each { |model| model.connection.execute("TRUNCATE TABLE #{model.table_name}") }
      options = {:group =>'category_id', :order => 'category_id'}
      options.merge!({:joins => :medium, :conditions => {'media.application_filter_id' => ApplicationFilter.application_filter.id}}) if !ApplicationFilter.application_filter.nil?
      MediaCategoryAssociation.find(:all, options).collect(&:category).each do |category|
        medium_ids = ApplicationFilter.application_filter.nil? ? MediaCategoryAssociation.find(:all, :conditions => {:category_id => category.id}).collect(&:medium_id) : MediaCategoryAssociation.find(:all, :joins => :medium, :conditions => {:category_id => category.id, 'media.application_filter_id' => ApplicationFilter.application_filter.id}).collect(&:medium_id)
        ([category] + category.ancestors).each{ |c| medium_ids.each { |medium_id| CumulativeMediaCategoryAssociation.create(:category => c, :medium_id => medium_id) if CumulativeMediaCategoryAssociation.find(:first, :conditions => {:category_id => c.id, :medium_id => medium_id}).nil? } }
      end
    end
    
    desc "Deletes view cache"
    task :view_cleanup do |t|
      ['categories', 'documents', 'media_objects', 'places'].each{ |folder| `rm -rf #{File.join('public', folder)}` }
      `rm -rf #{File.join('tmp', 'cache', 'views')}`
    end
  end
end
{'fraction_non_alphanumeric': 0.13839285714285715, 'fraction_numerical': 0.0, 'mean_word_length': 5.551656920077972, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1966122', 'n_tokens_mistral': 977, 'n_tokens_neox': 941, 'n_words': 195}