<commit_before>require 'whole_edition_translator'

namespace :editions do
  desc "Take old publications and convert to new top level editions"
  task :extract_to_whole_editions => :environment do
    WholeEdition.delete_all

    Publication.all.each do |publication|
      puts "Processing #{publication.class} #{publication.id}"
      if publication.panopticon_id.present?
        publication.editions.each do |edition|
          puts "  Into edition #{edition.id}"
          whole_edition = WholeEditionTranslator.new(publication, edition).run
          whole_edition.save!
        end
      else
        puts "No panopticon ID for #{publication.name} : #{publication.id}"
      end
    end
  end

  desc "denormalise associated users"
  task :denormalise => :environment do
    WholeEdition.all.each do |edition|
      begin
        puts "Processing #{edition.class} #{edition.id}"
        edition.denormalise_users and edition.save!
        puts "   Done!"
      rescue Exception => e
        puts "   [Err] Could not denormalise edition: #{e}"
      end
    end
  end

  desc "cache latest version number against editions"
  task :cache_version_numbers => :environment do
     WholeEdition.all.each do |edition|
      begin
        puts "Processing #{edition.class} #{edition.id}"
        if edition.subsequent_siblings.any?
          latest_edition = edition.subsequent_siblings.sort_by(&:version_number).last
          if latest_edition.in_progress?
            edition.update_sibling_in_progress(latest_edition.version_number)
          end
        end
        edition.save!
        puts "   Done!"
      rescue Exception => e
        puts "   [Err] Could not denormalise edition: #{e}"
      end
    end
  end
end<commit_msg>we should set nil when there is no siblings in progress<commit_after>require 'whole_edition_translator'

namespace :editions do
  desc "Take old publications and convert to new top level editions"
  task :extract_to_whole_editions => :environment do
    WholeEdition.delete_all

    Publication.all.each do |publication|
      puts "Processing #{publication.class} #{publication.id}"
      if publication.panopticon_id.present?
        publication.editions.each do |edition|
          puts "  Into edition #{edition.id}"
          whole_edition = WholeEditionTranslator.new(publication, edition).run
          whole_edition.save!
        end
      else
        puts "No panopticon ID for #{publication.name} : #{publication.id}"
      end
    end
  end

  desc "denormalise associated users"
  task :denormalise => :environment do
    WholeEdition.all.each do |edition|
      begin
        puts "Processing #{edition.class} #{edition.id}"
        edition.denormalise_users and edition.save!
        puts "   Done!"
      rescue Exception => e
        puts "   [Err] Could not denormalise edition: #{e}"
      end
    end
  end

  desc "cache latest version number against editions"
  task :cache_version_numbers => :environment do
     WholeEdition.all.each do |edition|
      begin
        puts "Processing #{edition.class} #{edition.id}"
        if edition.subsequent_siblings.any?
          latest_edition = edition.subsequent_siblings.sort_by(&:version_number).last
          if latest_edition.in_progress?
            edition.update_sibling_in_progress(latest_edition.version_number)
          else
            edition.update_sibling_in_progress(nil)
          end
        end
        edition.save!
        puts "   Done!"
      rescue Exception => e
        puts "   [Err] Could not denormalise edition: #{e}"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.08303047246295779, 'fraction_numerical': 0.0, 'mean_word_length': 2.636178861788618, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2607582', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1058, 'n_words': 275}