<commit_before>desc "Sanitize access limited data"
task sanitize_data: :environment do
  Tasks::DataSanitizer.delete_access_limited(STDOUT)
end<commit_msg>Restore links that were accidently erased<commit_after>desc "Sanitize access limited data"
task sanitize_data: :environment do
  Tasks::DataSanitizer.delete_access_limited(STDOUT)
end

task :restore_policy_links, [:action] => [:environment] do |_, args|
  dry_run = args["action"] != 'apply'
  puts "Dry run" if dry_run

  policy_query = Queries::GetContentCollection.new(
    document_type: 'policy',
    fields: %w(content_id),
    pagination: NullPagination.new
  )
  policy_content_ids = policy_query.call.map { |item| item["content_id"] }

  # Identify the destructive events
  policy_wipe_events = Event
    .where("created_at::date = '2016-03-14' and action = 'PatchLinkSet'")
    .where(content_id: policy_content_ids)
    .order("content_id, created_at asc")

  wipe_events_by_content_id = Hash.new { |h, k| h[k] = [] }

  puts "Found #{policy_wipe_events.size} destructive events"

  policy_wipe_events.each do |event|
    puts "#{event.content_id} #{event.action} #{event.created_at} #{event.payload[:links]}"

    wipe_events_by_content_id[event.content_id] << event.id
  end

  puts

  # Restore the links
  policy_wipe_events.each do |wipe_event|
    puts "Restoring #{wipe_event.content_id}..."

    policy_events = Event.where(
      content_id: wipe_event.content_id,
      action: %w(PutContentWithLinks PatchLinkSet)
    ).order("created_at desc")

    policy_log = []

    policy_events.each do |event|
      policy_log << event
      break if event.action == 'PutContentWithLinks'
    end

    raise "Event log error: no events found for #{wipe_event.content_id}" if policy_log.empty?

    if policy_log.last.action != 'PutContentWithLinks'
      # Policy created since Publishing API V2
      puts "New policy #{wipe_event.content_id}"
    end

    policy_log.reverse_each do |event|
      unless wipe_events_by_content_id[event.content_id].include?(event.id)
        puts "Reapplying #{event.created_at} #{event.action}: #{event.payload[:links]}"
        Commands::V2::PatchLinkSet.call(event.payload) unless dry_run
      end
    end

    puts
  end
end
{'fraction_non_alphanumeric': 0.11290322580645161, 'fraction_numerical': 0.004480286738351254, 'mean_word_length': 4.229508196721311, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3074673', 'n_tokens_mistral': 757, 'n_tokens_neox': 738, 'n_words': 162}