<commit_before>require 'gds_api/rummager'
require 'csv'

namespace :taxonomy do
  namespace :analysis do
    desc "Produces a CSV list of content items under a root taxon ID, and the taxons they are tagged to"
    task :taxons_tagged_to_content, [:root_taxon_id] => [:environment] do |_, args|
      root_taxon_id = args.fetch(:root_taxon_id)

      results_per_page = 1000
      page = 0
      content_items = []
      loop do
        results = GdsApi::Rummager.new(Plek.find('rummager')).search(
          filter_part_of_taxonomy_tree: root_taxon_id,
          fields: %w(link taxons content_id),
          start: page * results_per_page,
          count: results_per_page,
        )['results']

        break if results.empty?

        content_items += results
        page += 1
      end

      puts %w(content_id, base_path, number_of_taxons).to_csv

      content_items.each do |content_item|
        csv_row = [
          content_item['content_id'],
          content_item['link'],
          content_item['taxons'].length,
        ]

        content_item['taxons'].each { |taxon_id| csv_row << taxon_id }

        puts csv_row.to_csv
      end
    end
  end
end<commit_msg>Output organisations to taxonomy analysis<commit_after>require 'gds_api/rummager'
require 'csv'

namespace :taxonomy do
  namespace :analysis do
    desc "Produces a CSV list of content items under a root taxon ID, and the taxons they are tagged to"
    task :taxons_tagged_to_content, [:root_taxon_id] => [:environment] do |_, args|
      root_taxon_id = args.fetch(:root_taxon_id)

      results_per_page = 1000
      page = 0
      content_items = []
      loop do
        results = GdsApi::Rummager.new(Plek.find('rummager')).search(
          filter_part_of_taxonomy_tree: root_taxon_id,
          fields: %w(link taxons content_id organisations),
          start: page * results_per_page,
          count: results_per_page,
        )['results']

        break if results.empty?

        content_items += results
        page += 1
      end

      puts %w(organisations content_id base_path number_of_taxons).to_csv

      content_items.each do |content_item|
        organisations = content_item.fetch('organisations', [])
          .map { |org| org['title'] }
          .join('; ')

        csv_row = [
          organisations,
          content_item['content_id'],
          content_item['link'],
          content_item['taxons'].length,
        ]

        content_item['taxons'].each { |taxon_id| csv_row << taxon_id }

        puts csv_row.to_csv
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09633385335413416, 'fraction_numerical': 0.0046801872074883, 'mean_word_length': 2.508891928864569, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368910', 'n_tokens_mistral': 839, 'n_tokens_neox': 807, 'n_words': 188}