<commit_before>namespace :tariff do
  desc 'Installs Trade Tariff, creates relevant records, imports national data'
  task install: %w[environment
                   install:taric:sections
                   install:taric:section_notes
                   install:taric:chapter_notes
                   install:chief:static_national_data
                   install:chief:standing_data]

  desc 'Reindex relevant entities on ElasticSearch'
  task reindex: %w[environment] do
    TradeTariffBackend.reindex
  end


  task :assign_footnote_id_to_headings, [:footnote_id, :footnote_type_id, :heading_ids] => :environment do |t, args|
    footnote_id = args[:footnote_id]
    footnote_type_id = args[:footnote_type_id]
    heading_ids = args[:heading_ids].split(' ')
    puts "Arguments: "
    puts "Footnote id #{footnote_id}"
    puts "Footnote type id #{footnote_type_id}"
    puts "Heading ids #{heading_ids.join(', ')}"

    footnote = Footnote.where(footnote_id: footnote_id, footnote_type_id: footnote_type_id).first

    Heading.where(goods_nomenclatures__goods_nomenclature_item_id: heading_ids).each do |heading|
      next if heading.footnotes.include?(footnote)
      puts "Associating footnote #{footnote.inspect} with heading #{heading.inspect}"
      associate_footnote_with_goods_nomenclature(heading, footnote)

      heading.commodities.each do |commodity|
        associate_footnote_with_goods_nomenclature(commodity, footnote)
        puts "Associating footnote #{footnote.inspect} with Commodity #{commodity.inspect}"
      end
    end

    puts "SUCCESS"
  end

  desc 'Add commodity footnotes for ECO licences where these is an export restriction'
  task add_missing_commodity_footnote: :environment do
    measure_type_id = MeasureType.all.detect { |mt| mt.description == 'Export authorization (Dual use)' }.values[:measure_type_id]
    footnote = Footnote.filter(footnote_id: '002', footnote_type_id: '05').first
    sql = %Q{
      SELECT DISTINCT goods_nomenclatures.goods_nomenclature_sid, goods_nomenclatures.goods_nomenclature_item_id,
      goods_nomenclatures.validity_start_date, producline_suffix,
      goods_nomenclatures.validity_end_date, goods_nomenclatures.operation
      FROM goods_nomenclatures
      LEFT JOIN measures ON (measures.goods_nomenclature_item_id = goods_nomenclatures.goods_nomenclature_item_id)
         WHERE ((measures.measure_type_id = '#{measure_type_id}')
        )
      AND NOT EXISTS (
      select 1 from footnote_association_goods_nomenclatures
      where footnote_association_goods_nomenclatures.footnote_type = '05'
      and footnote_association_goods_nomenclatures.footnote_id = '002'
      and goods_nomenclature_item_id = goods_nomenclatures.goods_nomenclature_item_id
      )
    }
    GoodsNomenclature.fetch(sql).all.each do |c|
      if c.footnote != footnote
        FootnoteAssociationGoodsNomenclature.associate_footnote_with_goods_nomenclature(c, footnote)
      end
    end
  end

  def associate_footnote_with_goods_nomenclature(source, footnote)
    f = FootnoteAssociationGoodsNomenclature.new
    f.values[:goods_nomenclature_sid] = source.values[:goods_nomenclature_sid]
    f.values[:goods_nomenclature_item_id] = source.values[:goods_nomenclature_item_id]
    f.values[:productline_suffix] = source.values[:producline_suffix]
    f.values[:validity_start_date] = source.values[:validity_start_date]
    f.values[:validity_end_date] = source.values[:validity_end_date]
    f.values[:operation] = source.values[:operation]
    f.values[:footnote_id] = footnote.values[:footnote_id]
    f.values[:footnote_type] = footnote.values[:footnote_type_id]
    f.values[:national] = footnote.values[:national]
    f.save
  end

  desc 'Download and apply Taric and CHIEF data'
  task sync: %w[environment sync:apply]

  desc "Restore missing chief records files"
  task restore_missing_chief_records: :environment do
    require "csv"

    # Custom converter
    CSV::Converters[:null_to_nil] = lambda do |field|
      field && field == "NULL" ? nil : field
    end

    ["comm", "tamf", "tbl9", "mfcm", "tame"].each do |table_name|
      file_path = File.join(Rails.root, "data", "missing_chief_records", "#{table_name}.csv")

      rows = CSV.read(file_path, headers: true, header_converters: :symbol, converters: [:null_to_nil])

      rows.each do |line|
        "Chief::#{table_name.capitalize}".constantize.insert line.to_hash
      end

      puts "#{table_name} table processed"
    end<commit_msg>Remove redundant method<commit_after>namespace :tariff do
  desc 'Installs Trade Tariff, creates relevant records, imports national data'
  task install: %w[environment
                   install:taric:sections
                   install:taric:section_notes
                   install:taric:chapter_notes
                   install:chief:static_national_data
                   install:chief:standing_data]

  desc 'Reindex relevant entities on ElasticSearch'
  task reindex: %w[environment] do
    TradeTariffBackend.reindex
  end


  task :assign_footnote_id_to_headings, [:footnote_id, :footnote_type_id, :heading_ids] => :environment do |t, args|
    footnote_id = args[:footnote_id]
    footnote_type_id = args[:footnote_type_id]
    heading_ids = args[:heading_ids].split(' ')
    puts "Arguments: "
    puts "Footnote id #{footnote_id}"
    puts "Footnote type id #{footnote_type_id}"
    puts "Heading ids #{heading_ids.join(', ')}"

    footnote = Footnote.where(footnote_id: footnote_id, footnote_type_id: footnote_type_id).first

    Heading.where(goods_nomenclatures__goods_nomenclature_item_id: heading_ids).each do |heading|
      next if heading.footnotes.include?(footnote)
      puts "Associating footnote #{footnote.inspect} with heading #{heading.inspect}"
      associate_footnote_with_goods_nomenclature(heading, footnote)

      heading.commodities.each do |commodity|
        associate_footnote_with_goods_nomenclature(commodity, footnote)
        puts "Associating footnote #{footnote.inspect} with Commodity #{commodity.inspect}"
      end
    end

    puts "SUCCESS"
  end

  desc 'Add commodity footnotes for ECO licences where these is an export restriction'
  task add_missing_commodity_footnote: :environment do
    measure_type_id = MeasureType.all.detect { |mt| mt.description == 'Export authorization (Dual use)' }.values[:measure_type_id]
    footnote = Footnote.filter(footnote_id: '002', footnote_type_id: '05').first
    sql = %Q{
      SELECT DISTINCT goods_nomenclatures.goods_nomenclature_sid, goods_nomenclatures.goods_nomenclature_item_id,
      goods_nomenclatures.validity_start_date, producline_suffix,
      goods_nomenclatures.validity_end_date, goods_nomenclatures.operation
      FROM goods_nomenclatures
      LEFT JOIN measures ON (measures.goods_nomenclature_item_id = goods_nomenclatures.goods_nomenclature_item_id)
         WHERE ((measures.measure_type_id = '#{measure_type_id}')
        )
      AND NOT EXISTS (
      select 1 from footnote_association_goods_nomenclatures
      where footnote_association_goods_nomenclatures.footnote_type = '05'
      and footnote_association_goods_nomenclatures.footnote_id = '002'
      and goods_nomenclature_item_id = goods_nomenclatures.goods_nomenclature_item_id
      )
    }
    GoodsNomenclature.fetch(sql).all.each do |c|
      if c.footnote != footnote
        FootnoteAssociationGoodsNomenclature.associate_footnote_with_goods_nomenclature(c, footnote)
      end
    end
  end

  desc 'Download and apply Taric and CHIEF data'
  task sync: %w[environment sync:apply]

  desc "Restore missing chief records files"
  task restore_missing_chief_records: :environment do
    require "csv"

    # Custom converter
    CSV::Converters[:null_to_nil] = lambda do |field|
      field && field == "NULL" ? nil : field
    end

    ["comm", "tamf", "tbl9", "mfcm", "tame"].each do |table_name|
      file_path = File.join(Rails.root, "data", "missing_chief_records", "#{table_name}.csv")

      rows = CSV.read(file_path, headers: true, header_converters: :symbol, converters: [:null_to_nil])

      rows.each do |line|
        "Chief::#{table_name.capitalize}".constantize.insert line.to_hash
      end

      puts "#{table_name} table processed"
    end
{'fraction_non_alphanumeric': 0.09006928406466513, 'fraction_numerical': 0.002917223775373769, 'mean_word_length': 4.113735239279055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3909943', 'n_tokens_mistral': 2711, 'n_tokens_neox': 2554, 'n_words': 520}