<commit_before>TradeTariffBackend::DataMigrator.migration do
  name "Assign footnote to commodities"

  up do
    applicable { true }
    apply {

      footnote = Footnote.where(footnote_id: "425", footnote_type_id: "01").first

      codes = %w[2103909081 2103909089]

      codes.each do |code|
        Sequel::Model.db.fetch("SELECT goods_nomenclature_item_id FROM measures_oplog
                                WHERE goods_nomenclature_item_id LIKE '#{code}%'
                                GROUP BY goods_nomenclature_item_id").all do |measure|
          puts "Running code #{code} and Measure with gono_item_id #{measure[:goods_nomenclature_item_id]}"
          goods_nomenclature = GoodsNomenclature.where(goods_nomenclature_item_id: measure[:goods_nomenclature_item_id]).first

          if goods_nomenclature.nil?
            puts "GoodsNomenclature #{measure.inspect} does not exist"
            next
          end
          if goods_nomenclature.footnotes_dataset.where(footnotes__footnote_id: footnote.footnote_id,
                                                        footnotes__footnote_type_id: footnote.footnote_type_id).empty?
            puts "Creating association for Footnote #{footnote.inspect} and GoodsNomenclature #{goods_nomenclature.inspect}"
            FootnoteAssociationGoodsNomenclature.associate_footnote_with_goods_nomenclature(goods_nomenclature, footnote)
          else
            puts "Footnote is already associated with GoodsNomenclature #{measure.inspect}"
          end<commit_msg>Update migration: Use footnote "05853" for commodities 2103909081 and 2103909089<commit_after>TradeTariffBackend::DataMigrator.migration do
  name "Assign footnote to commodities"

  up do
    applicable { true }
    apply {

      footnote = Footnote.where(footnote_id: "853", footnote_type_id: "05").first

      codes = %w[2103909081 2103909089]

      codes.each do |code|
        Sequel::Model.db.fetch("SELECT goods_nomenclature_item_id FROM measures_oplog
                                WHERE goods_nomenclature_item_id LIKE '#{code}%'
                                GROUP BY goods_nomenclature_item_id").all do |measure|
          puts "Running code #{code} and Measure with gono_item_id #{measure[:goods_nomenclature_item_id]}"
          goods_nomenclature = GoodsNomenclature.where(goods_nomenclature_item_id: measure[:goods_nomenclature_item_id]).first

          if goods_nomenclature.nil?
            puts "GoodsNomenclature #{measure.inspect} does not exist"
            next
          end
          if goods_nomenclature.footnotes_dataset.where(footnotes__footnote_id: footnote.footnote_id,
                                                        footnotes__footnote_type_id: footnote.footnote_type_id).empty?
            puts "Creating association for Footnote #{footnote.inspect} and GoodsNomenclature #{goods_nomenclature.inspect}"
            FootnoteAssociationGoodsNomenclature.associate_footnote_with_goods_nomenclature(goods_nomenclature, footnote)
          else
            puts "Footnote is already associated with GoodsNomenclature #{measure.inspect}"
          end
{'fraction_non_alphanumeric': 0.06888817686638898, 'fraction_numerical': 0.024030759371996155, 'mean_word_length': 2.8353808353808354, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4626909', 'n_tokens_mistral': 975, 'n_tokens_neox': 837, 'n_words': 176}