<commit_before>class Ekylibre::BackupExchanger < ActiveExchanger::Base
  class Backup < Hash
    MODELS = [:account, :account_balance, :area, :asset, :asset_depreciation, :bank_statement, :cash, :cash_transfer, :contact, :cultivation, :custom_field, :custom_field_choice, :custom_field_datum, :delay, :department, :deposit, :deposit_line, :district, :document, :document_template, :entity, :entity_category, :entity_link, :entity_link_nature, :entity_nature, :establishment, :event, :event_nature, :financial_year, :incoming_delivery, :incoming_delivery_line, :incoming_delivery_mode, :incoming_payment, :incoming_payment_mode, :incoming_payment_use, :inventory, :inventory_line, :journal, :journal_entry, :journal_entry_line, :land_parcel, :land_parcel_group, :land_parcel_kinship, :listing, :listing_node, :listing_node_item, :mandate, :observation, :operation, :operation_line, :operation_nature, :operation_use, :outgoing_delivery, :outgoing_delivery_line, :outgoing_delivery_mode, :outgoing_payment, :outgoing_payment_mode, :outgoing_payment_use, :preference, :price, :product, :product_category, :product_component, :production_chain, :production_chain_conveyor, :production_chain_work_center, :production_chain_work_center_use, :profession, :purchase, :purchase_line, :purchase_nature, :role, :sale, :sale_line, :sale_nature, :sequence, :stock, :stock_move, :stock_transfer, :subscription, :subscription_nature, :tax, :tax_declaration, :tool, :tracking, :tracking_state, :transfer, :transport, :unit, :user, :warehouse]

    SCHEMA = YAML.load_file(Pathname.new(__FILE__).dirname.join('backup', 'schema-0.4.yml')).deep_symbolize_keys.freeze

    attr_reader :matchings

    def self.load(company)
      # x = SCHEMA.select do |table, columns|
      #   y = columns.keys.delete_if{|c| [:creator_id, :updater_id, :company_id].include?(c)}.detect{|c| c.to_s =~ /\_id$/}
      #   y
      # end.keys
      # # Root models: (MODELS.map{|m| m.to_s.pluralize.to_sym} - x)
      backup = new
      company.children.each do |rows|
        model = rows.attr(:model).to_sym
        table = model.to_s.pluralize.to_sym
        backup[model] = []
        rows.children.each do |row|
          backup[model] << row.attributes.inject({}) do |hash, pair|
            attribute = pair.first.to_sym
            value = pair.second.to_s
            if SCHEMA[table] and SCHEMA[table][attribute] and type = SCHEMA[table][attribute][:type].to_sym
              if type == :boolean
                value = (value == 'true' ? true : false)
              elsif type == :integer
                value = (value.blank? ? nil : value.to_i)
              elsif type == :decimal
                value = (value.blank? ? nil : value.to_d)
              elsif type == :date
                value = (value.blank? ? nil : value.to_date)
              elsif type == :datetime
                value = (value.blank? ? nil : value.to_datetime)
              end
            elsif attribute.to_s =~ /(^|\_)id$/
              value = value.to_i
              value = nil if value.zero?
            end
            # value = nil if value.blank?
            hash[attribute] = value unless [:company_id].include?(attribute)
            hash
          end.to_struct
        end
      end
      backup
    end

    def initialize(*args)
      super(*args)
      @matchings = {}.with_indifferent_access
      @indexes = {}.with_indifferent_access
    end

    def value_of(item, old_column, references = nil, converter = nil)
      value = nil
      if converter
        value = converter.call(item)
      else
        value = item[old_column]
        if references
          if references =~ /^\~/ and ref = item[references[1..-1]]
            ref = ref.underscore
            if ref.blank?
              value = nil
            elsif @matchings[ref]
              value = @matchings[ref][value]
            else
              fail "Cannot match #{ref.inspect} (polymorphic)"
            end
          else
            if @matchings[references]
              value = @matchings[references][value]
            else
              fail "Cannot match #{references.inspect}"
            end
          end
        end
      end
      value
    end

    def build_index(backup_model, keys)
      @indexes[backup_model] = []
      self[backup_model].each do |item|
        unique_code = keys.collect { |k| item.send(k) }.join('-')
        @indexes[backup_model] << unique_code
      end
    end

    def uniqify_doubles!(backup_model, keys, options = {})
      build_index(backup_model, keys)
      puts keys.inspect
      doubles = @indexes[backup_model].size - @indexes[backup_model].uniq.size
      puts keys.inspect
      undoubler = options[:undoubler] || keys.reject { |k| k.to_s =~ /(^|\_)id$/ }.last
      puts keys.inspect
      puts undoubler.inspect
      unless doubles.zero?
        puts ''
        puts "#{doubles} doubles detected on #{backup_model}".red
        dones = []
        separator = options[:undoubler_separator] || '-'
        self[backup_model].each do |item|
          unique_code = keys.collect { |k| item.send(k) }.join('-')
          if dones.include?(unique_code)
            puts unique_code.red
            prefix = item.send(undoubler)
            undoubler_value = prefix
            counter = 1
            loop do
              undoubler_value = prefix + separator + counter.to_s
              unique_code = keys.collect { |k| k == undoubler ? undoubler_value : item.send(k) }.join('-')
              break unless dones.include?(unique_code)
              counter += 1
            end
            puts unique_code.green
            item.send("#{undoubler}=", undoubler_value)
          end
          dones << unique_code
        end
      end
    end

    def import(backup_model, *args)
      # puts "Import #{backup_model.to_s.red}"
      # puts self[backup_model].first.inspect.yellow
      options = args.extract_options!
      keys = args
      keys << :name if keys.empty?
      columns = SCHEMA[backup_model.to_s.pluralize.to_sym].delete_if do |c|
        [:company_id, :creator_id, :updater_id, :id, :lock_version].include? c
      end
      keys.each do |key|
        fail "Invalid key for record identification: #{key}" unless columns.include? key<commit_msg>Removes unwanted puts in backup exchanger<commit_after>class Ekylibre::BackupExchanger < ActiveExchanger::Base
  class Backup < Hash
    MODELS = [:account, :account_balance, :area, :asset, :asset_depreciation, :bank_statement, :cash, :cash_transfer, :contact, :cultivation, :custom_field, :custom_field_choice, :custom_field_datum, :delay, :department, :deposit, :deposit_line, :district, :document, :document_template, :entity, :entity_category, :entity_link, :entity_link_nature, :entity_nature, :establishment, :event, :event_nature, :financial_year, :incoming_delivery, :incoming_delivery_line, :incoming_delivery_mode, :incoming_payment, :incoming_payment_mode, :incoming_payment_use, :inventory, :inventory_line, :journal, :journal_entry, :journal_entry_line, :land_parcel, :land_parcel_group, :land_parcel_kinship, :listing, :listing_node, :listing_node_item, :mandate, :observation, :operation, :operation_line, :operation_nature, :operation_use, :outgoing_delivery, :outgoing_delivery_line, :outgoing_delivery_mode, :outgoing_payment, :outgoing_payment_mode, :outgoing_payment_use, :preference, :price, :product, :product_category, :product_component, :production_chain, :production_chain_conveyor, :production_chain_work_center, :production_chain_work_center_use, :profession, :purchase, :purchase_line, :purchase_nature, :role, :sale, :sale_line, :sale_nature, :sequence, :stock, :stock_move, :stock_transfer, :subscription, :subscription_nature, :tax, :tax_declaration, :tool, :tracking, :tracking_state, :transfer, :transport, :unit, :user, :warehouse]

    SCHEMA = YAML.load_file(Pathname.new(__FILE__).dirname.join('backup', 'schema-0.4.yml')).deep_symbolize_keys.freeze

    attr_reader :matchings

    def self.load(company)
      # x = SCHEMA.select do |table, columns|
      #   y = columns.keys.delete_if{|c| [:creator_id, :updater_id, :company_id].include?(c)}.detect{|c| c.to_s =~ /\_id$/}
      #   y
      # end.keys
      # # Root models: (MODELS.map{|m| m.to_s.pluralize.to_sym} - x)
      backup = new
      company.children.each do |rows|
        model = rows.attr(:model).to_sym
        table = model.to_s.pluralize.to_sym
        backup[model] = []
        rows.children.each do |row|
          backup[model] << row.attributes.inject({}) do |hash, pair|
            attribute = pair.first.to_sym
            value = pair.second.to_s
            if SCHEMA[table] and SCHEMA[table][attribute] and type = SCHEMA[table][attribute][:type].to_sym
              if type == :boolean
                value = (value == 'true' ? true : false)
              elsif type == :integer
                value = (value.blank? ? nil : value.to_i)
              elsif type == :decimal
                value = (value.blank? ? nil : value.to_d)
              elsif type == :date
                value = (value.blank? ? nil : value.to_date)
              elsif type == :datetime
                value = (value.blank? ? nil : value.to_datetime)
              end
            elsif attribute.to_s =~ /(^|\_)id$/
              value = value.to_i
              value = nil if value.zero?
            end
            # value = nil if value.blank?
            hash[attribute] = value unless [:company_id].include?(attribute)
            hash
          end.to_struct
        end
      end
      backup
    end

    def initialize(*args)
      super(*args)
      @matchings = {}.with_indifferent_access
      @indexes = {}.with_indifferent_access
    end

    def value_of(item, old_column, references = nil, converter = nil)
      value = nil
      if converter
        value = converter.call(item)
      else
        value = item[old_column]
        if references
          if references =~ /^\~/ and ref = item[references[1..-1]]
            ref = ref.underscore
            if ref.blank?
              value = nil
            elsif @matchings[ref]
              value = @matchings[ref][value]
            else
              fail "Cannot match #{ref.inspect} (polymorphic)"
            end
          else
            if @matchings[references]
              value = @matchings[references][value]
            else
              fail "Cannot match #{references.inspect}"
            end
          end
        end
      end
      value
    end

    def build_index(backup_model, keys)
      @indexes[backup_model] = []
      self[backup_model].each do |item|
        unique_code = keys.collect { |k| item.send(k) }.join('-')
        @indexes[backup_model] << unique_code
      end
    end

    def uniqify_doubles!(backup_model, keys, options = {})
      build_index(backup_model, keys)
      doubles = @indexes[backup_model].size - @indexes[backup_model].uniq.size
      undoubler = options[:undoubler] || keys.reject { |k| k.to_s =~ /(^|\_)id$/ }.last
      unless doubles.zero?
        dones = []
        separator = options[:undoubler_separator] || '-'
        self[backup_model].each do |item|
          unique_code = keys.collect { |k| item.send(k) }.join('-')
          if dones.include?(unique_code)
            prefix = item.send(undoubler)
            undoubler_value = prefix
            counter = 1
            loop do
              undoubler_value = prefix + separator + counter.to_s
              unique_code = keys.collect { |k| k == undoubler ? undoubler_value : item.send(k) }.join('-')
              break unless dones.include?(unique_code)
              counter += 1
            end
            item.send("#{undoubler}=", undoubler_value)
          end
          dones << unique_code
        end
      end
    end

    def import(backup_model, *args)
      # puts "Import #{backup_model.to_s.red}"
      # puts self[backup_model].first.inspect.yellow
      options = args.extract_options!
      keys = args
      keys << :name if keys.empty?
      columns = SCHEMA[backup_model.to_s.pluralize.to_sym].delete_if do |c|
        [:company_id, :creator_id, :updater_id, :id, :lock_version].include? c
      end
      keys.each do |key|
        fail "Invalid key for record identification: #{key}" unless columns.include? key
{'fraction_non_alphanumeric': 0.11911537523375884, 'fraction_numerical': 0.0009756890804130417, 'mean_word_length': 2.531438415159345, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '516880', 'n_tokens_mistral': 3910, 'n_tokens_neox': 3816, 'n_words': 825}