<commit_before>require 'cenit_cmd/collection'

module Cenit
  class Actions
    class << self

      def pull_request(shared_collection, options = {})

        pull_parameters = options[:pull_parameters] || {}
        missing_parameters = []
        shared_collection.pull_parameters.each do |pull_parameter|
          unless pull_parameters[(param_id = pull_parameter.id.to_s)].present?
            missing_parameters << param_id
          end
        end
        if options[:auto_fill]
          missing_parameters.each { |param_id| pull_parameters[param_id] = FFaker::Lorem.word }
          missing_parameters.clear
        end

        new_records = Hash.new { |h, k| h[k] = [] }
        updated_records = Hash.new { |h, k| h[k] = [] }

        pull_data = shared_collection.data_with(pull_parameters)

        invariant_data = {}

        collection_data = { '_reset' => resetting = [] }
        collection = Setup::Collection.where(name: shared_collection.name).first
        fields = %w(readme title)
        unless collection && fields.all? { |field| collection.send(field) == shared_collection.send(field) }
          fields.each do |field|
            shared_value = shared_collection[field]
            unless collection && (collection[field] == shared_value)
              collection_data[field] = shared_value
              resetting << field
            end
          end
        end
        fields = %w(metadata)
        unless collection && fields.all? { |field| collection.send(field) == pull_data[field] }
          fields.each do |field|
            shared_value = pull_data[field]
            unless collection && (collection[field] == shared_value)
              collection_data[field] = shared_value
              resetting << field
            end
          end
        end

        Setup::Collection.reflect_on_all_associations(:has_and_belongs_to_many).each do |relation|
          entry = relation.name.to_s
          if (items = pull_data[entry])
            invariant_data[entry] = invariant_names = Set.new
            invariant_on_collection = 0
            items_data_type = relation.klass.data_type
            refs =
              items.collect do |item|
                criteria = ref_criteria = {}
                items_data_type.get_referenced_by.each do |field|
                  field = field.to_s
                  if %w(id _id).include?(field)
                    criteria['_id'] = item['_id'] || item['id']
                  else
                    criteria[field] = item[field]
                  end
                end
                criteria.delete_if { |_, value| value.nil? }
                criteria = Cenit::Utility.deep_remove(criteria, '_reference')
                unless (on_collection = (record = collection && Cenit::Utility.find_record(criteria, collection.send(relation.name))))
                  record = Cenit::Utility.find_record(criteria, relation.klass.all)
                end
                if record
                  share_hash_options = {}
                  if shared_collection.installed?
                    share_hash_options[:include_id] = ->(r) { r.is_a?(Setup::CrossOriginShared) && r.shared? }
                  end
                  record_hash = record.share_hash(share_hash_options)
                  record_hash = Cenit::Utility.stringfy(record_hash)
                  %w(id _id).each do |id_key|
                    record_hash[id_key] = item[id_key] if item.has_key?(id_key)
                  end
                  if item['_type']
                    record_hash['_type'] = record.class.to_s unless record_hash['_type']
                  end
                  record_hash['_reset'] ||= item['_reset'] if item['_reset']
                  record_hash.reject! { |key, _| !item.has_key?(key) }
                  invariant = Cenit::Utility.eql_content?(record_hash, item) do |record_value, item_value|
                    (record_value.nil? && item_value.blank?) || (item_value.nil? && record_value.blank?)
                  end
                  if invariant
                    invariant_names << ref_criteria
                    invariant_on_collection += 1 if on_collection
                    item = ref_criteria
                    item['_reference'] = true
                  else
                    updated_records[entry] <<
                      if options[:updated_records_ids]
                        record.id.to_s
                      else
                        record
                      end
                  end
                  item['id'] = record.id.to_s
                  check_embedded_items(item, record)
                else
                  new_records[entry] << item
                end
                item
              end
            unless (collection && collection.send(relation.name).count == invariant_on_collection) && (invariant_on_collection == items.size)
              collection_data[entry] = refs
              resetting << entry
            end
          elsif collection && collection.send(relation.name).present?
            resetting << entry
          end
        end

        if resetting.present? && !options[:discard_collection]
          if collection
            updated_records['collections'] <<
              if options[:updated_records_ids]
                collection.id.to_s
              else
                collection
              end
          else
            new_records['collections'] << { 'name' => shared_collection.name }
          end
        end

        invariant_data.each do |key, invariant_names|
          pull_data[key].delete_if do |item|
            criteria = { namespace: item['namespace'], name: item['name'] }
            criteria.delete_if { |_, value| value.nil? }
            invariant_names.include?(criteria)
          end
        end

        [collection_data, pull_data, updated_records].each { |hash| hash.each_key { |key| hash.delete(key) if hash[key].blank? } }

        {
          pull_parameters: pull_parameters,
          new_records: new_records,
          updated_records: updated_records,
          missing_parameters: missing_parameters,
          pull_data: pull_data,
          collection_data: collection_data,
          collection_discarded: options[:discard_collection].present?,
          updated_records_ids: options[:updated_records_ids].present?
        }
      end

      def pull(shared_collection, pull_request = {})
        pull_request = pull_request.with_indifferent_access
        pull_request = pull_request(shared_collection, pull_request) if pull_request[:pull_data].nil?
        errors = []
        if pull_request[:missing_parameters].blank?
          created_nss_ids = []
          begin
            collection_data = pull_request.delete(:collection_data)
            (collection_data['namespaces'] || []).each do |ns_hash|
              if (slug = ns_hash['slug']) && Setup::Namespace.where(slug: slug).present?
                ns_hash.delete('slug')
              end
              next if ns_hash.has_key?('id')
              if (ns = Setup::Namespace.create(ns_hash)).errors.blank?
                ns_hash['id'] = ns.id.to_s
                created_nss_ids << ns.id
              else
                fail "Error creating namespace for #{ns_hash.to_json}: #{ns.errors.full_messages.to_sentence}"
              end
            end
            collection = Setup::Collection.where(name: shared_collection.name).first
            attrs = (collection && collection.attributes.deep_dup) || {}
            attrs.delete('_id')
            collection = Setup::Collection.new(attrs)
            collection.from_json(collection_data, add_only: true)
            collection.events.each { |e| e[:activated] = false if e.is_a?(Setup::Scheduler) && e.new_record? }
            begin
              collection.name = BSON::ObjectId.new.to_s
            end while Setup::Collection.where(name: collection.name).present?
            unless Cenit::Utility.save(collection, bind_references: { if: ->(r) { r.instance_variable_get(:@_edi_parsed) } },
                                       create_collector: (create_collector = Set.new),
                                       saved_collector: (saved = Set.new))
              collection.errors.full_messages.each { |msg| errors << msg }
              collection.errors.clear
              if Cenit::Utility.save(collection, { create_collector: create_collector })
                pull_request[:fixed_errors] = errors
                errors = []
              else
                saved.each do |obj|
                  if (obj = (obj.reload rescue nil))
                    obj.delete
                  end
                end<commit_msg>Refactoring auto-fixed error descriptions<commit_after>require 'cenit_cmd/collection'

module Cenit
  class Actions
    class << self

      def pull_request(shared_collection, options = {})

        pull_parameters = options[:pull_parameters] || {}
        missing_parameters = []
        shared_collection.pull_parameters.each do |pull_parameter|
          unless pull_parameters[(param_id = pull_parameter.id.to_s)].present?
            missing_parameters << param_id
          end
        end
        if options[:auto_fill]
          missing_parameters.each { |param_id| pull_parameters[param_id] = FFaker::Lorem.word }
          missing_parameters.clear
        end

        new_records = Hash.new { |h, k| h[k] = [] }
        updated_records = Hash.new { |h, k| h[k] = [] }

        pull_data = shared_collection.data_with(pull_parameters)

        invariant_data = {}

        collection_data = { '_reset' => resetting = [] }
        collection = Setup::Collection.where(name: shared_collection.name).first
        fields = %w(readme title)
        unless collection && fields.all? { |field| collection.send(field) == shared_collection.send(field) }
          fields.each do |field|
            shared_value = shared_collection[field]
            unless collection && (collection[field] == shared_value)
              collection_data[field] = shared_value
              resetting << field
            end
          end
        end
        fields = %w(metadata)
        unless collection && fields.all? { |field| collection.send(field) == pull_data[field] }
          fields.each do |field|
            shared_value = pull_data[field]
            unless collection && (collection[field] == shared_value)
              collection_data[field] = shared_value
              resetting << field
            end
          end
        end

        Setup::Collection.reflect_on_all_associations(:has_and_belongs_to_many).each do |relation|
          entry = relation.name.to_s
          if (items = pull_data[entry])
            invariant_data[entry] = invariant_names = Set.new
            invariant_on_collection = 0
            items_data_type = relation.klass.data_type
            refs =
              items.collect do |item|
                criteria = ref_criteria = {}
                items_data_type.get_referenced_by.each do |field|
                  field = field.to_s
                  if %w(id _id).include?(field)
                    criteria['_id'] = item['_id'] || item['id']
                  else
                    criteria[field] = item[field]
                  end
                end
                criteria.delete_if { |_, value| value.nil? }
                criteria = Cenit::Utility.deep_remove(criteria, '_reference')
                unless (on_collection = (record = collection && Cenit::Utility.find_record(criteria, collection.send(relation.name))))
                  record = Cenit::Utility.find_record(criteria, relation.klass.all)
                end
                if record
                  share_hash_options = {}
                  if shared_collection.installed?
                    share_hash_options[:include_id] = ->(r) { r.is_a?(Setup::CrossOriginShared) && r.shared? }
                  end
                  record_hash = record.share_hash(share_hash_options)
                  record_hash = Cenit::Utility.stringfy(record_hash)
                  %w(id _id).each do |id_key|
                    record_hash[id_key] = item[id_key] if item.has_key?(id_key)
                  end
                  if item['_type']
                    record_hash['_type'] = record.class.to_s unless record_hash['_type']
                  end
                  record_hash['_reset'] ||= item['_reset'] if item['_reset']
                  record_hash.reject! { |key, _| !item.has_key?(key) }
                  invariant = Cenit::Utility.eql_content?(record_hash, item) do |record_value, item_value|
                    (record_value.nil? && item_value.blank?) || (item_value.nil? && record_value.blank?)
                  end
                  if invariant
                    invariant_names << ref_criteria
                    invariant_on_collection += 1 if on_collection
                    item = ref_criteria
                    item['_reference'] = true
                  else
                    updated_records[entry] <<
                      if options[:updated_records_ids]
                        record.id.to_s
                      else
                        record
                      end
                  end
                  item['id'] = record.id.to_s
                  check_embedded_items(item, record)
                else
                  new_records[entry] << item
                end
                item
              end
            unless (collection && collection.send(relation.name).count == invariant_on_collection) && (invariant_on_collection == items.size)
              collection_data[entry] = refs
              resetting << entry
            end
          elsif collection && collection.send(relation.name).present?
            resetting << entry
          end
        end

        if resetting.present? && !options[:discard_collection]
          if collection
            updated_records['collections'] <<
              if options[:updated_records_ids]
                collection.id.to_s
              else
                collection
              end
          else
            new_records['collections'] << { 'name' => shared_collection.name }
          end
        end

        invariant_data.each do |key, invariant_names|
          pull_data[key].delete_if do |item|
            criteria = { namespace: item['namespace'], name: item['name'] }
            criteria.delete_if { |_, value| value.nil? }
            invariant_names.include?(criteria)
          end
        end

        [collection_data, pull_data, updated_records].each { |hash| hash.each_key { |key| hash.delete(key) if hash[key].blank? } }

        {
          pull_parameters: pull_parameters,
          new_records: new_records,
          updated_records: updated_records,
          missing_parameters: missing_parameters,
          pull_data: pull_data,
          collection_data: collection_data,
          collection_discarded: options[:discard_collection].present?,
          updated_records_ids: options[:updated_records_ids].present?
        }
      end

      def pull(shared_collection, pull_request = {})
        pull_request = pull_request.with_indifferent_access
        pull_request = pull_request(shared_collection, pull_request) if pull_request[:pull_data].nil?
        errors = []
        if pull_request[:missing_parameters].blank?
          created_nss_ids = []
          begin
            collection_data = pull_request.delete(:collection_data)
            (collection_data['namespaces'] || []).each do |ns_hash|
              if (slug = ns_hash['slug']) && Setup::Namespace.where(slug: slug).present?
                ns_hash.delete('slug')
              end
              next if ns_hash.has_key?('id')
              if (ns = Setup::Namespace.create(ns_hash)).errors.blank?
                ns_hash['id'] = ns.id.to_s
                created_nss_ids << ns.id
              else
                fail "Error creating namespace for #{ns_hash.to_json}: #{ns.errors.full_messages.to_sentence}"
              end
            end
            collection = Setup::Collection.where(name: shared_collection.name).first
            attrs = (collection && collection.attributes.deep_dup) || {}
            attrs.delete('_id')
            collection = Setup::Collection.new(attrs)
            collection.from_json(collection_data, add_only: true)
            collection.events.each { |e| e[:activated] = false if e.is_a?(Setup::Scheduler) && e.new_record? }
            begin
              collection.name = BSON::ObjectId.new.to_s
            end while Setup::Collection.where(name: collection.name).present?
            unless Cenit::Utility.save(collection, bind_references: { if: ->(r) { r.instance_variable_get(:@_edi_parsed) } },
                                       create_collector: (create_collector = Set.new),
                                       saved_collector: (saved = Set.new))
              collection.errors.full_messages.each { |msg| errors << msg }
              collection.errors.clear
              if Cenit::Utility.save(collection, { create_collector: create_collector })
                pull_request[:fixed_errors] = errors.collect { |error| "Auto-fixed error: #{error}" }
                errors = []
              else
                saved.each do |obj|
                  if (obj = (obj.reload rescue nil))
                    obj.delete
                  end
                end
{'fraction_non_alphanumeric': 0.09761836441893831, 'fraction_numerical': 0.0002295552367288379, 'mean_word_length': 1.7206869633099142, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637539', 'n_tokens_mistral': 4596, 'n_tokens_neox': 4473, 'n_words': 850}