<commit_before>class Sequencer
  class Unit
    module Import
      module Zendesk
        module SubSequence
          module Base
            module ClassMethods

              def resource_klass
                @resource_klass ||= name.split('::').last.singularize
              end
            end

            def self.included(base)
              base.extend(ClassMethods)

              base.uses :dry_run, :import_job
            end

            private

            def default_params
              {
                dry_run:    dry_run,
                import_job: import_job,
              }
            end

            def resource_klass
              # base.instance_delegate [:resource_klass] => base
              # doesn't work since we are included and then inherited
              # there might be multiple inherited hooks which overwrite
              # each other :/
              self.class.resource_klass
            end

            def sequence_name
              "Import::Zendesk::#{resource_klass}"
            end

            def resource_iteration(&block)
              resource_collection.public_send(resource_iteration_method, &block)
            rescue ZendeskAPI::Error::NetworkError => e
              case e.response.status.to_s
              when '403'
                return if resource_klass.in?(%w[UserField OrganizationField])
              when /^5\d\d$/
                raise if (fail_count ||= 1) > 10

                logger.error e
                logger.info "Sleeping 10 seconds after ZendeskAPI::Error::NetworkError and retry (##{fail_count}/10)."
                sleep 10

                (fail_count += 1) && retry
              end

              raise
            end

            def resource_collection
              collection_provider.public_send(resource_collection_attribute)
            end

            def resource_iteration_method
              :all!
            end

            def resource_collection_attribute
              @resource_collection_attribute ||= resource_klass.pluralize.underscore
            end
          end
        end
      end
    end
  end
end
<commit_msg>Maintenance: Improve handling of Zendesk API hickups/temporary errors.<commit_after>class Sequencer
  class Unit
    module Import
      module Zendesk
        module SubSequence
          module Base
            module ClassMethods

              def resource_klass
                @resource_klass ||= name.split('::').last.singularize
              end
            end

            def self.included(base)
              base.extend(ClassMethods)

              base.uses :dry_run, :import_job
            end

            private

            def default_params
              {
                dry_run:    dry_run,
                import_job: import_job,
              }
            end

            def resource_klass
              # base.instance_delegate [:resource_klass] => base
              # doesn't work since we are included and then inherited
              # there might be multiple inherited hooks which overwrite
              # each other :/
              self.class.resource_klass
            end

            def sequence_name
              "Import::Zendesk::#{resource_klass}"
            end

            def resource_iteration(&block)
              resource_collection.public_send(resource_iteration_method, &block)
            rescue ZendeskAPI::Error::NetworkError => e
              status = e.response.status.to_s

              if status.match?(/^(4|5)\d\d$/)

                # #2262 Zendesk-Import fails for User & Organizations when 403 "access" denied
                return if status == '403' && resource_klass.in?(%w[UserField OrganizationField])

                raise if (fail_count ||= 1) > 10

                logger.error e
                logger.info "Sleeping 10 seconds after ZendeskAPI::Error::NetworkError and retry (##{fail_count}/10)."
                sleep 10

                (fail_count += 1) && retry
              end

              raise
            end

            def resource_collection
              collection_provider.public_send(resource_collection_attribute)
            end

            def resource_iteration_method
              :all!
            end

            def resource_collection_attribute
              @resource_collection_attribute ||= resource_klass.pluralize.underscore
            end
          end
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.061659192825112105, 'fraction_numerical': 0.008071748878923767, 'mean_word_length': 1.380469583778015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5321237', 'n_tokens_mistral': 1120, 'n_tokens_neox': 1035, 'n_words': 267}