<commit_before>module HarvesterCore
  class Base
    include HarvesterCore::Modifiers
    include HarvesterCore::OptionTransformers

    class_attribute :_base_urls
    class_attribute :_attribute_definitions
    class_attribute :_basic_auth
    class_attribute :_pagination_options
    class_attribute :_rejection_rules
    class_attribute :_throttle

    self._base_urls = {}
    self._attribute_definitions = {}
    self._basic_auth = {}
    self._pagination_options = {}
    self._rejection_rules = {}

    class << self
      def identifier
        @identifier ||= begin
          parent_adapter = self.ancestors[1].to_s.split("::")[1]
          "#{parent_adapter.underscore}_#{self.name.underscore}"
        end
      end

      def base_url(url)
        self._base_urls[self.identifier] ||= []
        self._base_urls[self.identifier] += [url]
      end

      def base_urls
        if self.basic_auth_credentials
          self._base_urls[self.identifier].map do |url|
            url.gsub("http://", "http://#{self.basic_auth_credentials[:username]}:#{self.basic_auth_credentials[:password]}@")
          end
        else
          self._base_urls[self.identifier]
        end
      end

      def basic_auth(username, password)
        self._basic_auth[self.identifier] = {username: username, password: password}
      end

      def basic_auth_credentials
        self._basic_auth[self.identifier]
      end

      def paginate(options={})
        self._pagination_options[self.identifier] = options
      end

      def pagination_options
        self._pagination_options[self.identifier]
      end

      def attribute(name, options={}, &block)
        self._attribute_definitions[self.identifier] ||= {}
        self._attribute_definitions[self.identifier][name] = options || {}

        self._attribute_definitions[self.identifier][name][:block] = block if block_given?
      end

      def attributes(*args, &block)
        options = args.extract_options!

        args.each do |attribute|
          self.attribute(attribute, options, &block)
        end
      end

      def attribute_definitions
        self._attribute_definitions[self.identifier] ||= {}
        self._attribute_definitions[self.identifier]
      end

      def with_options(options={}, &block)
        yield(HarvesterCore::Scope.new(self, options))
      end

      def custom_instance_methods
        self.instance_methods(false)
      end

      def reject_if(&block)
        self._rejection_rules[self.identifier] = block
      end

      def rejection_rules
        self._rejection_rules[self.identifier]
      end

      def clear_definitions
        self._base_urls[self.identifier] = nil
        self._attribute_definitions[self.identifier] = {}
        self._basic_auth[self.identifier] = nil
        self._pagination_options[self.identifier] = nil
        self._rejection_rules[self.identifier] = nil
      end

      def throttle(options={})
        self._throttle ||= []
        self._throttle << options
      end
    end

    attr_reader :original_attributes, :attributes, :errors

    def initialize(*args)
      @errors = {}
      @original_attributes = {}
    end

    def set_attribute_values
      self.class.attribute_definitions.each do |name, options|
        begin
          value = transformed_attribute_value(options, document)
          @original_attributes[name] ||= nil
          @original_attributes[name] = value if value.present?<commit_msg>Add ability to include shared modules and class_eval them in the context of the parser<commit_after>module HarvesterCore
  class Base
    include HarvesterCore::Modifiers
    include HarvesterCore::OptionTransformers

    class_attribute :_base_urls
    class_attribute :_attribute_definitions
    class_attribute :_basic_auth
    class_attribute :_pagination_options
    class_attribute :_rejection_rules
    class_attribute :_throttle

    self._base_urls = {}
    self._attribute_definitions = {}
    self._basic_auth = {}
    self._pagination_options = {}
    self._rejection_rules = {}

    class << self
      def identifier
        @identifier ||= begin
          parent_adapter = self.ancestors[1].to_s.split("::")[1]
          "#{parent_adapter.underscore}_#{self.name.underscore}"
        end
      end

      def base_url(url)
        self._base_urls[self.identifier] ||= []
        self._base_urls[self.identifier] += [url]
      end

      def base_urls
        if self.basic_auth_credentials
          self._base_urls[self.identifier].map do |url|
            url.gsub("http://", "http://#{self.basic_auth_credentials[:username]}:#{self.basic_auth_credentials[:password]}@")
          end
        else
          self._base_urls[self.identifier]
        end
      end

      def basic_auth(username, password)
        self._basic_auth[self.identifier] = {username: username, password: password}
      end

      def basic_auth_credentials
        self._basic_auth[self.identifier]
      end

      def paginate(options={})
        self._pagination_options[self.identifier] = options
      end

      def pagination_options
        self._pagination_options[self.identifier]
      end

      def attribute(name, options={}, &block)
        self._attribute_definitions[self.identifier] ||= {}
        self._attribute_definitions[self.identifier][name] = options || {}

        self._attribute_definitions[self.identifier][name][:block] = block if block_given?
      end

      def attributes(*args, &block)
        options = args.extract_options!

        args.each do |attribute|
          self.attribute(attribute, options, &block)
        end
      end

      def attribute_definitions
        self._attribute_definitions[self.identifier] ||= {}
        self._attribute_definitions[self.identifier]
      end

      def with_options(options={}, &block)
        yield(HarvesterCore::Scope.new(self, options))
      end

      def custom_instance_methods
        self.instance_methods(false)
      end

      def reject_if(&block)
        self._rejection_rules[self.identifier] = block
      end

      def rejection_rules
        self._rejection_rules[self.identifier]
      end

      def clear_definitions
        self._base_urls[self.identifier] = nil
        self._attribute_definitions[self.identifier] = {}
        self._basic_auth[self.identifier] = nil
        self._pagination_options[self.identifier] = nil
        self._rejection_rules[self.identifier] = nil
      end

      def throttle(options={})
        self._throttle ||= []
        self._throttle << options
      end

      def include_shared_module(name)
        if defined?(SharedModule)
          if shared_module = SharedModule.find_by_name(name)
            self.class_eval <<-METHOD, __FILE__, __LINE__ + 1
              #{shared_module.content}
            METHOD
          end
        end
      end
    end

    attr_reader :original_attributes, :attributes, :errors

    def initialize(*args)
      @errors = {}
      @original_attributes = {}
    end

    def set_attribute_values
      self.class.attribute_definitions.each do |name, options|
        begin
          value = transformed_attribute_value(options, document)
          @original_attributes[name] ||= nil
          @original_attributes[name] = value if value.present?
{'fraction_non_alphanumeric': 0.09451847781288639, 'fraction_numerical': 0.0006869075422448139, 'mean_word_length': 2.60932077342588, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4861816', 'n_tokens_mistral': 2073, 'n_tokens_neox': 2027, 'n_words': 372}