<commit_before>#coding: utf-8
require 'wombat/dsl/metadata'
require 'wombat/dsl/property'
require 'wombat/processing/parser'
require 'active_support'
require 'date'

module Wombat
  module Crawler
    include Processing::Parser
    extend ActiveSupport::Concern

    included do
      class << self
        attr_accessor :metadata
      end
      self.metadata = DSL::Metadata.new
    end

    def crawl(url = nil, &block)
      if block
        @metadata_dup = self.class.send(:metadata).clone
        instance_eval do
          alias :old_method_missing :method_missing
          def method_missing method, *args, &block
            @metadata_dup.send method, *args, &block
          end
        end
        self.instance_eval &block
        parsed = parse(@metadata_dup, url)
        instance_eval do
          alias :method_missing :old_method_missing
          remove_instance_variable :@metadata_dup
        end
        parsed<commit_msg>Remove unnecessary `old_method_missing` method after crawled<commit_after>#coding: utf-8
require 'wombat/dsl/metadata'
require 'wombat/dsl/property'
require 'wombat/processing/parser'
require 'active_support'
require 'date'

module Wombat
  module Crawler
    include Processing::Parser
    extend ActiveSupport::Concern

    included do
      class << self
        attr_accessor :metadata
      end
      self.metadata = DSL::Metadata.new
    end

    def crawl(url = nil, &block)
      if block
        @metadata_dup = self.class.send(:metadata).clone
        instance_eval do
          alias :old_method_missing :method_missing
          def method_missing method, *args, &block
            @metadata_dup.send method, *args, &block
          end
        end
        self.instance_eval &block
        parsed = parse(@metadata_dup, url)
        instance_eval do
          alias :method_missing :old_method_missing
          undef :old_method_missing
          remove_instance_variable :@metadata_dup
        end
        parsed
{'fraction_non_alphanumeric': 0.07255200405885337, 'fraction_numerical': 0.0010147133434804667, 'mean_word_length': 2.5467625899280577, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6920793', 'n_tokens_mistral': 563, 'n_tokens_neox': 550, 'n_words': 142}