<commit_before>#
#
# Author:  Michael 'entropie' Trommer <<EMAIL>>
#

module OY

  module Markup

    class << self
      attr_accessor :default_extension
    end

    # default extension without .
    self.default_extension = "textile"

    # returns extension string without leading .
    def self.normalize_extension(ext)
      ext = ext[1..-1] if ext[0] == ?.
      ext
    end
    
    # return default extension if +ext+ is nil or checks wheter +ext+ is valid
    def self.extension(ext)
      return default_extension unless ext
      if valid_extension?(ext)
        normalize_extension(ext)
      end
    end

    def self.real_markups
      @markups ||= Markups.to_a.select{|ext| not ext.is_virtual? }
    end
    
    def self.extensions
      @extensions ||= Markups.to_a.map{|mup| mup::extension }
    end

    def self.valid_extension?(ext)
      extensions.include?(normalize_extension(ext))
    end
    
    # selects markup class by extension (w/o dot)
    def self.choose_for(extension)
      ret = Markups.to_a.select{|mu|
        mu::extension == extension
      }.first
    end
    
    class Markups
      
      @markups = []

      def self.[](obj)
        if obj.kind_of?(Symbol)
          @markups.select{|m| m.to_s.split("::").last.downcase.to_sym == obj}.first
        else
          @markups[obj]
        end

      end

      def self.<<(obj)
        @markups << obj
      end

      def self.to_a
        @markups
      end
    end

    class Markup

      include OY
      
      attr_reader :data
      attr_reader :wiki
      
      class << self
        attr_accessor :extension
      end

      def self.is_virtual?
        false
      end
      
      def self.inherited(obj)
        Markups << obj
      end
      
      def initialize(data, wiki = nil)
        @wiki = wiki
        @data = data
      end
    end

    require "markup/global"
    require "markup/compare"
    require "markup/nokogiri"

<commit_msg>#measure added to Markup base class<commit_after>#
#
# Author:  Michael 'entropie' Trommer <<EMAIL>>
#

module OY

  module Markup

    class << self
      attr_accessor :default_extension
    end

    # default extension without .
    self.default_extension = "textile"

    # returns extension string without leading .
    def self.normalize_extension(ext)
      ext = ext[1..-1] if ext[0] == ?.
      ext
    end
    
    # return default extension if +ext+ is nil or checks wheter +ext+ is valid
    def self.extension(ext)
      return default_extension unless ext
      if valid_extension?(ext)
        normalize_extension(ext)
      end
    end

    def self.real_markups
      @markups ||= Markups.to_a.select{|ext| not ext.is_virtual? }
    end
    
    def self.extensions
      @extensions ||= Markups.to_a.map{|mup| mup::extension }
    end

    def self.valid_extension?(ext)
      extensions.include?(normalize_extension(ext))
    end
    
    # selects markup class by extension (w/o dot)
    def self.choose_for(extension)
      ret = Markups.to_a.select{|mu|
        mu::extension == extension
      }.first
    end
    
    class Markups
      
      @markups = []

      def self.[](obj)
        if obj.kind_of?(Symbol)
          @markups.select{|m| m.to_s.split("::").last.downcase.to_sym == obj}.first
        else
          @markups[obj]
        end

      end

      def self.<<(obj)
        @markups << obj
      end

      def self.to_a
        @markups
      end
    end

    class Markup

      include OY
      
      attr_reader :data
      attr_reader :wiki
      
      class << self
        attr_accessor :extension
      end

      def self.is_virtual?
        false
      end
      
      def self.inherited(obj)
        Markups << obj
      end
      
      def initialize(data, wiki = nil)
        @wiki = wiki
        @data = data
      end

      def measure
        t = Time.now
        yield
        Time.now - t
      end

    end

    require "markup/global"
    require "markup/compare"
    require "markup/nokogiri"


{'fraction_non_alphanumeric': 0.0924265072247135, 'fraction_numerical': 0.0014947683109118087, 'mean_word_length': 1.8057302585604473, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716530', 'n_tokens_mistral': 1300, 'n_tokens_neox': 1256, 'n_words': 315}