<commit_before>require 'yaml'

module Gherkin
  class I18n
    KEYWORD_KEYS = %w{name native feature background scenario scenario_outline examples given when then and but}
    LANGUAGES    = YAML.load_file(File.dirname(__FILE__) + '/i18n.yml')

    class << self
      # Used by code generators for other lexer tools like pygments lexer and textmate bundle
      def all
        LANGUAGES.keys.sort.map{|key| get(key)}
      end

      def get(key)
        languages[key] ||= new(key)
      end

      def languages
        @languages ||= {}
      end

      # Returns all keyword translations and aliases of +keywords+, escaped and joined with <tt>|</tt>.
      # This method is convenient for editor support and syntax highlighting engines for Gherkin, where
      # there is typically a code generation tool to generate regular expressions for recognising the
      # various I18n translations of Gherkin's keywords.
      #
      # The +keywords+ arguments can be one of <tt>:feature</tt>, <tt>:background</tt>, <tt>:scenario</tt>, 
      # <tt>:scenario_outline</tt>, <tt>:examples</tt>, <tt>:step</tt>.
      def keyword_regexp(*keywords)
        unique_sorted_keywords = all.map do |lang|
          keywords.map do |keyword|
            lang.__send__("#{keyword}_keywords".to_sym)
          end
        end
        
        unique_sorted_keywords.flatten.compact.sort.reverse.uniq.join('|').gsub(/'/, "\\\\'").gsub(/\*/, "\\\\*")
      end
    end

    attr_reader :key

    def initialize(key)<commit_msg>Don't escape single quotes<commit_after>require 'yaml'

module Gherkin
  class I18n
    KEYWORD_KEYS = %w{name native feature background scenario scenario_outline examples given when then and but}
    LANGUAGES    = YAML.load_file(File.dirname(__FILE__) + '/i18n.yml')

    class << self
      # Used by code generators for other lexer tools like pygments lexer and textmate bundle
      def all
        LANGUAGES.keys.sort.map{|key| get(key)}
      end

      def get(key)
        languages[key] ||= new(key)
      end

      def languages
        @languages ||= {}
      end

      # Returns all keyword translations and aliases of +keywords+, escaped and joined with <tt>|</tt>.
      # This method is convenient for editor support and syntax highlighting engines for Gherkin, where
      # there is typically a code generation tool to generate regular expressions for recognising the
      # various I18n translations of Gherkin's keywords.
      #
      # The +keywords+ arguments can be one of <tt>:feature</tt>, <tt>:background</tt>, <tt>:scenario</tt>, 
      # <tt>:scenario_outline</tt>, <tt>:examples</tt>, <tt>:step</tt>.
      def keyword_regexp(*keywords)
        unique_keywords = all.map do |lang|
          keywords.map do |keyword|
            lang.__send__("#{keyword}_keywords".to_sym)
          end
        end
        
        unique_keywords.flatten.compact.sort.reverse.uniq.join('|').gsub(/\*/, '\*')
      end
    end

    attr_reader :key

    def initialize(key)
{'fraction_non_alphanumeric': 0.11055611055611056, 'fraction_numerical': 0.003996003996003996, 'mean_word_length': 2.6768665850673194, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5481328', 'n_tokens_mistral': 926, 'n_tokens_neox': 853, 'n_words': 265}