<commit_before>require 'linguist/lexer'

require 'yaml'

module Linguist
  class Language
    @name_index      = {}
    @lexer_index     = {}
    @extension_index = {}

    def self.create(attributes = {})
      language = new(attributes)

      @name_index[language.name.downcase] = language

      if attributes[:default_lexer] || language.default_lexer?
        @lexer_index[language.lexer_name.downcase] = language
      end

      @lexer_index[language.lexer_name.downcase] ||= language

      language.extensions.each do |extension|
        @extension_index[extension] = language
        @extension_index[extension.sub(/^./, '')] = language
      end

      language
    end

    def self.[](name)
      find_by_name(name) || find_by_lexer(name)
    end

    def self.find_by_name(name)
      @name_index[name.downcase]
    end

    def self.find_by_extension(extension)<commit_msg>Warn about duplicate language names and extensions<commit_after>require 'linguist/lexer'

require 'yaml'

module Linguist
  class Language
    @name_index      = {}
    @lexer_index     = {}
    @extension_index = {}

    def self.create(attributes = {})
      language = new(attributes)

      if @name_index.key?(language.name.downcase)
        warn "Duplicate language name: #{language.name}"
      end

      @name_index[language.name.downcase] = language

      if attributes[:default_lexer] || language.default_lexer?
        @lexer_index[language.lexer_name.downcase] = language
      end

      @lexer_index[language.lexer_name.downcase] ||= language

      language.extensions.each do |extension|
        if @extension_index.key?(extension)
          warn "Duplicate extension: #{extension}"
        end

        @extension_index[extension] = language
        @extension_index[extension.sub(/^./, '')] = language
      end

      language
    end

    def self.[](name)
      find_by_name(name) || find_by_lexer(name)
    end

    def self.find_by_name(name)
      @name_index[name.downcase]
    end

    def self.find_by_extension(extension)
{'fraction_non_alphanumeric': 0.10936733692986758, 'fraction_numerical': 0.0, 'mean_word_length': 2.8563327032136105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6240418', 'n_tokens_mistral': 643, 'n_tokens_neox': 624, 'n_words': 112}