<commit_before>require 'uri'

module Licensee
  class InvalidLicense < ArgumentError; end
  class License
    @all = {}
    @keys_licenses = {}

    class << self
      # All license objects defined via Licensee (via choosealicense.com)
      #
      # Options:
      # - :hidden - boolean, return hidden licenses (default: false)
      # - :featured - boolean, return only (non)featured licenses (default: all)
      #
      # Returns an Array of License objects.
      def all(options = {})
        @all[options] ||= begin
          options = DEFAULT_OPTIONS.merge(options)
          output = licenses.dup
          output.reject!(&:hidden?) unless options[:hidden]
          output.reject!(&:pseudo_license?) unless options[:psuedo]
          return output if options[:featured].nil?
          output.select { |l| l.featured? == options[:featured] }
        end
      end

      def keys
        @keys ||= license_files.map do |license_file|
          ::File.basename(license_file, '.txt').downcase
        end + PSEUDO_LICENSES
      end

      def find(key, options = {})
        options = { hidden: true }.merge(options)
        keys_licenses(options)[key.downcase]
      end
      alias [] find
      alias find_by_key find

      # Given a license title or nickname, fuzzy match the license
      def find_by_title(title)
        License.all(hidden: true, psuedo: false).find do |license|
          title =~ /\A(the )?#{license.title_regex}( license)?\z/i
        end
      end

      def license_dir
        dir = ::File.dirname(__FILE__)
        ::File.expand_path '../../vendor/choosealicense.com/_licenses', dir
      end

      def license_files
        @license_files ||= Dir.glob("#{license_dir}/*.txt")
      end

      private

      def licenses
        @licenses ||= keys.map { |key| new(key) }
      end

      def keys_licenses(options = {})
        @keys_licenses[options] ||= all(options).map { |l| [l.key, l] }.to_h
      end
    end

    attr_reader :key

    # Preserved for backwards compatibility
    YAML_DEFAULTS = Licensee::LicenseMeta.members

    # Pseudo-license are license placeholders with no content
    #
    # `other` - The project had a license, but we were not able to detect it
    # `no-license` - The project is not licensed (e.g., all rights reserved)
    #
    # Note: A lack of detected license will be a nil license
    PSEUDO_LICENSES = %w[other no-license].freeze

    # Default options to use when retrieving licenses via #all
    DEFAULT_OPTIONS = {
      hidden:   false,
      featured: nil,
      psuedo:   true
    }.freeze

    ALT_TITLE_REGEX = {
      'bsd-2-clause'       => /bsd 2-clause(?: \"simplified\")?/i,
      'bsd-3-clause'       => /bsd 3-clause(?: \"new\" or \"revised\")?/i,
      'bsd-3-clause-clear' => /(?:clear bsd|bsd 3-clause(?: clear)?)/i
    }.freeze

    SOURCE_PREFIX = %r{https?://(?:www\.)?}i
    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|\/)(?:\?[^\s]*)?}i

    HASH_METHODS = %i[
      key spdx_id meta url rules fields other? gpl? lgpl? cc?
    ].freeze

    include Licensee::ContentHelper
    include Licensee::HashHelper
    extend Forwardable
    def_delegators :meta, *LicenseMeta.helper_methods

    def initialize(key)
      @key = key.downcase
    end

    # Path to vendored license file on disk
    def path
      @path ||= File.expand_path "#{@key}.txt", Licensee::License.license_dir
    end

    # License metadata from YAML front matter with defaults merged in
    def meta
      @meta ||= LicenseMeta.from_yaml(yaml)
    end

    # Returns the human-readable license name
    def name
      title ? title : key.capitalize
    end

    def name_without_version
      /(.+?)(( v?\d\.\d)|$)/.match(name)[1]
    end

    def title_regex
      return @title_regex if defined? @title_regex<commit_msg>NONE and NOASSERTION for 'no-license' and 'other'<commit_after>require 'uri'

module Licensee
  class InvalidLicense < ArgumentError; end
  class License
    @all = {}
    @keys_licenses = {}

    class << self
      # All license objects defined via Licensee (via choosealicense.com)
      #
      # Options:
      # - :hidden - boolean, return hidden licenses (default: false)
      # - :featured - boolean, return only (non)featured licenses (default: all)
      #
      # Returns an Array of License objects.
      def all(options = {})
        @all[options] ||= begin
          options = DEFAULT_OPTIONS.merge(options)
          output = licenses.dup
          output.reject!(&:hidden?) unless options[:hidden]
          output.reject!(&:pseudo_license?) unless options[:psuedo]
          return output if options[:featured].nil?
          output.select { |l| l.featured? == options[:featured] }
        end
      end

      def keys
        @keys ||= license_files.map do |license_file|
          ::File.basename(license_file, '.txt').downcase
        end + PSEUDO_LICENSES
      end

      def find(key, options = {})
        options = { hidden: true }.merge(options)
        keys_licenses(options)[key.downcase]
      end
      alias [] find
      alias find_by_key find

      # Given a license title or nickname, fuzzy match the license
      def find_by_title(title)
        License.all(hidden: true, psuedo: false).find do |license|
          title =~ /\A(the )?#{license.title_regex}( license)?\z/i
        end
      end

      def license_dir
        dir = ::File.dirname(__FILE__)
        ::File.expand_path '../../vendor/choosealicense.com/_licenses', dir
      end

      def license_files
        @license_files ||= Dir.glob("#{license_dir}/*.txt")
      end

      private

      def licenses
        @licenses ||= keys.map { |key| new(key) }
      end

      def keys_licenses(options = {})
        @keys_licenses[options] ||= all(options).map { |l| [l.key, l] }.to_h
      end
    end

    attr_reader :key

    # Preserved for backwards compatibility
    YAML_DEFAULTS = Licensee::LicenseMeta.members

    # Pseudo-license are license placeholders with no content
    #
    # `other` - The project had a license, but we were not able to detect it
    # `no-license` - The project is not licensed (e.g., all rights reserved)
    #
    # Note: A lack of detected license will be a nil license
    PSEUDO_LICENSES = %w[other no-license].freeze

    # Default options to use when retrieving licenses via #all
    DEFAULT_OPTIONS = {
      hidden:   false,
      featured: nil,
      psuedo:   true
    }.freeze

    ALT_TITLE_REGEX = {
      'bsd-2-clause'       => /bsd 2-clause(?: \"simplified\")?/i,
      'bsd-3-clause'       => /bsd 3-clause(?: \"new\" or \"revised\")?/i,
      'bsd-3-clause-clear' => /(?:clear bsd|bsd 3-clause(?: clear)?)/i
    }.freeze

    SOURCE_PREFIX = %r{https?://(?:www\.)?}i
    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|\/)(?:\?[^\s]*)?}i

    HASH_METHODS = %i[
      key spdx_id meta url rules fields other? gpl? lgpl? cc?
    ].freeze

    include Licensee::ContentHelper
    include Licensee::HashHelper
    extend Forwardable
    def_delegators :meta, *LicenseMeta.helper_methods

    def initialize(key)
      @key = key.downcase
    end

    # Path to vendored license file on disk
    def path
      @path ||= File.expand_path "#{@key}.txt", Licensee::License.license_dir
    end

    # License metadata from YAML front matter with defaults merged in
    def meta
      @meta ||= LicenseMeta.from_yaml(yaml)
    end

    def spdx_id
      return meta.spdx_id if meta.spdx_id
      return 'NOASSERTION' if key == 'other'
      return 'NONE' if key == 'no-license'
    end

    # Returns the human-readable license name
    def name
      title ? title : key.capitalize
    end

    def name_without_version
      /(.+?)(( v?\d\.\d)|$)/.match(name)[1]
    end

    def title_regex
      return @title_regex if defined? @title_regex
{'fraction_non_alphanumeric': 0.12295819935691318, 'fraction_numerical': 0.0018006430868167203, 'mean_word_length': 2.3618677042801557, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5967359', 'n_tokens_mistral': 2490, 'n_tokens_neox': 2408, 'n_words': 656}