<commit_before># typed: false
# frozen_string_literal: true

# Helper functions for updating PyPI resources.
#
# @api private
module PyPI
  extend T::Sig

  module_function

  PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/"
  private_constant :PYTHONHOSTED_URL_PREFIX

  @pipgrip_installed = nil

  # PyPI Package
  #
  # @api private
  class Package
    extend T::Sig

    attr_accessor :name
    attr_accessor :extras
    attr_accessor :version

    sig { params(package_string: String, is_url: T::Boolean).void }
    def initialize(package_string, is_url: false)
      @pypi_info = nil

      if is_url
        unless package_string.start_with?(PYTHONHOSTED_URL_PREFIX) &&
               match = File.basename(package_string).match(/^(.+)-([a-z\d.]+?)(?:.tar.gz|.zip)$/)
          raise ArgumentError, "package should be a valid PyPI url"
        end

        @name = match[1]
        @version = match[2]
        return
      end

      @name = package_string
      @name, @version = @name.split("==") if @name.include? "=="

      return unless match = @name.match(/^(.*?)\[(.+)\]$/)

      @name = match[1]
      @extras = match[2].split ","
    end

    # Get name, URL, SHA-256 checksum, and latest version for a given PyPI package.
    sig { params(version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
    def pypi_info(version: nil)
      return @pypi_info if @pypi_info.present? && version.blank?

      version ||= @version
      metadata_url = if version.present?
        "https://pypi.org/pypi/#{@name}/#{version}/json"
      else
        "https://pypi.org/pypi/#{@name}/json"
      end
      out, _, status = curl_output metadata_url, "--location"

      return unless status.success?

      begin
        json = JSON.parse out
      rescue JSON::ParserError
        return
      end

      sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
      return json["info"]["name"] if sdist.nil?

      @pypi_info = [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]]
    end

    sig { returns(T::Boolean) }
    def valid_pypi_package?
      info = pypi_info
      info.present? && info.is_a?(Array)
    end

    sig { returns(String) }
    def to_s
      out = @name
      out += "[#{@extras.join(",")}]" if @extras.present?
      out += "==#{@version}" if @version.present?
      out
    end

    sig { params(other: Package).returns(T::Boolean) }
    def same_package?(other)
      @name.downcase.tr("_", "-") == other.name.downcase.tr("_", "-")
    end

    # Compare only names so we can use .include? on a Package array
    sig { params(other: Package).returns(T::Boolean) }
    def ==(other)
      same_package?(other)
    end

    sig { params(other: Package).returns(T.nilable(Integer)) }
    def <=>(other)
      @name <=> other.name
    end
  end<commit_msg>utils/pypi: avoid creating new string objects<commit_after># typed: false
# frozen_string_literal: true

# Helper functions for updating PyPI resources.
#
# @api private
module PyPI
  extend T::Sig

  module_function

  PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/"
  private_constant :PYTHONHOSTED_URL_PREFIX

  @pipgrip_installed = nil

  # PyPI Package
  #
  # @api private
  class Package
    extend T::Sig

    attr_accessor :name
    attr_accessor :extras
    attr_accessor :version

    sig { params(package_string: String, is_url: T::Boolean).void }
    def initialize(package_string, is_url: false)
      @pypi_info = nil

      if is_url
        unless package_string.start_with?(PYTHONHOSTED_URL_PREFIX) &&
               match = File.basename(package_string).match(/^(.+)-([a-z\d.]+?)(?:.tar.gz|.zip)$/)
          raise ArgumentError, "package should be a valid PyPI url"
        end

        @name = match[1]
        @version = match[2]
        return
      end

      @name = package_string
      @name, @version = @name.split("==") if @name.include? "=="

      return unless match = @name.match(/^(.*?)\[(.+)\]$/)

      @name = match[1]
      @extras = match[2].split ","
    end

    # Get name, URL, SHA-256 checksum, and latest version for a given PyPI package.
    sig { params(version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
    def pypi_info(version: nil)
      return @pypi_info if @pypi_info.present? && version.blank?

      version ||= @version
      metadata_url = if version.present?
        "https://pypi.org/pypi/#{@name}/#{version}/json"
      else
        "https://pypi.org/pypi/#{@name}/json"
      end
      out, _, status = curl_output metadata_url, "--location"

      return unless status.success?

      begin
        json = JSON.parse out
      rescue JSON::ParserError
        return
      end

      sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
      return json["info"]["name"] if sdist.nil?

      @pypi_info = [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]]
    end

    sig { returns(T::Boolean) }
    def valid_pypi_package?
      info = pypi_info
      info.present? && info.is_a?(Array)
    end

    sig { returns(String) }
    def to_s
      out = @name
      out += "[#{@extras.join(",")}]" if @extras.present?
      out += "==#{@version}" if @version.present?
      out
    end

    sig { params(other: Package).returns(T::Boolean) }
    def same_package?(other)
      @name.tr("_", "-").casecmp(other.name.tr("_", "-")).zero?
    end

    # Compare only names so we can use .include? on a Package array
    sig { params(other: Package).returns(T::Boolean) }
    def ==(other)
      same_package?(other)
    end

    sig { params(other: Package).returns(T.nilable(Integer)) }
    def <=>(other)
      @name <=> other.name
    end
  end
{'fraction_non_alphanumeric': 0.1531951640759931, 'fraction_numerical': 0.0034542314335060447, 'mean_word_length': 2.728911783644559, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6620612', 'n_tokens_mistral': 1976, 'n_tokens_neox': 1914, 'n_words': 431}