<commit_before># frozen_string_literal: true
require "strscan"

# Some versions of the Bundler 1.1 RC series introduced corrupted
# lockfiles. There were two major problems:
#
# * multiple copies of the same GIT section appeared in the lockfile
# * when this happened, those sections got multiple copies of gems
#   in those sections.
#
# As a result, Bundler 1.1 contains code that fixes the earlier
# corruption. We will remove this fix-up code in Bundler 1.2.

module Bundler
  class LockfileParser
    attr_reader :sources, :dependencies, :specs, :platforms, :bundler_version, :ruby_version

    BUNDLED      = "BUNDLED WITH".freeze
    DEPENDENCIES = "DEPENDENCIES".freeze
    PLATFORMS    = "PLATFORMS".freeze
    RUBY         = "RUBY VERSION".freeze
    GIT          = "GIT".freeze
    GEM          = "GEM".freeze
    PATH         = "PATH".freeze
    PLUGIN       = "PLUGIN SOURCE".freeze
    SPECS        = "  specs:".freeze
    OPTIONS      = /^  ([a-z]+): (.*)$/i
    SOURCE       = [GIT, GEM, PATH, PLUGIN].freeze

    SECTIONS_BY_VERSION_INTRODUCED = {
      # The strings have to be dup'ed for old RG on Ruby 2.3+
      # TODO: remove dup in Bundler 2.0
      Gem::Version.create("1.0".dup) => [DEPENDENCIES, PLATFORMS, GIT, GEM, PATH].freeze,
      Gem::Version.create("1.10".dup) => [BUNDLED].freeze,
      Gem::Version.create("1.12".dup) => [RUBY].freeze,
      Gem::Version.create("1.13".dup) => [PLUGIN].freeze,
    }.freeze

    KNOWN_SECTIONS = SECTIONS_BY_VERSION_INTRODUCED.values.flatten.freeze

    ENVIRONMENT_VERSION_SECTIONS = [BUNDLED, RUBY].freeze

    def self.sections_in_lockfile(lockfile_contents)
      lockfile_contents.scan(/^\w[\w ]*$/).uniq
    end

    def self.unknown_sections_in_lockfile(lockfile_contents)
      sections_in_lockfile(lockfile_contents) - KNOWN_SECTIONS
    end

    def self.sections_to_ignore(base_version = nil)
      base_version &&= base_version.release
      base_version ||= Gem::Version.create("1.0".dup)
      attributes = []
      SECTIONS_BY_VERSION_INTRODUCED.each do |version, introduced|
        next if version <= base_version
        attributes += introduced
      end
      attributes
    end

    def initialize(lockfile)
      @platforms    = []
      @sources      = []
      @dependencies = []
      @state        = nil
      @specs        = {}

      @rubygems_aggregate = Source::Rubygems.new

      if lockfile.match(/<<<<<<<|=======|>>>>>>>|\|\|\|\|\|\|\|/)
        raise LockfileError, "Your #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} contains merge conflicts.\n" \
          "Run `git checkout HEAD -- #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` first to get a clean lock."
      end

      lockfile.split(/(?:\r?\n)+/).each do |line|
        if SOURCE.include?(line)
          @state = :source
          parse_source(line)
        elsif line == DEPENDENCIES
          @state = :dependency
        elsif line == PLATFORMS
          @state = :platform
        elsif line == RUBY
          @state = :ruby
        elsif line == BUNDLED
          @state = :bundled_with
        elsif line =~ /^[^\s]/
          @state = nil
        elsif @state
          send("parse_#{@state}", line)
        end
      end
      @sources << @rubygems_aggregate
      @specs = @specs.values
      warn_for_outdated_bundler_version
    rescue ArgumentError => e
      Bundler.ui.debug(e)
      raise LockfileError, "Your lockfile is unreadable. Run `rm #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` " \
        "and then `bundle install` to generate a new lockfile."
    end

    def warn_for_outdated_bundler_version
      return unless bundler_version
      prerelease_text = bundler_version.prerelease? ? " --pre" : ""
      current_version = Gem::Version.create(Bundler::VERSION)
      case current_version.segments.first <=> bundler_version.segments.first
      when -1
        raise LockfileError, "You must use Bundler #{bundler_version.segments.first} or greater with this lockfile."
      when 0
        if current_version < bundler_version
          Bundler.ui.warn "Warning: the running version of Bundler is older " \
               "than the version that created the lockfile. We suggest you " \
               "upgrade to the latest version of Bundler by running `gem " \
               "install bundler#{prerelease_text}`.\n"
        end
      end
    end

  private

    TYPES = {
      GIT    => Bundler::Source::Git,
      GEM    => Bundler::Source::Rubygems,
      PATH   => Bundler::Source::Path,
      PLUGIN => Bundler::Plugin,
    }.freeze

    def parse_source(line)
      case line
      when *SOURCE
        @current_source = nil
        @opts = {}
        @type = line
      when SPECS
        case @type
        when PATH
          @current_source = TYPES[@type].from_lock(@opts)
          @sources << @current_source
        when GIT
          @current_source = TYPES[@type].from_lock(@opts)
          # Strip out duplicate GIT sections
          if @sources.include?(@current_source)
            @current_source = @sources.find {|s| s == @current_source }
          else
            @sources << @current_source
          end
        when GEM
          Array(@opts["remote"]).each do |url|
            @rubygems_aggregate.add_remote(url)
          end
          @current_source = @rubygems_aggregate
        when PLUGIN
          @current_source = Plugin.source_from_lock(@opts)
          @sources << @current_source
        end
      when OPTIONS
        value = $2
        value = true if value == "true"
        value = false if value == "false"

        key = $1

        if @opts[key]
          @opts[key] = Array(@opts[key])
          @opts[key] << value
        else
          @opts[key] = value
        end
      else
        parse_spec(line)
      end
    end

    NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
    NAME_VERSION_2 = /^ {2}#{NAME_VERSION}(!)?$/
    NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
    NAME_VERSION_6 = /^ {6}#{NAME_VERSION}$/

    def parse_dependency(line)
      if line =~ NAME_VERSION_2
        name = $1
        version = $2
        pinned = $4
        version = version.split(",").map(&:strip) if version

        dep = Bundler::Dependency.new(name, version)

        if pinned && dep.name != "bundler"
          spec = @specs.find {|_, v| v.name == dep.name }
          dep.source = spec.last.source if spec<commit_msg>rubocop fix<commit_after># frozen_string_literal: true
require "strscan"

# Some versions of the Bundler 1.1 RC series introduced corrupted
# lockfiles. There were two major problems:
#
# * multiple copies of the same GIT section appeared in the lockfile
# * when this happened, those sections got multiple copies of gems
#   in those sections.
#
# As a result, Bundler 1.1 contains code that fixes the earlier
# corruption. We will remove this fix-up code in Bundler 1.2.

module Bundler
  class LockfileParser
    attr_reader :sources, :dependencies, :specs, :platforms, :bundler_version, :ruby_version

    BUNDLED      = "BUNDLED WITH".freeze
    DEPENDENCIES = "DEPENDENCIES".freeze
    PLATFORMS    = "PLATFORMS".freeze
    RUBY         = "RUBY VERSION".freeze
    GIT          = "GIT".freeze
    GEM          = "GEM".freeze
    PATH         = "PATH".freeze
    PLUGIN       = "PLUGIN SOURCE".freeze
    SPECS        = "  specs:".freeze
    OPTIONS      = /^  ([a-z]+): (.*)$/i
    SOURCE       = [GIT, GEM, PATH, PLUGIN].freeze

    SECTIONS_BY_VERSION_INTRODUCED = {
      # The strings have to be dup'ed for old RG on Ruby 2.3+
      # TODO: remove dup in Bundler 2.0
      Gem::Version.create("1.0".dup) => [DEPENDENCIES, PLATFORMS, GIT, GEM, PATH].freeze,
      Gem::Version.create("1.10".dup) => [BUNDLED].freeze,
      Gem::Version.create("1.12".dup) => [RUBY].freeze,
      Gem::Version.create("1.13".dup) => [PLUGIN].freeze,
    }.freeze

    KNOWN_SECTIONS = SECTIONS_BY_VERSION_INTRODUCED.values.flatten.freeze

    ENVIRONMENT_VERSION_SECTIONS = [BUNDLED, RUBY].freeze

    def self.sections_in_lockfile(lockfile_contents)
      lockfile_contents.scan(/^\w[\w ]*$/).uniq
    end

    def self.unknown_sections_in_lockfile(lockfile_contents)
      sections_in_lockfile(lockfile_contents) - KNOWN_SECTIONS
    end

    def self.sections_to_ignore(base_version = nil)
      base_version &&= base_version.release
      base_version ||= Gem::Version.create("1.0".dup)
      attributes = []
      SECTIONS_BY_VERSION_INTRODUCED.each do |version, introduced|
        next if version <= base_version
        attributes += introduced
      end
      attributes
    end

    def initialize(lockfile)
      @platforms    = []
      @sources      = []
      @dependencies = []
      @state        = nil
      @specs        = {}

      @rubygems_aggregate = Source::Rubygems.new

      if lockfile.match(/<<<<<<<|=======|>>>>>>>|\|\|\|\|\|\|\|/)
        raise LockfileError, "Your #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} contains merge conflicts.\n" \
          "Run `git checkout HEAD -- #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` first to get a clean lock."
      end

      lockfile.split(/(?:\r?\n)+/).each do |line|
        if SOURCE.include?(line)
          @state = :source
          parse_source(line)
        elsif line == DEPENDENCIES
          @state = :dependency
        elsif line == PLATFORMS
          @state = :platform
        elsif line == RUBY
          @state = :ruby
        elsif line == BUNDLED
          @state = :bundled_with
        elsif line =~ /^[^\s]/
          @state = nil
        elsif @state
          send("parse_#{@state}", line)
        end
      end
      @sources << @rubygems_aggregate
      @specs = @specs.values
      warn_for_outdated_bundler_version
    rescue ArgumentError => e
      Bundler.ui.debug(e)
      raise LockfileError, "Your lockfile is unreadable. Run `rm #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` " \
        "and then `bundle install` to generate a new lockfile."
    end

    def warn_for_outdated_bundler_version
      return unless bundler_version
      prerelease_text = bundler_version.prerelease? ? " --pre" : ""
      current_version = Gem::Version.create(Bundler::VERSION)
      case current_version.segments.first <=> bundler_version.segments.first
      when -1
        raise LockfileError, "You must use Bundler #{bundler_version.segments.first} or greater with this lockfile."
      when 0
        if current_version < bundler_version
          Bundler.ui.warn "Warning: the running version of Bundler is older " \
               "than the version that created the lockfile. We suggest you " \
               "upgrade to the latest version of Bundler by running `gem " \
               "install bundler#{prerelease_text}`.\n"
        end
      end
    end

  private

    TYPES = {
      GIT    => Bundler::Source::Git,
      GEM    => Bundler::Source::Rubygems,
      PATH   => Bundler::Source::Path,
      PLUGIN => Bundler::Plugin,
    }.freeze

    def parse_source(line)
      case line
      when SPECS
        case @type
        when PATH
          @current_source = TYPES[@type].from_lock(@opts)
          @sources << @current_source
        when GIT
          @current_source = TYPES[@type].from_lock(@opts)
          # Strip out duplicate GIT sections
          if @sources.include?(@current_source)
            @current_source = @sources.find {|s| s == @current_source }
          else
            @sources << @current_source
          end
        when GEM
          Array(@opts["remote"]).each do |url|
            @rubygems_aggregate.add_remote(url)
          end
          @current_source = @rubygems_aggregate
        when PLUGIN
          @current_source = Plugin.source_from_lock(@opts)
          @sources << @current_source
        end
      when OPTIONS
        value = $2
        value = true if value == "true"
        value = false if value == "false"

        key = $1

        if @opts[key]
          @opts[key] = Array(@opts[key])
          @opts[key] << value
        else
          @opts[key] = value
        end
      when *SOURCE
        @current_source = nil
        @opts = {}
        @type = line
      else
        parse_spec(line)
      end
    end

    NAME_VERSION = '(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?'.freeze
    NAME_VERSION_2 = /^ {2}#{NAME_VERSION}(!)?$/
    NAME_VERSION_4 = /^ {4}#{NAME_VERSION}$/
    NAME_VERSION_6 = /^ {6}#{NAME_VERSION}$/

    def parse_dependency(line)
      if line =~ NAME_VERSION_2
        name = $1
        version = $2
        pinned = $4
        version = version.split(",").map(&:strip) if version

        dep = Bundler::Dependency.new(name, version)

        if pinned && dep.name != "bundler"
          spec = @specs.find {|_, v| v.name == dep.name }
          dep.source = spec.last.source if spec
{'fraction_non_alphanumeric': 0.11940298507462686, 'fraction_numerical': 0.0057524875621890546, 'mean_word_length': 2.261916835699797, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6319857', 'n_tokens_mistral': 4180, 'n_tokens_neox': 4051, 'n_words': 950}