<commit_before>module Homebrew extend self
  def update
    abort "Please `brew install git' first." unless system "/usr/bin/which -s git"

    updater = RefreshBrew.new
    if updater.update_from_masterbrew!
      updater.report
    else
      puts "Already up-to-date."
    end
  end
end

class RefreshBrew
  REPOSITORY_URL = "http://github.com/mxcl/homebrew.git"
  FORMULA_DIR = 'Library/Formula/'
  EXAMPLE_DIR = 'Library/Contributions/examples/'

  attr_reader :added_formulae, :updated_formulae, :deleted_formulae, :installed_formulae
  attr_reader :added_examples, :updated_examples, :deleted_examples
  attr_reader :initial_revision, :current_revision

  def initialize
    @added_formulae, @updated_formulae, @deleted_formulae, @installed_formulae = [], [], [], []
    @added_examples, @updated_examples, @deleted_examples = [], [], []
    @initial_revision, @current_revision = nil
  end

  # Performs an update of the homebrew source. Returns +true+ if a newer
  # version was available, +false+ if already up-to-date.
  def update_from_masterbrew!
    HOMEBREW_REPOSITORY.cd do
      if git_repo?
        safe_system "git checkout -q master"
        @initial_revision = read_revision
        # originally we fetched by URL but then we decided that we should
        # use origin so that it's easier for forks to operate seamlessly
        unless `git remote`.split.include? 'origin'
          safe_system "git remote add origin #{REPOSITORY_URL}"
        end
      else
        begin
          safe_system "git init"
          safe_system "git remote add origin #{REPOSITORY_URL}"
          safe_system "git fetch origin"
          safe_system "git reset --hard FETCH_HEAD"
        rescue Exception
          safe_system "/bin/rm -rf .git"
          raise
        end
      end
      execute "git pull origin master"
      @current_revision = read_revision
    end

    if initial_revision && initial_revision != current_revision
      # hash with status characters for keys:
      # Added (A), Copied (C), Deleted (D), Modified (M), Renamed (R)
      @changes_map = Hash.new {|h,k| h[k] = [] }

      changes = HOMEBREW_REPOSITORY.cd do
        execute("git diff-tree -r --name-status -z #{initial_revision} #{current_revision}").split("\0")
      end

      while status = changes.shift
        file = changes.shift
        @changes_map[status] << file
      end

      if @changes_map.any?
        @added_formulae   = changed_items('A', FORMULA_DIR)
        @deleted_formulae = changed_items('D', FORMULA_DIR)
        @updated_formulae = changed_items('M', FORMULA_DIR)
        @added_examples   = changed_items('A', EXAMPLE_DIR)
        @deleted_examples = changed_items('D', EXAMPLE_DIR)
        @updated_examples = changed_items('M', EXAMPLE_DIR)

        @installed_formulae = HOMEBREW_CELLAR.children.
          select{ |pn| pn.directory? }.
          map{ |pn| pn.basename.to_s }.sort if HOMEBREW_CELLAR.directory?

        return true
      end
    end
    # assume nothing was updated
    return false
  end

  def git_repo?
    Dir['.git/*'].length > 0
  end

  def pending_formulae_changes?
    !@updated_formulae.empty?
  end

  def pending_new_formulae?
    !@added_formulae.empty?
  end

  def deleted_formulae?
    !@deleted_formulae.empty?
  end

  def pending_examples_changes?
    !@updated_examples.empty?
  end

  def pending_new_examples?
    !@added_examples.empty?
  end

  def deleted_examples?
    !@deleted_examples.empty?
  end

  def report
    puts "Updated Homebrew from #{initial_revision[0,8]} to #{current_revision[0,8]}."
    ## New Formulae
    if pending_new_formulae?
      ohai "The following formulae are new:"
      puts_columns added_formulae
    end
    ## Deleted Formulae
    if deleted_formulae?
      ohai "The following formulae were removed:"
      puts_columns deleted_formulae, installed_formulae
    end
    ## Updated Formulae
    if pending_formulae_changes?
      ohai "The following formulae were updated:"
      puts_columns updated_formulae, installed_formulae
    end
    ## New examples
    if pending_new_examples?
      ohai "The following external commands are new:"
      puts_columns added_examples
    end
    ## Deleted examples
    if deleted_examples?
      ohai "The following external commands were removed:"
      puts_columns deleted_examples
    end
    ## Updated Formulae
    if pending_examples_changes?
      ohai "The following external commands were updated:"
      puts_columns updated_examples
    end
  end

  private

  def read_revision
    execute("git rev-parse HEAD").chomp
  end

  def filter_by_directory(files, dir)
    files.select { |f| f.index(dir) == 0 }
  end

  def basenames(files)
    files.map { |f| File.basename(f, '.rb') }
  end

  # extracts items by status from @changes_map
  def changed_items(status, dir)
    basenames(filter_by_directory(@changes_map[status], dir)).sort
  end

  def execute(cmd)
    out = `#{cmd}`
    if $? && !$?.success?
      $stderr.puts out
      raise "Failed while executing #{cmd}"
    end
    ohai(cmd, out) if ARGV.verbose?
    out
  end
end
<commit_msg>Fix `brew update` after new install for real this time?<commit_after>module Homebrew extend self
  def update
    abort "Please `brew install git' first." unless system "/usr/bin/which -s git"

    updater = RefreshBrew.new
    if updater.update_from_masterbrew!
      updater.report
    else
      puts "Already up-to-date."
    end
  end
end

class RefreshBrew
  REPOSITORY_URL = "http://github.com/mxcl/homebrew.git"
  FORMULA_DIR = 'Library/Formula/'
  EXAMPLE_DIR = 'Library/Contributions/examples/'

  attr_reader :added_formulae, :updated_formulae, :deleted_formulae, :installed_formulae
  attr_reader :added_examples, :updated_examples, :deleted_examples
  attr_reader :initial_revision, :current_revision

  def initialize
    @added_formulae, @updated_formulae, @deleted_formulae, @installed_formulae = [], [], [], []
    @added_examples, @updated_examples, @deleted_examples = [], [], []
    @initial_revision, @current_revision = nil
  end

  # Performs an update of the homebrew source. Returns +true+ if a newer
  # version was available, +false+ if already up-to-date.
  def update_from_masterbrew!
    HOMEBREW_REPOSITORY.cd do
      if git_repo?
        safe_system "git checkout -q master"
        @initial_revision = read_revision
        # originally we fetched by URL but then we decided that we should
        # use origin so that it's easier for forks to operate seamlessly
        unless `git remote`.split.include? 'origin'
          safe_system "git remote add origin #{REPOSITORY_URL}"
        end
      else
        begin
          safe_system "git init"
          safe_system "git remote add origin #{REPOSITORY_URL}"
          safe_system "git fetch origin"
          safe_system "git reset --hard origin/master"
        rescue Exception
          safe_system "/bin/rm -rf .git"
          raise
        end
      end
      execute "git pull origin master"
      @current_revision = read_revision
    end

    if initial_revision && initial_revision != current_revision
      # hash with status characters for keys:
      # Added (A), Copied (C), Deleted (D), Modified (M), Renamed (R)
      @changes_map = Hash.new {|h,k| h[k] = [] }

      changes = HOMEBREW_REPOSITORY.cd do
        execute("git diff-tree -r --name-status -z #{initial_revision} #{current_revision}").split("\0")
      end

      while status = changes.shift
        file = changes.shift
        @changes_map[status] << file
      end

      if @changes_map.any?
        @added_formulae   = changed_items('A', FORMULA_DIR)
        @deleted_formulae = changed_items('D', FORMULA_DIR)
        @updated_formulae = changed_items('M', FORMULA_DIR)
        @added_examples   = changed_items('A', EXAMPLE_DIR)
        @deleted_examples = changed_items('D', EXAMPLE_DIR)
        @updated_examples = changed_items('M', EXAMPLE_DIR)

        @installed_formulae = HOMEBREW_CELLAR.children.
          select{ |pn| pn.directory? }.
          map{ |pn| pn.basename.to_s }.sort if HOMEBREW_CELLAR.directory?

        return true
      end
    end
    # assume nothing was updated
    return false
  end

  def git_repo?
    Dir['.git/*'].length > 0
  end

  def pending_formulae_changes?
    !@updated_formulae.empty?
  end

  def pending_new_formulae?
    !@added_formulae.empty?
  end

  def deleted_formulae?
    !@deleted_formulae.empty?
  end

  def pending_examples_changes?
    !@updated_examples.empty?
  end

  def pending_new_examples?
    !@added_examples.empty?
  end

  def deleted_examples?
    !@deleted_examples.empty?
  end

  def report
    puts "Updated Homebrew from #{initial_revision[0,8]} to #{current_revision[0,8]}."
    ## New Formulae
    if pending_new_formulae?
      ohai "The following formulae are new:"
      puts_columns added_formulae
    end
    ## Deleted Formulae
    if deleted_formulae?
      ohai "The following formulae were removed:"
      puts_columns deleted_formulae, installed_formulae
    end
    ## Updated Formulae
    if pending_formulae_changes?
      ohai "The following formulae were updated:"
      puts_columns updated_formulae, installed_formulae
    end
    ## New examples
    if pending_new_examples?
      ohai "The following external commands are new:"
      puts_columns added_examples
    end
    ## Deleted examples
    if deleted_examples?
      ohai "The following external commands were removed:"
      puts_columns deleted_examples
    end
    ## Updated Formulae
    if pending_examples_changes?
      ohai "The following external commands were updated:"
      puts_columns updated_examples
    end
  end

  private

  def read_revision
    execute("git rev-parse HEAD").chomp
  end

  def filter_by_directory(files, dir)
    files.select { |f| f.index(dir) == 0 }
  end

  def basenames(files)
    files.map { |f| File.basename(f, '.rb') }
  end

  # extracts items by status from @changes_map
  def changed_items(status, dir)
    basenames(filter_by_directory(@changes_map[status], dir)).sort
  end

  def execute(cmd)
    out = `#{cmd}`
    if $? && !$?.success?
      $stderr.puts out
      raise "Failed while executing #{cmd}"
    end
    ohai(cmd, out) if ARGV.verbose?
    out
  end
end

{'fraction_non_alphanumeric': 0.08932228181374939, 'fraction_numerical': 0.0013651877133105802, 'mean_word_length': 2.924990432453119, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5500099', 'n_tokens_mistral': 3343, 'n_tokens_neox': 3248, 'n_words': 863}