<commit_before>require 'chef/knife'
require 'chef/knife/ec_base'

class Chef
  class Knife
    class EcBackup < Chef::Knife

      include Knife::EcBase

      banner "knife ec backup DIRECTORY"

      deps do
        require 'chef/chef_fs/config'
        require 'chef/chef_fs/file_system'
        require 'chef/chef_fs/file_pattern'
        require 'chef/chef_fs/parallelizer'
        require 'chef/server'
        require 'fileutils'
      end

      def run
        set_dest_dir_from_args!
        set_client_config!
        set_skip_user_acl!
        ensure_webui_key_exists!

        for_each_user do |username, url|
          download_user(username, url)
          if config[:skip_useracl]
            ui.warn("Skipping user ACL download for #{username}. To download this ACL, remove --skip-useracl or upgrade your Enterprise Chef Server.")
          else
            download_user_acl(username)
          end
        end

        if config[:with_user_sql] || config[:with_key_sql]
          export_from_sql
        end

        ensure_dir("#{dest_dir}/organizations")
        for_each_organization do |org_object|
          name = org_object['name']
          write_org_object_to_disk(org_object)
          download_org_data(name)
          download_org_members(name)
          download_org_invitations(name)
        end
      end

      def for_each_user
        rest.get('/users').each_pair do |name, url|
          yield name, url
        end
      end

      def for_each_organization
        rest.get('/organizations').each_pair do |name, url|
          next unless (config[:org].nil? || config[:org] == name)
          ui.msg "Downloading organization object for #{name} from #{url}"
          org = rest.get(url)
          # Enterprise Chef 11 and below uses a pool of precreated
          # organizations to account for slow organization creation
          # using CouchDB. Thus, on server versions < 12 we want to
          # skip any of these precreated organizations by checking if
          # they have been assigned or not.  The Chef 12 API does not
          # return an assigned_at field.
          if org['assigned_at'] || server.version >= Gem::Version.new("12")
            yield org
          else
            ui.msg "Skipping pre-created org #{name}"
          end
        end
      end

      def download_user(username, url)
        ensure_dir("#{dest_dir}/users")
        File.open("#{dest_dir}/users/#{username}.json", 'w') do |file|
          file.write(Chef::JSONCompat.to_json_pretty(rest.get(url)))
        end
      end

      def download_user_acl(username)
        ensure_dir("#{dest_dir}/user_acls")
        File.open("#{dest_dir}/user_acls/#{username}.json", 'w') do |file|
          file.write(Chef::JSONCompat.to_json_pretty(user_acl_rest.get("users/#{username}/_acl")))
        end
      end

      def export_from_sql
        require 'chef/knife/ec_key_export'
        Chef::Knife::EcKeyExport.deps
        k = Chef::Knife::EcKeyExport.new
        k.name_args = ["#{dest_dir}/key_dump.json", "#{dest_dir}/key_table_dump.json"]
        k.config[:sql_host] = config[:sql_host]
        k.config[:sql_port] = config[:sql_port]<commit_msg>support purge users on backup<commit_after>require 'chef/knife'
require 'chef/knife/ec_base'

class Chef
  class Knife
    class EcBackup < Chef::Knife

      include Knife::EcBase

      banner "knife ec backup DIRECTORY"

      deps do
        require 'chef/chef_fs/config'
        require 'chef/chef_fs/file_system'
        require 'chef/chef_fs/file_pattern'
        require 'chef/chef_fs/parallelizer'
        require 'chef/server'
        require 'fileutils'
      end

      def run
        set_dest_dir_from_args!
        set_client_config!
        set_skip_user_acl!
        ensure_webui_key_exists!

        ensure_dir("#{dest_dir}/users")
        ensure_dir("#{dest_dir}/user_acls") unless config[:skip_useracl]
        ui.msg "Downloading Users"
        remote_users = []
        for_each_user do |username, url|
          remote_users.push(username.to_sym)
          download_user(username, url)
          if config[:skip_useracl]
            ui.warn("Skipping user ACL download for #{username}. To download this ACL, remove --skip-useracl or upgrade your Enterprise Chef Server.")
          else
            download_user_acl(username)
          end
        end
        purge_users(remote_users) if config[:purge]

        if config[:with_user_sql] || config[:with_key_sql]
          export_from_sql
        end

        ensure_dir("#{dest_dir}/organizations")
        for_each_organization do |org_object|
          name = org_object['name']
          write_org_object_to_disk(org_object)
          download_org_data(name)
          download_org_members(name)
          download_org_invitations(name)
        end
      end

      def purge_users(src)
        local_users = Dir.glob("#{dest_dir}/users/*\.json").map { |u| File.basename(u, '.json').to_sym }
        purge_list = local_users - src
        # failsafe - don't delete pivotal
        purge_list -= [:pivotal]
        purge_list.collect(&:to_s).each do |user|
          ui.msg "Deleting local user #{user} from backup (purge in on)"
          begin
            File.delete("#{dest_dir}/users/#{user}.json")
            File.delete("#{dest_dir}/user_acls/#{user}.json")
          rescue Errno::ENOENT
            true
          end
        end
      end

      def for_each_user
        rest.get('/users').each_pair do |name, url|
          yield name, url
        end
      end

      def for_each_organization
        rest.get('/organizations').each_pair do |name, url|
          next unless (config[:org].nil? || config[:org] == name)
          ui.msg "Downloading organization object for #{name} from #{url}"
          org = rest.get(url)
          # Enterprise Chef 11 and below uses a pool of precreated
          # organizations to account for slow organization creation
          # using CouchDB. Thus, on server versions < 12 we want to
          # skip any of these precreated organizations by checking if
          # they have been assigned or not.  The Chef 12 API does not
          # return an assigned_at field.
          if org['assigned_at'] || server.version >= Gem::Version.new("12")
            yield org
          else
            ui.msg "Skipping pre-created org #{name}"
          end
        end
      end

      def download_user(username, url)
        File.open("#{dest_dir}/users/#{username}.json", 'w') do |file|
          file.write(Chef::JSONCompat.to_json_pretty(rest.get(url)))
        end
      end

      def download_user_acl(username)
        File.open("#{dest_dir}/user_acls/#{username}.json", 'w') do |file|
          file.write(Chef::JSONCompat.to_json_pretty(user_acl_rest.get("users/#{username}/_acl")))
        end
      end

      def export_from_sql
        require 'chef/knife/ec_key_export'
        Chef::Knife::EcKeyExport.deps
        k = Chef::Knife::EcKeyExport.new
        k.name_args = ["#{dest_dir}/key_dump.json", "#{dest_dir}/key_table_dump.json"]
        k.config[:sql_host] = config[:sql_host]
        k.config[:sql_port] = config[:sql_port]
{'fraction_non_alphanumeric': 0.10334218990351, 'fraction_numerical': 0.0022374493077891204, 'mean_word_length': 2.239130434782609, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5871279', 'n_tokens_mistral': 2218, 'n_tokens_neox': 2142, 'n_words': 491}