<commit_before>require 'ffi_yajl'
require 'chef_zero/endpoints/rest_object_endpoint'
require 'chef_zero/rest_error_response'
require 'chef_zero/chef_data/data_normalizer'
require 'chef_zero/data_store/data_not_found_error'

module ChefZero
  module Endpoints
    # /organizations/ORG/cookbooks/NAME/VERSION
    class CookbookVersionEndpoint < RestObjectEndpoint
      def get(request)
        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
        end
        super(request)
      end

      def put(request)
        name = request.rest_path[3]
        version = request.rest_path[4]
        existing_cookbook = get_data(request, request.rest_path, :nil)

        # Honor frozen
        if existing_cookbook
          existing_cookbook_json = FFI_Yajl::Parser.parse(existing_cookbook, :create_additions => false)
          if existing_cookbook_json['frozen?']
            if request.query_params['force'] != "true"
              raise RestErrorResponse.new(409, "The cookbook #{name} at version #{version} is frozen. Use the 'force' option to override.")
            end
            # For some reason, you are forever unable to modify "frozen?" on a frozen cookbook.
            request_body = FFI_Yajl::Parser.parse(request.body, :create_additions => false)
            if !request_body['frozen?']
              request_body['frozen?'] = true
              request.body = FFI_Yajl::Encoder.encode(request_body, :pretty => true)
            end
          end
        end

        # Set the cookbook
        set_data(request, request.rest_path, request.body, :create_dir, :create)

        # If the cookbook was updated, check for deleted files and clean them up
        if existing_cookbook
          missing_checksums = get_checksums(existing_cookbook) - get_checksums(request.body)
          if missing_checksums.size > 0
            hoover_unused_checksums(missing_checksums, request)
          end
        end

        already_json_response(existing_cookbook ? 200 : 201, populate_defaults(request, request.body))
      end

      def delete(request)
        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
        end

        deleted_cookbook = get_data(request)

        response = super(request)
        # Last one out turns out the lights: delete /organizations/ORG/cookbooks/NAME if it no longer has versions
        cookbook_path = request.rest_path[0..3]
        if exists_data_dir?(request, cookbook_path) && list_data(request, cookbook_path).size == 0
          delete_data_dir(request, cookbook_path)
        end

        # Hoover deleted files, if they exist
        hoover_unused_checksums(get_checksums(deleted_cookbook), request)
        response
      end

      def get_checksums(cookbook)
        result = []
        FFI_Yajl::Parser.parse(cookbook, :create_additions => false).each_pair do |key, value|
          if value.is_a?(Array)
            value.each do |file|
              if file.is_a?(Hash) && file.has_key?('checksum')
                result << file['checksum']
              end
            end
          end
        end
        result.uniq
      end

      private

      def hoover_unused_checksums(deleted_checksums, request)
        %w(cookbooks cookbook_artifacts).each do |cookbook_type|
          data_store.list(request.rest_path[0..1] + [cookbook_type]).each do |cookbook_name|
            data_store.list(request.rest_path[0..1] + [cookbook_type, cookbook_name]).each do |version|
              cookbook = data_store.get(request.rest_path[0..1] + [cookbook_type, cookbook_name, version], request)
              deleted_checksums = deleted_checksums - get_checksums(cookbook)
            end
          end
        end
        deleted_checksums.each do |checksum|
          # There can be a race here if multiple cookbooks are uploading.<commit_msg>Support Chef versions that don't support cookbook_artifacts<commit_after>require 'ffi_yajl'
require 'chef_zero/endpoints/rest_object_endpoint'
require 'chef_zero/rest_error_response'
require 'chef_zero/chef_data/data_normalizer'
require 'chef_zero/data_store/data_not_found_error'

module ChefZero
  module Endpoints
    # /organizations/ORG/cookbooks/NAME/VERSION
    class CookbookVersionEndpoint < RestObjectEndpoint
      def get(request)
        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
        end
        super(request)
      end

      def put(request)
        name = request.rest_path[3]
        version = request.rest_path[4]
        existing_cookbook = get_data(request, request.rest_path, :nil)

        # Honor frozen
        if existing_cookbook
          existing_cookbook_json = FFI_Yajl::Parser.parse(existing_cookbook, :create_additions => false)
          if existing_cookbook_json['frozen?']
            if request.query_params['force'] != "true"
              raise RestErrorResponse.new(409, "The cookbook #{name} at version #{version} is frozen. Use the 'force' option to override.")
            end
            # For some reason, you are forever unable to modify "frozen?" on a frozen cookbook.
            request_body = FFI_Yajl::Parser.parse(request.body, :create_additions => false)
            if !request_body['frozen?']
              request_body['frozen?'] = true
              request.body = FFI_Yajl::Encoder.encode(request_body, :pretty => true)
            end
          end
        end

        # Set the cookbook
        set_data(request, request.rest_path, request.body, :create_dir, :create)

        # If the cookbook was updated, check for deleted files and clean them up
        if existing_cookbook
          missing_checksums = get_checksums(existing_cookbook) - get_checksums(request.body)
          if missing_checksums.size > 0
            hoover_unused_checksums(missing_checksums, request)
          end
        end

        already_json_response(existing_cookbook ? 200 : 201, populate_defaults(request, request.body))
      end

      def delete(request)
        if request.rest_path[4] == "_latest" || request.rest_path[4] == "latest"
          request.rest_path[4] = latest_version(list_data(request, request.rest_path[0..3]))
        end

        deleted_cookbook = get_data(request)

        response = super(request)
        # Last one out turns out the lights: delete /organizations/ORG/cookbooks/NAME if it no longer has versions
        cookbook_path = request.rest_path[0..3]
        if exists_data_dir?(request, cookbook_path) && list_data(request, cookbook_path).size == 0
          delete_data_dir(request, cookbook_path)
        end

        # Hoover deleted files, if they exist
        hoover_unused_checksums(get_checksums(deleted_cookbook), request)
        response
      end

      def get_checksums(cookbook)
        result = []
        FFI_Yajl::Parser.parse(cookbook, :create_additions => false).each_pair do |key, value|
          if value.is_a?(Array)
            value.each do |file|
              if file.is_a?(Hash) && file.has_key?('checksum')
                result << file['checksum']
              end
            end
          end
        end
        result.uniq
      end

      private

      def hoover_unused_checksums(deleted_checksums, request)
        %w(cookbooks cookbook_artifacts).each do |cookbook_type|
          begin
            cookbooks = data_store.list(request.rest_path[0..1] + [cookbook_type])
          rescue ChefZero::DataStore::DataNotFoundError
            # Not all chef versions support cookbook_artifacts
            raise unless cookbook_type == "cookbook_artifacts"
            cookbooks = []
          end
          cookbooks.each do |cookbook_name|
            data_store.list(request.rest_path[0..1] + [cookbook_type, cookbook_name]).each do |version|
              cookbook = data_store.get(request.rest_path[0..1] + [cookbook_type, cookbook_name, version], request)
              deleted_checksums = deleted_checksums - get_checksums(cookbook)
            end
          end
        end
        deleted_checksums.each do |checksum|
          # There can be a race here if multiple cookbooks are uploading.
{'fraction_non_alphanumeric': 0.09022556390977443, 'fraction_numerical': 0.007399451008473565, 'mean_word_length': 2.585793752674369, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3429892', 'n_tokens_mistral': 2519, 'n_tokens_neox': 2444, 'n_words': 538}