v3(services): refactor delete service instance job

[#176032428](https://www.pivotaltracker.com/story/show/176032428)

Co-authored-by: George Blue <gblue@pivotal.io>

Author: FelisiaM

app/actions/v3/service_instance_delete.rb

@@ -1,39 +1,138 @@
 require 'jobs/v3/delete_service_instance_job'
+require 'actions/services/locks/deleter_lock'
+require 'cloud_controller/errors/api_error'
 
 module VCAP::CloudController
   module V3
     class ServiceInstanceDelete
-      class AssociationNotEmptyError < StandardError; end
+      class AssociationNotEmptyError < StandardError
+      end
+
+      class InstanceSharedError < StandardError
+      end
+
+      class DeleteFailed < StandardError
+      end
 
-      class InstanceSharedError < StandardError; end
+      DeleteStatus = Struct.new(:finished, :operation).freeze
+      DeleteStarted = ->(operation) { DeleteStatus.new(false, operation) }
+      DeleteComplete = DeleteStatus.new(true, nil).freeze
 
-      def initialize(event_repo)
+      PollingStatus = Struct.new(:finished, :retry_after).freeze
+      PollingFinished = PollingStatus.new(true, nil).freeze
+      ContinuePolling = ->(retry_after) { PollingStatus.new(false, retry_after) }
+
+      def initialize(service_instance, event_repo)
+        @service_instance = service_instance
         @service_event_repository = event_repo
       end
 
-      def delete(service_instance)
-        association_not_empty! if service_instance.has_bindings? || service_instance.has_keys? || service_instance.has_routes?
+      def delete
+        operation_in_progress! if service_instance.operation_in_progress? && service_instance.last_operation.type != 'create'
+
+        result = send_deprovison_to_broker
+        if result[:finished]
+          destroy
+          record_delete_event
+        else
+          update_last_operation_with_operation_id(result[:operation])
+          record_start_delete_event
+        end
 
+        result
+      rescue => e
+        update_last_operation_with_failure(e.message) unless service_instance.operation_in_progress?
+        raise e
+      end
+
+      def delete_checks
+        association_not_empty! if service_instance.has_bindings? || service_instance.has_keys? || service_instance.has_routes?
         cannot_delete_shared_instances! if service_instance.shared?
+      end
 
-        lock = DeleterLock.new(service_instance)
+      def poll
+        result = client.fetch_service_instance_last_operation(service_instance)
+        case result[:last_operation][:state]
+        when 'in progress'
+          update_last_operation_with_description(result[:last_operation][:description])
+          ContinuePolling.call(result[:retry_after])
+        when 'succeeded'
+          destroy
+          PollingFinished
+        else
+          delete_failed!(result[:last_operation][:description])
+        end
+      rescue DeleteFailed => e
+        update_last_operation_with_failure(e.message)
+        raise CloudController::Errors::ApiError.new_from_details('UnableToPerform', 'delete', e.message)
+      rescue => e
+        update_last_operation_with_failure(e.message)
+        ContinuePolling.call(nil)
+      end
+
+      def update_last_operation_with_failure(message)
+        service_instance.save_with_new_operation(
+          {},
+          {
+            type: 'delete',
+            state: 'failed',
+            description: message,
+          }
+        )
+      end
 
+      private
+
+      attr_reader :service_event_repository, :service_instance
+
+      def client
+        VCAP::Services::ServiceClientProvider.provide(instance: service_instance)
+      end
+
+      def send_deprovison_to_broker
+        result = client.deprovision(service_instance, accepts_incomplete: true)
+        return DeleteComplete if result[:last_operation][:state] == 'succeeded'
+
+        DeleteStarted.call(result[:last_operation][:broker_provided_operation])
+      end
+
+      def record_delete_event
         case service_instance
-        when ManagedServiceInstance
-          return false
-        when UserProvidedServiceInstance
-          lock.lock!
-          synchronous_destroy(service_instance, lock)
-          return true
+        when VCAP::CloudController::ManagedServiceInstance
+          service_event_repository.record_service_instance_event(:delete, service_instance)
+        when VCAP::CloudController::UserProvidedServiceInstance
+          service_event_repository.record_user_provided_service_instance_event(:delete, service_instance)
         end
       end
 
-      private
+      def record_start_delete_event
+        service_event_repository.record_service_instance_event(:start_delete, service_instance)
+      end
 
-      def synchronous_destroy(service_instance, lock)
-        lock.unlock_and_destroy!
-        service_event_repository.record_user_provided_service_instance_event(:delete, service_instance)
-        nil
+      def destroy
+        ServiceInstance.db.transaction do
+          service_instance.lock!
+          service_instance.last_operation&.destroy
+          service_instance.destroy
+        end
+      end
+
+      def update_last_operation_with_operation_id(operation_id)
+        service_instance.save_with_new_operation(
+          {},
+          {
+            type: 'delete',
+            state: 'in progress',
+            broker_provided_operation: operation_id
+          }
+        )
+      end
+
+      def update_last_operation_with_description(description)
+        lo = service_instance.last_operation.to_hash
+        lo[:broker_provided_operation] = service_instance.last_operation.broker_provided_operation
+        lo[:description] = description
+        service_instance.save_with_new_operation({}, lo)
       end
 
       def association_not_empty!
@@ -44,7 +143,13 @@ def cannot_delete_shared_instances!
         raise InstanceSharedError
       end
 
-      attr_reader :service_event_repository
+      def operation_in_progress!
+        raise CloudController::Errors::ApiError.new_from_details('AsyncServiceInstanceOperationInProgress', service_instance.name)
+      end
+
+      def delete_failed!(message)
+        raise DeleteFailed.new(message)
+      end
     end
   end
 end

app/controllers/v3/service_instances_controller.rb

@@ -112,13 +112,16 @@ def destroy
       return [:no_content, nil]
     end
 
-    deleted = V3::ServiceInstanceDelete.new(service_event_repository).delete(service_instance)
+    delete_action = V3::ServiceInstanceDelete.new(service_instance, service_event_repository)
+    delete_action.delete_checks
 
-    if deleted
-      head :no_content
-    else
+    case service_instance
+    when VCAP::CloudController::ManagedServiceInstance
       job_guid = enqueue_delete_job(service_instance)
       head :accepted, 'Location' => url_builder.build_url(path: "/v3/jobs/#{job_guid}")
+    when VCAP::CloudController::UserProvidedServiceInstance
+      delete_action.delete
+      head :no_content
     end
   rescue V3::ServiceInstanceDelete::AssociationNotEmptyError
     associations_not_empty!

app/jobs/v3/delete_service_instance_job.rb

@@ -3,28 +3,37 @@
 
 module VCAP::CloudController
   module V3
-    class DeprovisionBadResponse < StandardError
-    end
+    class DeleteServiceInstanceJob < VCAP::CloudController::Jobs::ReoccurringJob
+      attr_reader :resource_guid
 
-    class DeleteServiceInstanceJob < ServiceInstanceAsyncJob
-      def initialize(guid, audit_info)
-        super
-        @request_attr = nil
+      def initialize(guid, user_audit_info)
+        super()
+        @resource_guid = guid
+        @user_audit_info = user_audit_info
       end
 
-      def send_broker_request(client)
-        deprovision_response = client.deprovision(service_instance, { accepts_incomplete: true })
+      def perform
+        return finish unless service_instance
 
-        @request_failed = false
+        self.maximum_duration_seconds = service_instance.service_plan.try(:maximum_polling_duration)
 
-        deprovision_response
-      rescue VCAP::Services::ServiceBrokers::V2::Errors::ServiceBrokerBadResponse => err
-        @request_failed = true
-        raise DeprovisionBadResponse.new(err.message)
-      rescue CloudController::Errors::ApiError => err
-        raise OperationCancelled.new('The service broker rejected the request') if err.name == 'AsyncServiceInstanceOperationInProgress'
+        unless delete_in_progress?
+          result = action.delete
+          return finish if result[:finished]
+        end
+
+        result = action.poll
+        return finish if result[:finished]
 
+        self.polling_interval_seconds = result[:retry_after].to_i if result[:retry_after]
+      rescue CloudController::Errors::ApiError => err
         raise err
+      rescue => err
+        raise CloudController::Errors::ApiError.new_from_details('UnableToPerform', operation_type, err.message)
+      end
+
+      def handle_timeout
+        action.update_last_operation_with_failure("Service Broker failed to #{operation} within the required time.")
       end
 
       def operation
@@ -35,46 +44,29 @@ def operation_type
         'delete'
       end
 
-      def gone!
-        finish
+      def resource_type
+        'service_instance'
       end
 
-      def restart_on_failure?
-        true
+      def display_name
+        "#{resource_type}.#{operation_type}"
       end
 
-      def pollable_job_state
-        return PollableJobModel::PROCESSING_STATE if @request_failed
-
-        PollableJobModel::POLLING_STATE
-      end
+      private
 
-      def restart_job(msg)
-        super
-        logger.info("could not complete the operation: #{msg}. Triggering orphan mitigation")
-      end
+      attr_reader :user_audit_info
 
-      def fail!(err)
-        case err
-        when DeprovisionBadResponse
-          trigger_orphan_mitigation(err)
-        else
-          super
-        end
+      def service_instance
+        ManagedServiceInstance.first(guid: resource_guid)
       end
 
-      private
-
-      def operation_succeeded
-        ServiceInstance.db.transaction do
-          service_instance.lock!
-          service_instance.last_operation&.destroy
-          service_instance.destroy
-        end
+      def delete_in_progress?
+        service_instance.last_operation&.type == 'delete' &&
+          service_instance.last_operation&.state == 'in progress'
       end
 
-      def trigger_orphan_mitigation(err)
-        restart_job(err.message)
+      def action
+        ServiceInstanceDelete.new(service_instance, Repositories::ServiceEventRepository.new(user_audit_info))
       end
     end
   end