v3(services) Make sure the LO state is failed when job fails (cloudfoundry#1903)

* v3(services) Make sure the LO state is failed when job fails

Sometimes the Job can fail for unexpected reasons. In any failure we should make sure to leave the LO state as 'failed'

[#175264299](https://www.pivotaltracker.com/story/show/175264299)

* Renamed the BindingGone error to BindingNotFound

BindingGone could have been missunderstood as response to 410 error

* Revert accidental rename

Author: FelisiaM

app/jobs/v3/create_binding_async_job.rb

@@ -7,6 +7,8 @@
 module VCAP::CloudController
   module V3
     class CreateBindingAsyncJob < Jobs::ReoccurringJob
+      class BindingNotFound < CloudController::Errors::ApiError; end
+
       def initialize(type, precursor_guid, parameters:, user_audit_info:, audit_hash:)
         super()
         @type = type
@@ -50,7 +52,7 @@ def resource_type
       end
 
       def perform
-        gone! unless resource
+        not_found! unless resource
 
         compute_maximum_duration
 
@@ -70,9 +72,12 @@ def perform
         if polling_status[:retry_after].present?
           self.polling_interval_seconds = polling_status[:retry_after]
         end
+      rescue BindingNotFound => e
+        raise e
       rescue ServiceBindingCreate::BindingNotRetrievable
         raise CloudController::Errors::ApiError.new_from_details('ServiceBindingInvalid', 'The broker responded asynchronously but does not support fetching binding data')
       rescue => e
+        save_failure(e.message)
         raise CloudController::Errors::ApiError.new_from_details('UnableToPerform', 'bind', e.message)
       end
 
@@ -98,8 +103,21 @@ def compute_maximum_duration
         self.maximum_duration_seconds = max_poll_duration_on_plan
       end
 
-      def gone!
-        raise CloudController::Errors::ApiError.new_from_details('ResourceNotFound', "The binding could not be found: #{@resource_guid}")
+      def not_found!
+        raise BindingNotFound.new_from_details('ResourceNotFound', "The binding could not be found: #{@resource_guid}")
+      end
+
+      def save_failure(error_message)
+        if resource.reload.last_operation.state != 'failed'
+          resource.save_with_attributes_and_new_operation(
+            {},
+            {
+              type: operation_type,
+              state: 'failed',
+              description: error_message,
+            }
+          )
+        end
       end
     end
   end

spec/support/shared_examples/jobs/create_binding_job.rb

@@ -180,7 +180,7 @@ def test_retry_after(value, expected)
         binding.destroy
 
         expect { subject.perform }.to raise_error(
-          CloudController::Errors::ApiError,
+          VCAP::CloudController::V3::CreateBindingAsyncJob::BindingNotFound,
           /The binding could not be found/,
         )
       end
@@ -194,6 +194,10 @@ def test_retry_after(value, expected)
           CloudController::Errors::ApiError,
           'bind could not be completed: StandardError',
         )
+
+        binding.reload
+        expect(binding.last_operation.type).to eq('create')
+        expect(binding.last_operation.state).to eq('failed')
       end
     end
 
@@ -205,13 +209,18 @@ def test_retry_after(value, expected)
           CloudController::Errors::ApiError,
           'bind could not be completed: StandardError',
         )
+
+        binding.reload
+        expect(binding.last_operation.type).to eq('create')
+        expect(binding.last_operation.state).to eq('failed')
       end
     end
   end
 
   describe '#handle_timeout' do
     it 'updates the last operation to failed' do
       subject.handle_timeout
+
       binding.reload
       expect(binding.last_operation.type).to eq('create')
       expect(binding.last_operation.state).to eq('failed')