retry 3 times on k8s route update 409s

[finishes #173583970]

fixes cloudfoundry/capi-k8s-release#48

Co-authored-by: Connor Braa <cbraa@pivotal.io>
Co-authored-by: Matt Royal <mroyal@pivotal.io>

Author: cwlbraa

lib/kubernetes/api_client.rb

@@ -2,6 +2,9 @@
 
 module Kubernetes
   class ApiClient
+    class Error < StandardError; end
+    class ConflictError < Error; end
+
     def initialize(build_kube_client:, kpack_kube_client:, route_kube_client:)
       @build_kube_client = build_kube_client
       @kpack_kube_client = kpack_kube_client
@@ -64,6 +67,9 @@ def update_route(resource_config)
       @route_kube_client.update_route(resource_config)
     rescue Kubeclient::HttpError => e
       logger.error('update_route', error: e.inspect, response: e.response, backtrace: e.backtrace)
+
+      raise ConflictError.new("Conflict on update of #{resource_name(resource_config)}") if e.error_code == 409
+
       error = CloudController::Errors::ApiError.new_from_details('KubernetesRouteResourceError', resource_name(resource_config), e.message, e.response)
       error.set_backtrace(e.backtrace)
       raise error

lib/kubernetes/route_resource_manager.rb

@@ -2,6 +2,8 @@
 
 module Kubernetes
   class RouteResourceManager
+    UPDATE_DESTINATION_CONFLICT_RETRIES = 3
+
     def initialize(kube_client)
       @client = kube_client
     end
@@ -37,29 +39,40 @@ def create_route(route)
 
       @client.create_route(Kubeclient::Resource.new(route_resource_hash))
     rescue => e
-      Steno.logger('cc.action.route_create').info("Failed to Create Route CRD: #{e}")
+      logger.info("Failed to Create Route CRD: #{e}")
       raise
     end
 
     def update_destinations(route)
-      route_resource = @client.get_route(route.guid, 'cf-workloads')
-      route_resource.spec.destinations = get_destinations(route)
-
-      @client.update_route(route_resource)
-    rescue => e
-      Steno.logger('cc.action.route_update').info("Failed to Update Route CRD: #{e}")
-      raise
+      remaining_retries = UPDATE_DESTINATION_CONFLICT_RETRIES
+      begin
+        route_resource = @client.get_route(route.guid, 'cf-workloads')
+        route_resource.spec.destinations = get_destinations(route)
+        @client.update_route(route_resource)
+      rescue ApiClient::ConflictError
+        remaining_retries -= 1
+        retry if remaining_retries.positive?
+        logger.info("Failed to Update Route CRD after #{UPDATE_DESTINATION_CONFLICT_RETRIES}: #{e}")
+        raise
+      rescue => e
+        logger.info("Failed to Update Route CRD: #{e}")
+        raise
+      end
     end
 
     def delete_route(route)
       @client.delete_route(route.guid, VCAP::CloudController::Config.config.kubernetes_workloads_namespace)
     rescue => e
-      Steno.logger('cc.action.route_create').info("Failed to Delete Route CRD: #{e}")
+      logger.info("Failed to Delete Route CRD: #{e}")
       raise
     end
 
     private
 
+    def logger
+      Steno.logger('kubernetes.route_resource_manager')
+    end
+
     def get_destinations(route)
       route.route_mappings.map do |route_mapping|
         destination = {

spec/unit/lib/kubernetes/api_client_spec.rb

@@ -248,6 +248,17 @@
           end
         end
 
+        context 'when the error is a 409' do
+          let(:error) { Kubeclient::HttpError.new(409, 'foo', 'bar') }
+
+          it 'raises as an ApiError that includes the resource name' do
+            expect {
+              subject.update_route(resource_config)
+            }.to raise_error(Kubernetes::ApiClient::ConflictError)
+            expect(logger).to have_received(:error).with('update_route', error: /status code/, response: error.response, backtrace: error.backtrace)
+          end
+        end
+
         context 'when the config is a hash with symbol keys' do
           let(:resource_config) { config_hash.symbolize_keys }
 

spec/unit/lib/kubernetes/route_resource_manager_spec.rb

@@ -224,6 +224,24 @@
           }.to raise_error(error)
         end
       end
+
+      context 'when there is a k8s conflict error' do
+        let(:error) { ::Kubernetes::ApiClient::ConflictError.new('boom') }
+
+        before do
+          # raise a 409 twice, then succeed
+          expect(k8s_client).to receive(:update_route).once.with(any_args).and_raise(error)
+          expect(k8s_client).to receive(:update_route).once.with(any_args).and_raise(error)
+          expect(k8s_client).to receive(:update_route).once.with(any_args)
+        end
+
+        it 'retries 3 times, fetching the route to patch each time' do
+          expect {
+            subject.update_destinations(route)
+          }.not_to raise_error
+          expect(k8s_client).to have_received(:get_route).exactly(3).times
+        end
+      end
     end
   end
 end