Choose config schema based on its contents

- Use the kubernetes config schema if the kubernetes.host_url key has a
  value. Otherwise use the VM config schema.

- Most contexts inherit from the Base config schema, and then add
  additional config options for the VM or Kubernetes configuration. Some
  contexts only exist in VM deployments, so they don't use Base

- Before we were just validating against parent schema and child schema
  separately; however, splitting the schemas up without merging breaks
  how we access config objects via `Config.#get`

- The `delegate` definitions in each of the child schemas was also
  incorrect and fixed that

- Change the cloud_controller.yml used in test so it represent a VMs
  deployment, instead of a hybrid of VMs and k8s.

- Removed Kpack::Stager logic to use blobstores, since that's no longer
  supported and it was causing spec failures

- Deleted some now unused test fixtures

[#175806825]

Co-authored-by: Matt Royal <mroyal@vmware.com>
Co-authored-by: Piyali Banerjee <pbanerjee@pivotal.io>
Co-authored-by: Jaskanwal Pawar <jpawar@pivotal.io>

Author: 3 people

config/cloud_controller.yml

@@ -293,16 +293,16 @@ opi:
   url: http://opi.service.cf.internal
   opi_staging: false
 
-kubernetes:
-  host_url: https://master.default.svc.cluster-domain.example
-  service_account:
-    token_file: "spec/fixtures/service_accounts/k8s.token"
-  ca_file: "spec/fixtures/certs/kubernetes_ca.crt"
-  workloads_namespace: 'cf-workloads'
-  kpack:
-    builder_namespace: 'cf-workloads-staging'
-    registry_service_account_name: 'fake-registry-service-account'
-    registry_tag_base: 'gcr.io/fake-image-repository'
+#kubernetes:
+#  host_url: https://master.default.svc.cluster-domain.example
+#  service_account:
+#    token_file: "spec/fixtures/service_accounts/k8s.token"
+#  ca_file: "spec/fixtures/certs/kubernetes_ca.crt"
+#  workloads_namespace: 'cf-workloads'
+#  kpack:
+#    builder_namespace: 'cf-workloads-staging'
+#    registry_service_account_name: 'fake-registry-service-account'
+#    registry_tag_base: 'gcr.io/fake-image-repository'
 
 directories:
   tmpdir: /tmp

lib/cloud_controller/config.rb

@@ -5,13 +5,18 @@
 require 'cloud_controller/index_stopper'
 require 'cloud_controller/backends/instances_reporters'
 require 'repositories/service_event_repository'
-require 'cloud_controller/config_schemas/api_schema'
-require 'cloud_controller/config_schemas/clock_schema'
-require 'cloud_controller/config_schemas/migrate_schema'
-require 'cloud_controller/config_schemas/route_syncer_schema'
-require 'cloud_controller/config_schemas/worker_schema'
-require 'cloud_controller/config_schemas/deployment_updater_schema'
-require 'cloud_controller/config_schemas/rotatate_database_key_schema'
+require 'cloud_controller/config_schemas/kubernetes/api_schema'
+require 'cloud_controller/config_schemas/vms/api_schema'
+require 'cloud_controller/config_schemas/kubernetes/clock_schema'
+require 'cloud_controller/config_schemas/vms/clock_schema'
+require 'cloud_controller/config_schemas/kubernetes/migrate_schema'
+require 'cloud_controller/config_schemas/vms/migrate_schema'
+require 'cloud_controller/config_schemas/vms/route_syncer_schema'
+require 'cloud_controller/config_schemas/kubernetes/worker_schema'
+require 'cloud_controller/config_schemas/vms/worker_schema'
+require 'cloud_controller/config_schemas/kubernetes/deployment_updater_schema'
+require 'cloud_controller/config_schemas/vms/deployment_updater_schema'
+require 'cloud_controller/config_schemas/vms/rotate_database_key_schema'
 require 'utils/hash_utils'
 require 'cloud_controller/internal_api'
 
@@ -25,25 +30,41 @@ class KubernetesApiNotConfigured < StandardError
 
     class << self
       def load_from_file(file_name, context: :api, secrets_hash: {})
-        schema_class = schema_class_for_context(context)
-        config_from_file = schema_class.from_file(file_name, secrets_hash: secrets_hash)
-        hash = merge_defaults(config_from_file)
+        config = VCAP::CloudController::YAMLConfig.safe_load_file(file_name)
+        config = deep_symbolize_keys_except_in_arrays(config)
+        secrets_hash = deep_symbolize_keys_except_in_arrays(secrets_hash)
+        config = config.deep_merge(secrets_hash)
+
+        schema_class = schema_class_for_context(context, config)
+        schema_class.validate(config)
+
+        hash = merge_defaults(config)
         @instance = new(hash, context: context)
       end
 
       def config
         @instance
       end
 
-      def schema_class_for_context(context)
-        const_get("VCAP::CloudController::ConfigSchemas::#{context.to_s.camelize}Schema")
+      def schema_class_for_context(context, config)
+        module_name = config.dig(:kubernetes, :host_url).present? ? 'Kubernetes' : 'Vms'
+        const_get("VCAP::CloudController::ConfigSchemas::#{module_name}::#{context.to_s.camelize}Schema")
       end
 
       delegate :kubernetes_api_configured?, to: :config
 
       private
 
-      def merge_defaults(config)
+      def deep_symbolize_keys_except_in_arrays(hash)
+        return hash unless hash.is_a? Hash
+
+        hash.each.with_object({}) do |(k, v), new_hash|
+          new_hash[k.to_sym] = deep_symbolize_keys_except_in_arrays(v)
+        end
+      end
+
+      def merge_defaults(orig_config)
+        config = orig_config.dup
         config[:db] ||= {}
         ensure_config_has_database_parts(config)
         sanitize(config)
@@ -85,7 +106,7 @@ def valid_in_userinfo?(value)
 
     def initialize(config_hash, context: :api)
       @config_hash = config_hash
-      @schema_class = self.class.schema_class_for_context(context)
+      @schema_class = self.class.schema_class_for_context(context, config_hash)
     end
 
     def configure_components
@@ -129,6 +150,8 @@ def valid_config_path?(keys, some_schema)
 
     def kubernetes_api_configured?
       get(:kubernetes, :host_url).present?
+    rescue InvalidConfigPath
+      false
     end
 
     def kubernetes_host_url