ate-setup: reconcile ate-api-server ConfigMaps and roll out on config changes

cmd/ate-setup/commands.md

@@ -30,7 +30,7 @@ every setting with its environment variable.
 | `--egress-gateway-extra-spiffe-ids IDS` | Comma-separated SPIFFE IDs that hold ate-api-server's `egress_gateway` role next to atenet-egress, such as a credential provider that mints actor JWTs. ate-setup passes atenet-egress's ID and then each listed ID as repeated `--egress-gateway-spiffe-id` flags; unset passes none, leaving ate-api-server's default. `ATE_EGRESS_GATEWAY_EXTRA_SPIFFE_IDS` when the flag is absent |
 | `--experimental-additional-egress-extproc-service NS/SVC:PORT` | External processor authorization filter |
 | `--credential-provider JSON` | Required by `deploy ate-system` and `deploy atenet`. A JSON object: `{"name":"k8s.io"}` deploys and uses the bundled Kubernetes Secrets provider, with a NetworkPolicy that admits only the egress gateway; `{"enabled":false}` turns egress credential injection off; `{"name":"<provider>","address":"<host>:<port>"}` uses a provider you deploy yourself. See [`docs/guides/egress-credential-injection.md`](../../docs/guides/egress-credential-injection.md). `ATE_CREDENTIAL_PROVIDER` when the flag is absent |
-| `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). The server picks up a change without restarting. `ATE_API_LOG_LEVEL` when the flag is absent |
+| `--ateapi-log-level LEVEL` | ate-api-server log level: `debug`, `info`, `warn`, or `error`. Written to the `ate-api-config` ConfigMap on every deploy when set; unset keeps the installed level (`info` on a new install). Changing the level updates the env-hash annotation and rolls `ate-api-server`. `ATE_API_LOG_LEVEL` when the flag is absent |
 | `--otlp-endpoint URL` | Send control plane telemetry to `URL` instead of the cluster default (see [`benchmarking/telemetry/README.md`](../../benchmarking/telemetry/README.md)). `ATE_OTLP_ENDPOINT` when the flag is absent |
 | `--preview GATES` | Preview gates passed as `--preview` to every substrate component: comma-separated gate names, or `ALL` for all. `ATE_PREVIEW` or `config.preview` when the flag is absent. An explicit `--preview=''` turns them all off, even when `ATE_PREVIEW` is set |
 | `--context NAME` | Kubeconfig context. `KUBECTL_CONTEXT` when the flag is absent |
@@ -155,6 +155,8 @@ Individual secrets and config that `deploy ate-system` creates automatically.
 - `create api-authentication-config`
 - `create api-config`
 
+`create api-server-env-vars`, `create api-authentication-config`, and `create api-config` also update `Deployment/ate-api-server`'s env-hash annotation and roll any running pods onto the new configuration.
+
 ## Setup
 
 - `setup csi [driver]`

cmd/ate-setup/internal/cmd/create.go

@@ -67,7 +67,7 @@ func init() {
 		},
 		{
 			use:   "api-server-env-vars",
-			short: "Create the ate-api-server environment ConfigMap and Secret",
+			short: "Create the ate-api-server environment ConfigMap and Secret, rolling any running deployment",
 			run: func(e *steps.Env, ctx context.Context) error {
 				// The full deploy needs no guard: it updates the Deployment in
 				// the same run.
@@ -79,13 +79,13 @@ func init() {
 		},
 		{
 			use:   "api-authentication-config",
-			short: "Create the default ate-api-server authentication config",
-			run:   (*steps.Env).CreateAPIAuthenticationConfig,
+			short: "Create the default ate-api-server authentication config, rolling any running deployment",
+			run:   (*steps.Env).UpdateAPIAuthenticationConfig,
 		},
 		{
 			use:   "api-config",
-			short: "Create the ate-api-server config file, including its log level",
-			run:   (*steps.Env).CreateAPIConfig,
+			short: "Create the ate-api-server config file, including its log level, rolling any running deployment",
+			run:   (*steps.Env).UpdateAPIConfig,
 		},
 	} {
 		run := sub.run

cmd/ate-setup/internal/kube/resources.go

@@ -216,6 +216,9 @@ func (c *Client) ServiceAccountAnnotation(ctx context.Context, namespace, name,
 // An empty string means the endpoint is unavailable or has no issuer, which
 // callers treat as "fall back to the in-cluster default".
 func (c *Client) OIDCIssuer(ctx context.Context) string {
+	if c == nil || c.Typed == nil || c.Typed.Discovery() == nil || c.Typed.Discovery().RESTClient() == nil {
+		return ""
+	}
 	raw, err := c.Typed.Discovery().RESTClient().
 		Get().AbsPath("/.well-known/openid-configuration").
 		DoRaw(ctx)

cmd/ate-setup/internal/steps/apienv.go

@@ -457,12 +457,20 @@ func stampAPIServerEnvHash(objs []*unstructured.Unstructured, hash string) error
 	return fmt.Errorf("the manifest has no deployment/ate-api-server to stamp with %s", envHashAnnotation)
 }
 
-// apiServerEnvHash digests the apiserver's environment as it is in the cluster.
+// apiServerEnvHash digests the apiserver's environment and configuration sources as they are in the cluster.
 func (e *Env) apiServerEnvHash(ctx context.Context) (string, error) {
 	cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIEnvVars)
 	if err != nil {
 		return "", err
 	}
+	authn, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIAuthn)
+	if err != nil {
+		return "", err
+	}
+	apiConfig, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIConfig)
+	if err != nil {
+		return "", err
+	}
 	secret, err := e.Kube.GetSecret(ctx, e.Namespace(), SecretAPIEnvVars)
 	if err != nil {
 		return "", err
@@ -471,23 +479,36 @@ func (e *Env) apiServerEnvHash(ctx context.Context) (string, error) {
 	if cm != nil {
 		cmData = cm.Data
 	}
+	var authnData map[string]string
+	if authn != nil {
+		authnData = authn.Data
+	}
+	var apiConfigData map[string]string
+	if apiConfig != nil {
+		apiConfigData = apiConfig.Data
+	}
 	var secretData map[string][]byte
 	if secret != nil {
 		secretData = secret.Data
 	}
-	return envHash(cmData, secretData), nil
+	return envHash(cmData, authnData, apiConfigData, secretData), nil
 }
 
 // envHash digests the apiserver's environment sources. Only changes matter, so
 // the digest is an opaque value rather than a defined format; it does not
 // agree with the one the shell installer computed, so the first install after
 // the move to ate-setup rolls ate-api-server once.
-func envHash(configMap map[string]string, secret map[string][]byte) string {
+func envHash(configMap, authn, apiConfig map[string]string, secret map[string][]byte) string {
 	h := sha256.New()
-	fmt.Fprint(h, "configmap\n")
-	for _, k := range slices.Sorted(maps.Keys(configMap)) {
-		fmt.Fprintf(h, "%s=%s\n", k, configMap[k])
+	hashConfigMap := func(label string, m map[string]string) {
+		fmt.Fprintf(h, "%s\n", label)
+		for _, k := range slices.Sorted(maps.Keys(m)) {
+			fmt.Fprintf(h, "%s=%s\n", k, m[k])
+		}
 	}
+	hashConfigMap("configmap", configMap)
+	hashConfigMap("authn", authn)
+	hashConfigMap("apiconfig", apiConfig)
 	fmt.Fprint(h, "secret\n")
 	for _, k := range slices.Sorted(maps.Keys(secret)) {
 		fmt.Fprintf(h, "%s=%s\n", k, secret[k])

cmd/ate-setup/internal/steps/apienv_test.go

@@ -29,27 +29,39 @@ import (
 	"go.ate.dev/substrate/pkg/postgressetup"
 )
 
-// The digest exists to turn an envFrom change into a rollout, so what matters
-// is that it moves when a value does and holds still otherwise.
+// The digest exists to turn an environment or configuration change into a
+// rollout, so what matters is that it moves when a value does and holds still
+// otherwise.
 func TestEnvHash(t *testing.T) {
 	cm := map[string]string{"A": "1", "B": "2"}
+	authn := map[string]string{"authentication.yaml": "jwtProviders: []"}
+	apiConfig := map[string]string{"apiconfig.textproto": `logging { level: "info" }`}
 	secret := map[string][]byte{"DSN": []byte("postgresql://h/atepg")}
 
-	base := envHash(cm, secret)
-	if base != envHash(map[string]string{"B": "2", "A": "1"}, secret) {
+	base := envHash(cm, authn, apiConfig, secret)
+	if base != envHash(map[string]string{"B": "2", "A": "1"}, authn, apiConfig, secret) {
 		t.Error("envHash() depends on map iteration order")
 	}
-	if base == envHash(cm, map[string][]byte{"DSN": []byte("postgresql://other/atepg")}) {
+	if base == envHash(cm, authn, apiConfig, map[string][]byte{"DSN": []byte("postgresql://other/atepg")}) {
 		t.Error("envHash() did not change when the DSN did")
 	}
-	if base == envHash(map[string]string{"A": "1"}, secret) {
+	if base == envHash(map[string]string{"A": "1"}, authn, apiConfig, secret) {
 		t.Error("envHash() did not change when a ConfigMap key was removed")
 	}
-	// The two sources are hashed into the same stream, so they need a
-	// separator to stay distinguishable.
-	if envHash(map[string]string{"X": "1"}, nil) == envHash(nil, map[string][]byte{"X": []byte("1")}) {
+	if base == envHash(cm, map[string]string{"authentication.yaml": "jwtProviders: [- name: other]"}, apiConfig, secret) {
+		t.Error("envHash() did not change when authentication.yaml did")
+	}
+	if base == envHash(cm, authn, map[string]string{"apiconfig.textproto": `logging { level: "debug" }`}, secret) {
+		t.Error("envHash() did not change when apiconfig.textproto did")
+	}
+	// The sources are hashed into the same stream, so they need
+	// separators to stay distinguishable.
+	if envHash(map[string]string{"X": "1"}, nil, nil, nil) == envHash(nil, nil, nil, map[string][]byte{"X": []byte("1")}) {
 		t.Error("envHash() does not distinguish the ConfigMap from the Secret")
 	}
+	if envHash(nil, map[string]string{"X": "1"}, nil, nil) == envHash(nil, nil, map[string]string{"X": "1"}, nil) {
+		t.Error("envHash() does not distinguish authn from apiConfig")
+	}
 }
 
 func TestCreateAPIServerEnvVarsPostgresIdentities(t *testing.T) {
@@ -532,7 +544,7 @@ func TestAnnotateAPIServerEnvHash(t *testing.T) {
 		if err != nil {
 			t.Fatalf("GetDeployment() error = %v", err)
 		}
-		want := envHash(nil, secret.Data)
+		want := envHash(nil, nil, nil, secret.Data)
 		if got := dep.Spec.Template.Annotations[envHashAnnotation]; got != want {
 			t.Errorf("%s = %q, want %q", envHashAnnotation, got, want)
 		}

cmd/ate-setup/internal/steps/create.go

@@ -158,6 +158,16 @@ func (e *Env) CreateAPIConfig(ctx context.Context) error {
 	})
 }
 
+// UpdateAPIConfig is CreateAPIConfig for an install that is not being
+// redeployed: it also stamps the new digest on the running Deployment,
+// which rolls ate-api-server onto the new config.
+func (e *Env) UpdateAPIConfig(ctx context.Context) error {
+	if err := e.CreateAPIConfig(ctx); err != nil {
+		return err
+	}
+	return e.annotateAPIServerEnvHash(ctx)
+}
+
 // buildAPIConfig renders apiconfig.textproto.
 func buildAPIConfig(logLevel string) (string, error) {
 	b, err := prototext.MarshalOptions{Multiline: true}.Marshal(&ateconfigpb.APIConfig{
@@ -189,6 +199,16 @@ func (e *Env) CreateAPIAuthenticationConfig(ctx context.Context) error {
 	})
 }
 
+// UpdateAPIAuthenticationConfig is CreateAPIAuthenticationConfig for an install
+// that is not being redeployed: it also stamps the new digest on the running
+// Deployment, which rolls ate-api-server onto the new config.
+func (e *Env) UpdateAPIAuthenticationConfig(ctx context.Context) error {
+	if err := e.CreateAPIAuthenticationConfig(ctx); err != nil {
+		return err
+	}
+	return e.annotateAPIServerEnvHash(ctx)
+}
+
 // jwtIssuer determines the service account token issuer to trust.
 //
 // ate-api-server accepts a token only if its iss claim equals this string

cmd/ate-setup/internal/steps/prereqs.go

@@ -19,6 +19,8 @@ import (
 	"fmt"
 
 	"go.ate.dev/substrate/cmd/ate-setup/internal/log"
+	"go.ate.dev/substrate/internal/apiconfig"
+	"go.ate.dev/substrate/internal/ateapiauth"
 )
 
 // trustBundleNames are the ClusterTrustBundles the podcertificate controller
@@ -58,28 +60,53 @@ func (e *Env) EnsureAPIServerPrerequisites(ctx context.Context) error {
 		return err
 	}
 
-	exists, err := e.Kube.ConfigMapExists(ctx, e.Namespace(), ConfigMapAPIAuthn)
+	if err := e.ensureAPIAuthenticationConfig(ctx); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ensureAPIAuthenticationConfig writes the ate-api-server authentication config
+// when it is missing, when an expected JWT issuer was asked for explicitly, or
+// when the stored configuration fails schema validation (e.g. after an upgrade
+// changes the config schema or a rollback leaves unsupported fields).
+// Otherwise an existing valid config, possibly configured with additional JWT
+// providers or a custom issuer, is kept.
+func (e *Env) ensureAPIAuthenticationConfig(ctx context.Context) error {
+	if e.Cfg.ExpectedJWTIssuer != "" {
+		return e.CreateAPIAuthenticationConfig(ctx)
+	}
+	cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIAuthn)
 	if err != nil {
 		return err
 	}
-	if !exists {
+	if cm == nil {
+		return e.CreateAPIAuthenticationConfig(ctx)
+	}
+	if _, err := ateapiauth.ParseAuthenticationConfig([]byte(cm.Data["authentication.yaml"])); err != nil {
+		log.Warnf("Rewriting %s/%s: existing authentication.yaml is invalid (%v)", e.Namespace(), ConfigMapAPIAuthn, err)
 		return e.CreateAPIAuthenticationConfig(ctx)
 	}
 	return nil
 }
 
-// ensureAPIConfig writes the ate-api-server config file when it is missing or
-// a log level was asked for explicitly. Otherwise an existing config, possibly
+// ensureAPIConfig writes the ate-api-server config file when it is missing,
+// when a log level was asked for explicitly, or when the stored configuration
+// fails schema or log-level validation. Otherwise an existing config, possibly
 // edited by hand, is kept.
 func (e *Env) ensureAPIConfig(ctx context.Context) error {
 	if e.Cfg.APILogLevel != "" {
 		return e.CreateAPIConfig(ctx)
 	}
-	exists, err := e.Kube.ConfigMapExists(ctx, e.Namespace(), ConfigMapAPIConfig)
+	cm, err := e.Kube.GetConfigMap(ctx, e.Namespace(), ConfigMapAPIConfig)
 	if err != nil {
 		return err
 	}
-	if !exists {
+	if cm == nil {
+		return e.CreateAPIConfig(ctx)
+	}
+	if _, err := apiconfig.ParseTextProto([]byte(cm.Data["apiconfig.textproto"])); err != nil {
+		log.Warnf("Rewriting %s/%s: existing apiconfig.textproto is invalid (%v)", e.Namespace(), ConfigMapAPIConfig, err)
 		return e.CreateAPIConfig(ctx)
 	}
 	return nil