fix(agent): reuse managed certificates across restarts

certificate-agent-config.yaml

@@ -32,6 +32,15 @@ certificates:
       renew-before-expiry: "1d" # When to start checking for renewal before expiration
       status-check-interval: "6h" # How often to check certificate status and renewal needs
 
+      # State defaults to <certificate.path>.infisical-state.json and contains no private key.
+      # Keep it across restarts so the Agent can resume pending requests and repair missing outputs.
+      # Pre-upgrade recovery requires read access to the certificate profile.
+      # An override and its .lock file must also be allowed in systemd write paths.
+      # state-path: "/etc/ssl/infisical/certificate-state.json"
+
+      # An unknown request outcome after a crash stops automatic issuance to avoid duplicate orders.
+      # Check Infisical before restoring or removing the state file.
+
     # Post-hooks for automation
     post-hooks:
       on-issuance:

e2e/agent/agent_helpers.go

@@ -603,6 +603,7 @@ type agentCertificateFileOutput struct {
 type agentFileOutputEntry struct {
 	Path       string `yaml:"path"`
 	Permission string `yaml:"permission,omitempty"`
+	OmitRoot   *bool  `yaml:"omit-root,omitempty"`
 }
 
 type agentCertificatePostHooks struct {
@@ -621,6 +622,11 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin
 
 	var certs []agentCertificateConfig
 	for _, cert := range opts.Certificates {
+		var omitRoot *bool
+		if cert.IncludeRootInChain {
+			omit := false
+			omitRoot = &omit
+		}
 		c := agentCertificateConfig{
 			ProjectSlug:     cert.ProjectSlug,
 			ApplicationName: cert.ApplicationName,
@@ -645,7 +651,7 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin
 			FileOutput: agentCertificateFileOutput{
 				Certificate: agentFileOutputEntry{Path: cert.CertPath, Permission: cert.CertPermission},
 				PrivateKey:  agentFileOutputEntry{Path: cert.KeyPath, Permission: cert.KeyPermission},
-				Chain:       agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission},
+				Chain:       agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission, OmitRoot: omitRoot},
 			},
 		}
 
@@ -724,6 +730,7 @@ type CertificateConfigEntry struct {
 	CertPermission      string
 	KeyPermission       string
 	ChainPermission     string
+	IncludeRootInChain  bool
 	PostHookOnFailure   string
 	CertificateID       string
 	ReplaceOnRenewals   bool

e2e/agent/certificate_test.go

@@ -3,6 +3,7 @@ package agent_test
 import (
 	"context"
 	"crypto/x509"
+	"encoding/json"
 	"encoding/pem"
 	"fmt"
 	"log/slog"
@@ -122,6 +123,7 @@ func certAgent_BasicCertificateIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -162,6 +164,75 @@ func certAgent_BasicCertificateIssuance(t *testing.T) {
 	agentHelpers.VerifyCertificateCommonName(t, certPath, "test.example.com")
 }
 
+func certAgent_DefaultRootOmissionSurvivesRestart(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	helper := setupCertAgentTest(t, ctx)
+	certDir := filepath.Join(helper.TempDir, "certs")
+	require.NoError(t, os.MkdirAll(certDir, 0755))
+	certPath, keyPath, chainPath := agentHelpers.CertFilePaths(certDir)
+	clientIDPath, clientSecretPath := helper.WriteCredentialFiles()
+	configPath := helper.GenerateAgentConfig(agentHelpers.AgentConfigOptions{
+		ClientIDPath: clientIDPath, ClientSecretPath: clientSecretPath,
+		Certificates: []agentHelpers.CertificateConfigEntry{{
+			ApplicationName: helper.ApplicationName, ProfileSlug: helper.ProfileSlug,
+			CommonName: "default-chain.example.com", TTL: "1h", RenewBeforeExpiry: "10m", StatusCheckInterval: "5s",
+			CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath,
+		}},
+	})
+	config, err := os.ReadFile(configPath)
+	require.NoError(t, err)
+	require.NotContains(t, string(config), "omit-root:")
+	runAgent := func() *helpers.Command {
+		cmd := &helpers.Command{Test: t, Args: []string{"cert-manager", "agent", "--config", configPath, "--verbose"}, Env: map[string]string{}}
+		cmd.Start(ctx)
+		t.Cleanup(func() {
+			if t.Failed() {
+				t.Logf("Agent stderr:\n%s", cmd.Stderr())
+			}
+			cmd.Stop()
+		})
+		return cmd
+	}
+	first := runAgent()
+	result := helpers.WaitForStderr(t, helpers.WaitForStderrOptions{
+		EnsureCmdRunning: first, ExpectedString: "certificate issued successfully", Timeout: 120 * time.Second, Interval: 2 * time.Second,
+	})
+	require.Equal(t, helpers.WaitSuccess, result, "initial issuance failed: %s", first.Stderr())
+	agentHelpers.VerifyCertificateFile(t, certPath)
+	agentHelpers.VerifyPrivateKeyFile(t, keyPath)
+	chain, err := os.ReadFile(chainPath)
+	require.NoError(t, err)
+	require.Empty(t, chain)
+	initialSerial := readCertificateSerial(t, certPath)
+	statePath := certPath + ".infisical-state.json"
+	stateWrittenAt := fileModTime(t, statePath)
+	first.Stop()
+	second := runAgent()
+	require.Eventually(t, func() bool {
+		info, err := os.Stat(statePath)
+		if err != nil || !info.ModTime().After(stateWrittenAt) {
+			return false
+		}
+		data, err := os.ReadFile(statePath)
+		if err != nil {
+			return false
+		}
+		var state struct {
+			State struct {
+				Status string `json:"status"`
+			} `json:"state"`
+		}
+		return json.Unmarshal(data, &state) == nil && state.State.Status == "active"
+	}, 120*time.Second, 2*time.Second, "restart recovery failed: %s", second.Stderr())
+	require.True(t, second.IsRunning())
+	require.Equal(t, initialSerial, readCertificateSerial(t, certPath))
+	require.NotContains(t, second.Stderr(), "issuing certificate")
+	chain, err = os.ReadFile(chainPath)
+	require.NoError(t, err)
+	require.Empty(t, chain)
+}
+
 func certAgent_CertificateRenewal(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
@@ -349,6 +420,7 @@ func certAgent_MultipleCertificates(t *testing.T) {
 				CertPath:            certPath1,
 				KeyPath:             keyPath1,
 				ChainPath:           chainPath1,
+				IncludeRootInChain:  true,
 			},
 			{
 				ApplicationName:     helper.ApplicationName,
@@ -360,6 +432,7 @@ func certAgent_MultipleCertificates(t *testing.T) {
 				CertPath:            certPath2,
 				KeyPath:             keyPath2,
 				ChainPath:           chainPath2,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -435,6 +508,7 @@ func certAgent_FilePermissions(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				CertPermission:      "0644",
 				KeyPermission:       "0640",
 				ChainPermission:     "0644",
@@ -502,6 +576,7 @@ func certAgent_AltNames(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				AltNames:            expectedAltNames,
 			},
 		},
@@ -570,6 +645,7 @@ func certAgent_CSRBasedIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				CSR:                 csrPEM,
 			},
 		},
@@ -647,6 +723,7 @@ func certAgent_CSRPathBasedIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				CSRPath:             csrFilePath,
 			},
 		},
@@ -714,6 +791,7 @@ func certAgent_AcmeCA_CertificateIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -757,6 +835,38 @@ func certAgent_AcmeCA_CertificateIssuance(t *testing.T) {
 	if _, err := os.Stat(chainPath); err == nil {
 		agentHelpers.VerifyChainFile(t, chainPath)
 	}
+	initialSerial := readCertificateSerial(t, certPath)
+	statePath := certPath + ".infisical-state.json"
+	stateWrittenAt := fileModTime(t, statePath)
+	cmd.Stop()
+	restarted := &helpers.Command{Test: t, Args: []string{"cert-manager", "agent", "--config", configPath, "--verbose"}, Env: map[string]string{}}
+	restarted.Start(ctx)
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("Restarted Agent stderr:\n%s", restarted.Stderr())
+		}
+		restarted.Stop()
+	})
+	require.Eventually(t, func() bool {
+		info, err := os.Stat(statePath)
+		if err != nil || !info.ModTime().After(stateWrittenAt) {
+			return false
+		}
+		data, err := os.ReadFile(statePath)
+		if err != nil {
+			return false
+		}
+		var state struct {
+			State struct {
+				Status string `json:"status"`
+			} `json:"state"`
+		}
+		return json.Unmarshal(data, &state) == nil && state.State.Status == "active"
+	}, 120*time.Second, 2*time.Second, "ACME restart recovery failed: %s", restarted.Stderr())
+	require.True(t, restarted.IsRunning())
+	require.Equal(t, initialSerial, readCertificateSerial(t, certPath))
+	require.NotContains(t, restarted.Stderr(), "issuing certificate")
+	require.NotContains(t, restarted.Stderr(), "renewing certificate")
 }
 
 func setupAcmeCertAgentTest(t *testing.T, ctx context.Context, certCount ...int) (*agentHelpers.CertAgentTestHelper, string) { //nolint:unparam
@@ -1044,6 +1154,7 @@ func certAgent_AcmeCA_MultipleCertificates(t *testing.T) {
 				CertPath:            certPath1,
 				KeyPath:             keyPath1,
 				ChainPath:           chainPath1,
+				IncludeRootInChain:  true,
 			},
 			{
 				ApplicationName:     helper.ApplicationName,
@@ -1055,6 +1166,7 @@ func certAgent_AcmeCA_MultipleCertificates(t *testing.T) {
 				CertPath:            certPath2,
 				KeyPath:             keyPath2,
 				ChainPath:           chainPath2,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -1208,6 +1320,7 @@ func certAgent_AcmeCA_CSRBasedIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				CSR:                 csrPEM,
 			},
 		},
@@ -1360,6 +1473,7 @@ func certAgent_CertificateWithFullAttributes(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				KeyAlgorithm:        "RSA_2048",
 				KeyUsages:           []string{"digital_signature", "key_encipherment"},
 				ExtendedKeyUsages:   []string{"server_auth"},
@@ -1873,6 +1987,7 @@ func certAgent_SignatureAlgorithm(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 				KeyAlgorithm:        "RSA_2048",
 				SignatureAlgorithm:  "RSA-SHA256",
 			},
@@ -1944,6 +2059,7 @@ func certAgent_V1LegacyIssuance(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -2011,6 +2127,7 @@ func certAgent_V1ValidationRejectsAppOnly(t *testing.T) {
 				CertPath:            certPath,
 				KeyPath:             keyPath,
 				ChainPath:           chainPath,
+				IncludeRootInChain:  true,
 			},
 		},
 	})
@@ -2537,6 +2654,7 @@ func normalizeSerial(serial string) string {
 
 func TestCertAgent_InternalCA(t *testing.T) {
 	t.Run("BasicCertificateIssuance", certAgent_BasicCertificateIssuance)
+	t.Run("DefaultRootOmissionSurvivesRestart", certAgent_DefaultRootOmissionSurvivesRestart)
 	t.Run("CertificateRenewal", certAgent_CertificateRenewal)
 	t.Run("PostHookExecution", certAgent_PostHookExecution)
 	t.Run("MultipleCertificates", certAgent_MultipleCertificates)

packages/api/api.go

@@ -1376,6 +1376,22 @@ func CallRetrieveCertificate(httpClient *resty.Client, certificateId string) (*R
 	return &resBody, nil
 }
 
+func CallListCertificateProfileCertificates(httpClient *resty.Client, profileID, search string, offset int) (*ListCertificateProfileCertificatesResponse, error) {
+	var resBody ListCertificateProfileCertificatesResponse
+	response, err := httpClient.R().
+		SetResult(&resBody).
+		SetHeader("User-Agent", USER_AGENT).
+		SetQueryParams(map[string]string{"search": search, "offset": strconv.Itoa(offset), "limit": "100"}).
+		Get(fmt.Sprintf("%v/v1/cert-manager/certificate-profiles/%s/certificates", config.INFISICAL_URL, profileID))
+	if err != nil {
+		return nil, NewGenericRequestError("CallListCertificateProfileCertificates", err)
+	}
+	if response.IsError() {
+		return nil, NewAPIErrorWithResponse("CallListCertificateProfileCertificates", response, nil)
+	}
+	return &resBody, nil
+}
+
 func CallGetCertificateBundle(httpClient *resty.Client, certificateId string) (*CertificateBundleResponse, error) {
 	var resBody CertificateBundleResponse
 	response, err := httpClient.

packages/api/model.go

@@ -1207,6 +1207,13 @@ type CertificateBundleResponse struct {
 	SerialNumber     string `json:"serialNumber"`
 }
 
+type ListCertificateProfileCertificatesResponse struct {
+	Certificates []struct {
+		ID           string `json:"id"`
+		SerialNumber string `json:"serialNumber"`
+	} `json:"certificates"`
+}
+
 type RenewCertificateRequest struct {
 	RemoveRootsFromChain bool `json:"removeRootsFromChain,omitempty"`
 }