From bf6f8ccb981962fbe0118176c2bc1fa4c89e2e35 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:40:31 +0000 Subject: [PATCH 1/8] fix(agent): resume managed certificates across restarts Co-Authored-By: ashwin --- certificate-agent-config.yaml | 9 + packages/api/api.go | 16 + packages/api/model.go | 7 + packages/cmd/agent.go | 471 ++++++-------------- packages/cmd/agent_cert_state.go | 591 ++++++++++++++++++++++++++ packages/cmd/agent_cert_state_test.go | 498 ++++++++++++++++++++++ 6 files changed, 1253 insertions(+), 339 deletions(-) create mode 100644 packages/cmd/agent_cert_state.go create mode 100644 packages/cmd/agent_cert_state_test.go diff --git a/certificate-agent-config.yaml b/certificate-agent-config.yaml index 360756b07..d903fce9e 100644 --- a/certificate-agent-config.yaml +++ b/certificate-agent-config.yaml @@ -32,6 +32,15 @@ certificates: renew-before-expiry: "1d" # When to start checking for renewal before expiration status-check-interval: "6h" # How often to check certificate status and renewal needs + # State defaults to .infisical-state.json and contains no private key. + # Keep it across restarts so the Agent can resume pending requests and repair missing outputs. + # Pre-upgrade recovery requires read access to the certificate profile. + # An override and its .lock file must also be allowed in systemd write paths. + # state-path: "/etc/ssl/infisical/certificate-state.json" + + # An unknown request outcome after a crash stops automatic issuance to avoid duplicate orders. + # Check Infisical before restoring or removing the state file. + # Post-hooks for automation post-hooks: on-issuance: diff --git a/packages/api/api.go b/packages/api/api.go index 40ed2b755..3cba757ec 100644 --- a/packages/api/api.go +++ b/packages/api/api.go @@ -1376,6 +1376,22 @@ func CallRetrieveCertificate(httpClient *resty.Client, certificateId string) (*R return &resBody, nil } +func CallListCertificateProfileCertificates(httpClient *resty.Client, profileID, search string, offset int) (*ListCertificateProfileCertificatesResponse, error) { + var resBody ListCertificateProfileCertificatesResponse + response, err := httpClient.R(). + SetResult(&resBody). + SetHeader("User-Agent", USER_AGENT). + SetQueryParams(map[string]string{"search": search, "offset": strconv.Itoa(offset), "limit": "100"}). + Get(fmt.Sprintf("%v/v1/cert-manager/certificate-profiles/%s/certificates", config.INFISICAL_URL, profileID)) + if err != nil { + return nil, NewGenericRequestError("CallListCertificateProfileCertificates", err) + } + if response.IsError() { + return nil, NewAPIErrorWithResponse("CallListCertificateProfileCertificates", response, nil) + } + return &resBody, nil +} + func CallGetCertificateBundle(httpClient *resty.Client, certificateId string) (*CertificateBundleResponse, error) { var resBody CertificateBundleResponse response, err := httpClient. diff --git a/packages/api/model.go b/packages/api/model.go index 73521a6f7..c4c654bfe 100644 --- a/packages/api/model.go +++ b/packages/api/model.go @@ -1207,6 +1207,13 @@ type CertificateBundleResponse struct { SerialNumber string `json:"serialNumber"` } +type ListCertificateProfileCertificatesResponse struct { + Certificates []struct { + ID string `json:"id"` + SerialNumber string `json:"serialNumber"` + } `json:"certificates"` +} + type RenewCertificateRequest struct { RemoveRootsFromChain bool `json:"removeRootsFromChain,omitempty"` } diff --git a/packages/cmd/agent.go b/packages/cmd/agent.go index 95e63864a..d9a5b4d6f 100644 --- a/packages/cmd/agent.go +++ b/packages/cmd/agent.go @@ -123,6 +123,7 @@ type CertificateState struct { CertificateID string `json:"certificate_id"` CertificateRequestID string `json:"certificate_request_id,omitempty"` SerialNumber string `json:"serial_number"` + CertificateSHA256 string `json:"certificate_sha256,omitempty"` CommonName string `json:"common_name"` IssuedAt time.Time `json:"issued_at"` ExpiresAt time.Time `json:"expires_at"` @@ -206,6 +207,7 @@ type Template struct { } type CertificateLifecycleConfig struct { + StatePath string `yaml:"state-path,omitempty"` RenewBeforeExpiry string `yaml:"renew-before-expiry"` StatusCheckInterval string `yaml:"status-check-interval"` FailureRetryInterval string `yaml:"failure-retry-interval,omitempty"` @@ -1125,6 +1127,8 @@ type AgentManager struct { templates []TemplateWithID certificates []CertificateWithID certificateStates map[int]*CertificateState + certificateContext context.Context + certificatePollers sync.WaitGroup dynamicSecretLeases *DynamicSecretLeaseManager cacheManager *CacheManager authConfigBytes []byte @@ -2147,6 +2151,9 @@ func validateCertificateSourceConfig(version string, certificates *[]AgentCertif if len(*certificates) == 0 { return nil } + if err := validateCertificateStatePaths(*certificates); err != nil { + return err + } switch version { case AgentConfigVersionV1, AgentConfigVersionV2: @@ -2283,7 +2290,17 @@ func buildCertificateAttributes(certificate *AgentCertificateConfig) *api.Certif } func (tm *AgentManager) createAuthenticatedClient() (*resty.Client, error) { - return newAuthenticatedClient(tm.GetToken()) + client, err := newAuthenticatedClient(tm.GetToken()) + if err != nil { + return nil, err + } + if tm.certificateContext != nil { + client.OnBeforeRequest(func(_ *resty.Client, request *resty.Request) error { + request.SetContext(tm.certificateContext) + return nil + }) + } + return client, nil } func newAuthenticatedClient(token string) (*resty.Client, error) { @@ -2564,8 +2581,19 @@ func (tm *AgentManager) IssueCertificate(certificateId int, certificate *AgentCe return err } + previous := *state + state.Status = "requesting_issuance" + if err := preflightCertificateOutputs(certificate); err != nil { + *state = previous + return err + } + state.CertificateRequestID = "" + if err := persistCertificateState(certificate, state); err != nil { + return err + } response, err := api.CallIssueCertificate(httpClient, request) if err != nil { + err = restoreRejectedCertificateRequest(certificate, state, previous, err) state.Status = "failed" state.LastError = err.Error() state.RetryCount++ @@ -2575,66 +2603,20 @@ func (tm *AgentManager) IssueCertificate(certificateId int, certificate *AgentCe return fmt.Errorf("failed to issue certificate: %v", err) } - setCommonName := func() { - if certificate.Attributes != nil { - state.CommonName = certificate.Attributes.CommonName - } - - if state.CommonName == "" && request.Attributes != nil { - state.CommonName = request.Attributes.CommonName - } - } - if response.Certificate != nil { - state.CertificateID = response.Certificate.CertificateID - state.SerialNumber = response.Certificate.SerialNumber - setCommonName() - state.IssuedAt = time.Now() - state.Status = "active" - state.LastError = "" - state.RetryCount = 0 - } else { - state.CertificateRequestID = response.CertificateRequestID - setCommonName() - state.Status = "pending_issuance" - state.LastError = "" - state.RetryCount = 0 - - go tm.PollCertificateRequest(certificateId, certificate) - return nil - } - - if ttlDuration, err := parseDurationWithDays(tm.getCertificateTTL(certificate)); err == nil { - state.ExpiresAt = state.IssuedAt.Add(ttlDuration) - } else { - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Warn().Str("Certificate", displayName).Msg("unable to parse TTL") - state.ExpiresAt = state.IssuedAt.Add(24 * time.Hour) + return tm.completeManagedCertificate(certificateId, certificate, response, false) } - - if renewBeforeDuration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { - state.NextRenewalCheck = state.ExpiresAt.Add(-renewBeforeDuration) - } else { - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Warn().Str("Certificate", displayName).Msg("unable to parse lifecycle.renew-before-expiry") - state.NextRenewalCheck = state.ExpiresAt.Add(-24 * time.Hour) + if response.CertificateRequestID == "" { + return fmt.Errorf("issuance response has neither a certificate nor a request ID") } - - err = tm.WriteCertificateFiles(certificate, response) - if err != nil { - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Error().Str("Certificate", displayName).Msgf("failed to write certificate files: %v", err) - state.Status = "failed" - state.LastError = fmt.Sprintf("failed to write files: %v", err) + state.CertificateRequestID = response.CertificateRequestID + state.Status = "pending_issuance" + state.LastError = "" + state.RetryCount = 0 + if err := persistCertificateState(certificate, state); err != nil { return err } - - log.Info().Str("Certificate", displayName).Str("serial", response.Certificate.SerialNumber).Msg("certificate issued successfully") - - if certificate.PostHooks.OnIssuance.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnIssuance.Command, certificate.PostHooks.OnIssuance.Timeout, "issuance", certificateId, certificate) - } - + tm.startCertificatePolling(certificateId, certificate) return nil } @@ -2643,6 +2625,9 @@ func (tm *AgentManager) PollCertificateRequest(certificateId int, certificate *A pollingInterval := EXTERNAL_CA_INITIAL_POLLING_INTERVAL for { + if tm.certificateContext != nil && tm.certificateContext.Err() != nil { + return + } if err := tm.checkCertificateRequestStatus(certificateId, certificate); err != nil { log.Error().Str("Certificate", displayName).Msgf("failed to check certificate request status: %v", err) } @@ -2664,7 +2649,9 @@ func (tm *AgentManager) PollCertificateRequest(certificateId int, certificate *A } log.Info().Str("Certificate", displayName).Msgf("waiting %s before next polling attempt", pollingInterval) - time.Sleep(pollingInterval) + if !tm.waitForCertificatePoll(pollingInterval) { + return + } pollingInterval *= 2 if pollingInterval > EXTERNAL_CA_MAX_POLLING_INTERVAL { @@ -2712,28 +2699,9 @@ func (tm *AgentManager) checkCertificateRequestStatus(certificateId int, certifi return nil } - state.CertificateID = *response.CertificateID - state.SerialNumber = *response.SerialNumber - state.IssuedAt = time.Now() - state.Status = "active" - state.LastError = "" - state.RetryCount = 0 - - if ttlDuration, err := parseDurationWithDays(tm.getCertificateTTL(certificate)); err == nil { - state.ExpiresAt = state.IssuedAt.Add(ttlDuration) - } else { - state.ExpiresAt = state.IssuedAt.Add(24 * time.Hour) - } - - if renewBeforeDuration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { - state.NextRenewalCheck = state.ExpiresAt.Add(-renewBeforeDuration) - } else { - state.NextRenewalCheck = state.ExpiresAt.Add(-24 * time.Hour) - } - certData := api.CertificateData{ Certificate: *response.Certificate, - CertificateID: state.CertificateRequestID, + CertificateID: *response.CertificateID, SerialNumber: *response.SerialNumber, } if response.IssuingCaCertificate != nil { @@ -2750,7 +2718,7 @@ func (tm *AgentManager) checkCertificateRequestStatus(certificateId int, certifi Certificate: &certData, } - if err := tm.WriteCertificateFiles(certificate, certResponse); err != nil { + if err := tm.completeManagedCertificate(certificateId, certificate, certResponse, state.Status == "renewing"); err != nil { displayName := tm.getCertificateDisplayName(certificateId, certificate) log.Error().Str("Certificate", displayName).Msgf("failed to write certificate files: %v", err) state.Status = "failed" @@ -2761,16 +2729,13 @@ func (tm *AgentManager) checkCertificateRequestStatus(certificateId int, certifi displayName := tm.getCertificateDisplayName(certificateId, certificate) log.Info().Str("Certificate", displayName).Str("serial", *response.SerialNumber).Msg("certificate issued successfully") - if certificate.PostHooks.OnIssuance.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnIssuance.Command, certificate.PostHooks.OnIssuance.Timeout, "issuance", certificateId, certificate) - } - case "failed": errorMsg := "unknown error" if response.ErrorMessage != nil { errorMsg = *response.ErrorMessage } tm.handleFailedCertificateRequest(certificateId, errorMsg) + return persistCertificateState(certificate, state) case "pending": // Still waiting, no action needed @@ -2954,6 +2919,16 @@ func (tm *AgentManager) MonitorCertificates(ctx context.Context) { } log.Info().Msg("starting certificate monitoring") + ctx, cancel := context.WithCancel(ctx) + tm.certificateContext = ctx + var unlocks []func() + defer func() { + cancel() + tm.certificatePollers.Wait() + for _, unlock := range unlocks { + unlock() + } + }() var monitoringInterval time.Duration = DEFAULT_MONITORING_INTERVAL for _, cert := range tm.certificates { @@ -2976,9 +2951,25 @@ func (tm *AgentManager) MonitorCertificates(ctx context.Context) { } continue } - if err := tm.IssueCertificate(cert.ID, &cert.Certificate); err != nil { + unlock, err := lockCertificateState(&cert.Certificate) + if err != nil { + tm.mutex.Lock() + tm.certificateStates[cert.ID].Status = "blocked" + tm.mutex.Unlock() + log.Error().Str("Certificate", tm.getCertificateDisplayName(cert.ID, &cert.Certificate)).Msgf("certificate state unavailable: %v", err) + continue + } + unlocks = append(unlocks, unlock) + if err := tm.initializeManagedCertificate(cert.ID, &cert.Certificate); err != nil { displayName := tm.getCertificateDisplayName(cert.ID, &cert.Certificate) - log.Error().Str("Certificate", displayName).Msgf("initial certificate issuance failed: %v", err) + log.Error().Str("Certificate", displayName).Msgf("initial certificate recovery failed: %v", err) + tm.mutex.Lock() + state := tm.certificateStates[cert.ID] + state.Status = "failed" + state.LastError = err.Error() + state.LastRetry = time.Now() + state.RetryCount++ + tm.mutex.Unlock() } } @@ -3059,6 +3050,30 @@ func (tm *AgentManager) CheckCertificateRenewals() { continue } + if state.Status == "failed" { + if state.RetryCount >= effectiveMaxFailureRetries(&cert.Certificate) { + if now.Sub(state.LastRetry) < failureRetryCooldownFor(&cert.Certificate) { + continue + } + state.RetryCount = 0 + } + if now.Sub(state.LastRetry) < failureRetryIntervalFor(&cert.Certificate) { + continue + } + retryCount := state.RetryCount + tm.mutex.Unlock() + err := tm.initializeManagedCertificate(cert.ID, &cert.Certificate) + tm.mutex.Lock() + state = tm.certificateStates[cert.ID] + if err != nil { + state.Status = "failed" + state.LastError = err.Error() + state.LastRetry = now + state.RetryCount = retryCount + 1 + log.Error().Str("Certificate", tm.getCertificateDisplayName(cert.ID, &cert.Certificate)).Msgf("certificate recovery retry failed: %v", err) + } + continue + } if cert.Certificate.CSR != "" || cert.Certificate.CSRPath != "" { continue } @@ -3074,9 +3089,26 @@ func (tm *AgentManager) CheckCertificateRenewals() { tm.mutex.Unlock() if err := tm.CheckCertificateStatus(cert.ID, state.CertificateID); err != nil { log.Error().Str("Certificate", displayName).Msgf("failed to check status: %v", err) + tm.mutex.Lock() + state.NextRenewalCheck = now.Add(statusCheckIntervalFor(&cert.Certificate)) + continue } tm.mutex.Lock() } + state.NextRenewalCheck = now.Add(statusCheckIntervalFor(&cert.Certificate)) + if state.Status == "expired" { + tm.mutex.Unlock() + err := tm.initializeManagedCertificate(cert.ID, &cert.Certificate) + tm.mutex.Lock() + if err != nil { + state.Status = "failed" + state.LastError = err.Error() + state.LastRetry = now + state.RetryCount++ + log.Error().Str("Certificate", displayName).Msgf("failed to replace expired certificate: %v", err) + } + continue + } if tm.ShouldRenewCertificate(cert.ID) { log.Info().Str("Certificate", displayName).Msg("renewing certificate") if err := tm.RenewCertificate(cert.ID, &cert.Certificate); err != nil { @@ -3214,8 +3246,19 @@ func (tm *AgentManager) RenewCertificate(certificateId int, certificate *AgentCe request := api.RenewCertificateRequest{ RemoveRootsFromChain: removeRoots, } + previous := *state + state.Status = "requesting_renewal" + if err := preflightCertificateOutputs(certificate); err != nil { + *state = previous + return err + } + state.CertificateRequestID = "" + if err := persistCertificateState(certificate, state); err != nil { + return err + } response, err := api.CallRenewCertificate(httpClient, state.CertificateID, request) if err != nil { + err = restoreRejectedCertificateRequest(certificate, state, previous, err) return fmt.Errorf("failed to renew certificate: %v", err) } @@ -3228,7 +3271,10 @@ func (tm *AgentManager) RenewCertificate(certificateId int, certificate *AgentCe state.LastError = "" state.RetryCount = 0 - go tm.PollCertificateRequestForRenewal(certificateId, certificate, response.CertificateRequestID) + if err := persistCertificateState(certificate, state); err != nil { + return err + } + tm.startCertificatePolling(certificateId, certificate) return nil } @@ -3237,23 +3283,6 @@ func (tm *AgentManager) RenewCertificate(certificateId int, certificate *AgentCe } func (tm *AgentManager) handleImmediateRenewalResponse(certificateId int, certificate *AgentCertificateConfig, response *api.RenewCertificateResponse) error { - state := tm.certificateStates[certificateId] - - state.CertificateID = response.CertificateID - state.SerialNumber = response.SerialNumber - state.IssuedAt = time.Now() - state.Status = "active" - state.LastError = "" - state.RetryCount = 0 - - if ttlDuration, err := parseDurationWithDays(tm.getCertificateTTL(certificate)); err == nil { - state.ExpiresAt = state.IssuedAt.Add(ttlDuration) - } - - if renewBeforeDuration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { - state.NextRenewalCheck = state.ExpiresAt.Add(-renewBeforeDuration) - } - certResponse := &api.CertificateResponse{ Certificate: &api.CertificateData{ Certificate: response.Certificate, @@ -3265,243 +3294,7 @@ func (tm *AgentManager) handleImmediateRenewalResponse(certificateId int, certif }, } - err := tm.WriteCertificateFiles(certificate, certResponse) - if err != nil { - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Error().Str("Certificate", displayName).Msgf("failed to write renewed certificate files: %v", err) - state.Status = "failed" - state.LastError = fmt.Sprintf("failed to write files: %v", err) - return err - } - - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Info().Str("Certificate", displayName).Str("serial", response.SerialNumber).Msg("certificate renewed successfully") - - if certificate.PostHooks.OnRenewal.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnRenewal.Command, certificate.PostHooks.OnRenewal.Timeout, "renewal", certificateId, certificate) - } - - return nil -} - -func (tm *AgentManager) PollCertificateRequestForRenewal(certificateId int, certificate *AgentCertificateConfig, requestID string) { - pollingInterval := EXTERNAL_CA_INITIAL_POLLING_INTERVAL - displayName := tm.getCertificateDisplayName(certificateId, certificate) - - for { - status, certResponse, err := tm.checkCertificateRequestStatusByID(requestID) - if err != nil { - log.Error().Str("Certificate", displayName).Msgf("failed to check renewal status: %v", err) - - func() { - tm.mutex.Lock() - defer tm.mutex.Unlock() - state := tm.certificateStates[certificateId] - state.LastError = fmt.Sprintf("polling error: %v", err) - state.RetryCount++ - }() - - log.Info().Str("Certificate", displayName).Msgf("waiting %s before next renewal polling attempt", pollingInterval) - time.Sleep(pollingInterval) - - pollingInterval *= 2 - if pollingInterval > EXTERNAL_CA_MAX_POLLING_INTERVAL { - pollingInterval = EXTERNAL_CA_MAX_POLLING_INTERVAL - } - continue - } - - var shouldReturn bool - var shouldContinue bool - - func() { - tm.mutex.Lock() - defer tm.mutex.Unlock() - - state := tm.certificateStates[certificateId] - - switch status { - case "issued": - if certResponse == nil { - shouldContinue = true - return - } - - if certResponse.Certificate == nil { - log.Error().Str("Certificate", displayName).Msg("certificate renewal failed: no certificate data received") - tm.handleFailedCertificateRenewal(certificateId, certificate, "no certificate data in issued response") - shouldReturn = true - return - } - - log.Info().Str("Certificate", displayName).Msg("certificate renewed successfully") - - state.CertificateID = certResponse.Certificate.CertificateID - state.SerialNumber = certResponse.Certificate.SerialNumber - state.IssuedAt = time.Now() - state.Status = "active" - state.LastError = "" - state.RetryCount = 0 - state.CertificateRequestID = requestID - - if ttlDuration, err := parseDurationWithDays(tm.getCertificateTTL(certificate)); err == nil { - state.ExpiresAt = state.IssuedAt.Add(ttlDuration) - } - - if renewBeforeDuration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { - state.NextRenewalCheck = state.ExpiresAt.Add(-renewBeforeDuration) - } - - case "failed": - log.Error().Str("Certificate", displayName).Msg("certificate renewal failed") - tm.handleFailedCertificateRenewal(certificateId, certificate, "external CA renewal failed") - shouldReturn = true - return - - case "pending": - // Continue polling - - default: - log.Warn().Str("Certificate", displayName).Msg("unknown renewal status") - } - }() - - if shouldReturn { - return - } - - if shouldContinue { - time.Sleep(pollingInterval) - continue - } - - if status == "issued" && certResponse != nil && certResponse.Certificate != nil { - if err := tm.WriteCertificateFiles(certificate, certResponse); err != nil { - log.Error().Str("Certificate", displayName).Msgf("failed to write renewed certificate files: %v", err) - - func() { - tm.mutex.Lock() - defer tm.mutex.Unlock() - state := tm.certificateStates[certificateId] - state.Status = "failed" - state.LastError = fmt.Sprintf("failed to write files: %v", err) - }() - return - } - - log.Info().Str("Certificate", displayName).Str("serial", certResponse.Certificate.SerialNumber).Msg("successfully renewed certificate") - - if certificate.PostHooks.OnRenewal.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnRenewal.Command, certificate.PostHooks.OnRenewal.Timeout, "renewal", certificateId, certificate) - } - return - } - - log.Info().Str("Certificate", displayName).Msgf("waiting %s before next renewal polling attempt", pollingInterval) - time.Sleep(pollingInterval) - - pollingInterval *= 2 - if pollingInterval > EXTERNAL_CA_MAX_POLLING_INTERVAL { - pollingInterval = EXTERNAL_CA_MAX_POLLING_INTERVAL - } - } -} - -func (tm *AgentManager) handleFailedCertificateRenewal(certificateId int, certificate *AgentCertificateConfig, reason string) { - state := tm.certificateStates[certificateId] - state.Status = "failed" - state.LastError = fmt.Sprintf("renewal failed: %s", reason) - state.RetryCount++ - state.CertificateRequestID = "" - - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Error().Str("Certificate", displayName).Msgf("renewal failed: %s", reason) - - if certificate.PostHooks.OnFailure.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnFailure.Command, certificate.PostHooks.OnFailure.Timeout, "failure", certificateId, certificate) - } - - if state.RetryCount < certificate.Lifecycle.MaxFailureRetries { - state.LastRetry = time.Now() - log.Info().Str("Certificate", displayName).Msg("scheduling retry for certificate renewal") - } else { - log.Error().Str("Certificate", displayName).Msg("max retries exceeded for certificate renewal") - } -} - -func (tm *AgentManager) handleImmediateRenewalResponseFromIssuance(certificateId int, certificate *AgentCertificateConfig, response *api.CertificateResponse) error { - state := tm.certificateStates[certificateId] - - state.CertificateID = response.Certificate.CertificateID - state.SerialNumber = response.Certificate.SerialNumber - state.IssuedAt = time.Now() - state.Status = "active" - state.LastError = "" - state.RetryCount = 0 - - if ttlDuration, err := parseDurationWithDays(tm.getCertificateTTL(certificate)); err == nil { - state.ExpiresAt = state.IssuedAt.Add(ttlDuration) - } - - if renewBeforeDuration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { - state.NextRenewalCheck = state.ExpiresAt.Add(-renewBeforeDuration) - } - - err := tm.WriteCertificateFiles(certificate, response) - if err != nil { - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Error().Str("Certificate", displayName).Msgf("failed to write renewed certificate files: %v", err) - state.Status = "failed" - state.LastError = fmt.Sprintf("failed to write files: %v", err) - return err - } - - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Info().Str("Certificate", displayName).Str("serial", response.Certificate.SerialNumber).Msg("successfully renewed certificate") - - if certificate.PostHooks.OnRenewal.Command != "" { - tm.ExecutePostHook(certificate.PostHooks.OnRenewal.Command, certificate.PostHooks.OnRenewal.Timeout, "renewal", certificateId, certificate) - } - - return nil -} - -func (tm *AgentManager) checkCertificateRequestStatusByID(requestID string) (string, *api.CertificateResponse, error) { - httpClient, err := tm.createAuthenticatedClient() - if err != nil { - return "", nil, err - } - - response, err := api.CallGetCertificateRequest(httpClient, requestID) - if err != nil { - return "", nil, fmt.Errorf("failed to get certificate request status: %v", err) - } - - if response.Status == "issued" { - if response.Certificate == nil || response.SerialNumber == nil || response.CertificateID == nil { - return response.Status, nil, nil - } - - certData := &api.CertificateData{ - Certificate: *response.Certificate, - SerialNumber: *response.SerialNumber, - CertificateID: *response.CertificateID, - } - - if response.IssuingCaCertificate != nil { - certData.IssuingCaCertificate = *response.IssuingCaCertificate - } - if response.CertificateChain != nil { - certData.CertificateChain = *response.CertificateChain - } - if response.PrivateKey != nil { - certData.PrivateKey = *response.PrivateKey - } - - return response.Status, &api.CertificateResponse{Certificate: certData}, nil - } - - return response.Status, nil, nil + return tm.completeManagedCertificate(certificateId, certificate, certResponse, true) } // runCmd represents the run command diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go new file mode 100644 index 000000000..228491b04 --- /dev/null +++ b/packages/cmd/agent_cert_state.go @@ -0,0 +1,591 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "math/big" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/Infisical/infisical-merge/packages/config" + "github.com/Infisical/infisical-merge/packages/util" + "github.com/gofrs/flock" +) + +type persistedCertificateState struct { + Version int `json:"version"` + Fingerprint string `json:"fingerprint"` + State CertificateState `json:"state"` +} + +func validPersistedCertificateState(saved persistedCertificateState) bool { + if saved.Version != 1 || saved.Fingerprint == "" { + return false + } + switch saved.State.Status { + case "active": + return saved.State.CertificateID != "" && saved.State.CertificateRequestID == "" + case "pending_issuance", "renewing": + return saved.State.CertificateRequestID != "" + case "failed": + return saved.State.CertificateID != "" || saved.State.CertificateRequestID != "" + case "requesting_issuance", "requesting_renewal", "pending": + return true + default: + return false + } +} + +func restoreRejectedCertificateRequest(certificate *AgentCertificateConfig, state *CertificateState, previous CertificateState, requestErr error) error { + var apiErr *api.APIError + // A timeout, transport error, or server error can occur after the CA has accepted the request. + if errors.As(requestErr, &apiErr) && apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 && apiErr.StatusCode != 408 { + *state = previous + if state.CertificateID == "" && state.CertificateRequestID == "" { + state.Status = "pending" + } + return errors.Join(requestErr, persistCertificateState(certificate, state)) + } + return requestErr +} + +func validateCertificateStatePaths(certificates []AgentCertificateConfig) error { + outputs := make(map[string]bool) + for _, certificate := range certificates { + for _, output := range []string{certificate.FileConfig.Certificate.Path, certificate.FileConfig.PrivateKey.Path, certificate.FileConfig.Chain.Path} { + if output == "" { + continue + } + absolute, err := filepath.Abs(output) + if err != nil { + return err + } + outputs[absolute] = true + } + } + states := make(map[string]bool) + for _, certificate := range certificates { + if certificate.HasCertificateID() { + if certificate.Lifecycle.StatePath != "" { + return fmt.Errorf("lifecycle.state-path is only supported for profile-based issuance") + } + continue + } + if certificate.FileConfig.Certificate.Path == "" && certificate.Lifecycle.StatePath == "" { + continue + } + for _, statePath := range []string{certificateStatePath(&certificate), certificateStatePath(&certificate) + ".lock"} { + absolute, err := filepath.Abs(statePath) + if err != nil { + return err + } + if outputs[absolute] || states[absolute] { + return fmt.Errorf("certificate state or lock path %s overlaps an output or another certificate's state", statePath) + } + states[absolute] = true + } + } + return nil +} + +func certificateStatePath(certificate *AgentCertificateConfig) string { + if certificate.Lifecycle.StatePath != "" { + return certificate.Lifecycle.StatePath + } + return certificate.FileConfig.Certificate.Path + ".infisical-state.json" +} + +func preflightCertificateOutputs(certificate *AgentCertificateConfig) error { + if certificate.FileConfig.Certificate.Path == "" { + return fmt.Errorf("certificate.path is required in file-output configuration") + } + for _, output := range []string{certificate.FileConfig.Certificate.Path, certificate.FileConfig.PrivateKey.Path, certificate.FileConfig.Chain.Path} { + if output == "" { + continue + } + if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil { + return fmt.Errorf("cannot prepare certificate output %s: %w", output, err) + } + file, err := os.OpenFile(output, os.O_WRONLY, 0) + if err == nil { + _ = file.Close() + continue + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot write certificate output %s: %w", output, err) + } + probe, err := os.CreateTemp(filepath.Dir(output), ".infisical-output-*") + if err != nil { + return fmt.Errorf("cannot write certificate output directory for %s: %w", output, err) + } + _ = probe.Close() + _ = os.Remove(probe.Name()) + } + return nil +} + +func certificateFingerprint(certificate *AgentCertificateConfig) (string, error) { + attributes := buildCertificateAttributes(certificate) + if attributes != nil { + attributes.AltNames = slices.Clone(attributes.AltNames) + slices.SortFunc(attributes.AltNames, func(a, b api.AltName) int { return strings.Compare(a.Value, b.Value) }) + attributes.RemoveRootsFromChain = false + attributes.KeyUsages = slices.Clone(attributes.KeyUsages) + slices.Sort(attributes.KeyUsages) + attributes.ExtendedKeyUsages = slices.Clone(attributes.ExtendedKeyUsages) + slices.Sort(attributes.ExtendedKeyUsages) + } + data, err := json.Marshal(struct { + Address string + ProfileID string + ApplicationID string + CSR string + Attributes *api.CertificateAttributes + }{config.INFISICAL_URL, certificate.ProfileID, certificate.ApplicationID, certificate.CSR, attributes}) + if err != nil { + return "", err + } + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]), nil +} + +func lockCertificateState(certificate *AgentCertificateConfig) (func(), error) { + if err := validateCertificateStatePaths([]AgentCertificateConfig{*certificate}); err != nil { + return nil, err + } + if certificate.FileConfig.Certificate.Path == "" { + return nil, fmt.Errorf("certificate.path is required in file-output configuration") + } + statePath := certificateStatePath(certificate) + if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil { + return nil, fmt.Errorf("cannot create certificate state directory: %w", err) + } + lockFile, err := os.OpenFile(statePath+".lock", os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, fmt.Errorf("cannot create certificate state lock: %w", err) + } + _ = lockFile.Close() + lock := flock.New(statePath + ".lock") + locked, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("cannot lock certificate state: %w", err) + } + if !locked { + return nil, fmt.Errorf("another Agent is managing certificate state at %s", statePath) + } + return func() { _ = lock.Unlock() }, nil +} + +func persistCertificateState(certificate *AgentCertificateConfig, state *CertificateState) error { + fingerprint, err := certificateFingerprint(certificate) + if err != nil { + return err + } + data, err := json.Marshal(persistedCertificateState{Version: 1, Fingerprint: fingerprint, State: *state}) + if err != nil { + return err + } + statePath := certificateStatePath(certificate) + if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil { + return fmt.Errorf("cannot create certificate state directory: %w", err) + } + if err := util.WriteFileAtomic(statePath, data, 0600); err != nil { + return fmt.Errorf("cannot persist certificate state at %s: %w", statePath, err) + } + file, err := os.Open(statePath) + if err != nil { + return err + } + err = file.Sync() + _ = file.Close() + if err != nil { + return err + } + if runtime.GOOS != "windows" { + directory, err := os.Open(filepath.Dir(statePath)) + if err != nil { + return err + } + err = directory.Sync() + _ = directory.Close() + return err + } + return nil +} + +func parseAgentCertificate(data []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(data) + if block == nil || block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("invalid certificate PEM") + } + return x509.ParseCertificate(block.Bytes) +} + +func readAgentCertificate(certificate *AgentCertificateConfig) (*x509.Certificate, []byte, error) { + data, err := os.ReadFile(certificate.FileConfig.Certificate.Path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("cannot read existing certificate: %w", err) + } + leaf, err := parseAgentCertificate(data) + if err != nil { + return nil, data, nil + } + return leaf, data, nil +} + +func certificateSerialMatches(serial string, leaf *x509.Certificate) bool { + value, ok := new(big.Int).SetString(strings.TrimPrefix(strings.ToLower(serial), "0x"), 16) + return ok && leaf != nil && value.Cmp(leaf.SerialNumber) == 0 +} + +func certificateMatchesIdentity(leaf *x509.Certificate, certificate *AgentCertificateConfig) bool { + commonName := "" + var altNames []string + if certificate.CSR != "" { + block, _ := pem.Decode([]byte(certificate.CSR)) + if block == nil { + return false + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil || csr.CheckSignature() != nil { + return false + } + key, err := x509.MarshalPKIXPublicKey(csr.PublicKey) + if err != nil || !bytes.Equal(key, leaf.RawSubjectPublicKeyInfo) { + return false + } + commonName, altNames = csr.Subject.CommonName, csr.DNSNames + } else if certificate.Attributes != nil { + commonName, altNames = certificate.Attributes.CommonName, certificate.Attributes.AltNames + algorithm := certificate.Attributes.KeyAlgorithm + if algorithm != "" { + actualAlgorithm := "" + switch key := leaf.PublicKey.(type) { + case *rsa.PublicKey: + actualAlgorithm = fmt.Sprintf("RSA_%d", key.N.BitLen()) + case *ecdsa.PublicKey: + actualAlgorithm = map[int]string{256: "EC_prime256v1", 384: "EC_secp384r1", 521: "EC_secp521r1"}[key.Curve.Params().BitSize] + } + if algorithm != actualAlgorithm { + return false + } + } + } + if commonName != "" && commonName != leaf.Subject.CommonName { + return false + } + actual := slices.Clone(leaf.DNSNames) + actual = append(actual, leaf.EmailAddresses...) + for _, ip := range leaf.IPAddresses { + actual = append(actual, ip.String()) + } + for _, uri := range leaf.URIs { + actual = append(actual, uri.String()) + } + for _, name := range altNames { + if !slices.Contains(actual, name) { + return false + } + } + return true +} + +func (tm *AgentManager) recoverCertificateID(certificate *AgentCertificateConfig, leaf *x509.Certificate) (string, error) { + httpClient, err := tm.createAuthenticatedClient() + if err != nil { + return "", err + } + serial := leaf.SerialNumber.Text(16) + for offset := 0; ; offset += 100 { + response, err := api.CallListCertificateProfileCertificates(httpClient, certificate.ProfileID, serial, offset) + if err != nil { + return "", fmt.Errorf("cannot recover existing certificate identity: %w", err) + } + for _, candidate := range response.Certificates { + if !certificateSerialMatches(candidate.SerialNumber, leaf) { + continue + } + bundle, err := api.CallGetCertificateBundle(httpClient, candidate.ID) + if err != nil { + return "", err + } + remoteLeaf, err := parseAgentCertificate([]byte(bundle.Certificate)) + if err != nil { + return "", err + } + if bytes.Equal(leaf.Raw, remoteLeaf.Raw) { + return candidate.ID, nil + } + } + if len(response.Certificates) < 100 { + return "", nil + } + } +} + +func (tm *AgentManager) initializeManagedCertificate(certificateID int, certificate *AgentCertificateConfig) error { + data, readErr := os.ReadFile(certificateStatePath(certificate)) + if readErr != nil && !errors.Is(readErr, os.ErrNotExist) { + return fmt.Errorf("cannot read certificate state: %w", readErr) + } + var saved persistedCertificateState + if readErr == nil { + if err := json.Unmarshal(data, &saved); err != nil || !validPersistedCertificateState(saved) { + return fmt.Errorf("certificate state at %s is invalid; inspect the existing certificate and any pending request before restoring or removing it", certificateStatePath(certificate)) + } + fingerprint, err := certificateFingerprint(certificate) + if err != nil { + return err + } + if saved.State.Status == "requesting_issuance" || saved.State.Status == "requesting_renewal" { + return fmt.Errorf("a previous certificate request has an unknown outcome; check Infisical before restoring or removing state at %s", certificateStatePath(certificate)) + } + if saved.Fingerprint != fingerprint { + if saved.State.CertificateRequestID != "" { + return fmt.Errorf("issuance configuration changed while a certificate request is pending; restore the original configuration to resume it") + } + return tm.IssueCertificate(certificateID, certificate) + } + tm.mutex.Lock() + restored := saved.State + if restored.CertificateRequestID == "" { + restored.Status = "restoring" + } + tm.certificateStates[certificateID] = &restored + tm.mutex.Unlock() + if saved.State.CertificateRequestID != "" { + if saved.State.Status == "failed" { + return fmt.Errorf("previous certificate request failed: %s", saved.State.LastError) + } + tm.startCertificatePolling(certificateID, certificate) + return nil + } + } + + leaf, certificatePEM, err := readAgentCertificate(certificate) + if err != nil { + return err + } + if saved.State.CertificateID == "" { + if leaf == nil || !certificateMatchesIdentity(leaf, certificate) { + return tm.IssueCertificate(certificateID, certificate) + } + id, err := tm.recoverCertificateID(certificate, leaf) + if err != nil { + return err + } + if id == "" { + return tm.IssueCertificate(certificateID, certificate) + } + saved.State.CertificateID = id + hash := sha256.Sum256(leaf.Raw) + saved.State.CertificateSHA256 = hex.EncodeToString(hash[:]) + } + httpClient, err := tm.createAuthenticatedClient() + if err != nil { + return err + } + metadata, err := api.CallRetrieveCertificate(httpClient, saved.State.CertificateID) + if err != nil { + return fmt.Errorf("cannot restore certificate metadata: %w", err) + } + if metadata.Certificate.LatestRenewalCertificateID != "" && metadata.Certificate.LatestRenewalCertificateID != metadata.Certificate.ID { + metadata, err = api.CallRetrieveCertificate(httpClient, metadata.Certificate.LatestRenewalCertificateID) + if err != nil { + return err + } + } + if metadata.Certificate.Status == "revoked" { + return fmt.Errorf("existing certificate is revoked; refusing automatic issuance or renewal") + } + if metadata.Certificate.Status == "expired" || !metadata.Certificate.NotAfter.After(time.Now()) { + return tm.IssueCertificate(certificateID, certificate) + } + if metadata.Certificate.ID == "" || metadata.Certificate.Status != "active" { + return fmt.Errorf("existing certificate is not active") + } + + repair := leaf == nil || !certificateSerialMatches(metadata.Certificate.SerialNumber, leaf) || saved.State.CertificateSHA256 == "" + if leaf != nil && saved.State.CertificateSHA256 != "" { + hash := sha256.Sum256(leaf.Raw) + repair = repair || hex.EncodeToString(hash[:]) != saved.State.CertificateSHA256 + } + if !repair { + if !certificateMatchesIdentity(leaf, certificate) { + return fmt.Errorf("existing certificate does not match the configured identity") + } + for _, output := range []string{certificate.FileConfig.PrivateKey.Path, certificate.FileConfig.Chain.Path} { + if output == "" { + continue + } + outputData, err := os.ReadFile(output) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot read certificate output %s: %w", output, err) + } + if len(outputData) == 0 { + repair = true + } else if output == certificate.FileConfig.PrivateKey.Path { + if _, err := tls.X509KeyPair(certificatePEM, outputData); err != nil { + repair = true + } + } else if _, err := parseAgentCertificate(outputData); err != nil { + repair = true + } + } + } + if repair { + bundle, err := api.CallGetCertificateBundle(httpClient, metadata.Certificate.ID) + if err != nil { + return err + } + leaf, err = parseAgentCertificate([]byte(bundle.Certificate)) + if err != nil || !certificateSerialMatches(metadata.Certificate.SerialNumber, leaf) || !certificateMatchesIdentity(leaf, certificate) { + return fmt.Errorf("stored certificate bundle does not match the certificate identity") + } + if certificate.FileConfig.PrivateKey.Path != "" { + key := []byte(bundle.PrivateKey) + if len(key) == 0 { + key, err = os.ReadFile(certificate.FileConfig.PrivateKey.Path) + if err != nil { + return fmt.Errorf("cannot restore the certificate's locally managed private key: %w", err) + } + } + if _, err := tls.X509KeyPair([]byte(bundle.Certificate), key); err != nil { + return fmt.Errorf("cannot restore a matching private key: %w", err) + } + } + if certificate.FileConfig.Chain.Path != "" && bundle.CertificateChain == "" { + return fmt.Errorf("cannot restore configured chain output: certificate bundle has no chain") + } + response := &api.CertificateResponse{Certificate: &api.CertificateData{ + CertificateID: metadata.Certificate.ID, SerialNumber: bundle.SerialNumber, + Certificate: bundle.Certificate, PrivateKey: bundle.PrivateKey, CertificateChain: bundle.CertificateChain, + }} + if err := tm.WriteCertificateFiles(certificate, response); err != nil { + return err + } + } + if time.Now().Before(leaf.NotBefore) { + return fmt.Errorf("existing certificate is not valid yet") + } + if !time.Now().Before(leaf.NotAfter) { + return tm.IssueCertificate(certificateID, certificate) + } + tm.mutex.Lock() + defer tm.mutex.Unlock() + state := tm.certificateStates[certificateID] + state.CertificateID = metadata.Certificate.ID + setManagedCertificateState(state, leaf, certificate) + if err := persistCertificateState(certificate, state); err != nil { + return err + } + if certificate.CSR == "" && tm.ShouldRenewCertificate(certificateID) { + return tm.RenewCertificate(certificateID, certificate) + } + return nil +} + +func setManagedCertificateState(state *CertificateState, leaf *x509.Certificate, certificate *AgentCertificateConfig) { + hash := sha256.Sum256(leaf.Raw) + state.CertificateSHA256 = hex.EncodeToString(hash[:]) + state.SerialNumber = leaf.SerialNumber.Text(16) + state.CommonName = leaf.Subject.CommonName + state.IssuedAt = leaf.NotBefore + state.ExpiresAt = leaf.NotAfter + state.Status = "active" + state.LastError = "" + state.RetryCount = 0 + state.CertificateRequestID = "" + state.NextRenewalCheck = time.Now().Add(statusCheckIntervalFor(certificate)) + if duration, err := parseDurationWithDays(certificate.Lifecycle.RenewBeforeExpiry); err == nil { + renewAt := leaf.NotAfter.Add(-duration) + if renewAt.After(time.Now()) && renewAt.Before(state.NextRenewalCheck) { + state.NextRenewalCheck = renewAt + } + } +} + +func (tm *AgentManager) completeManagedCertificate(certificateID int, certificate *AgentCertificateConfig, response *api.CertificateResponse, renewal bool) error { + if response.Certificate == nil || response.Certificate.CertificateID == "" { + return fmt.Errorf("issued certificate response has no certificate identity") + } + state := tm.certificateStates[certificateID] + state.CertificateID = response.Certificate.CertificateID + state.CertificateRequestID = "" + state.Status = "active" + // Save the identity before touching output files so a partial delivery can be repaired without another CA request. + if err := persistCertificateState(certificate, state); err != nil { + return err + } + leaf, err := parseAgentCertificate([]byte(response.Certificate.Certificate)) + if err != nil { + return fmt.Errorf("cannot parse issued certificate: %w", err) + } + if response.Certificate.PrivateKey != "" { + if _, err := tls.X509KeyPair([]byte(response.Certificate.Certificate), []byte(response.Certificate.PrivateKey)); err != nil { + return fmt.Errorf("issued certificate does not match its private key: %w", err) + } + } else if certificate.FileConfig.PrivateKey.Path != "" { + key, err := os.ReadFile(certificate.FileConfig.PrivateKey.Path) + if err != nil { + return fmt.Errorf("issued certificate has no private key and no locally managed key is available: %w", err) + } + if _, err := tls.X509KeyPair([]byte(response.Certificate.Certificate), key); err != nil { + return fmt.Errorf("issued certificate does not match the locally managed private key: %w", err) + } + } + setManagedCertificateState(state, leaf, certificate) + if err := persistCertificateState(certificate, state); err != nil { + return err + } + if err := tm.WriteCertificateFiles(certificate, response); err != nil { + return err + } + if renewal { + tm.ExecutePostHook(certificate.PostHooks.OnRenewal.Command, certificate.PostHooks.OnRenewal.Timeout, "renewal", certificateID, certificate) + } else { + tm.ExecutePostHook(certificate.PostHooks.OnIssuance.Command, certificate.PostHooks.OnIssuance.Timeout, "issuance", certificateID, certificate) + } + return nil +} + +func (tm *AgentManager) startCertificatePolling(certificateID int, certificate *AgentCertificateConfig) { + tm.certificatePollers.Add(1) + go func() { + defer tm.certificatePollers.Done() + tm.PollCertificateRequest(certificateID, certificate) + }() +} + +func (tm *AgentManager) waitForCertificatePoll(interval time.Duration) bool { + ctx := tm.certificateContext + if ctx == nil { + ctx = context.Background() + } + timer := time.NewTimer(interval) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go new file mode 100644 index 000000000..5a2b474b7 --- /dev/null +++ b/packages/cmd/agent_cert_state_test.go @@ -0,0 +1,498 @@ +package cmd + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Infisical/infisical-merge/packages/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func agentTestCertificate(t *testing.T, serial int64, expires time.Time) *api.CertificateData { + t.Helper() + publicKey, key, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + leaf := &x509.Certificate{ + SerialNumber: big.NewInt(serial), Subject: pkix.Name{CommonName: "restart.example.test"}, + DNSNames: []string{"restart.example.test"}, NotBefore: time.Now().Add(-24 * time.Hour), NotAfter: expires, + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, leaf, leaf, publicKey, key) + require.NoError(t, err) + certificatePEM := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + return &api.CertificateData{ + CertificateID: fmt.Sprintf("cert-%d", serial), SerialNumber: fmt.Sprintf("%x", serial), + Certificate: certificatePEM, CertificateChain: certificatePEM, + PrivateKey: string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})), + } +} + +func agentTestConfig(t *testing.T) *AgentCertificateConfig { + t.Helper() + dir := t.TempDir() + certificate := &AgentCertificateConfig{ + ApplicationID: "application", ProfileID: "profile", + Attributes: &CertificateAttributes{CommonName: "restart.example.test", AltNames: []string{"restart.example.test"}, TTL: "90d"}, + Lifecycle: CertificateLifecycleConfig{RenewBeforeExpiry: "15d", StatusCheckInterval: "6h"}, + } + certificate.FileConfig.Certificate.Path = filepath.Join(dir, "certificate.pem") + certificate.FileConfig.PrivateKey.Path = filepath.Join(dir, "key.pem") + certificate.FileConfig.Chain.Path = filepath.Join(dir, "chain.pem") + return certificate +} + +func agentTestManager(certificate *AgentCertificateConfig) *AgentManager { + return &AgentManager{ + accessToken: "mock-token", certificates: []CertificateWithID{{ID: 1, Certificate: *certificate}}, + certificateStates: map[int]*CertificateState{1: {Status: "pending"}}, + } +} + +type agentTestAPI struct { + onIssue func() + issueError atomic.Int32 + issueCalls, renewCalls, pollCalls, listCalls, bundleCalls, statusCalls atomic.Int32 + pending, pollIssued, revoked atomic.Bool + statusError atomic.Int32 + old, next *api.CertificateData +} + +func newAgentTestAPI(t *testing.T, old, next *api.CertificateData) *agentTestAPI { + t.Helper() + backend := &agentTestAPI{old: old, next: next} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + const prefix = "/v1/cert-manager/" + switch { + case r.Method == http.MethodPost && r.URL.Path == prefix+"certificates": + backend.issueCalls.Add(1) + if backend.onIssue != nil { + backend.onIssue() + } + if status := backend.issueError.Load(); status != 0 { + w.WriteHeader(int(status)) + return + } + if backend.pending.Load() { + _ = json.NewEncoder(w).Encode(api.CertificateResponse{CertificateRequestID: "request-1"}) + } else { + _ = json.NewEncoder(w).Encode(api.CertificateResponse{Certificate: backend.next}) + } + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/renew"): + backend.renewCalls.Add(1) + if backend.pending.Load() { + _ = json.NewEncoder(w).Encode(api.RenewCertificateResponse{CertificateRequestID: "request-1"}) + } else { + _ = json.NewEncoder(w).Encode(api.RenewCertificateResponse{ + CertificateID: backend.next.CertificateID, SerialNumber: backend.next.SerialNumber, + Certificate: backend.next.Certificate, CertificateChain: backend.next.CertificateChain, PrivateKey: backend.next.PrivateKey, + }) + } + case r.URL.Path == prefix+"certificates/certificate-requests/request-1": + backend.pollCalls.Add(1) + response := api.GetCertificateRequestResponse{Status: "pending"} + if backend.pollIssued.Load() { + response.Status = "issued" + response.CertificateID = &backend.next.CertificateID + response.SerialNumber = &backend.next.SerialNumber + response.Certificate = &backend.next.Certificate + response.CertificateChain = &backend.next.CertificateChain + response.PrivateKey = &backend.next.PrivateKey + } + _ = json.NewEncoder(w).Encode(response) + case r.URL.Path == prefix+"certificate-profiles/profile/certificates": + backend.listCalls.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{"certificates": []any{map[string]string{"id": backend.old.CertificateID, "serialNumber": backend.old.SerialNumber}}}) + case strings.HasSuffix(r.URL.Path, "/bundle"): + backend.bundleCalls.Add(1) + certificate := backend.old + if strings.Contains(r.URL.Path, backend.next.CertificateID+"/") { + certificate = backend.next + } + _ = json.NewEncoder(w).Encode(api.CertificateBundleResponse{ + Certificate: certificate.Certificate, PrivateKey: certificate.PrivateKey, + SerialNumber: certificate.SerialNumber, CertificateChain: certificate.CertificateChain, + }) + case strings.HasPrefix(r.URL.Path, prefix+"certificates/"): + backend.statusCalls.Add(1) + if status := backend.statusError.Load(); status != 0 { + w.WriteHeader(int(status)) + return + } + certificate := backend.old + if strings.HasSuffix(r.URL.Path, backend.next.CertificateID) { + certificate = backend.next + } + leaf, err := parseAgentCertificate([]byte(certificate.Certificate)) + if err != nil { + w.WriteHeader(500) + return + } + var response api.RetrieveCertificateResponse + response.Certificate.ID, response.Certificate.SerialNumber = certificate.CertificateID, certificate.SerialNumber + response.Certificate.Status = "active" + if time.Now().After(leaf.NotAfter) { + response.Certificate.Status = "expired" + } + if backend.revoked.Load() { + response.Certificate.Status = "revoked" + } + response.Certificate.NotBefore, response.Certificate.NotAfter = leaf.NotBefore, leaf.NotAfter + _ = json.NewEncoder(w).Encode(response) + default: + w.WriteHeader(404) + } + })) + t.Cleanup(server.Close) + withMockInfisicalURL(t, server.URL) + return backend +} + +func seedAgentCertificate(t *testing.T, certificate *AgentCertificateConfig, data *api.CertificateData, saveState bool) { + t.Helper() + manager := agentTestManager(certificate) + require.NoError(t, manager.WriteCertificateFiles(certificate, &api.CertificateResponse{Certificate: data})) + if saveState { + leaf, err := parseAgentCertificate([]byte(data.Certificate)) + require.NoError(t, err) + state := &CertificateState{CertificateID: data.CertificateID} + setManagedCertificateState(state, leaf, certificate) + require.NoError(t, persistCertificateState(certificate, state)) + } +} + +func runAgentMonitor(t *testing.T, certificate *AgentCertificateConfig, complete func(*AgentManager) bool) *AgentManager { + t.Helper() + manager := agentTestManager(certificate) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); manager.MonitorCertificates(ctx) }() + t.Cleanup(cancel) + require.Eventually(t, func() bool { + manager.mutex.Lock() + defer manager.mutex.Unlock() + return complete(manager) + }, 5*time.Second, 10*time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("certificate monitor did not stop and release its state lock") + } + return manager +} + +func TestManagedCertificateRestartReusesExistingCertificate(t *testing.T) { + for _, saved := range []bool{false, true} { + t.Run(fmt.Sprintf("persisted=%v", saved), func(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, saved) + modTime := time.Now().Add(-time.Hour).Truncate(time.Second) + require.NoError(t, os.Chtimes(certificate.FileConfig.Certificate.Path, modTime, modTime)) + for i := 0; i < 3; i++ { + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.Equal(t, old.CertificateID, manager.certificateStates[1].CertificateID) + assert.WithinDuration(t, time.Now().Add(90*24*time.Hour), manager.certificateStates[1].ExpiresAt, 2*time.Second) + } + assert.Zero(t, backend.issueCalls.Load()) + assert.Zero(t, backend.renewCalls.Load()) + info, err := os.Stat(certificate.FileConfig.Certificate.Path) + require.NoError(t, err) + assert.Equal(t, modTime, info.ModTime()) + if saved { + assert.Zero(t, backend.listCalls.Load()) + } else { + assert.EqualValues(t, 1, backend.listCalls.Load()) + } + }) + } +} + +func TestManagedCertificateStartupRenewsUsingActualExpiry(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(10*24*time.Hour)) + next := agentTestCertificate(t, 2, time.Now().Add(70*24*time.Hour)) + backend := newAgentTestAPI(t, old, next) + seedAgentCertificate(t, certificate, old, true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { + return manager.certificateStates[1].CertificateID == next.CertificateID && manager.certificateStates[1].Status == "active" + }) + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.Zero(t, backend.issueCalls.Load()) + assert.EqualValues(t, 1, backend.renewCalls.Load()) + leaf, err := parseAgentCertificate([]byte(next.Certificate)) + require.NoError(t, err) + assert.Equal(t, leaf.NotAfter, manager.certificateStates[1].ExpiresAt) +} + +func TestManagedCertificateRepairsOutputsWithoutIssuance(t *testing.T) { + for _, broken := range []string{"missing certificate", "invalid certificate", "missing key", "invalid key", "mismatched key", "missing chain", "invalid chain"} { + t.Run(broken, func(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + other := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, other) + seedAgentCertificate(t, certificate, old, true) + output := certificate.FileConfig.Certificate.Path + if strings.Contains(broken, "key") { + output = certificate.FileConfig.PrivateKey.Path + } + if strings.Contains(broken, "chain") { + output = certificate.FileConfig.Chain.Path + } + if strings.HasPrefix(broken, "missing") { + require.NoError(t, os.Remove(output)) + } else if broken == "mismatched key" { + require.NoError(t, os.WriteFile(output, []byte(other.PrivateKey), 0600)) + } else { + require.NoError(t, os.WriteFile(output, []byte("corrupt"), 0600)) + } + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.Zero(t, backend.issueCalls.Load()) + assert.Zero(t, backend.renewCalls.Load()) + assert.EqualValues(t, 1, backend.bundleCalls.Load()) + certPEM, err := os.ReadFile(certificate.FileConfig.Certificate.Path) + require.NoError(t, err) + keyPEM, err := os.ReadFile(certificate.FileConfig.PrivateKey.Path) + require.NoError(t, err) + assert.Equal(t, old.Certificate, string(certPEM)) + assert.Equal(t, old.PrivateKey, string(keyPEM)) + }) + } +} + +func TestManagedCertificateIssuesWhenMissingInvalidOrExpired(t *testing.T) { + for _, condition := range []string{"missing", "invalid", "expired"} { + t.Run(condition, func(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(-time.Hour)) + next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, next) + if condition == "expired" { + seedAgentCertificate(t, certificate, old, true) + } + if condition == "invalid" { + require.NoError(t, os.WriteFile(certificate.FileConfig.Certificate.Path, []byte("invalid"), 0600)) + } + for i := 0; i < 3; i++ { + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + } + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.Zero(t, backend.renewCalls.Load()) + data, err := os.ReadFile(certificateStatePath(certificate)) + require.NoError(t, err) + assert.NotContains(t, string(data), "PRIVATE KEY") + info, err := os.Stat(certificateStatePath(certificate)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) + }) + } +} + +func TestManagedCertificateResumesPendingRequestsOnRestart(t *testing.T) { + for _, renewal := range []bool{false, true} { + t.Run(fmt.Sprintf("renewal=%v", renewal), func(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(10*24*time.Hour)) + next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, next) + backend.pending.Store(true) + if renewal { + seedAgentCertificate(t, certificate, old, true) + } + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return backend.pollCalls.Load() > 0 }) + backend.pollIssued.Store(true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + if renewal { + assert.Zero(t, backend.issueCalls.Load()) + assert.EqualValues(t, 1, backend.renewCalls.Load()) + } else { + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.Zero(t, backend.renewCalls.Load()) + } + data, err := os.ReadFile(certificateStatePath(certificate)) + require.NoError(t, err) + var saved persistedCertificateState + require.NoError(t, json.Unmarshal(data, &saved)) + assert.Empty(t, saved.State.CertificateRequestID) + assert.Equal(t, next.CertificateID, saved.State.CertificateID) + }) + } +} + +func TestManagedCertificateDoesNotIssueOnAmbiguousStateOrStatusErrors(t *testing.T) { + for _, condition := range []string{"corrupt state", "requesting_issuance", "requesting_renewal", "revoked", "403", "500"} { + t.Run(condition, func(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + switch condition { + case "corrupt state": + require.NoError(t, os.WriteFile(certificateStatePath(certificate), []byte("{"), 0600)) + case "requesting_issuance", "requesting_renewal": + require.NoError(t, persistCertificateState(certificate, &CertificateState{Status: condition})) + case "revoked": + backend.revoked.Store(true) + case "403": + backend.statusError.Store(403) + case "500": + backend.statusError.Store(500) + } + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.NotEmpty(t, manager.certificateStates[1].LastError) + assert.Zero(t, backend.issueCalls.Load()) + assert.Zero(t, backend.renewCalls.Load()) + }) + } +} + +func TestManagedCertificateStateLockPreventsConcurrentAgents(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + unlock, err := lockCertificateState(certificate) + require.NoError(t, err) + defer unlock() + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "blocked" }) + assert.Zero(t, backend.issueCalls.Load()) +} + +func TestManagedCertificateRenewsDuringPeriodicCheck(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + manager := agentTestManager(certificate) + require.NoError(t, manager.initializeManagedCertificate(1, certificate)) + manager.certificateStates[1].NextRenewalCheck = time.Now().Add(-time.Hour) + manager.CheckCertificateRenewals() + assert.Zero(t, backend.renewCalls.Load()) + manager.certificates[0].Certificate.Lifecycle.RenewBeforeExpiry = "91d" + manager.certificateStates[1].NextRenewalCheck = time.Now().Add(-time.Hour) + manager.CheckCertificateRenewals() + assert.EqualValues(t, 1, backend.renewCalls.Load()) + assert.Zero(t, backend.issueCalls.Load()) +} + +func TestManagedCertificateDoesNotDuplicateRequestsAfterServerErrors(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + backend.issueError.Store(500) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + backend.issueError.Store(0) + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.Contains(t, manager.certificateStates[1].LastError, "unknown outcome") +} + +func TestManagedCertificateRetriesDefinitivelyRejectedRequests(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + backend.issueError.Store(403) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + backend.issueError.Store(0) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.EqualValues(t, 2, backend.issueCalls.Load()) +} + +func TestManagedCertificateRecoversInterruptedFileDelivery(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, next) + backend.onIssue = func() { _ = os.Mkdir(certificate.FileConfig.Chain.Path, 0700) } + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + require.NoError(t, os.Remove(certificate.FileConfig.Chain.Path)) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.EqualValues(t, 1, backend.bundleCalls.Load()) + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + require.NoError(t, err) + assert.Equal(t, next.CertificateChain, string(chain)) +} + +func TestManagedCertificateConfigChangesIssueOnlyOnce(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + certificate.ProfileID = "other-profile" + for i := 0; i < 3; i++ { + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + } + assert.EqualValues(t, 1, backend.issueCalls.Load()) +} + +func TestManagedCertificateDoesNotDiscardPendingRequestOnConfigChange(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + backend.pending.Store(true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return backend.pollCalls.Load() > 0 }) + certificate.ProfileID = "other-profile" + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.Contains(t, manager.certificateStates[1].LastError, "configuration changed") +} + +func TestCertificateStatePathValidation(t *testing.T) { + certificate := agentTestConfig(t) + certificate.Lifecycle.StatePath = certificate.FileConfig.PrivateKey.Path + _, err := lockCertificateState(certificate) + require.ErrorContains(t, err, "overlaps") + certificate.Lifecycle.StatePath = certificate.FileConfig.Certificate.Path + ".state" + require.NoError(t, validateCertificateStatePaths([]AgentCertificateConfig{*certificate})) + require.ErrorContains(t, validateCertificateStatePaths([]AgentCertificateConfig{*certificate, *certificate}), "overlaps") + certificate.CertificateID = "pinned" + require.ErrorContains(t, validateCertificateStatePaths([]AgentCertificateConfig{*certificate}), "profile-based") +} + +func TestManagedCertificateStateCanBeStoredInConfiguredDirectory(t *testing.T) { + certificate := agentTestConfig(t) + certificate.Lifecycle.StatePath = filepath.Join(t.TempDir(), "state.json") + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, false) + for i := 0; i < 3; i++ { + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + } + assert.Zero(t, backend.issueCalls.Load()) + _, err := os.Stat(certificate.Lifecycle.StatePath) + require.NoError(t, err) + _, err = os.Stat(certificate.FileConfig.Certificate.Path + ".infisical-state.json") + require.True(t, os.IsNotExist(err)) +} + +func TestManagedCertificateDoesNotIssueWhenCertificateIsUnreadable(t *testing.T) { + certificate := agentTestConfig(t) + require.NoError(t, os.Mkdir(certificate.FileConfig.Certificate.Path, 0700)) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.Zero(t, backend.issueCalls.Load()) + assert.Contains(t, manager.certificateStates[1].LastError, "cannot read existing certificate") +} From 19ccc24839cdca55d2827d50e50d561b8307f07e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:55:24 +0000 Subject: [PATCH 2/8] fix(agent): harden certificate state recovery Co-Authored-By: ashwin --- packages/cmd/agent.go | 36 ++++- packages/cmd/agent_cert_state.go | 206 +++++++++++++++++++++++--- packages/cmd/agent_cert_state_test.go | 124 ++++++++++++++++ 3 files changed, 345 insertions(+), 21 deletions(-) diff --git a/packages/cmd/agent.go b/packages/cmd/agent.go index d9a5b4d6f..0dd991cc8 100644 --- a/packages/cmd/agent.go +++ b/packages/cmd/agent.go @@ -124,6 +124,9 @@ type CertificateState struct { CertificateRequestID string `json:"certificate_request_id,omitempty"` SerialNumber string `json:"serial_number"` CertificateSHA256 string `json:"certificate_sha256,omitempty"` + ChainSHA256 string `json:"chain_sha256,omitempty"` + ChainOmitRoot bool `json:"chain_omit_root"` + DeliveryHook string `json:"delivery_hook,omitempty"` CommonName string `json:"common_name"` IssuedAt time.Time `json:"issued_at"` ExpiresAt time.Time `json:"expires_at"` @@ -1129,6 +1132,7 @@ type AgentManager struct { certificateStates map[int]*CertificateState certificateContext context.Context certificatePollers sync.WaitGroup + certificateUnlocks map[int]func() dynamicSecretLeases *DynamicSecretLeaseManager cacheManager *CacheManager authConfigBytes []byte @@ -2735,6 +2739,7 @@ func (tm *AgentManager) checkCertificateRequestStatus(certificateId int, certifi errorMsg = *response.ErrorMessage } tm.handleFailedCertificateRequest(certificateId, errorMsg) + state.CertificateRequestID = "" return persistCertificateState(certificate, state) case "pending": @@ -2921,11 +2926,11 @@ func (tm *AgentManager) MonitorCertificates(ctx context.Context) { log.Info().Msg("starting certificate monitoring") ctx, cancel := context.WithCancel(ctx) tm.certificateContext = ctx - var unlocks []func() + tm.certificateUnlocks = make(map[int]func()) defer func() { cancel() tm.certificatePollers.Wait() - for _, unlock := range unlocks { + for _, unlock := range tm.certificateUnlocks { unlock() } }() @@ -2955,11 +2960,12 @@ func (tm *AgentManager) MonitorCertificates(ctx context.Context) { if err != nil { tm.mutex.Lock() tm.certificateStates[cert.ID].Status = "blocked" + tm.certificateStates[cert.ID].NextRenewalCheck = time.Now().Add(failureRetryIntervalFor(&cert.Certificate)) tm.mutex.Unlock() log.Error().Str("Certificate", tm.getCertificateDisplayName(cert.ID, &cert.Certificate)).Msgf("certificate state unavailable: %v", err) continue } - unlocks = append(unlocks, unlock) + tm.certificateUnlocks[cert.ID] = unlock if err := tm.initializeManagedCertificate(cert.ID, &cert.Certificate); err != nil { displayName := tm.getCertificateDisplayName(cert.ID, &cert.Certificate) log.Error().Str("Certificate", displayName).Msgf("initial certificate recovery failed: %v", err) @@ -3050,6 +3056,30 @@ func (tm *AgentManager) CheckCertificateRenewals() { continue } + if state.Status == "blocked" { + if now.Before(state.NextRenewalCheck) { + continue + } + tm.mutex.Unlock() + unlock, err := lockCertificateState(&cert.Certificate) + tm.mutex.Lock() + state.NextRenewalCheck = now.Add(failureRetryIntervalFor(&cert.Certificate)) + if err != nil { + state.LastError = err.Error() + continue + } + tm.certificateUnlocks[cert.ID] = unlock + tm.mutex.Unlock() + err = tm.initializeManagedCertificate(cert.ID, &cert.Certificate) + tm.mutex.Lock() + if err != nil { + state = tm.certificateStates[cert.ID] + state.Status = "failed" + state.LastError = err.Error() + state.LastRetry = now + } + continue + } if state.Status == "failed" { if state.RetryCount >= effectiveMaxFailureRetries(&cert.Certificate) { if now.Sub(state.LastRetry) < failureRetryCooldownFor(&cert.Certificate) { diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go index 228491b04..e9d06a7ca 100644 --- a/packages/cmd/agent_cert_state.go +++ b/packages/cmd/agent_cert_state.go @@ -18,6 +18,7 @@ import ( "path/filepath" "runtime" "slices" + "strconv" "strings" "time" @@ -43,7 +44,7 @@ func validPersistedCertificateState(saved persistedCertificateState) bool { case "pending_issuance", "renewing": return saved.State.CertificateRequestID != "" case "failed": - return saved.State.CertificateID != "" || saved.State.CertificateRequestID != "" + return saved.State.CertificateID != "" || saved.State.CertificateRequestID != "" || (saved.State.LastError != "" && !saved.State.LastRetry.IsZero()) case "requesting_issuance", "requesting_renewal", "pending": return true default: @@ -64,6 +65,39 @@ func restoreRejectedCertificateRequest(certificate *AgentCertificateConfig, stat return requestErr } +func canonicalCertificatePath(path string) (string, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(absolute) + if err == nil { + return resolved, nil + } + if !errors.Is(err, os.ErrNotExist) { + return "", err + } + if info, statErr := os.Lstat(absolute); statErr == nil && info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(absolute) + if err != nil { + return "", err + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(absolute), target) + } + return canonicalCertificatePath(target) + } + parent := filepath.Dir(absolute) + if parent == absolute { + return "", err + } + resolvedParent, err := canonicalCertificatePath(parent) + if err != nil { + return "", err + } + return filepath.Join(resolvedParent, filepath.Base(absolute)), nil +} + func validateCertificateStatePaths(certificates []AgentCertificateConfig) error { outputs := make(map[string]bool) for _, certificate := range certificates { @@ -71,7 +105,7 @@ func validateCertificateStatePaths(certificates []AgentCertificateConfig) error if output == "" { continue } - absolute, err := filepath.Abs(output) + absolute, err := canonicalCertificatePath(output) if err != nil { return err } @@ -90,7 +124,7 @@ func validateCertificateStatePaths(certificates []AgentCertificateConfig) error continue } for _, statePath := range []string{certificateStatePath(&certificate), certificateStatePath(&certificate) + ".lock"} { - absolute, err := filepath.Abs(statePath) + absolute, err := canonicalCertificatePath(statePath) if err != nil { return err } @@ -139,6 +173,34 @@ func preflightCertificateOutputs(certificate *AgentCertificateConfig) error { return nil } +func certificateChainOutput(certificate *AgentCertificateConfig, chain string) (string, string, error) { + var output []byte + var der []byte + rest := []byte(chain) + for len(bytes.TrimSpace(rest)) > 0 { + block, remaining := pem.Decode(rest) + if block == nil || block.Type != "CERTIFICATE" { + return "", "", fmt.Errorf("certificate chain contains invalid PEM") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return "", "", err + } + rest = remaining + if chainOmitRoot(certificate) && cert.IsCA && bytes.Equal(cert.RawIssuer, cert.RawSubject) && cert.CheckSignatureFrom(cert) == nil { + continue + } + output = append(output, pem.EncodeToMemory(block)...) + der = append(der, cert.Raw...) + } + hash := sha256.Sum256(der) + return string(output), hex.EncodeToString(hash[:]), nil +} + +func chainOmitRoot(certificate *AgentCertificateConfig) bool { + return certificate.FileConfig.Chain.OmitRoot == nil || *certificate.FileConfig.Chain.OmitRoot +} + func certificateFingerprint(certificate *AgentCertificateConfig) (string, error) { attributes := buildCertificateAttributes(certificate) if attributes != nil { @@ -171,7 +233,10 @@ func lockCertificateState(certificate *AgentCertificateConfig) (func(), error) { if certificate.FileConfig.Certificate.Path == "" { return nil, fmt.Errorf("certificate.path is required in file-output configuration") } - statePath := certificateStatePath(certificate) + statePath, err := canonicalCertificatePath(certificateStatePath(certificate)) + if err != nil { + return nil, err + } if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil { return nil, fmt.Errorf("cannot create certificate state directory: %w", err) } @@ -192,6 +257,9 @@ func lockCertificateState(certificate *AgentCertificateConfig) (func(), error) { } func persistCertificateState(certificate *AgentCertificateConfig, state *CertificateState) error { + if err := validateCertificateStatePaths([]AgentCertificateConfig{*certificate}); err != nil { + return err + } fingerprint, err := certificateFingerprint(certificate) if err != nil { return err @@ -200,7 +268,10 @@ func persistCertificateState(certificate *AgentCertificateConfig, state *Certifi if err != nil { return err } - statePath := certificateStatePath(certificate) + statePath, err := canonicalCertificatePath(certificateStatePath(certificate)) + if err != nil { + return err + } if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil { return fmt.Errorf("cannot create certificate state directory: %w", err) } @@ -273,6 +344,13 @@ func certificateMatchesIdentity(leaf *x509.Certificate, certificate *AgentCertif return false } commonName, altNames = csr.Subject.CommonName, csr.DNSNames + altNames = append(slices.Clone(altNames), csr.EmailAddresses...) + for _, ip := range csr.IPAddresses { + altNames = append(altNames, ip.String()) + } + for _, uri := range csr.URIs { + altNames = append(altNames, uri.String()) + } } else if certificate.Attributes != nil { commonName, altNames = certificate.Attributes.CommonName, certificate.Attributes.AltNames algorithm := certificate.Attributes.KeyAlgorithm @@ -364,6 +442,18 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific } return tm.IssueCertificate(certificateID, certificate) } + if saved.State.Status == "failed" && saved.State.CertificateRequestID == "" { + interval := failureRetryIntervalFor(certificate) + if saved.State.RetryCount >= effectiveMaxFailureRetries(certificate) { + interval = failureRetryCooldownFor(certificate) + } + if time.Since(saved.State.LastRetry) < interval { + tm.mutex.Lock() + tm.certificateStates[certificateID] = &saved.State + tm.mutex.Unlock() + return nil + } + } tm.mutex.Lock() restored := saved.State if restored.CertificateRequestID == "" { @@ -428,11 +518,34 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific hash := sha256.Sum256(leaf.Raw) repair = repair || hex.EncodeToString(hash[:]) != saved.State.CertificateSHA256 } + var bundle *api.CertificateBundleResponse + chainHash := saved.State.ChainSHA256 + if certificate.FileConfig.Chain.Path != "" { + if chainHash == "" || saved.State.ChainOmitRoot != chainOmitRoot(certificate) { + bundle, err = api.CallGetCertificateBundle(httpClient, metadata.Certificate.ID) + if err != nil { + return err + } + _, chainHash, err = certificateChainOutput(certificate, bundle.CertificateChain) + if err != nil { + return err + } + } + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot read certificate chain: %w", err) + } + actualConfig := *certificate + omitRoot := false + actualConfig.FileConfig.Chain.OmitRoot = &omitRoot + _, actualHash, parseErr := certificateChainOutput(&actualConfig, string(chain)) + repair = repair || err != nil || parseErr != nil || actualHash != chainHash + } if !repair { if !certificateMatchesIdentity(leaf, certificate) { return fmt.Errorf("existing certificate does not match the configured identity") } - for _, output := range []string{certificate.FileConfig.PrivateKey.Path, certificate.FileConfig.Chain.Path} { + for _, output := range []string{certificate.FileConfig.PrivateKey.Path} { if output == "" { continue } @@ -446,15 +559,15 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific if _, err := tls.X509KeyPair(certificatePEM, outputData); err != nil { repair = true } - } else if _, err := parseAgentCertificate(outputData); err != nil { - repair = true } } } if repair { - bundle, err := api.CallGetCertificateBundle(httpClient, metadata.Certificate.ID) - if err != nil { - return err + if bundle == nil { + bundle, err = api.CallGetCertificateBundle(httpClient, metadata.Certificate.ID) + if err != nil { + return err + } } leaf, err = parseAgentCertificate([]byte(bundle.Certificate)) if err != nil || !certificateSerialMatches(metadata.Certificate.SerialNumber, leaf) || !certificateMatchesIdentity(leaf, certificate) { @@ -472,14 +585,30 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific return fmt.Errorf("cannot restore a matching private key: %w", err) } } - if certificate.FileConfig.Chain.Path != "" && bundle.CertificateChain == "" { - return fmt.Errorf("cannot restore configured chain output: certificate bundle has no chain") + chain, hash, err := certificateChainOutput(certificate, bundle.CertificateChain) + if err != nil { + return err } + chainHash = hash response := &api.CertificateResponse{Certificate: &api.CertificateData{ CertificateID: metadata.Certificate.ID, SerialNumber: bundle.SerialNumber, - Certificate: bundle.Certificate, PrivateKey: bundle.PrivateKey, CertificateChain: bundle.CertificateChain, + Certificate: bundle.Certificate, PrivateKey: bundle.PrivateKey, CertificateChain: chain, }} - if err := tm.WriteCertificateFiles(certificate, response); err != nil { + delivery := saved.State + if delivery.DeliveryHook == "" { + delivery.DeliveryHook = "issuance" + if saved.State.CertificateID != "" && saved.State.CertificateID != metadata.Certificate.ID { + delivery.DeliveryHook = "renewal" + } + } + delivery.CertificateID = metadata.Certificate.ID + setManagedCertificateState(&delivery, leaf, certificate) + delivery.ChainSHA256, delivery.ChainOmitRoot = chainHash, chainOmitRoot(certificate) + if err := persistCertificateState(certificate, &delivery); err != nil { + return err + } + saved.State = delivery + if err := tm.writeManagedCertificateFiles(certificate, response); err != nil { return err } } @@ -494,9 +623,14 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific state := tm.certificateStates[certificateID] state.CertificateID = metadata.Certificate.ID setManagedCertificateState(state, leaf, certificate) + state.ChainSHA256, state.ChainOmitRoot = chainHash, chainOmitRoot(certificate) + state.DeliveryHook = saved.State.DeliveryHook if err := persistCertificateState(certificate, state); err != nil { return err } + if err := tm.finishManagedCertificateDelivery(certificateID, certificate, state); err != nil { + return err + } if certificate.CSR == "" && tm.ShouldRenewCertificate(certificateID) { return tm.RenewCertificate(certificateID, certificate) } @@ -531,6 +665,10 @@ func (tm *AgentManager) completeManagedCertificate(certificateID int, certificat state.CertificateID = response.Certificate.CertificateID state.CertificateRequestID = "" state.Status = "active" + state.DeliveryHook = "issuance" + if renewal { + state.DeliveryHook = "renewal" + } // Save the identity before touching output files so a partial delivery can be repaired without another CA request. if err := persistCertificateState(certificate, state); err != nil { return err @@ -553,16 +691,48 @@ func (tm *AgentManager) completeManagedCertificate(certificateID int, certificat } } setManagedCertificateState(state, leaf, certificate) + certificateData := *response.Certificate + chain, hash, err := certificateChainOutput(certificate, certificateData.CertificateChain) + if err != nil { + return err + } + certificateData.CertificateChain = chain + state.ChainSHA256, state.ChainOmitRoot = hash, chainOmitRoot(certificate) if err := persistCertificateState(certificate, state); err != nil { return err } - if err := tm.WriteCertificateFiles(certificate, response); err != nil { + if err := tm.writeManagedCertificateFiles(certificate, &api.CertificateResponse{Certificate: &certificateData}); err != nil { return err } - if renewal { + return tm.finishManagedCertificateDelivery(certificateID, certificate, state) +} + +func (tm *AgentManager) finishManagedCertificateDelivery(certificateID int, certificate *AgentCertificateConfig, state *CertificateState) error { + switch state.DeliveryHook { + case "renewal": tm.ExecutePostHook(certificate.PostHooks.OnRenewal.Command, certificate.PostHooks.OnRenewal.Timeout, "renewal", certificateID, certificate) - } else { + case "issuance": tm.ExecutePostHook(certificate.PostHooks.OnIssuance.Command, certificate.PostHooks.OnIssuance.Timeout, "issuance", certificateID, certificate) + default: + return nil + } + state.DeliveryHook = "" + return persistCertificateState(certificate, state) +} + +func (tm *AgentManager) writeManagedCertificateFiles(certificate *AgentCertificateConfig, response *api.CertificateResponse) error { + if err := tm.WriteCertificateFiles(certificate, response); err != nil { + return err + } + if certificate.FileConfig.Chain.Path != "" && response.Certificate.CertificateChain == "" { + permission := os.FileMode(0600) + if configured, err := strconv.ParseUint(certificate.FileConfig.Chain.Permission, 8, 32); err == nil { + permission = os.FileMode(configured) + } + if err := os.MkdirAll(filepath.Dir(certificate.FileConfig.Chain.Path), 0755); err != nil { + return err + } + return os.WriteFile(certificate.FileConfig.Chain.Path, nil, permission) } return nil } diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go index 5a2b474b7..c453f8969 100644 --- a/packages/cmd/agent_cert_state_test.go +++ b/packages/cmd/agent_cert_state_test.go @@ -45,6 +45,18 @@ func agentTestCertificate(t *testing.T, serial int64, expires time.Time) *api.Ce } } +func agentTestRootCertificate(t *testing.T) string { + t.Helper() + publicKey, key, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + root := &x509.Certificate{SerialNumber: big.NewInt(100), Subject: pkix.Name{CommonName: "Test Root"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(365 * 24 * time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + der, err := x509.CreateCertificate(rand.Reader, root, root, publicKey, key) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) +} + func agentTestConfig(t *testing.T) *AgentCertificateConfig { t.Helper() dir := t.TempDir() @@ -67,6 +79,7 @@ func agentTestManager(certificate *AgentCertificateConfig) *AgentManager { } type agentTestAPI struct { + pollFailed atomic.Bool onIssue func() issueError atomic.Int32 issueCalls, renewCalls, pollCalls, listCalls, bundleCalls, statusCalls atomic.Int32 @@ -109,6 +122,11 @@ func newAgentTestAPI(t *testing.T, old, next *api.CertificateData) *agentTestAPI case r.URL.Path == prefix+"certificates/certificate-requests/request-1": backend.pollCalls.Add(1) response := api.GetCertificateRequestResponse{Status: "pending"} + if backend.pollFailed.Load() { + response.Status = "failed" + message := "provider rejected request" + response.ErrorMessage = &message + } if backend.pollIssued.Load() { response.Status = "issued" response.CertificateID = &backend.next.CertificateID @@ -120,6 +138,10 @@ func newAgentTestAPI(t *testing.T, old, next *api.CertificateData) *agentTestAPI _ = json.NewEncoder(w).Encode(response) case r.URL.Path == prefix+"certificate-profiles/profile/certificates": backend.listCalls.Add(1) + if r.URL.Query().Get("search") != backend.old.SerialNumber || r.URL.Query().Get("offset") != "0" || r.URL.Query().Get("limit") != "100" { + w.WriteHeader(http.StatusBadRequest) + return + } _ = json.NewEncoder(w).Encode(map[string]any{"certificates": []any{map[string]string{"id": backend.old.CertificateID, "serialNumber": backend.old.SerialNumber}}}) case strings.HasSuffix(r.URL.Path, "/bundle"): backend.bundleCalls.Add(1) @@ -421,6 +443,8 @@ func TestManagedCertificateRetriesDefinitivelyRejectedRequests(t *testing.T) { func TestManagedCertificateRecoversInterruptedFileDelivery(t *testing.T) { certificate := agentTestConfig(t) + hookOutput := filepath.Join(t.TempDir(), "hook-ran") + certificate.PostHooks.OnIssuance.Command = fmt.Sprintf("touch %q", hookOutput) old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) backend := newAgentTestAPI(t, old, next) @@ -433,6 +457,7 @@ func TestManagedCertificateRecoversInterruptedFileDelivery(t *testing.T) { chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) require.NoError(t, err) assert.Equal(t, next.CertificateChain, string(chain)) + require.Eventually(t, func() bool { _, err := os.Stat(hookOutput); return err == nil }, 2*time.Second, 10*time.Millisecond) } func TestManagedCertificateConfigChangesIssueOnlyOnce(t *testing.T) { @@ -496,3 +521,102 @@ func TestManagedCertificateDoesNotIssueWhenCertificateIsUnreadable(t *testing.T) assert.Zero(t, backend.issueCalls.Load()) assert.Contains(t, manager.certificateStates[1].LastError, "cannot read existing certificate") } + +func TestManagedCertificateRetriesFailedAsyncRequestAfterRetryInterval(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + backend.pending.Store(true) + backend.pollFailed.Store(true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + backend.pending.Store(false) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + data, err := os.ReadFile(certificateStatePath(certificate)) + require.NoError(t, err) + var saved persistedCertificateState + require.NoError(t, json.Unmarshal(data, &saved)) + saved.State.LastRetry = time.Now().Add(-48 * time.Hour) + require.NoError(t, persistCertificateState(certificate, &saved.State)) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.EqualValues(t, 2, backend.issueCalls.Load()) +} + +func TestManagedCertificateDoesNotRequestWhenOutputsAreUnwritable(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permission checks require an unprivileged user") + } + certificate := agentTestConfig(t) + dir := t.TempDir() + certificate.FileConfig.Chain.Path = filepath.Join(dir, "chain.pem") + require.NoError(t, os.Chmod(dir, 0500)) + t.Cleanup(func() { _ = os.Chmod(dir, 0700) }) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + manager := runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + assert.Zero(t, backend.issueCalls.Load()) + assert.Contains(t, manager.certificateStates[1].LastError, "cannot write certificate output") +} + +func TestManagedCertificateRepairsValidButWrongChain(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + wrong := agentTestCertificate(t, 99, time.Now().Add(90*24*time.Hour)) + require.NoError(t, os.WriteFile(certificate.FileConfig.Chain.Path, []byte(wrong.Certificate), 0600)) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + require.NoError(t, err) + assert.Equal(t, old.CertificateChain, string(chain)) + assert.Zero(t, backend.issueCalls.Load()) +} + +func TestCertificateStatePathRejectsSymlinkAliasToOutput(t *testing.T) { + certificate := agentTestConfig(t) + aliasRoot := t.TempDir() + alias := filepath.Join(aliasRoot, "outputs") + require.NoError(t, os.Symlink(filepath.Dir(certificate.FileConfig.PrivateKey.Path), alias)) + certificate.Lifecycle.StatePath = filepath.Join(alias, filepath.Base(certificate.FileConfig.PrivateKey.Path)) + require.ErrorContains(t, validateCertificateStatePaths([]AgentCertificateConfig{*certificate}), "overlaps") +} + +func TestManagedCertificateRetriesContendedStateLock(t *testing.T) { + certificate := agentTestConfig(t) + certificate.Lifecycle.FailureRetryInterval = "10ms" + certificate.Lifecycle.StatusCheckInterval = "10ms" + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + unlock, err := lockCertificateState(certificate) + require.NoError(t, err) + released := false + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { + if !released && manager.certificateStates[1].Status == "blocked" { + unlock() + released = true + return false + } + return manager.certificateStates[1].Status == "active" + }) + assert.True(t, released) + assert.Zero(t, backend.issueCalls.Load()) +} + +func TestManagedCertificateAppliesChangedChainRootPolicy(t *testing.T) { + certificate := agentTestConfig(t) + includeRoot := false + certificate.FileConfig.Chain.OmitRoot = &includeRoot + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + old.CertificateChain = agentTestRootCertificate(t) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + omitRoot := true + certificate.FileConfig.Chain.OmitRoot = &omitRoot + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + require.NoError(t, err) + assert.Empty(t, chain) + assert.Zero(t, backend.issueCalls.Load()) +} From d5d0fee283fbbb5d54d1afbe5d809e68275547e5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:05:14 +0000 Subject: [PATCH 3/8] fix(agent): reject absent chains and preserve hook semantics Co-Authored-By: ashwin --- packages/cmd/agent_cert_state.go | 21 +++++++++++---------- packages/cmd/agent_cert_state_test.go | 23 ++++++++++++++++++++++- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go index e9d06a7ca..8f1b9d893 100644 --- a/packages/cmd/agent_cert_state.go +++ b/packages/cmd/agent_cert_state.go @@ -174,6 +174,13 @@ func preflightCertificateOutputs(certificate *AgentCertificateConfig) error { } func certificateChainOutput(certificate *AgentCertificateConfig, chain string) (string, string, error) { + if certificate.FileConfig.Chain.Path != "" && strings.TrimSpace(chain) == "" { + return "", "", fmt.Errorf("configured certificate chain is missing from the remote bundle") + } + return formatCertificateChain(chain, chainOmitRoot(certificate)) +} + +func formatCertificateChain(chain string, omitRoot bool) (string, string, error) { var output []byte var der []byte rest := []byte(chain) @@ -187,7 +194,7 @@ func certificateChainOutput(certificate *AgentCertificateConfig, chain string) ( return "", "", err } rest = remaining - if chainOmitRoot(certificate) && cert.IsCA && bytes.Equal(cert.RawIssuer, cert.RawSubject) && cert.CheckSignatureFrom(cert) == nil { + if omitRoot && cert.IsCA && bytes.Equal(cert.RawIssuer, cert.RawSubject) && cert.CheckSignatureFrom(cert) == nil { continue } output = append(output, pem.EncodeToMemory(block)...) @@ -535,10 +542,7 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("cannot read certificate chain: %w", err) } - actualConfig := *certificate - omitRoot := false - actualConfig.FileConfig.Chain.OmitRoot = &omitRoot - _, actualHash, parseErr := certificateChainOutput(&actualConfig, string(chain)) + _, actualHash, parseErr := formatCertificateChain(string(chain), false) repair = repair || err != nil || parseErr != nil || actualHash != chainHash } if !repair { @@ -595,11 +599,8 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific Certificate: bundle.Certificate, PrivateKey: bundle.PrivateKey, CertificateChain: chain, }} delivery := saved.State - if delivery.DeliveryHook == "" { - delivery.DeliveryHook = "issuance" - if saved.State.CertificateID != "" && saved.State.CertificateID != metadata.Certificate.ID { - delivery.DeliveryHook = "renewal" - } + if delivery.DeliveryHook == "" && saved.State.CertificateID != "" && saved.State.CertificateID != metadata.Certificate.ID { + delivery.DeliveryHook = "renewal" } delivery.CertificateID = metadata.Certificate.ID setManagedCertificateState(&delivery, leaf, certificate) diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go index c453f8969..8cc55be0f 100644 --- a/packages/cmd/agent_cert_state_test.go +++ b/packages/cmd/agent_cert_state_test.go @@ -79,6 +79,7 @@ func agentTestManager(certificate *AgentCertificateConfig) *AgentManager { } type agentTestAPI struct { + missingChain atomic.Bool pollFailed atomic.Bool onIssue func() issueError atomic.Int32 @@ -149,9 +150,13 @@ func newAgentTestAPI(t *testing.T, old, next *api.CertificateData) *agentTestAPI if strings.Contains(r.URL.Path, backend.next.CertificateID+"/") { certificate = backend.next } + chain := certificate.CertificateChain + if backend.missingChain.Load() { + chain = "" + } _ = json.NewEncoder(w).Encode(api.CertificateBundleResponse{ Certificate: certificate.Certificate, PrivateKey: certificate.PrivateKey, - SerialNumber: certificate.SerialNumber, CertificateChain: certificate.CertificateChain, + SerialNumber: certificate.SerialNumber, CertificateChain: chain, }) case strings.HasPrefix(r.URL.Path, prefix+"certificates/"): backend.statusCalls.Add(1) @@ -560,6 +565,8 @@ func TestManagedCertificateDoesNotRequestWhenOutputsAreUnwritable(t *testing.T) func TestManagedCertificateRepairsValidButWrongChain(t *testing.T) { certificate := agentTestConfig(t) + hookOutput := filepath.Join(t.TempDir(), "issuance-hook") + certificate.PostHooks.OnIssuance.Command = fmt.Sprintf("touch %q", hookOutput) old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) seedAgentCertificate(t, certificate, old, true) @@ -570,6 +577,20 @@ func TestManagedCertificateRepairsValidButWrongChain(t *testing.T) { require.NoError(t, err) assert.Equal(t, old.CertificateChain, string(chain)) assert.Zero(t, backend.issueCalls.Load()) + assert.Never(t, func() bool { _, err := os.Stat(hookOutput); return err == nil }, 200*time.Millisecond, 10*time.Millisecond) +} + +func TestManagedCertificateDoesNotEraseChainWhenBundleIsEmpty(t *testing.T) { + certificate := agentTestConfig(t) + old := agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)) + backend := newAgentTestAPI(t, old, agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour))) + seedAgentCertificate(t, certificate, old, true) + backend.missingChain.Store(true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + require.NoError(t, err) + assert.Equal(t, old.CertificateChain, string(chain)) + assert.Zero(t, backend.issueCalls.Load()) } func TestCertificateStatePathRejectsSymlinkAliasToOutput(t *testing.T) { From e84e3f9f926d976097af289a8d35a1ba7471bc5a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:29:27 +0000 Subject: [PATCH 4/8] fix(agent): recover root-omitted chains and retain lifecycle logs Co-Authored-By: ashwin --- e2e/agent/agent_helpers.go | 5 ++- packages/cmd/agent.go | 3 -- packages/cmd/agent_cert_state.go | 27 ++++++++++++++- packages/cmd/agent_cert_state_test.go | 47 +++++++++++++++++++++++++-- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/e2e/agent/agent_helpers.go b/e2e/agent/agent_helpers.go index 6b1554a47..a545827c9 100644 --- a/e2e/agent/agent_helpers.go +++ b/e2e/agent/agent_helpers.go @@ -603,6 +603,7 @@ type agentCertificateFileOutput struct { type agentFileOutputEntry struct { Path string `yaml:"path"` Permission string `yaml:"permission,omitempty"` + OmitRoot *bool `yaml:"omit-root,omitempty"` } type agentCertificatePostHooks struct { @@ -621,6 +622,8 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin var certs []agentCertificateConfig for _, cert := range opts.Certificates { + // These fixtures assert a nonempty chain for certificates signed directly by a root CA. + omitRoot := false c := agentCertificateConfig{ ProjectSlug: cert.ProjectSlug, ApplicationName: cert.ApplicationName, @@ -645,7 +648,7 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin FileOutput: agentCertificateFileOutput{ Certificate: agentFileOutputEntry{Path: cert.CertPath, Permission: cert.CertPermission}, PrivateKey: agentFileOutputEntry{Path: cert.KeyPath, Permission: cert.KeyPermission}, - Chain: agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission}, + Chain: agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission, OmitRoot: &omitRoot}, }, } diff --git a/packages/cmd/agent.go b/packages/cmd/agent.go index 0dd991cc8..13d88042b 100644 --- a/packages/cmd/agent.go +++ b/packages/cmd/agent.go @@ -2730,9 +2730,6 @@ func (tm *AgentManager) checkCertificateRequestStatus(certificateId int, certifi return err } - displayName := tm.getCertificateDisplayName(certificateId, certificate) - log.Info().Str("Certificate", displayName).Str("serial", *response.SerialNumber).Msg("certificate issued successfully") - case "failed": errorMsg := "unknown error" if response.ErrorMessage != nil { diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go index 8f1b9d893..2abc6cda9 100644 --- a/packages/cmd/agent_cert_state.go +++ b/packages/cmd/agent_cert_state.go @@ -26,6 +26,7 @@ import ( "github.com/Infisical/infisical-merge/packages/config" "github.com/Infisical/infisical-merge/packages/util" "github.com/gofrs/flock" + "github.com/rs/zerolog/log" ) type persistedCertificateState struct { @@ -693,6 +694,21 @@ func (tm *AgentManager) completeManagedCertificate(certificateID int, certificat } setManagedCertificateState(state, leaf, certificate) certificateData := *response.Certificate + if certificate.FileConfig.Chain.Path != "" && strings.TrimSpace(certificateData.CertificateChain) == "" { + httpClient, err := tm.createAuthenticatedClient() + if err != nil { + return err + } + bundle, err := api.CallGetCertificateBundle(httpClient, state.CertificateID) + if err != nil { + return err + } + bundleLeaf, err := parseAgentCertificate([]byte(bundle.Certificate)) + if err != nil || !bytes.Equal(bundleLeaf.Raw, leaf.Raw) { + return fmt.Errorf("stored certificate bundle does not match the issued certificate") + } + certificateData.CertificateChain = bundle.CertificateChain + } chain, hash, err := certificateChainOutput(certificate, certificateData.CertificateChain) if err != nil { return err @@ -705,7 +721,16 @@ func (tm *AgentManager) completeManagedCertificate(certificateID int, certificat if err := tm.writeManagedCertificateFiles(certificate, &api.CertificateResponse{Certificate: &certificateData}); err != nil { return err } - return tm.finishManagedCertificateDelivery(certificateID, certificate, state) + if err := tm.finishManagedCertificateDelivery(certificateID, certificate, state); err != nil { + return err + } + event := log.Info().Str("Certificate", tm.getCertificateDisplayName(certificateID, certificate)).Str("serial", state.SerialNumber) + if renewal { + event.Msg("certificate renewed successfully") + } else { + event.Msg("certificate issued successfully") + } + return nil } func (tm *AgentManager) finishManagedCertificateDelivery(certificateID int, certificate *AgentCertificateConfig, state *CertificateState) error { diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go index 8cc55be0f..bbbe0a299 100644 --- a/packages/cmd/agent_cert_state_test.go +++ b/packages/cmd/agent_cert_state_test.go @@ -46,6 +46,12 @@ func agentTestCertificate(t *testing.T, serial int64, expires time.Time) *api.Ce } func agentTestRootCertificate(t *testing.T) string { + t.Helper() + _, _, certificate := agentTestRoot(t) + return certificate +} + +func agentTestRoot(t *testing.T) (*x509.Certificate, ed25519.PrivateKey, string) { t.Helper() publicKey, key, err := ed25519.GenerateKey(rand.Reader) require.NoError(t, err) @@ -54,7 +60,22 @@ func agentTestRootCertificate(t *testing.T) string { IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} der, err := x509.CreateCertificate(rand.Reader, root, root, publicKey, key) require.NoError(t, err) - return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + parsed, err := x509.ParseCertificate(der) + require.NoError(t, err) + return parsed, key, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) +} + +func agentTestRootSignedCertificate(t *testing.T, serial int64) *api.CertificateData { + t.Helper() + certificate := agentTestCertificate(t, serial, time.Now().Add(90*24*time.Hour)) + leaf, err := parseAgentCertificate([]byte(certificate.Certificate)) + require.NoError(t, err) + root, key, rootPEM := agentTestRoot(t) + der, err := x509.CreateCertificate(rand.Reader, leaf, root, leaf.PublicKey, key) + require.NoError(t, err) + certificate.Certificate = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + certificate.CertificateChain = rootPEM + return certificate } func agentTestConfig(t *testing.T) *AgentCertificateConfig { @@ -79,6 +100,7 @@ func agentTestManager(certificate *AgentCertificateConfig) *AgentManager { } type agentTestAPI struct { + omitResponseChain atomic.Bool missingChain atomic.Bool pollFailed atomic.Bool onIssue func() @@ -108,7 +130,11 @@ func newAgentTestAPI(t *testing.T, old, next *api.CertificateData) *agentTestAPI if backend.pending.Load() { _ = json.NewEncoder(w).Encode(api.CertificateResponse{CertificateRequestID: "request-1"}) } else { - _ = json.NewEncoder(w).Encode(api.CertificateResponse{Certificate: backend.next}) + certificate := *backend.next + if backend.omitResponseChain.Load() { + certificate.CertificateChain = "" + } + _ = json.NewEncoder(w).Encode(api.CertificateResponse{Certificate: &certificate}) } case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/renew"): backend.renewCalls.Add(1) @@ -593,6 +619,23 @@ func TestManagedCertificateDoesNotEraseChainWhenBundleIsEmpty(t *testing.T) { assert.Zero(t, backend.issueCalls.Load()) } +func TestManagedCertificateDeliversRootOmittedEmptyResponseChain(t *testing.T) { + certificate := agentTestConfig(t) + next := agentTestRootSignedCertificate(t, 2) + backend := newAgentTestAPI(t, agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)), next) + backend.omitResponseChain.Store(true) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + chain, err := os.ReadFile(certificate.FileConfig.Chain.Path) + require.NoError(t, err) + assert.Empty(t, chain) + cert, err := os.ReadFile(certificate.FileConfig.Certificate.Path) + require.NoError(t, err) + assert.Equal(t, next.Certificate, string(cert)) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.EqualValues(t, 1, backend.bundleCalls.Load()) +} + func TestCertificateStatePathRejectsSymlinkAliasToOutput(t *testing.T) { certificate := agentTestConfig(t) aliasRoot := t.TempDir() From 42a812bc56b7fbe4a0c34a989bd83ec844ab8585 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:35:37 +0000 Subject: [PATCH 5/8] test(agent): cover default root omission across restarts Co-Authored-By: ashwin --- e2e/agent/agent_helpers.go | 10 ++-- e2e/agent/certificate_test.go | 86 +++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/e2e/agent/agent_helpers.go b/e2e/agent/agent_helpers.go index a545827c9..44b7260c6 100644 --- a/e2e/agent/agent_helpers.go +++ b/e2e/agent/agent_helpers.go @@ -622,8 +622,11 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin var certs []agentCertificateConfig for _, cert := range opts.Certificates { - // These fixtures assert a nonempty chain for certificates signed directly by a root CA. - omitRoot := false + var omitRoot *bool + if cert.IncludeRootInChain { + omit := false + omitRoot = &omit + } c := agentCertificateConfig{ ProjectSlug: cert.ProjectSlug, ApplicationName: cert.ApplicationName, @@ -648,7 +651,7 @@ func (h *CertAgentTestHelper) GenerateAgentConfig(opts AgentConfigOptions) strin FileOutput: agentCertificateFileOutput{ Certificate: agentFileOutputEntry{Path: cert.CertPath, Permission: cert.CertPermission}, PrivateKey: agentFileOutputEntry{Path: cert.KeyPath, Permission: cert.KeyPermission}, - Chain: agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission, OmitRoot: &omitRoot}, + Chain: agentFileOutputEntry{Path: cert.ChainPath, Permission: cert.ChainPermission, OmitRoot: omitRoot}, }, } @@ -727,6 +730,7 @@ type CertificateConfigEntry struct { CertPermission string KeyPermission string ChainPermission string + IncludeRootInChain bool PostHookOnFailure string CertificateID string ReplaceOnRenewals bool diff --git a/e2e/agent/certificate_test.go b/e2e/agent/certificate_test.go index 9b429fb1b..dc231fb07 100644 --- a/e2e/agent/certificate_test.go +++ b/e2e/agent/certificate_test.go @@ -3,6 +3,7 @@ package agent_test import ( "context" "crypto/x509" + "encoding/json" "encoding/pem" "fmt" "log/slog" @@ -122,6 +123,7 @@ func certAgent_BasicCertificateIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, }, }, }) @@ -162,6 +164,75 @@ func certAgent_BasicCertificateIssuance(t *testing.T) { agentHelpers.VerifyCertificateCommonName(t, certPath, "test.example.com") } +func certAgent_DefaultRootOmissionSurvivesRestart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + helper := setupCertAgentTest(t, ctx) + certDir := filepath.Join(helper.TempDir, "certs") + require.NoError(t, os.MkdirAll(certDir, 0755)) + certPath, keyPath, chainPath := agentHelpers.CertFilePaths(certDir) + clientIDPath, clientSecretPath := helper.WriteCredentialFiles() + configPath := helper.GenerateAgentConfig(agentHelpers.AgentConfigOptions{ + ClientIDPath: clientIDPath, ClientSecretPath: clientSecretPath, + Certificates: []agentHelpers.CertificateConfigEntry{{ + ApplicationName: helper.ApplicationName, ProfileSlug: helper.ProfileSlug, + CommonName: "default-chain.example.com", TTL: "1h", RenewBeforeExpiry: "10m", StatusCheckInterval: "5s", + CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + }}, + }) + config, err := os.ReadFile(configPath) + require.NoError(t, err) + require.NotContains(t, string(config), "omit-root:") + runAgent := func() *helpers.Command { + cmd := &helpers.Command{Test: t, Args: []string{"cert-manager", "agent", "--config", configPath, "--verbose"}, Env: map[string]string{}} + cmd.Start(ctx) + t.Cleanup(func() { + if t.Failed() { + t.Logf("Agent stderr:\n%s", cmd.Stderr()) + } + cmd.Stop() + }) + return cmd + } + first := runAgent() + result := helpers.WaitForStderr(t, helpers.WaitForStderrOptions{ + EnsureCmdRunning: first, ExpectedString: "certificate issued successfully", Timeout: 120 * time.Second, Interval: 2 * time.Second, + }) + require.Equal(t, helpers.WaitSuccess, result, "initial issuance failed: %s", first.Stderr()) + agentHelpers.VerifyCertificateFile(t, certPath) + agentHelpers.VerifyPrivateKeyFile(t, keyPath) + chain, err := os.ReadFile(chainPath) + require.NoError(t, err) + require.Empty(t, chain) + initialSerial := readCertificateSerial(t, certPath) + statePath := certPath + ".infisical-state.json" + stateWrittenAt := fileModTime(t, statePath) + first.Stop() + second := runAgent() + require.Eventually(t, func() bool { + info, err := os.Stat(statePath) + if err != nil || !info.ModTime().After(stateWrittenAt) { + return false + } + data, err := os.ReadFile(statePath) + if err != nil { + return false + } + var state struct { + State struct { + Status string `json:"status"` + } `json:"state"` + } + return json.Unmarshal(data, &state) == nil && state.State.Status == "active" + }, 120*time.Second, 2*time.Second, "restart recovery failed: %s", second.Stderr()) + require.True(t, second.IsRunning()) + require.Equal(t, initialSerial, readCertificateSerial(t, certPath)) + require.NotContains(t, second.Stderr(), "issuing certificate") + chain, err = os.ReadFile(chainPath) + require.NoError(t, err) + require.Empty(t, chain) +} + func certAgent_CertificateRenewal(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -349,6 +420,7 @@ func certAgent_MultipleCertificates(t *testing.T) { CertPath: certPath1, KeyPath: keyPath1, ChainPath: chainPath1, + IncludeRootInChain: true, }, { ApplicationName: helper.ApplicationName, @@ -360,6 +432,7 @@ func certAgent_MultipleCertificates(t *testing.T) { CertPath: certPath2, KeyPath: keyPath2, ChainPath: chainPath2, + IncludeRootInChain: true, }, }, }) @@ -435,6 +508,7 @@ func certAgent_FilePermissions(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, CertPermission: "0644", KeyPermission: "0640", ChainPermission: "0644", @@ -502,6 +576,7 @@ func certAgent_AltNames(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, AltNames: expectedAltNames, }, }, @@ -570,6 +645,7 @@ func certAgent_CSRBasedIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, CSR: csrPEM, }, }, @@ -647,6 +723,7 @@ func certAgent_CSRPathBasedIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, CSRPath: csrFilePath, }, }, @@ -714,6 +791,7 @@ func certAgent_AcmeCA_CertificateIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, }, }, }) @@ -1044,6 +1122,7 @@ func certAgent_AcmeCA_MultipleCertificates(t *testing.T) { CertPath: certPath1, KeyPath: keyPath1, ChainPath: chainPath1, + IncludeRootInChain: true, }, { ApplicationName: helper.ApplicationName, @@ -1055,6 +1134,7 @@ func certAgent_AcmeCA_MultipleCertificates(t *testing.T) { CertPath: certPath2, KeyPath: keyPath2, ChainPath: chainPath2, + IncludeRootInChain: true, }, }, }) @@ -1208,6 +1288,7 @@ func certAgent_AcmeCA_CSRBasedIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, CSR: csrPEM, }, }, @@ -1360,6 +1441,7 @@ func certAgent_CertificateWithFullAttributes(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, KeyAlgorithm: "RSA_2048", KeyUsages: []string{"digital_signature", "key_encipherment"}, ExtendedKeyUsages: []string{"server_auth"}, @@ -1873,6 +1955,7 @@ func certAgent_SignatureAlgorithm(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, KeyAlgorithm: "RSA_2048", SignatureAlgorithm: "RSA-SHA256", }, @@ -1944,6 +2027,7 @@ func certAgent_V1LegacyIssuance(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, }, }, }) @@ -2011,6 +2095,7 @@ func certAgent_V1ValidationRejectsAppOnly(t *testing.T) { CertPath: certPath, KeyPath: keyPath, ChainPath: chainPath, + IncludeRootInChain: true, }, }, }) @@ -2537,6 +2622,7 @@ func normalizeSerial(serial string) string { func TestCertAgent_InternalCA(t *testing.T) { t.Run("BasicCertificateIssuance", certAgent_BasicCertificateIssuance) + t.Run("DefaultRootOmissionSurvivesRestart", certAgent_DefaultRootOmissionSurvivesRestart) t.Run("CertificateRenewal", certAgent_CertificateRenewal) t.Run("PostHookExecution", certAgent_PostHookExecution) t.Run("MultipleCertificates", certAgent_MultipleCertificates) From 4c6a8d5ca0049a55713de632772101730e5c4b19 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:01:30 +0000 Subject: [PATCH 6/8] fix(agent): preserve externally managed CSR private keys Co-Authored-By: ashwin --- packages/cmd/agent_cert_state.go | 53 +++++++++++++++------------ packages/cmd/agent_cert_state_test.go | 47 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go index 2abc6cda9..40e60b5ee 100644 --- a/packages/cmd/agent_cert_state.go +++ b/packages/cmd/agent_cert_state.go @@ -394,6 +394,26 @@ func certificateMatchesIdentity(leaf *x509.Certificate, certificate *AgentCertif return true } +func validateManagedCertificateKey(certificate *AgentCertificateConfig, certificatePEM, key []byte) error { + if len(key) == 0 { + if certificate.FileConfig.PrivateKey.Path == "" { + return nil + } + var err error + key, err = os.ReadFile(certificate.FileConfig.PrivateKey.Path) + if errors.Is(err, os.ErrNotExist) && certificate.CSR != "" { + return nil + } + if err != nil { + return fmt.Errorf("certificate private key is unavailable: %w", err) + } + } + if _, err := tls.X509KeyPair(certificatePEM, key); err != nil { + return fmt.Errorf("certificate does not match its private key: %w", err) + } + return nil +} + func (tm *AgentManager) recoverCertificateID(certificate *AgentCertificateConfig, leaf *x509.Certificate) (string, error) { httpClient, err := tm.createAuthenticatedClient() if err != nil { @@ -555,6 +575,9 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific continue } outputData, err := os.ReadFile(output) + if errors.Is(err, os.ErrNotExist) && certificate.CSR != "" { + continue + } if err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("cannot read certificate output %s: %w", output, err) } @@ -578,17 +601,8 @@ func (tm *AgentManager) initializeManagedCertificate(certificateID int, certific if err != nil || !certificateSerialMatches(metadata.Certificate.SerialNumber, leaf) || !certificateMatchesIdentity(leaf, certificate) { return fmt.Errorf("stored certificate bundle does not match the certificate identity") } - if certificate.FileConfig.PrivateKey.Path != "" { - key := []byte(bundle.PrivateKey) - if len(key) == 0 { - key, err = os.ReadFile(certificate.FileConfig.PrivateKey.Path) - if err != nil { - return fmt.Errorf("cannot restore the certificate's locally managed private key: %w", err) - } - } - if _, err := tls.X509KeyPair([]byte(bundle.Certificate), key); err != nil { - return fmt.Errorf("cannot restore a matching private key: %w", err) - } + if err := validateManagedCertificateKey(certificate, []byte(bundle.Certificate), []byte(bundle.PrivateKey)); err != nil { + return err } chain, hash, err := certificateChainOutput(certificate, bundle.CertificateChain) if err != nil { @@ -679,18 +693,11 @@ func (tm *AgentManager) completeManagedCertificate(certificateID int, certificat if err != nil { return fmt.Errorf("cannot parse issued certificate: %w", err) } - if response.Certificate.PrivateKey != "" { - if _, err := tls.X509KeyPair([]byte(response.Certificate.Certificate), []byte(response.Certificate.PrivateKey)); err != nil { - return fmt.Errorf("issued certificate does not match its private key: %w", err) - } - } else if certificate.FileConfig.PrivateKey.Path != "" { - key, err := os.ReadFile(certificate.FileConfig.PrivateKey.Path) - if err != nil { - return fmt.Errorf("issued certificate has no private key and no locally managed key is available: %w", err) - } - if _, err := tls.X509KeyPair([]byte(response.Certificate.Certificate), key); err != nil { - return fmt.Errorf("issued certificate does not match the locally managed private key: %w", err) - } + if !certificateMatchesIdentity(leaf, certificate) { + return fmt.Errorf("issued certificate does not match the configured identity") + } + if err := validateManagedCertificateKey(certificate, []byte(response.Certificate.Certificate), []byte(response.Certificate.PrivateKey)); err != nil { + return err } setManagedCertificateState(state, leaf, certificate) certificateData := *response.Certificate diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go index bbbe0a299..618a5e755 100644 --- a/packages/cmd/agent_cert_state_test.go +++ b/packages/cmd/agent_cert_state_test.go @@ -636,6 +636,53 @@ func TestManagedCertificateDeliversRootOmittedEmptyResponseChain(t *testing.T) { assert.EqualValues(t, 1, backend.bundleCalls.Load()) } +func TestManagedCertificateCSRWithExternalPrivateKey(t *testing.T) { + for _, mode := range []string{"missing external key", "existing external key", "mismatched external key", "mismatched CSR"} { + t.Run(mode, func(t *testing.T) { + certificate := agentTestConfig(t) + next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + privateKey := next.PrivateKey + csrKey := privateKey + if mode == "mismatched CSR" { + csrKey = agentTestCertificate(t, 3, time.Now().Add(90*24*time.Hour)).PrivateKey + } + block, _ := pem.Decode([]byte(csrKey)) + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + require.NoError(t, err) + leaf, err := parseAgentCertificate([]byte(next.Certificate)) + require.NoError(t, err) + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: leaf.Subject, DNSNames: leaf.DNSNames}, key) + require.NoError(t, err) + certificate.CSR = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csr})) + if mode == "existing external key" { + require.NoError(t, os.WriteFile(certificate.FileConfig.PrivateKey.Path, []byte(privateKey), 0600)) + } + if mode == "mismatched external key" { + require.NoError(t, os.WriteFile(certificate.FileConfig.PrivateKey.Path, []byte(agentTestCertificate(t, 3, time.Now().Add(90*24*time.Hour)).PrivateKey), 0600)) + } + next.PrivateKey = "" + backend := newAgentTestAPI(t, agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)), next) + if strings.HasPrefix(mode, "mismatched") { + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "failed" }) + } else { + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + require.NoError(t, os.Remove(certificate.FileConfig.Certificate.Path)) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + key, err := os.ReadFile(certificate.FileConfig.PrivateKey.Path) + if mode == "missing external key" { + assert.ErrorIs(t, err, os.ErrNotExist) + } else { + require.NoError(t, err) + assert.Equal(t, privateKey, string(key)) + } + assert.EqualValues(t, 1, backend.bundleCalls.Load()) + } + assert.EqualValues(t, 1, backend.issueCalls.Load()) + }) + } +} + func TestCertificateStatePathRejectsSymlinkAliasToOutput(t *testing.T) { certificate := agentTestConfig(t) aliasRoot := t.TempDir() From f4534caad1e537ea7d71796c61f04a3aacb5f7d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:38:37 +0000 Subject: [PATCH 7/8] fix(agent): recognize SAN-only certificate identities Co-Authored-By: ashwin --- packages/cmd/agent_cert_state.go | 9 ++++-- packages/cmd/agent_cert_state_test.go | 40 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/packages/cmd/agent_cert_state.go b/packages/cmd/agent_cert_state.go index 40e60b5ee..8ad1b5d1c 100644 --- a/packages/cmd/agent_cert_state.go +++ b/packages/cmd/agent_cert_state.go @@ -375,9 +375,6 @@ func certificateMatchesIdentity(leaf *x509.Certificate, certificate *AgentCertif } } } - if commonName != "" && commonName != leaf.Subject.CommonName { - return false - } actual := slices.Clone(leaf.DNSNames) actual = append(actual, leaf.EmailAddresses...) for _, ip := range leaf.IPAddresses { @@ -386,6 +383,12 @@ func certificateMatchesIdentity(leaf *x509.Certificate, certificate *AgentCertif for _, uri := range leaf.URIs { actual = append(actual, uri.String()) } + if commonName != "" && commonName != leaf.Subject.CommonName { + // Some CAs omit the subject common name and encode the requested identity only in SANs. + if leaf.Subject.CommonName != "" || !slices.Contains(actual, commonName) { + return false + } + } for _, name := range altNames { if !slices.Contains(actual, name) { return false diff --git a/packages/cmd/agent_cert_state_test.go b/packages/cmd/agent_cert_state_test.go index 618a5e755..78568b5cd 100644 --- a/packages/cmd/agent_cert_state_test.go +++ b/packages/cmd/agent_cert_state_test.go @@ -683,6 +683,46 @@ func TestManagedCertificateCSRWithExternalPrivateKey(t *testing.T) { } } +func TestManagedCertificateReusesSANOnlyIdentity(t *testing.T) { + for _, csrMode := range []bool{false, true} { + t.Run(fmt.Sprintf("CSR=%t", csrMode), func(t *testing.T) { + certificate := agentTestConfig(t) + next := agentTestCertificate(t, 2, time.Now().Add(90*24*time.Hour)) + leaf, err := parseAgentCertificate([]byte(next.Certificate)) + require.NoError(t, err) + block, _ := pem.Decode([]byte(next.PrivateKey)) + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + require.NoError(t, err) + if csrMode { + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: leaf.Subject, DNSNames: leaf.DNSNames}, key) + require.NoError(t, err) + certificate.CSR = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csr})) + } + leaf.Subject.CommonName = "" + leaf.RawSubject = nil + assert.True(t, certificateMatchesIdentity(leaf, certificate)) + unrelated := *leaf + unrelated.DNSNames = []string{"unrelated.example.test"} + assert.False(t, certificateMatchesIdentity(&unrelated, certificate)) + unrelated = *leaf + unrelated.Subject.CommonName = "unrelated.example.test" + assert.False(t, certificateMatchesIdentity(&unrelated, certificate)) + der, err := x509.CreateCertificate(rand.Reader, leaf, leaf, leaf.PublicKey, key) + require.NoError(t, err) + next.Certificate = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + issued, err := parseAgentCertificate([]byte(next.Certificate)) + require.NoError(t, err) + assert.Empty(t, issued.Subject.CommonName) + next.CertificateChain = next.Certificate + backend := newAgentTestAPI(t, agentTestCertificate(t, 1, time.Now().Add(90*24*time.Hour)), next) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + runAgentMonitor(t, certificate, func(manager *AgentManager) bool { return manager.certificateStates[1].Status == "active" }) + assert.EqualValues(t, 1, backend.issueCalls.Load()) + assert.EqualValues(t, 0, backend.bundleCalls.Load()) + }) + } +} + func TestCertificateStatePathRejectsSymlinkAliasToOutput(t *testing.T) { certificate := agentTestConfig(t) aliasRoot := t.TempDir() From dcdbdb5a79f0ed4fc156ffbe8542949bec3ff78b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:40:04 +0000 Subject: [PATCH 8/8] test(agent): verify ACME certificate reuse after restart Co-Authored-By: ashwin --- e2e/agent/certificate_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/e2e/agent/certificate_test.go b/e2e/agent/certificate_test.go index dc231fb07..4da094378 100644 --- a/e2e/agent/certificate_test.go +++ b/e2e/agent/certificate_test.go @@ -835,6 +835,38 @@ func certAgent_AcmeCA_CertificateIssuance(t *testing.T) { if _, err := os.Stat(chainPath); err == nil { agentHelpers.VerifyChainFile(t, chainPath) } + initialSerial := readCertificateSerial(t, certPath) + statePath := certPath + ".infisical-state.json" + stateWrittenAt := fileModTime(t, statePath) + cmd.Stop() + restarted := &helpers.Command{Test: t, Args: []string{"cert-manager", "agent", "--config", configPath, "--verbose"}, Env: map[string]string{}} + restarted.Start(ctx) + t.Cleanup(func() { + if t.Failed() { + t.Logf("Restarted Agent stderr:\n%s", restarted.Stderr()) + } + restarted.Stop() + }) + require.Eventually(t, func() bool { + info, err := os.Stat(statePath) + if err != nil || !info.ModTime().After(stateWrittenAt) { + return false + } + data, err := os.ReadFile(statePath) + if err != nil { + return false + } + var state struct { + State struct { + Status string `json:"status"` + } `json:"state"` + } + return json.Unmarshal(data, &state) == nil && state.State.Status == "active" + }, 120*time.Second, 2*time.Second, "ACME restart recovery failed: %s", restarted.Stderr()) + require.True(t, restarted.IsRunning()) + require.Equal(t, initialSerial, readCertificateSerial(t, certPath)) + require.NotContains(t, restarted.Stderr(), "issuing certificate") + require.NotContains(t, restarted.Stderr(), "renewing certificate") } func setupAcmeCertAgentTest(t *testing.T, ctx context.Context, certCount ...int) (*agentHelpers.CertAgentTestHelper, string) { //nolint:unparam