OCPEDGE-3133: certs metrics dropin

cmd/generate-config/config/config-openapi-spec.json

@@ -11,6 +11,7 @@
     "ingress",
     "kubelet",
     "manifests",
+    "metrics",
     "network",
     "node",
     "storage",
@@ -1056,6 +1057,30 @@
         }
       }
     },
+    "metrics": {
+      "description": "Metrics configures the MicroShift-owned local metrics endpoint, independently\nof telemetry uploads and the optional OpenTelemetry Collector package.",
+      "type": "object",
+      "required": [
+        "listenAddress",
+        "status"
+      ],
+      "properties": {
+        "listenAddress": {
+          "description": "IP address and port for HTTP /metrics. Only literal loopback addresses are\nallowed (for example, 127.0.0.1:9090 or [::1]:9090). Local access is unauthenticated.",
+          "type": "string",
+          "default": "127.0.0.1:9090"
+        },
+        "status": {
+          "description": "Enable the local metrics endpoint. Changes require a MicroShift restart.",
+          "type": "string",
+          "default": "Disabled",
+          "enum": [
+            "Enabled",
+            "Disabled"
+          ]
+        }
+      }
+    },
     "network": {
       "type": "object",
       "required": [

docs/contributor/architecture.md

@@ -133,6 +133,30 @@ activation is recovered before startup uses the files. See
 [Certificate Renewal Transactions](./certificates.md) for the private on-disk
 layout, validation, and recovery protocol.
 
+## Local Metrics Service
+
+`pkg/metrics` owns an isolated Prometheus registry and a loopback HTTP listener.
+The service is registered before network/control-plane services, with no readiness
+dependencies. It starts after PKI initialization, not before startup data recovery.
+Bind/serve failures are non-fatal; existing core-service failure policy is unchanged.
+The listener accepts two concurrent scrapes, with a five-second scrape/header
+timeout, ten-second read/write timeouts, and a 16 KiB header limit.
+
+The first collector periodically calls `pkg/admin/certificates.Observe`, using the
+same chain builder and `StatusAt()` as the CLI. It takes a non-creating,
+non-blocking shared operation lock, reads public certificates and pending metadata,
+and never invokes CLI recovery. Scrapes read immutable cached metrics, not disk.
+Unavailable series are dropped while the last-success timestamp is retained.
+
+Pass collectors to `NewService(cfg, collectors...)`. Each implements `Collector`,
+which embeds `prometheus.Collector` and adds `Run(context.Context)` for background
+collection. The service registers and starts each collector independently, canceling
+their contexts on shutdown. `Collect` must only expose cached data.
+Do not register the Kubernetes/global registry or add Kubernetes authentication
+calls. The optional OTel collector is a consumer,
+not a dependency of this service. See [local certificate metrics](../user/howto_certificates.md#local-certificate-metrics)
+for configuration, labels, access limitations, and failure semantics.
+
 ## Deployment Modes
 
 MicroShift supports two deployment models:

docs/contributor/observability_metric_exporters.md

@@ -1,4 +1,4 @@
-# Observability: Scraping the Optional Metric Exporters
+# Observability: Scraping MicroShift and Optional Metric Exporters
 
 MicroShift ships the `microshift-observability` RPM, which runs the Red Hat build of the
 OpenTelemetry Collector as a host `systemd` service, plus three optional metric-exporter
@@ -14,7 +14,8 @@ sync. Admin-facing usage of the OpenTelemetry Collector lives in **openshift-doc
 
 ## The `scrape.d` drop-in mechanism
 
-Each optional metrics RPM installs a Prometheus scrape-config drop-in into
+The observability RPM installs a scrape-config drop-in for MicroShift's local
+endpoint. Each optional metrics RPM installs its own drop-in into
 `/etc/microshift/observability/scrape.d/`. All three collector presets
 (`packaging/observability/opentelemetry-collector-{small,medium,large}.yaml`) load whatever
 drop-ins are present at startup via a `prometheus` receiver:
@@ -25,31 +26,40 @@ receivers:
     config:
       scrape_config_files:
         - /etc/microshift/observability/scrape.d/*.yaml
+processors:
+  resourcedetection/system:
+    detectors: [ "system" ]
+    system:
+      hostname_sources: [ "os" ]
 service:
   pipelines:
     metrics/scrape:
       receivers: [ prometheus ]
-      processors: [ batch ]
+      processors: [ resourcedetection/system, batch ]
       exporters: [ otlp ]
 ```
 
 Because the receiver globs only the files that are on disk, the design satisfies three
 cases with no extra wiring:
 
-- **Observability without metrics** — `scrape.d/` is empty; the receiver loads no configs
-  and reports no error.
-- **Observability with a subset of exporters** — only the installed exporters are scraped.
+- **Observability without optional exporters** — only the local MicroShift target
+  is configured. If the endpoint is disabled or unavailable, its scrape reports `up=0`;
+  this does not prevent other receivers or MicroShift from running.
+- **Observability with a subset of exporters** — the local target and only the
+  installed optional exporters are scraped.
 - **Exporters without observability** — the drop-in files exist on disk but are never read,
   so no scraping occurs (the exporter workload still deploys and serves its endpoint).
 
 ### Package ownership
 
-The `microshift-observability` subpackage owns the `scrape.d/` **directory**; each metrics
-subpackage owns its **drop-in file** as `%config(noreplace)` (see
+The `microshift-observability` subpackage owns the `scrape.d/` **directory** and
+`microshift.yaml`; each optional metrics subpackage owns its **drop-in file**.
+All drop-in files are packaged as `%config(noreplace)` (see
 `packaging/rpm/microshift.spec`).
 
 | Exporter | RPM package | Drop-in file (`.../scrape.d/`) | Service (ns `openshift-monitoring`) | Port(s) | Exposes | Auth |
 |---|---|---|---|---|---|---|
+| MicroShift | `microshift-observability` | `microshift.yaml` | host process (no Kubernetes Service) | 9090 (`http`, loopback only) | certificate status, pending renewal, collection freshness | none (local access) |
 | node-exporter | `microshift-metrics-node-exporter` | `node-exporter.yaml` | `node-exporter` | 9100 (`https`) | host CPU/memory/disk/network/OS metrics | kube-rbac-proxy static cert |
 | kube-state-metrics | `microshift-metrics-kube-state` | `kube-state-metrics.yaml` | `kube-state-metrics` | 8443 (`https-main`), 9443 (`https-self`) | Kubernetes object-state metrics | kube-rbac-proxy static cert |
 | metrics-server | `microshift-metrics-server` | `metrics-server.yaml` | `metrics-server` | 443 (`https`) | metrics-server's own serving metrics | Kubernetes RBAC |
@@ -58,7 +68,7 @@ The exporter RPMs are `ExclusiveArch: x86_64 aarch64`.
 
 ### Authentication model
 
-The drop-ins authenticate with the `openshift-observability-client` cert (issued from the
+The optional-exporter drop-ins authenticate with the `openshift-observability-client` cert (issued from the
 `admin-kubeconfig-signer` CA) against the service-CA-signed exporter endpoints.
 
 - **kube-state-metrics** and **node-exporter** sit behind `kube-rbac-proxy` configured with
@@ -73,16 +83,54 @@ The drop-ins authenticate with the `openshift-observability-client` cert (issued
 > metrics. Node/pod resource metrics for `kubectl top` are served through the aggregated
 > Metrics API (`metrics.k8s.io`), **not** through this scrape.
 
+### Local MicroShift target
+
+`microshift.yaml` scrapes `http://127.0.0.1:9090/metrics` once per minute with a
+five-second timeout. Enable the endpoint separately as described in
+[Local Certificate Metrics](../user/howto_certificates.md#local-certificate-metrics).
+Package installation does not enable metrics or restart MicroShift. No API
+discovery, client certificate, CA bundle, or insecure TLS bypass is needed for
+this HTTP loopback target.
+
+All three presets add the OS hostname as the OTel resource attribute `host.name`
+using the existing `resourcedetection/system` processor. Keep hostnames stable and
+unique within the fleet, or add a customer-owned node identifier in the collection
+pipeline. A backend must preserve resource identity: `127.0.0.1:9090` alone cannot
+distinguish nodes. Prometheus remote-write exporters can use
+`resource_to_telemetry_conversion.enabled: true` to retain `host_name` as a label.
+
+For a custom port or IPv6 listener, edit the existing drop-in's target to match
+`metrics.listenAddress` (for example, `'[::1]:9090'`) and restart
+`microshift-observability`. To opt out of this scrape, keep the file with
+`scrape_configs: []` and restart the collector; disabling the MicroShift listener
+alone leaves an unavailable target. Avoid adding a second job for the same endpoint.
+
+Upgrades preserve edited collector configuration and drop-ins. Existing custom
+configurations must include the Prometheus receiver, the `scrape_config_files`
+glob, and the `metrics/scrape` pipeline shown above, including host resource
+detection. Merge these additions into the active configuration without replacing
+customer exporters or credentials, then restart the collector. A customer-managed
+host collector can use the same wiring and drop-in without the bundled service;
+its distribution must include the Prometheus receiver and resource detection processor.
+
+Certificate renewal does not introduce scrape credentials to refresh for this
+target. Preparing renewal leaves active certificate series unchanged; activation
+on MicroShift restart updates them on subsequent observations. Existing
+authenticated receivers still depend on their own credentials. The bundled
+collector retains its systemd dependency on MicroShift; it is not an independent
+diagnostics service when MicroShift exits.
+
 ### Source-of-truth files
 
 Keep the collector presets, the drop-ins, and the RPM packaging in sync with these:
 
-- `packaging/observability/scrape.d/{node-exporter,kube-state-metrics,metrics-server}.yaml`
+- `packaging/observability/scrape.d/{microshift,node-exporter,kube-state-metrics,metrics-server}.yaml`
 - `packaging/observability/opentelemetry-collector-{small,medium,large}.yaml`
 - `assets/optional/{node-exporter,kube-state-metrics,metrics-server}/04-service.yaml`
 - `assets/optional/observability/02-cluster-role.yaml`
 - `packaging/rpm/microshift.spec` (subpackages and `scrape.d` file ownership)
 
 End-to-end behavior is exercised by `test/suites/optional/observability.robot`
 (USHIFT-7407), which enables the exporters alongside observability and asserts that
-kube-state-metrics, node-exporter, and metrics-server series appear in the collector output.
+kube-state-metrics, node-exporter, metrics-server, and MicroShift certificate series
+appear in the collector output and backend, with host identity.

docs/user/howto_certificates.md

@@ -72,6 +72,74 @@ keys. Go consumers can use the exported types and `AddToScheme` in
 `github.com/openshift/microshift/pkg/apis/certificates/v1alpha1` to decode the
 versioned documents.
 
+## Local Certificate Metrics
+
+MicroShift can expose certificate health at a dedicated, local Prometheus-format
+endpoint that an OpenTelemetry Prometheus receiver can scrape. It is disabled by
+default. To enable it, add this to `/etc/microshift/config.yaml` or a config drop-in
+and restart MicroShift:
+
+```yaml
+metrics:
+  status: Enabled
+  listenAddress: 127.0.0.1:9090
+```
+
+```bash
+curl --noproxy '*' http://127.0.0.1:9090/metrics
+```
+
+Only literal loopback addresses are accepted. Use `[::1]:9090` for IPv6 instead;
+one configured address is served, not both families automatically. The endpoint
+uses HTTP without authentication: local users and host-networked workloads can
+read it. Loopback binding is not authentication. No Kubernetes Service, Route,
+or remote export is created by MicroShift itself.
+
+Collection runs immediately and once per minute. Metrics describe **active files
+on disk**, just like `certs status`, not certificates cached in running components
+or verified through TLS connections. Status uses encoded validity dates and the
+existing rotation policy. Preparing renewal changes the pending signal, not the
+active certificate dates; activation is still performed by normal startup.
+
+All signals are gauges. Per-certificate series have `name`, `service`, `role`,
+and `rotation_policy` labels; status adds the `status` label. Paths, serial
+numbers, fingerprints, private keys, and raw errors are not exposed.
+
+| Metric | Meaning |
+|--------|---------|
+| `microshift_certificate_not_before_timestamp_seconds` | Active certificate validity start, in Unix seconds. |
+| `microshift_certificate_not_after_timestamp_seconds` | Active certificate expiry, in Unix seconds. |
+| `microshift_certificate_status` | Five series per certificate: `Healthy`, `ExpiresSoon`, `ExpirationImminent`, `Expired`, `NotYetValid`. Exactly one is 1. |
+| `microshift_certificate_renewal_pending` | 1 when valid pending metadata is present, otherwise 0. This does not verify the staged files or guarantee successful activation. |
+| `microshift_certificate_inventory_collection_success` | 1 when the latest collection read both the active inventory and pending metadata successfully; otherwise 0. |
+| `microshift_certificate_inventory_last_success_timestamp_seconds` | Time of the last complete successful observation; 0 until the first success. |
+
+A failed inventory read removes certificate series. A failed pending-metadata
+read omits the pending gauge rather than reporting a misleading zero; a readable
+active inventory can still be reported independently. Lock contention or an
+interrupted activation makes both unavailable. The last-success time remains
+unchanged on failure. Alert on collection success and freshness as well as
+certificate status; endpoint reachability alone does not prove certificate health.
+Collection never reads private keys, repairs transactions, or renews certificates.
+
+The listener starts after certificate initialization, before waiting for etcd or
+kube-apiserver readiness. It shares MicroShift's process lifetime and is unavailable
+if initialization fails or MicroShift exits. Listener failures are logged without
+stopping the control plane; restart MicroShift after fixing a bind failure.
+Configuration changes require a restart.
+
+The optional `microshift-observability` RPM installs
+`/etc/microshift/observability/scrape.d/microshift.yaml`. All three bundled presets
+scrape the default endpoint once per minute and attach the host's name as an OTel
+resource attribute. Enable the endpoint as shown above and restart
+`microshift-observability` to load the drop-in. If you choose a different loopback
+address or port, update the drop-in's target to match. Customized collector
+configurations must load the drop-ins and include host identity; see the
+[scrape integration guide](../contributor/observability_metric_exporters.md#local-microshift-target).
+An unavailable endpoint reports `up=0`; it does not prevent other collection.
+To disable this scrape, set `scrape_configs: []` in the drop-in and restart the
+collector. Package upgrades preserve edits to that file.
+
 ## Renewing Certificates
 
 Select exactly one renewal mode:

docs/user/howto_config.md

@@ -148,6 +148,9 @@ ingress:
 kubelet:
 manifests:
     kustomizePaths: []
+metrics:
+    listenAddress: ""
+    status: ""
 network:
     clusterNetwork: []
     cniPlugin: ""
@@ -325,6 +328,9 @@ manifests:
         - /usr/lib/microshift/manifests.d/*
         - /etc/microshift/manifests
         - /etc/microshift/manifests.d/*
+metrics:
+    listenAddress: 127.0.0.1:9090
+    status: Disabled
 network:
     clusterNetwork:
         - 10.42.0.0/16

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/opencontainers/selinux v1.15.1
 	github.com/openshift/cluster-policy-controller v0.0.0-20261019223900-81e222007c2c
 	github.com/openshift/route-controller-manager v0.0.0-20260909174728-a158fff7ce38
+	github.com/prometheus/client_golang v1.24.0
 	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.70.0
 	github.com/prometheus/prometheus v0.302.1
@@ -145,7 +146,6 @@ require (
 	github.com/openshift/apiserver-library-go v0.0.0-20261001145254-218f2c488d90 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
-	github.com/prometheus/client_golang v1.24.0 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect

packaging/microshift/config.yaml

@@ -704,6 +704,14 @@ manifests:
         - /usr/lib/microshift/manifests.d/*
         - /etc/microshift/manifests
         - /etc/microshift/manifests.d/*
+# Metrics configures the MicroShift-owned local metrics endpoint, independently
+# of telemetry uploads and the optional OpenTelemetry Collector package.
+metrics:
+    # IP address and port for HTTP /metrics. Only literal loopback addresses are
+    # allowed (for example, 127.0.0.1:9090 or [::1]:9090). Local access is unauthenticated.
+    listenAddress: 127.0.0.1:9090
+    # Enable the local metrics endpoint. Changes require a MicroShift restart.
+    status: Disabled
 network:
     # IP address pool to use for pod IPs.
     # This field is immutable after installation.

packaging/observability/opentelemetry-collector-large.yaml

@@ -4,7 +4,7 @@
 # - Host's CPU, memory, disk, and network usage metrics
 # - System journals for selected MicroShift services and dependencies, priority < Info
 # - Metrics exposed by Pods that have "prometheus.io/scrape": "true" annotation
-# - Metrics from optional component scrape configs (scrape.d drop-ins)
+# - MicroShift and optional component metrics (scrape.d drop-ins)
 
 receivers:
   kubeletstats:
@@ -114,7 +114,7 @@ service:
       exporters: [ otlp ]
     metrics/scrape:
       receivers: [ prometheus ]
-      processors: [ batch ]
+      processors: [ resourcedetection/system, batch ]
       exporters: [ otlp ]
   telemetry:
     metrics:

packaging/observability/opentelemetry-collector-medium.yaml

@@ -2,7 +2,7 @@
 # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node
 # - Kubernetes Events
 # - System journal for MicroShift service, priority < Info
-# - Metrics from optional component scrape configs (scrape.d drop-ins)
+# - MicroShift and optional component metrics (scrape.d drop-ins)
 
 receivers:
   prometheus:
@@ -55,7 +55,7 @@ service:
   pipelines:
     metrics/scrape:
       receivers: [ prometheus ]
-      processors: [ batch ]
+      processors: [ resourcedetection/system, batch ]
       exporters: [ otlp ]
     metrics/kubeletstats:
       receivers: [ kubeletstats ]

packaging/observability/opentelemetry-collector-small.yaml

@@ -1,7 +1,7 @@
 # Following file contains OpenTelemetry Collector configuration that exports:
 # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node
 # - Kubernetes Events
-# - Metrics from optional component scrape configs (scrape.d drop-ins)
+# - MicroShift and optional component metrics (scrape.d drop-ins)
 
 receivers:
   prometheus:
@@ -50,7 +50,7 @@ service:
   pipelines:
     metrics/scrape:
       receivers: [ prometheus ]
-      processors: [ batch ]
+      processors: [ resourcedetection/system, batch ]
       exporters: [ otlp ]
     metrics/kubeletstats:
       receivers: [ kubeletstats ]

packaging/observability/scrape.d/microshift.yaml

@@ -0,0 +1,9 @@
+# Enable metrics in MicroShift and keep this target aligned with metrics.listenAddress.
+scrape_configs:
+  - job_name: microshift
+    scrape_interval: 1m
+    scrape_timeout: 5s
+    scheme: http
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['127.0.0.1:9090']