Add background monitoring

.github/workflows/ci.yml

@@ -34,4 +34,4 @@ jobs:
         run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
 
       - name: Engine integration tests (raw sockets in a namespace)
-        run: unshare -Urn sh -c 'ip link set lo up && go test -count=1 ./internal/trace/...'
+        run: unshare -Urn sh -c 'ip link set lo up && go test -count=1 ./internal/trace/... ./internal/backend/...'

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
     types: [published]
 
 permissions:
-  contents: write # attach artifacts to the release
+  contents: read # checkout only; artifacts ship to the APT repo, GHCR and OCI
   packages: write # push images and charts to ghcr.io
 
 jobs:
@@ -90,11 +90,6 @@ jobs:
           done
           ls -l *.deb
 
-      - name: Attach to release
-        uses: softprops/action-gh-release@v3
-        with:
-          files: "*.deb"
-
       - name: Upload to the APT repository
         uses: yeti-switch/upload-debian-pkg-action@v2
         with:

Makefile

@@ -43,12 +43,13 @@ proto:
 test:
 	go test ./...
 
-## test-trace: the engine's integration tests, which need raw sockets.
+## test-trace: the raw-socket integration tests (trace engine + the backend
+## monitoring end-to-end test), which need raw sockets.
 ## Runs them as root inside an unprivileged user+network namespace, so no
 ## sudo and no capability on the binary is required. This is what CI runs.
 .PHONY: test-trace
 test-trace:
-	unshare -Urn sh -c 'ip link set lo up && go test -count=1 ./internal/trace/...'
+	unshare -Urn sh -c 'ip link set lo up && go test -count=1 ./internal/trace/... ./internal/backend/...'
 
 .PHONY: check
 check:

api/proto/prober/v1/agent.proto

@@ -117,8 +117,14 @@ message Assignment {
 message Monitor {
   string id = 1;
   uint32 interval_s = 2;
+  // labels are operator-defined dimensions (e.g. supplier, supplier_id) carried
+  // through to the metrics and log records the backend emits for this monitor.
+  map<string, string> labels = 3;
   oneof spec {
+    // A ping monitor is a TraceSpec with mode = PING; sip_options is its own
+    // spec. Both reuse the interactive engines on the agent.
     TraceSpec trace = 10;
+    SipOptionsSpec sip_options = 11;
   }
 }
 

charts/prober-backend/values.yaml

@@ -42,6 +42,31 @@ config:
   #     token: "..."
   auth:
     enabled: false
+  # Predefined monitoring (stage 2). Pushed to agents as a versioned assignment;
+  # results become prober_monitor_* metrics and (trace/mtr) VictoriaLogs records.
+  # This whole block flows through the config verbatim.
+  monitors: []
+  #   - id: supplier1-sip
+  #     kind: sip            # trace | ping | sip
+  #     target: 46.19.210.7
+  #     interval_s: 10
+  #     sites: [fra, ams]    # empty = every agent
+  #     labels: { supplier: supplier1 }
+  #     sip: { transport: udp, port: 5060, timeout_ms: 5000 }
+  #   - id: supplier1-mtr
+  #     kind: trace
+  #     target: 46.19.210.7
+  #     interval_s: 60
+  #     trace: { protocol: icmp, family: ipv4, max_ttl: 30, cycles: 1 }
+  victorialogs:
+    url: ""                  # e.g. http://vls-victoria-logs:9428 ; empty disables shipping
+    # username: ""
+    # password: ""
+    # account_id: 0
+    # project_id: 0
+    stream_fields: [site, monitor]
+    batch_max: 500
+    flush_interval_ms: 2000
 
 # The TLS keypair the gRPC gateway presents to agents. Three ways, in order of
 # preference for a cluster with certificate-expiry monitoring:

cmd/prober-backend/main.go

@@ -75,9 +75,37 @@ func run() error {
 	if err != nil {
 		return err
 	}
+	go watchReload(ctx, *configPath, srv, log)
 	return srv.Run(ctx)
 }
 
+// watchReload re-reads the config file on SIGHUP and applies the monitor set
+// (only that section is hot-reloaded). A bad reload is logged and the running
+// monitor set is kept.
+func watchReload(ctx context.Context, path string, srv *backend.Server, log *slog.Logger) {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP)
+	defer signal.Stop(ch)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ch:
+			var cfg backend.Config
+			if err := loadYAML(path, &cfg); err != nil {
+				log.Error("reload: config read failed", "err", err)
+				continue
+			}
+			if err := cfg.Validate(); err != nil {
+				log.Error("reload: config invalid", "err", err)
+				continue
+			}
+			log.Info("SIGHUP: reloading monitors")
+			srv.ReloadMonitors(cfg.Monitors)
+		}
+	}
+}
+
 func loadYAML(path string, v any) error {
 	if path == "" {
 		return fmt.Errorf("a -config file is required")

go.mod

@@ -8,10 +8,13 @@ tool (
 )
 
 require (
+	github.com/coreos/go-oidc/v3 v3.21.0
+	github.com/emiago/sipgo v1.6.0
 	github.com/go-chi/chi/v5 v5.3.2
 	github.com/google/uuid v1.6.0
 	github.com/prometheus/client_golang v1.24.1
 	golang.org/x/net v0.59.0
+	golang.org/x/oauth2 v0.37.0
 	golang.org/x/sync v0.23.0
 	golang.org/x/sys v0.48.0
 	google.golang.org/grpc v1.83.2
@@ -22,18 +25,17 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/coreos/go-oidc/v3 v3.21.0 // indirect
-	github.com/emiago/sipgo v1.6.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.2 // indirect
 	github.com/icholy/digest v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.70.1 // indirect
 	github.com/prometheus/procfs v0.21.1 // indirect
-	golang.org/x/oauth2 v0.37.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 // indirect

go.sum

@@ -4,6 +4,7 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
 github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emiago/sipgo v1.6.0 h1:6EuOP7c6f0VRatKYTPEYNezt4hslBEsaCzZZOhT2n3s=
@@ -32,6 +33,10 @@ github.com/icholy/digest v1.1.0 h1:HfGg9Irj7i+IX1o1QAmPfIBNu/Q5A5Tu3n/MED9k9H4=
 github.com/icholy/digest v1.1.0/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y=
 github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
 github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -46,6 +51,8 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi
 github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
 github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
 github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@@ -85,8 +92,10 @@ google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 h1:rgSNvqscFZ1JgV/4wH5GOsZF
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2/go.mod h1:iMEtFwDlAhjDU9L5mY6U1XLwlIId/G3h+QcBHDIvrJ8=
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=