microvm: forward sysctls and lift the guest's privileged-port limit to match gVisor

cmd/ateworker-microvm/internal/kata/specconv.go

@@ -93,9 +93,10 @@ func SpecToAgentPB(s *specs.Spec) *agentpb.Spec {
 			CgroupsPath:   s.Linux.CgroupsPath,
 			MaskedPaths:   s.Linux.MaskedPaths,
 			ReadonlyPaths: s.Linux.ReadonlyPaths,
+			Sysctl:        s.Linux.Sysctl,
 		}
 		// TODO: forward the remaining OCI security knobs the kata-agent supports
-		// for parity with the OCI spec — Linux.Seccomp and Linux.Sysctl here, and
+		// for parity with the OCI spec — Linux.Seccomp here, and
 		// Process.ApparmorProfile / Process.SelinuxLabel above. The MVP runs the
 		// actor with kata's defaults for these.
 		for _, ns := range s.Linux.Namespaces {

cmd/ateworker-microvm/internal/kata/specconv_linux_test.go

@@ -162,3 +162,15 @@ func TestSpecToAgentPB_NonPositiveCPUValuesAreDropped(t *testing.T) {
 		})
 	}
 }
+
+// The agent applies sysctls from the spec it is sent; a dropped map leaves
+// the guest at its defaults.
+func TestSpecToAgentPB_ForwardsSysctl(t *testing.T) {
+	const key = "net.ipv4.ip_unprivileged_port_start"
+	got := SpecToAgentPB(&specs.Spec{
+		Linux: &specs.Linux{Sysctl: map[string]string{key: "0"}},
+	})
+	if got.Linux == nil || got.Linux.Sysctl[key] != "0" {
+		t.Fatalf("Linux.Sysctl = %v, want %s=0 carried through", got.GetLinux().GetSysctl(), key)
+	}
+}

docs/api-guide.md

@@ -303,6 +303,8 @@ Each container runs with a default set of Linux capabilities — `AUDIT_WRITE`,
 
 - **`ALL` in `add` is rejected.** Kubernetes accepts it in the API and relies on PodSecurity admission to deny it; Substrate has no equivalent policy layer yet, so it is refused at admission instead. Name the capabilities the container needs.
 - **Ambient capabilities are not supported** ([gvisor#3166](https://github.com/google/gvisor/issues/3166)).
+- **Low ports.** A container may bind a port below 1024 without `NET_BIND_SERVICE` on both runtimes: gVisor's network stack has no privileged-port limit, and the micro-VM guest sets `net.ipv4.ip_unprivileged_port_start=0` to match.
+- **Setuid binaries work.** Neither runtime sets `no_new_privileges`, as for a Kubernetes container with `allowPrivilegeEscalation: true`; gVisor runs with `--allow-suid`.
 
 The sandbox — gVisor or microVM — remains the isolation boundary; capabilities constrain the workload *inside* it.
 

internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl

@@ -14,7 +14,7 @@
 
 # The capabilities fixture's ActorTemplates: protojson-shaped
 # ateapi.ActorTemplate documents (NOT the ate.dev/v1 CRD), created
-# through the ate API by e2e.DeploySubstrateFixture. Two templates run the
+# through the ate API by e2e.DeploySubstrateFixture. Three templates run the
 # same probe image and differ only in securityContext.capabilities, so the
 # suite can compare what the kernel reports inside the sandbox against what
 # the template asked for.
@@ -45,9 +45,8 @@ snapshotConfig:
 
 ---
 
-# drop ALL + add: the resulting set is exact rather than relative, so asserting
-# it proves both halves at once — everything default was dropped, and only the
-# named capability was granted back.
+# The set is exact rather than relative, so asserting it proves the drop and
+# the add at once.
 metadata:
   atespace: ate-e2e${FIXTURE_SUFFIX}
   name: caps-exact
@@ -58,9 +57,6 @@ containers:
 - name: probe
   image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe
   command: ["/ko-app/probe"]
-  # The probe binds :80, which is exactly what NET_BIND_SERVICE permits, so
-  # this template also proves the granted capability is usable and not merely
-  # present in the mask: without it the container could not become ready.
   securityContext:
     capabilities:
       drop: ["ALL"]
@@ -76,3 +72,31 @@ resources:
     memoryBytes: 536870912 # 512Mi
 snapshotConfig:
   storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-exact${FIXTURE_SUFFIX}/
+---
+
+# No capabilities at all, and the probe still binds :80: both runtimes lift
+# the privileged-port limit, so the golden bake fails if that regresses.
+metadata:
+  atespace: ate-e2e${FIXTURE_SUFFIX}
+  name: caps-none
+workerSelector:
+  matchLabels:
+    workload: caps
+containers:
+- name: probe
+  image: ko://go.ate.dev/substrate/internal/e2e/fixtures/probe
+  command: ["/ko-app/probe"]
+  securityContext:
+    capabilities:
+      drop: ["ALL"]
+  wakeupProbe:
+    httpGet:
+      path: /healthz
+      port: 80
+    timeoutSeconds: 60
+resources:
+  limits:
+    cpuMillicores: 1000
+    memoryBytes: 536870912 # 512Mi
+snapshotConfig:
+  storageLocation: gs://${BUCKET_NAME}/ate-e2e-caps-none${FIXTURE_SUFFIX}/

internal/e2e/fixtures/probe/main.go

@@ -80,6 +80,7 @@ type capabilitiesResponse struct {
 	Permitted   []string `json:"permitted"`
 	Inheritable []string `json:"inheritable"`
 	Ambient     []string `json:"ambient"`
+	NoNewPrivs  bool     `json:"noNewPrivs"`
 	// Error carries a read/parse failure so a failing assertion explains itself
 	// instead of just showing empty sets.
 	Error string `json:"error,omitempty"`
@@ -130,6 +131,10 @@ func readProcCapabilities() (*capabilitiesResponse, error) {
 		if !found {
 			continue
 		}
+		if key == "NoNewPrivs:" {
+			out.NoNewPrivs = strings.TrimSpace(value) == "1"
+			continue
+		}
 		dst, ok := targets[strings.TrimSpace(key)]
 		if !ok {
 			continue

internal/e2e/sandbox_test.go

@@ -58,7 +58,7 @@ var substrateFixtures = []struct {
 	{SubstrateFixtureManifests{
 		Pool:     "internal/e2e/fixtures/capabilities/capabilities.yaml.tmpl",
 		Template: "internal/e2e/fixtures/capabilities/capabilities-templates.yaml.tmpl",
-	}, 2, nil},
+	}, 3, nil},
 	{SubstrateFixtureManifests{
 		Pool:     "internal/e2e/fixtures/testserver/websocket.yaml.tmpl",
 		Template: "internal/e2e/fixtures/testserver/websocket-template.yaml.tmpl",

internal/e2e/suites/capabilities/capabilities_test.go

@@ -39,6 +39,7 @@ type capabilitiesResponse struct {
 	Permitted   []string `json:"permitted"`
 	Inheritable []string `json:"inheritable"`
 	Ambient     []string `json:"ambient"`
+	NoNewPrivs  bool     `json:"noNewPrivs"`
 	Error       string   `json:"error"`
 }
 
@@ -76,6 +77,13 @@ func TestActorCapabilities(t *testing.T) {
 		name:     "drop ALL plus add yields exactly the added capability",
 		template: "caps-exact",
 		want:     []string{"NET_BIND_SERVICE"},
+	}, {
+		// The golden bake in deployFixture is what proves port 80 binds without
+		// NET_BIND_SERVICE; this row checks the restored actor still answers
+		// with an empty set.
+		name:     "drop ALL still binds port 80",
+		template: "caps-none",
+		want:     nil,
 	}}
 
 	for _, tt := range tests {
@@ -110,6 +118,10 @@ func TestActorCapabilities(t *testing.T) {
 			if len(got.Ambient) != 0 {
 				t.Errorf("ambient = %v, want empty (ambient capabilities are not supported)", got.Ambient)
 			}
+			// Neither runtime sets no_new_privileges; gVisor runs with --allow-suid.
+			if got.NoNewPrivs {
+				t.Error("no_new_privs = 1, want 0")
+			}
 		})
 	}
 }
@@ -128,8 +140,8 @@ func assertSameCapabilities(t *testing.T, set string, got, want []string) {
 
 // deployFixture installs the fixture for the sandbox class under test and
 // returns its atespace (which also names the namespace it created, carrying
-// the class suffix so the gVisor and micro-VM lanes never share one). Both
-// templates are golden-snapshotted when this returns; a template whose
+// the class suffix so the gVisor and micro-VM lanes never share one). Every
+// template is golden-snapshotted when this returns; a template whose
 // container cannot start — for example because a needed capability was
 // dropped — fails the deploy with the template's error message rather than
 // timing out per subtest.

internal/ocispec/microvm.go

@@ -57,9 +57,9 @@ type MicroVMOptions struct {
 }
 
 // ShapeMicroVM replaces host system mounts with guest mounts, repoints volume
-// bind mounts to guest share paths, and fills in kata's default resources. It
-// must run on an unshaped spec, and errors on a bind it cannot place in the
-// guest.
+// bind mounts to guest share paths, fills in kata's default resources, and
+// lifts the guest's privileged-port limit. It must run on an unshaped spec,
+// and errors on a bind it cannot place in the guest.
 func ShapeMicroVM(spec *specs.Spec, o MicroVMOptions) error {
 	// Translate volume bind mounts into guest share paths.
 	volumes := make([]specs.Mount, 0, len(spec.Mounts))
@@ -86,6 +86,13 @@ func ShapeMicroVM(spec *specs.Spec, o MicroVMOptions) error {
 	if spec.Linux == nil {
 		spec.Linux = &specs.Linux{}
 	}
+	// gVisor has no privileged-port limit, and runsc ignores this key.
+	if spec.Linux.Sysctl == nil {
+		spec.Linux.Sysctl = map[string]string{}
+	}
+	if _, ok := spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"]; !ok {
+		spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"] = "0"
+	}
 	// The container's own declared limits survive the merge; StartRootfsContainer
 	// sets CgroupsPath.
 	spec.Linux.Resources = mergeKataResources(spec.Linux.Resources)

internal/ocispec/microvm_test.go

@@ -171,3 +171,27 @@ func TestShapeMicroVM_LeavesUndeclaredContainerUnlimited(t *testing.T) {
 		t.Errorf("cpu quota = %d, want unset for a container that declared none", *c.Quota)
 	}
 }
+
+func TestShapeMicroVM_AllowsLowPortsInTheGuest(t *testing.T) {
+	const portStart = "net.ipv4.ip_unprivileged_port_start"
+
+	spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}})
+	if _, ok := spec.Linux.Sysctl[portStart]; ok {
+		t.Errorf("Build() set %s; it must be micro-VM only", portStart)
+	}
+	if err := ShapeMicroVM(spec, MicroVMOptions{ActorDirs: parityActorDirs, ContainerName: "app"}); err != nil {
+		t.Fatalf("ShapeMicroVM() = %v", err)
+	}
+	if v := spec.Linux.Sysctl[portStart]; v != "0" {
+		t.Errorf("%s = %q, want \"0\"", portStart, v)
+	}
+
+	preset := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}})
+	preset.Linux.Sysctl = map[string]string{portStart: "1024"}
+	if err := ShapeMicroVM(preset, MicroVMOptions{ActorDirs: parityActorDirs, ContainerName: "app"}); err != nil {
+		t.Fatalf("ShapeMicroVM() = %v", err)
+	}
+	if v := preset.Linux.Sysctl[portStart]; v != "1024" {
+		t.Errorf("%s = %q, want the caller's value kept", portStart, v)
+	}
+}