catalog: list disabled sizes in mz_cluster_replica_sizes

console/src/api/materialize/cluster/__snapshots__/availableClusterSizes.test.ts.snap

@@ -12,3 +12,17 @@ exports[`queries > buildAvailableClusterReplicaSizesQuery produces the expected
         END, "credits_per_hour"",
 }
 `;
+
+exports[`queries > filters out disabled sizes on environments with the disabled column 1`] = `
+{
+  "parameters": [
+    "mz_%",
+    false,
+  ],
+  "sql": "select "size" from "mz_cluster_replica_sizes" where "size" not like $1 and "disabled" = $2 order by CASE
+          WHEN size ~ '^[0-9]+([a-zA-Z]*cc[xX]?|C)$' THEN '0_legacy'
+          WHEN size LIKE '%-%' THEN '1_' || split_part(size, '-', 1)
+          ELSE '2_tshirt'
+        END, "credits_per_hour"",
+}
+`;

console/src/api/materialize/cluster/availableClusterSizes.test.ts

@@ -7,12 +7,22 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-import { buildAvailableClusterReplicaSizesQuery } from "./availableClusterSizes";
+import {
+  buildAvailableClusterReplicaSizesQuery,
+  DISABLED_REPLICA_SIZES_MIN_VERSION,
+} from "./availableClusterSizes";
 
 describe("queries", () => {
   it("buildAvailableClusterReplicaSizesQuery produces the expected query", () => {
     const { sql, parameters } =
       buildAvailableClusterReplicaSizesQuery().compile();
     expect({ sql, parameters }).toMatchSnapshot();
   });
+
+  it("filters out disabled sizes on environments with the disabled column", () => {
+    const { sql, parameters } = buildAvailableClusterReplicaSizesQuery(
+      DISABLED_REPLICA_SIZES_MIN_VERSION,
+    ).compile();
+    expect({ sql, parameters }).toMatchSnapshot();
+  });
 });

console/src/api/materialize/cluster/availableClusterSizes.ts

@@ -10,21 +10,33 @@
 import { QueryKey } from "@tanstack/react-query";
 import { sql } from "kysely";
 
+import { extractEnvironmentVersion } from "~/api/buildQueryKeySchema";
 import { executeSqlV2, queryBuilder } from "~/api/materialize";
+import { semverParseGte } from "~/api/materialize/versioning";
+
+// TODO: set to the first release that includes the `disabled` column, and
+// drop the gate once every environment is on it.
+export const DISABLED_REPLICA_SIZES_MIN_VERSION = "26.50.0-dev";
 
 export const buildAllowedClusterReplicaSizesQuery = () => {
   return sql<{
     allowed_cluster_replica_sizes: string;
   }>`SHOW allowed_cluster_replica_sizes`;
 };
 
-export const buildAvailableClusterReplicaSizesQuery = () => {
+export const buildAvailableClusterReplicaSizesQuery = (
+  environmentVersion?: string,
+) => {
+  const hasDisabledColumn =
+    environmentVersion !== undefined &&
+    semverParseGte(environmentVersion, DISABLED_REPLICA_SIZES_MIN_VERSION);
   return (
     queryBuilder
       .selectFrom("mz_cluster_replica_sizes")
       .select("size")
       // We don't want to show the mz_probe size in the UI
       .where("size", "not like", "mz_%")
+      .$if(hasDisabledColumn, (qb) => qb.where("disabled", "=", false))
       // Group sizes by family (legacy cc, t-shirt, M.1, D.1, ...) so related
       // sizes stay contiguous, then sort by cost within each family.
       .orderBy(
@@ -49,7 +61,9 @@ export default async function fetchAvailableClusterSizes({
 }) {
   const compiledQueries = [
     buildAllowedClusterReplicaSizesQuery().compile(queryBuilder),
-    buildAvailableClusterReplicaSizesQuery().compile(),
+    buildAvailableClusterReplicaSizesQuery(
+      extractEnvironmentVersion(queryKey),
+    ).compile(),
   ] as const;
 
   let sizes: string[] | null = null;

console/types/materialize.d.ts

@@ -541,6 +541,10 @@ export interface MzClusterReplicaSizes {
    * The number of compute credits consumed per hour.
    */
   credits_per_hour: Generated<Numeric>;
+  /**
+   * Whether the size is disabled. Existing replicas keep running on a disabled size, but new clusters and replicas cannot choose it.
+   */
+  disabled: Generated<boolean>;
   /**
    * The disk allocation per process.
    */

doc/user/content/sql/system-catalog/mz_catalog.md

@@ -95,6 +95,8 @@ At this time, we do not make any guarantees about the freshness of these numbers
 
 The `mz_cluster_replica_sizes` table contains a mapping of logical sizes
 (e.g. `100cc`) to physical sizes (number of processes, and CPU and memory allocations per process).
+It includes disabled sizes, which existing replicas may still use; to list the
+sizes available for new clusters and replicas, filter on `NOT disabled`.
 
 This table was previously in the `mz_internal` schema. All queries previously referencing
 `mz_internal.mz_cluster_replica_sizes` should now reference `mz_catalog.mz_cluster_replica_sizes`.
@@ -114,6 +116,7 @@ any kind of capacity planning.
 | `memory_bytes`         | [`uint8`]   | The RAM allocation per process, in bytes.                                                                                                                    |
 | `disk_bytes`           | [`uint8`]   | The disk allocation per process.                                                                                                                             |
 | `credits_per_hour`     | [`numeric`] | The number of compute credits consumed per hour.                                                                                                             |
+| `disabled`             | [`boolean`] | Whether the size is disabled. Existing replicas keep running on a disabled size, but new clusters and replicas cannot choose it.                             |
 
 ### `mz_cluster_replicas`
 

src/adapter/src/catalog/builtin_table_updates.rs

@@ -483,12 +483,7 @@ impl CatalogState {
 
             // The disk column of mz_clusters / mz_cluster_replicas MVs needs
             // `swap_enabled` and `disk_bytes`; expose them through a parallel
-            // internal table. Unlike the public sizes table below, we write
-            // here unconditionally — `cluster_replica_size_has_disk` previously
-            // indexed the in-memory map without checking `disabled`, so a
-            // managed cluster pinned to a disabled size still resolved its
-            // `disk` column from the real allocation. Writing disabled rows
-            // here preserves that behavior.
+            // internal table.
             let internal_row = Row::pack_slice(&[
                 size.as_str().into(),
                 Datum::from(alloc.swap_enabled),
@@ -500,10 +495,6 @@ impl CatalogState {
                 Diff::ONE,
             ));
 
-            if alloc.disabled {
-                continue;
-            }
-
             let cpu_limit = alloc.cpu_limit.unwrap_or(CpuLimit::MAX);
             let MemoryLimit(ByteSize(memory_bytes)) =
                 (alloc.memory_limit).unwrap_or(MemoryLimit::MAX);
@@ -516,6 +507,7 @@ impl CatalogState {
                 memory_bytes.into(),
                 disk_bytes.into(),
                 (alloc.credits_per_hour).into(),
+                alloc.disabled.into(),
             ]);
 
             updates.push(BuiltinTableUpdate::row(

src/adapter/src/catalog/open/builtin_schema_migration.rs

@@ -570,6 +570,14 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
             MZ_CATALOG_SCHEMA,
             "mz_index_columns",
         ),
+        // Adding the `disabled` column to mz_cluster_replica_sizes. The table
+        // is rewritten at every startup, so replacing the shard loses nothing.
+        MigrationStep::replacement(
+            "26.50.0-dev.0",
+            CatalogItemType::Table,
+            MZ_CATALOG_SCHEMA,
+            "mz_cluster_replica_sizes",
+        ),
     ]
 });
 

src/catalog/src/builtin/mz_catalog.rs

@@ -2832,11 +2832,7 @@ pub static MZ_CLUSTERS: LazyLock<BuiltinMaterializedView> = LazyLock::new(|| {
         // `disk` is computed as in
         // `CatalogState::cluster_replica_size_has_disk`:
         // `NOT swap_enabled AND disk_bytes != 0`. Both flags are sourced from
-        // `mz_internal.mz_cluster_replica_size_internal`, which the catalog
-        // populates for every size — including ones flagged `disabled`, which
-        // the public `mz_cluster_replica_sizes` table omits. Using the internal
-        // table directly preserves the prior behavior of indexing the in-memory
-        // size map regardless of `disabled`.
+        // `mz_internal.mz_cluster_replica_size_internal`.
         //
         // `availability_zones` is normalised to NULL when empty, matching
         // `pack_cluster_update`'s `None` for an empty `Vec`. `list_agg` over
@@ -3145,6 +3141,7 @@ pub static MZ_CLUSTER_REPLICA_SIZES: LazyLock<BuiltinTable> = LazyLock::new(|| B
             "credits_per_hour",
             SqlScalarType::Numeric { max_scale: None }.nullable(false),
         )
+        .with_column("disabled", SqlScalarType::Bool.nullable(false))
         .finish(),
     column_comments: BTreeMap::from_iter([
         ("size", "The human-readable replica size."),
@@ -3163,12 +3160,16 @@ pub static MZ_CLUSTER_REPLICA_SIZES: LazyLock<BuiltinTable> = LazyLock::new(|| B
             "credits_per_hour",
             "The number of compute credits consumed per hour.",
         ),
+        (
+            "disabled",
+            "Whether the size is disabled. Existing replicas keep running on a disabled size, but new clusters and replicas cannot choose it.",
+        ),
     ]),
     is_retained_metrics_object: true,
     access: vec![PUBLIC_SELECT],
     ontology: Some(Ontology {
         entity_name: "replica_size",
-        description: "Available cluster replica sizes with CPU, memory, and credit cost",
+        description: "Cluster replica sizes with CPU, memory, and credit cost",
         links: &const { [] },
         column_semantic_types: &const {
             [

src/catalog/src/builtin/mz_internal.rs

@@ -1127,13 +1127,6 @@ WHERE
 /// so this table is locked down with `access: vec![]` (same pattern as
 /// `mz_catalog_raw`): builtin MVs read it at bootstrap, but direct user
 /// `SELECT` is denied.
-///
-/// Unlike `mz_cluster_replica_sizes`, this table includes rows for sizes
-/// flagged `disabled` — `CatalogState::cluster_replica_size_has_disk`
-/// indexed the in-memory map without checking `disabled`, so a managed
-/// cluster pinned to a disabled size still resolved its `disk` column from
-/// the size's real `swap_enabled` / `disk_limit`. Including disabled sizes
-/// here preserves that behavior.
 pub static MZ_CLUSTER_REPLICA_SIZE_INTERNAL: LazyLock<BuiltinTable> = LazyLock::new(|| {
     BuiltinTable {
         name: "mz_cluster_replica_size_internal",

test/cluster/mzcompose.py

@@ -7652,8 +7652,8 @@ def workflow_test_sql_cluster_disk(c: Composition) -> None:
     # writes every size to `mz_cluster_replica_size_internal`, including
     # disabled ones, so `disk` for the existing `noswap_nonzero` cluster and
     # its replica should still resolve to `true`. The public
-    # `mz_cluster_replica_sizes` table drops the disabled row, which the
-    # final assertion checks.
+    # `mz_cluster_replica_sizes` table lists the disabled row as disabled,
+    # which the final assertion checks.
     c.kill("materialized")
     sizes_with_disabled = dict(sizes)
     sizes_with_disabled["noswap,nonzero"] = {
@@ -7685,8 +7685,8 @@ def workflow_test_sql_cluster_disk(c: Composition) -> None:
                 noswap_zero    false
                 noswap_nonzero true
 
-                > SELECT 'noswap,nonzero' IN (SELECT size FROM mz_cluster_replica_sizes)
-                false
+                > SELECT disabled FROM mz_cluster_replica_sizes WHERE size = 'noswap,nonzero'
+                true
                 """))
 
 

test/restart/mzcompose.py

@@ -401,6 +401,10 @@ def workflow_disabled_cluster_replica_size_survives_restart(c: Composition) -> N
                 > SHOW CLUSTER REPLICAS WHERE cluster = 'test'
                 test r1 {size} true ""
 
+                # The disabled size stays listed, so its replicas still join to it.
+                > SELECT r.name, s.disabled FROM mz_cluster_replicas r JOIN mz_cluster_replica_sizes s USING (size) JOIN mz_clusters c ON r.cluster_id = c.id WHERE c.name = 'test'
+                r1 true
+
                 # Creating a new replica of the disabled size is rejected.
                 ! CREATE CLUSTER REPLICA test.r2 SIZE '{size}'
                 contains:unknown cluster replica size {size}

test/sqllogictest/autogenerated/mz_catalog.slt

@@ -79,6 +79,7 @@ cpu_nano_cores  uint8  The␠CPU␠allocation␠per␠process,␠in␠billionths
 memory_bytes  uint8  The␠RAM␠allocation␠per␠process,␠in␠bytes.
 disk_bytes  uint8  The␠disk␠allocation␠per␠process.
 credits_per_hour  numeric  The␠number␠of␠compute␠credits␠consumed␠per␠hour.
+disabled  boolean  Whether␠the␠size␠is␠disabled.␠Existing␠replicas␠keep␠running␠on␠a␠disabled␠size,␠but␠new␠clusters␠and␠replicas␠cannot␠choose␠it.
 
 query TTT
 SELECT name, type, comment FROM objects WHERE schema = 'mz_catalog' AND object = 'mz_cluster_replicas' ORDER BY position

test/sqllogictest/catalog_server_explain.slt

@@ -649,7 +649,7 @@ mz_internal.mz_cluster_replica_history:
       →Arrange (#0)
         →Union
           →Fused with Child Map/Filter/Project
-            Project: #0, #6, #7
+            Project: #0, #6, #8
             Map: true
               →Arranged mz_catalog.mz_cluster_replica_sizes
                 Key: (#0{size})
@@ -4930,7 +4930,7 @@ mz_catalog.mz_columns:
                               →Read l1
       →Differential Join %1:mz_catalog_raw[#1{schema_name}, #2{name}] » %0[#0{schema_name}, #1{relation_name}]
         →Arrange (#0{schema_name}, #1{relation_name})
-          →Constant (1937 rows)
+          →Constant (1938 rows)
         →Arrange (#1{schema_name}, #2{name})
           →Fused with Child Map/Filter/Project
             Project: #6, #4, #3
@@ -8796,7 +8796,7 @@ query T multiline
 EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_columns";
 ----
 Explained Query (fast path):
-  →Constant (1937 rows)
+  →Constant (1938 rows)
 
 Target cluster: mz_catalog_server
 

test/sqllogictest/mz_catalog_server_index_accounting.slt

@@ -335,6 +335,7 @@ mz_cluster_replica_size_internal  size
 mz_cluster_replica_size_internal  swap_enabled
 mz_cluster_replica_sizes  cpu_nano_cores
 mz_cluster_replica_sizes  credits_per_hour
+mz_cluster_replica_sizes  disabled
 mz_cluster_replica_sizes  disk_bytes
 mz_cluster_replica_sizes  memory_bytes
 mz_cluster_replica_sizes  processes

test/sqllogictest/mz_columns.slt

@@ -95,6 +95,7 @@ ORDER BY columns.position
 5  memory_bytes  uint8  false  NULL  uint8  -1
 6  disk_bytes  uint8  true  NULL  uint8  -1
 7  credits_per_hour  numeric  false  NULL  numeric  -1
+8  disabled  boolean  false  NULL  bool  -1
 
 # A log, a source, a materialized view, a view and the generated views that
 # report the builtins, including the one that reports these columns.

test/sqllogictest/system-cluster.slt

@@ -410,9 +410,9 @@ ORDER BY r.id;
 ----
 Explained Query:
   Finish order_by=[#0{id} asc nulls_last] output=[#0..=#5]
-    Project (#11{id}, #12{name}, #0{id}, #14{size}, #1{name}, #30)
-      Map (((uint8_to_double(#25{memory_bytes}) / uint8_to_double(case when (0 = uint8_to_numeric(#19{memory_bytes})) then null else #19{memory_bytes} end)) * 100))
-        Join on=(#0{id} = #13{cluster_id} AND #11{id} = #22{replica_id} AND #14{size} = #15{size}) type=delta
+    Project (#11{id}, #12{name}, #0{id}, #14{size}, #1{name}, #31)
+      Map (((uint8_to_double(#26{memory_bytes}) / uint8_to_double(case when (0 = uint8_to_numeric(#19{memory_bytes})) then null else #19{memory_bytes} end)) * 100))
+        Join on=(#0{id} = #13{cluster_id} AND #11{id} = #23{replica_id} AND #14{size} = #15{size}) type=delta
           ArrangeBy keys=[[#0{id}]]
             ReadIndex on=mz_clusters mz_clusters_ind=[delta join 1st input (full scan)]
           ArrangeBy keys=[[#0{id}], [#2{cluster_id}], [#3{size}]]