Convert mz_index_columns to a builtin materialized view

doc/user/content/sql/system-catalog/mz_catalog.md

@@ -262,8 +262,8 @@ The `mz_index_columns` table contains a row for each column in each index in the
 system. For example, an index on `(a, b + 1)` would have two rows in this
 table, one for each of the two columns in the index.
 
-For a given row, if `field_number` is null then `expression` will be nonnull, or
-vice-versa.
+For a given row, if `on_position` is null then `on_expression` will be nonnull,
+or vice-versa.
 
 <!-- RELATION_SPEC mz_catalog.mz_index_columns -->
 Field            | Type        | Meaning

doc/user/content/sql/system-catalog/mz_internal.md

@@ -1633,6 +1633,7 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys
 
 <!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_activity_log_thinned -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_columns -->
+<!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_index_columns -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_indexes -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_log_indexes -->
 <!-- RELATION_SPEC_UNDOCUMENTED mz_internal.mz_builtin_materialized_views -->

misc/python/materialize/checks/all_checks/builtin_version_pin.py

@@ -46,7 +46,7 @@
     "mz_egress_ips": None,
     "mz_functions": None,
     "mz_iceberg_sinks": MzVersion.parse_mz("v26.38.0"),
-    "mz_index_columns": None,
+    "mz_index_columns": MzVersion.parse_mz("v26.49.0"),
     "mz_kafka_connections": MzVersion.parse_mz("v26.37.0"),
     "mz_kafka_sinks": MzVersion.parse_mz("v26.38.0"),
     "mz_kafka_sources": MzVersion.parse_mz("v26.34.0"),

src/adapter/src/catalog/builtin_table_updates.rs

@@ -15,14 +15,13 @@ use mz_audit_log::VersionedStorageUsage;
 use mz_catalog::SYSTEM_CONN_ID;
 use mz_catalog::builtin::{
     BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL,
-    MZ_CLUSTER_REPLICA_SIZES, MZ_EGRESS_IPS, MZ_FUNCTIONS, MZ_INDEX_COLUMNS, MZ_LICENSE_KEYS,
-    MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_ROLE_AUTH, MZ_SESSIONS,
+    MZ_CLUSTER_REPLICA_SIZES, MZ_EGRESS_IPS, MZ_FUNCTIONS, MZ_LICENSE_KEYS, MZ_LIST_TYPES,
+    MZ_MAP_TYPES, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_ROLE_AUTH, MZ_SESSIONS,
     MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, MZ_TYPE_PG_METADATA, MZ_TYPES,
     MZ_WEBHOOKS_SOURCES,
 };
 use mz_catalog::memory::error::Error;
-use mz_catalog::memory::objects::{CatalogItem, DataSourceDesc, Func, Index, Type};
-use mz_expr::MirScalarExpr;
+use mz_catalog::memory::objects::{CatalogItem, DataSourceDesc, Func, Type};
 use mz_license_keys::ValidatedLicenseKey;
 use mz_orchestrator::{CpuLimit, DiskLimit, MemoryLimit};
 use mz_ore::cast::CastFrom;
@@ -31,8 +30,7 @@ use mz_persist_client::batch::ProtoBatch;
 use mz_repr::adt::array::ArrayDimension;
 use mz_repr::adt::mz_acl_item::PrivilegeMap;
 use mz_repr::role_id::RoleId;
-use mz_repr::{CatalogItemId, Datum, Diff, GlobalId, ReprColumnType, Row, RowPacker};
-use mz_sql::ast::{CreateIndexStatement, Statement};
+use mz_repr::{CatalogItemId, Datum, Diff, GlobalId, Row, RowPacker};
 use mz_sql::catalog::{CatalogType, TypeCategory};
 use mz_sql::func::FuncImplCatalogDetails;
 use mz_sql::names::SchemaSpecifier;
@@ -137,7 +135,6 @@ impl CatalogState {
         let privileges_row = self.pack_privilege_array_row(entry.privileges());
         let privileges = privileges_row.unpack_first();
         let updates = match entry.item() {
-            CatalogItem::Index(index) => self.pack_index_update(id, index, diff),
             CatalogItem::Source(source) => {
                 match &source.data_source {
                     DataSourceDesc::Webhook { .. } => {
@@ -163,13 +160,14 @@ impl CatalogState {
             CatalogItem::Func(func) => {
                 self.pack_func_update(id, schema_id, name, owner_id, func, diff)
             }
-            // Tables, views, materialized views, and metric sinks are exposed
-            // through materialized views derived from `mz_catalog_raw`, and
-            // logs and secrets never had builtin-table rows, so none pack a row
-            // here.
+            // Tables, views, materialized views, indexes, and metric sinks
+            // are exposed through materialized views derived from
+            // `mz_catalog_raw`, and logs and secrets never had builtin-table
+            // rows, so none pack a row here.
             CatalogItem::Table(_)
             | CatalogItem::View(_)
             | CatalogItem::MaterializedView(_)
+            | CatalogItem::Index(_)
             | CatalogItem::Log(_)
             | CatalogItem::Secret(_)
             | CatalogItem::MetricSink(_) => vec![],
@@ -183,70 +181,6 @@ impl CatalogState {
         updates
     }
 
-    fn pack_index_update(
-        &self,
-        id: CatalogItemId,
-        index: &Index,
-        diff: Diff,
-    ) -> Vec<BuiltinTableUpdate<&'static BuiltinTable>> {
-        let mut updates = vec![];
-
-        let create_stmt = mz_sql::parse::parse(&index.create_sql)
-            .unwrap_or_else(|e| {
-                panic!(
-                    "create_sql cannot be invalid: `{}` --- error: `{}`",
-                    index.create_sql, e
-                )
-            })
-            .into_element()
-            .ast;
-
-        let key_sqls = match &create_stmt {
-            Statement::CreateIndex(CreateIndexStatement { key_parts, .. }) => key_parts
-                .as_ref()
-                .expect("key_parts is filled in during planning"),
-            _ => unreachable!(),
-        };
-
-        let on_entry = self.get_entry_by_global_id(&index.on);
-        let on_desc = on_entry
-            .relation_desc()
-            .expect("can only create indexes on items with a valid description");
-        let repr_col_types: Vec<ReprColumnType> = on_desc
-            .typ()
-            .column_types
-            .iter()
-            .map(ReprColumnType::from)
-            .collect();
-        for (i, key) in index.keys.iter().enumerate() {
-            let nullable = key.typ(&repr_col_types).nullable;
-            let seq_in_index = u64::cast_from(i + 1);
-            let key_sql = key_sqls
-                .get(i)
-                .expect("missing sql information for index key")
-                .to_ast_string_simple();
-            let (field_number, expression) = match key {
-                MirScalarExpr::Column(col, _) => {
-                    (Datum::UInt64(u64::cast_from(*col + 1)), Datum::Null)
-                }
-                _ => (Datum::Null, Datum::String(&key_sql)),
-            };
-            updates.push(BuiltinTableUpdate::row(
-                &*MZ_INDEX_COLUMNS,
-                Row::pack_slice(&[
-                    Datum::String(&id.to_string()),
-                    Datum::UInt64(seq_in_index),
-                    field_number,
-                    expression,
-                    Datum::from(nullable),
-                ]),
-                diff,
-            ));
-        }
-
-        updates
-    }
-
     fn pack_type_update(
         &self,
         id: CatalogItemId,

src/adapter/src/catalog/open/builtin_schema_migration.rs

@@ -561,6 +561,15 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
             MZ_CATALOG_SCHEMA,
             "mz_columns",
         ),
+        // Converting mz_index_columns from a builtin table to a
+        // materialized view over mz_catalog_raw changes its catalog
+        // fingerprint.
+        MigrationStep::replacement(
+            "26.49.0-dev.0",
+            CatalogItemType::MaterializedView,
+            MZ_CATALOG_SCHEMA,
+            "mz_index_columns",
+        ),
     ]
 });
 

src/catalog-decode/src/create_sql.rs

@@ -552,6 +552,13 @@ pub fn item_details(a: &str) -> Result<serde_json::Value, String> {
                 stmt.with_options.iter().map(|o| (&o.name, &o.value)),
                 IndexOptionName::RetainHistory,
             )?;
+            let key_parts: Vec<_> = stmt
+                .key_parts
+                .iter()
+                .flatten()
+                .map(|expr| json!(expr.to_ast_string_simple()))
+                .collect();
+            info.insert("key_parts", json!(key_parts));
             "index"
         }
         CreateType(_) => "type",
@@ -2066,6 +2073,16 @@ mod tests {
         mz_repr::adt::jsonb::Jsonb::from_serde_json(out).expect("packs as jsonb");
     }
 
+    #[mz_ore::test]
+    fn item_details_reports_index_key_parts() {
+        let sql = r#"CREATE INDEX "t_idx" IN CLUSTER [u1] ON [u4 AS "materialize"."public"."t"] ("pg_catalog"."abs"("a"), "a" + 1, COALESCE("a", 0), 2)"#;
+        let out = super::item_details(sql).expect("ok");
+        assert_eq!(
+            out["key_parts"],
+            json!(["pg_catalog.abs(a)", "a + 1", "COALESCE(a, 0)", "2"])
+        );
+    }
+
     /// Every error here is fatal to the whole of `mz_tables`/`mz_views`, not to
     /// one row: the MVs call this function inside their `WHERE` clause, so an
     /// item the parser rejects makes the relation unreadable for everyone.

src/catalog/src/builtin.rs

@@ -1194,7 +1194,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::MaterializedView(&MZ_SCHEMAS),
         Builtin::MaterializedView(&MZ_COLUMNS),
         Builtin::MaterializedView(&MZ_INDEXES),
-        Builtin::Table(&MZ_INDEX_COLUMNS),
+        Builtin::MaterializedView(&MZ_INDEX_COLUMNS),
         Builtin::MaterializedView(&MZ_TABLES),
         Builtin::MaterializedView(&MZ_SOURCES),
         Builtin::MaterializedView(&MZ_SOURCE_REFERENCES),