diff --git a/docs/commands/extract.md b/docs/commands/extract.md index c4cce83e..352c4fb1 100644 --- a/docs/commands/extract.md +++ b/docs/commands/extract.md @@ -159,7 +159,7 @@ For every key, the value semantics are: ### API sub-filters To restrict which child resources of an API are extracted (operations, -diagnostics, schemas, releases), use the **nested object entry** form under +diagnostics, schemas, releases, tag descriptions), use the **nested object entry** form under `apis:`. Each entry becomes a `name: { subKey: [...] }` map: ```yaml @@ -175,9 +175,13 @@ apis: releases: [] ``` -Supported API sub-filter keys: `operations`, `diagnostics`, `schemas`, `releases`. +Supported API sub-filter keys: `operations`, `diagnostics`, `schemas`, `releases`, `tagDescriptions`. Same value semantics as top-level keys (omitted = include all, `[]` = include none, list = allowlist). +The key of a nested entry can be a wildcard pattern (`'*'`, `'v2-*'`), so one +sub-filter applies to every matching API. An entry keyed by an API's name takes +precedence over a pattern. See [One sub-filter for many APIs](../guides/filtering-resources.md#one-sub-filter-for-many-apis). + ### Workspace sub-filters Workspaces accept the same nested object entry form. Each workspace can have @@ -197,6 +201,10 @@ Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups`, `loggers`, `namedValues`, `policyFragments`, `products`, `schemas`, `subscriptions`, `tags`, `versionSets`. +As for APIs, the key can be a wildcard pattern (`'team-*'`), and an entry keyed +by the workspace's name takes precedence over a pattern. See +[Workspace sub-resource filters](../guides/filtering-resources.md#workspace-sub-resource-filters). + ### Transitive dependencies When extracting with a filter, transitive dependencies are **included automatically**. For example, extracting `echo-api` also extracts any backends, named values, and policy fragments that `echo-api` references in its policies. diff --git a/docs/guides/filtering-resources.md b/docs/guides/filtering-resources.md index 4b08c976..22e13407 100644 --- a/docs/guides/filtering-resources.md +++ b/docs/guides/filtering-resources.md @@ -228,7 +228,7 @@ apis: ### API sub-resource filters -For APIs, you can control which sub-resources (operations, diagnostics, schemas, releases) are extracted. Use an object entry instead of a plain string: +For APIs, you can control which sub-resources (operations, diagnostics, schemas, releases, tag descriptions) are extracted. Use an object entry instead of a plain string: ```yaml apis: @@ -242,6 +242,7 @@ apis: schemas: [] # Empty = exclude ALL schemas releases: - v1-release + tagDescriptions: [] # By name, as under apis//tagDescriptions/ ``` **Sub-filter rules:** @@ -249,6 +250,20 @@ apis: - If a sub-resource key is an **empty array** (`[]`), all sub-resources of that type are excluded - If a sub-resource key lists **names**, only those sub-resources are included +#### One sub-filter for many APIs + +The key of a nested entry can be a wildcard pattern (`*`, `?`) instead of an API name. Like a plain pattern, it selects the matching APIs, and its sub-filter applies to each of them. For example, to extract every API without its operations, schemas or tag descriptions: + +```yaml +apis: + - '*': + operations: [] + schemas: [] + tagDescriptions: [] +``` + +An entry keyed by an API's name takes precedence over a pattern, even one with no sub-resource lists (`- orders: {}` keeps all of `orders`' operations); otherwise the first matching pattern applies. Quote patterns that start with `*` so YAML reads them as strings. + During extraction, the `operations` filter also applies to the API's OpenAPI specification (`specification.yaml` or `specification.json`, including Swagger 2.0). Excluded operations are removed from the specification, and paths with no remaining operations are removed. @@ -275,6 +290,10 @@ workspaces: Supported workspace sub-filter keys: `apis`, `backends`, `diagnostics`, `groups`, `loggers`, `namedValues`, `policyFragments`, `products`, `schemas`, `subscriptions`, `tags`, `versionSets`. +As for APIs, the key can be a wildcard pattern: `- 'team-*': { apis: [orders] }` selects every matching workspace and applies the sub-filter to each. An entry keyed by the workspace's name takes precedence over a pattern; otherwise the first matching pattern applies. + +A sub-filter can't be attached to an exclusion (`- '!team-c': {...}`), for APIs or workspaces: the exclusion still applies, but its sub-filter is ignored with a warning. + --- ## All 17 Filterable Resource Types diff --git a/schemas/v1/extractor-config.schema.json b/schemas/v1/extractor-config.schema.json index 1ea17a9b..c34d7a59 100644 --- a/schemas/v1/extractor-config.schema.json +++ b/schemas/v1/extractor-config.schema.json @@ -105,7 +105,7 @@ }, { "type": "object", - "description": "A single API name mapped to sub-resource filters for that API.", + "description": "A single API name, or a wildcard pattern such as *, mapped to sub-resource filters for the matching APIs. A name takes precedence over a pattern.", "minProperties": 1, "maxProperties": 1, "patternProperties": { @@ -124,7 +124,7 @@ }, { "type": "object", - "description": "A single workspace name mapped to sub-resource filters for that workspace.", + "description": "A single workspace name, or a wildcard pattern such as team-*, mapped to sub-resource filters for the matching workspaces. A name takes precedence over a pattern.", "minProperties": 1, "maxProperties": 1, "patternProperties": { @@ -156,6 +156,10 @@ "releases": { "$ref": "#/definitions/resourcePatternArray", "description": "Releases to extract for this API. Matching is case-insensitive and supports * and ? wildcards." + }, + "tagDescriptions": { + "$ref": "#/definitions/resourcePatternArray", + "description": "TagDescriptions to extract for this API. Matching is case-insensitive and supports * and ? wildcards." } } }, diff --git a/scripts/generate-schemas.mjs b/scripts/generate-schemas.mjs index 524a563f..73284398 100644 --- a/scripts/generate-schemas.mjs +++ b/scripts/generate-schemas.mjs @@ -192,7 +192,7 @@ const extractorSchema = { { type: 'object', description: - 'A single API name mapped to sub-resource filters for that API.', + 'A single API name, or a wildcard pattern such as *, mapped to sub-resource filters for the matching APIs. A name takes precedence over a pattern.', minProperties: 1, maxProperties: 1, patternProperties: { @@ -208,7 +208,7 @@ const extractorSchema = { { type: 'object', description: - 'A single workspace name mapped to sub-resource filters for that workspace.', + 'A single workspace name, or a wildcard pattern such as team-*, mapped to sub-resource filters for the matching workspaces. A name takes precedence over a pattern.', minProperties: 1, maxProperties: 1, patternProperties: { diff --git a/src/lib/config-loader.ts b/src/lib/config-loader.ts index 74669367..6a380802 100644 --- a/src/lib/config-loader.ts +++ b/src/lib/config-loader.ts @@ -67,16 +67,19 @@ function parseFilterArrayWithNested( names.push(name); const subConfig = item[name]; - if (isPlainObject(subConfig)) { + // An entry with no sub-resource lists (`- orders: {}`, `- orders:`) is + // kept as an empty sub-filter: it includes everything, and as an entry + // keyed by the name it takes precedence over a wildcard pattern's. + if (subConfig === undefined || subConfig === null) { + subFilters[name] = {}; + } else if (isPlainObject(subConfig)) { const subFilter: Record = {}; for (const [subKey, subValue] of Object.entries(subConfig)) { if (subValue === undefined || subValue === null) continue; subFilter[subKey] = assertStringArray(subValue, `${fieldName}[${i}].${name}.${subKey}`); } - if (Object.keys(subFilter).length > 0) { - subFilters[name] = subFilter; - } - } else if (subConfig !== undefined && subConfig !== null) { + subFilters[name] = subFilter; + } else { logger.warn( `${fieldName}[${i}]: nested value for '${name}' should be an object with sub-resource arrays; ignoring.` ); @@ -163,8 +166,12 @@ export async function loadFilterConfig(filePath: string): Promise 0) { config.apiSubFilters = {}; - const validApiSubKeys = ['operations', 'diagnostics', 'schemas', 'releases'] as const; + const validApiSubKeys = ['operations', 'diagnostics', 'schemas', 'releases', 'tagDescriptions'] as const; for (const [apiName, sf] of Object.entries(subFilters)) { + if (apiName.startsWith('!')) { + logger.warn(`Sub-filter for exclusion '${apiName}' in '${sourceKey}' has no effect; ignoring.`); + continue; + } const apiSub: ApiSubFilter = {}; for (const subKey of validApiSubKeys) { if (subKey in sf) { @@ -188,6 +195,10 @@ export async function loadFilterConfig(filePath: string): Promise key.toLowerCase() === rootName - ); - if (!subFilterKey || filter?.apiSubFilters?.[subFilterKey].operations === undefined) { + if (findApiSubFilter(rootName, filter?.apiSubFilters)?.operations === undefined) { return content; } diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index 4fbc7fa2..bd939d52 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -33,7 +33,7 @@ import { } from '../lib/resource-path.js'; import { isAutoGeneratedId, isPortalGeneratedSchemaId } from '../lib/auto-generated.js'; import { normalizeWsdlXsdImportLocations } from '../lib/wsdl-normalizer.js'; -import { resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; +import { findApiSubFilter, resolveWorkspaceFilter, shouldIncludeResource } from './filter-service.js'; /** * API child resource types that should be published after the API itself @@ -225,9 +225,7 @@ export async function planApiPublication( const effectiveFilter = apiDescriptor.workspace ? resolveWorkspaceFilter(apiDescriptor.workspace, config.filter) : config.filter; - const subFilter = Object.entries(effectiveFilter?.apiSubFilters ?? {}).find( - ([name]) => name.toLowerCase() === apiName.toLowerCase() - )?.[1]; + const subFilter = findApiSubFilter(apiName, effectiveFilter?.apiSubFilters); const allowed = allowedDescriptors ? new Set(allowedDescriptors.map(getResourceDescriptorKey)) : undefined; diff --git a/src/services/filter-service.ts b/src/services/filter-service.ts index a73ca28b..6a019f3b 100644 --- a/src/services/filter-service.ts +++ b/src/services/filter-service.ts @@ -68,6 +68,7 @@ const API_SUB_FILTER_KEY_MAP: Partial> [ResourceType.ApiDiagnostic]: 'diagnostics', [ResourceType.ApiSchema]: 'schemas', [ResourceType.ApiRelease]: 'releases', + [ResourceType.ApiTagDescription]: 'tagDescriptions', }; /** @@ -139,37 +140,7 @@ export function shouldReconcileResource( return false; } - const workspaceFilter = getWorkspaceFilter(descriptor.workspace, filter); - return shouldIncludeResource(descriptor, workspaceFilter); -} - -function getWorkspaceFilter( - workspaceName: string, - filter?: FilterConfig -): FilterConfig | undefined { - const matchingKey = Object.keys(filter?.workspaceSubFilters ?? {}).find( - key => key.toLowerCase() === workspaceName.toLowerCase() - ); - if (!matchingKey) { - return undefined; - } - - const subFilter = filter?.workspaceSubFilters?.[matchingKey]; - return subFilter ? { - apis: subFilter.apis, - apiSubFilters: subFilter.apiSubFilters, - backends: subFilter.backends, - diagnostics: subFilter.diagnostics, - groups: subFilter.groups, - loggers: subFilter.loggers, - namedValues: subFilter.namedValues, - policyFragments: subFilter.policyFragments, - products: subFilter.products, - schemas: subFilter.schemas, - subscriptions: subFilter.subscriptions, - tags: subFilter.tags, - versionSets: subFilter.versionSets, - } : undefined; + return shouldIncludeResource(descriptor, resolveWorkspaceFilter(descriptor.workspace, filter)); } /** @@ -180,11 +151,9 @@ export function resolveWorkspaceFilter( workspaceName: string, filter?: FilterConfig ): FilterConfig | undefined { - const lowerName = workspaceName.toLowerCase(); - const matchingKey = Object.keys(filter?.workspaceSubFilters ?? {}).find( - (key) => key.toLowerCase() === lowerName - ); - const subFilter = matchingKey ? filter?.workspaceSubFilters?.[matchingKey] : undefined; + // A sub-filter keyed by a pattern (for example `team-*`) applies to every + // workspace it matches; one keyed by the workspace's name takes precedence. + const subFilter = findSubFilter(workspaceName, filter?.workspaceSubFilters); return subFilter ? workspaceSubFilterToFilterConfig(subFilter) : undefined; } @@ -225,18 +194,12 @@ function matchesApiSubFilter( parentApiName: string, apiSubFilters: Record ): boolean { - // Find matching sub-filter using case-insensitive API name - const lowerParent = parentApiName.toLowerCase(); - const matchingKey = Object.keys(apiSubFilters).find( - (k) => k.toLowerCase() === lowerParent - ); - - if (!matchingKey) { + const subFilter = findApiSubFilter(parentApiName, apiSubFilters); + if (!subFilter) { // No sub-filter for this API — include all children return true; } - const subFilter = apiSubFilters[matchingKey]; const subFilterKey = API_SUB_FILTER_KEY_MAP[descriptor.type]; if (!subFilterKey) { @@ -295,6 +258,32 @@ export function extractRootApiName(name: string): string { return semiIndex >= 0 ? name.substring(0, semiIndex) : name; } +/** + * The sub-filter that applies to a named API or workspace: the one keyed by + * its name (case-insensitive), otherwise the first one keyed by a wildcard + * pattern that matches it (for example `*`). Exclusion keys (`!name`) never + * apply. + */ +function findSubFilter(name: string, subFilters: Record | undefined): T | undefined { + if (!subFilters) { + return undefined; + } + const lowerName = name.toLowerCase(); + const keys = Object.keys(subFilters).filter((key) => !key.startsWith('!')); + const key = + keys.find((k) => k.toLowerCase() === lowerName) ?? + keys.find((k) => isWildcardPattern(k) && matchesFilter(name, [k])); + return key === undefined ? undefined : subFilters[key]; +} + +/** The sub-filter that applies to an API (see {@link findSubFilter}). */ +export function findApiSubFilter( + apiName: string, + apiSubFilters: Record | undefined +): ApiSubFilter | undefined { + return findSubFilter(apiName, apiSubFilters); +} + /** * Check if a pattern contains wildcard characters (* or ?). */ diff --git a/tests/unit/lib/config-loader.test.ts b/tests/unit/lib/config-loader.test.ts index 47fd3e54..a6736f50 100644 --- a/tests/unit/lib/config-loader.test.ts +++ b/tests/unit/lib/config-loader.test.ts @@ -3,6 +3,8 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { loadFilterConfig, loadOverrideConfig } from '../../../src/lib/config-loader.js'; import { logger } from '../../../src/lib/logger.js'; +import { resolveWorkspaceFilter, shouldIncludeResource } from '../../../src/services/filter-service.js'; +import { ResourceType } from '../../../src/models/resource-types.js'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; import * as os from 'node:os'; @@ -201,6 +203,76 @@ apis: }); }); + it('should load a wildcard API sub-filter with tagDescriptions (#316, #307)', async () => { + const content = ` +apis: + - '*': + operations: [] + schemas: [] + tagDescriptions: [] +`; + const filePath = path.join(tmpDir, 'wildcard-api-filter.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadFilterConfig(filePath); + + expect(config!.apis).toEqual(['*']); + expect(config!.apiSubFilters).toEqual({ + '*': { operations: [], schemas: [], tagDescriptions: [] }, + }); + }); + + it('should keep an empty entry keyed by a name, so it takes precedence over a pattern', async () => { + const content = ` +apis: + - '*': + operations: [] + - orders: {} + - billing: +workspaces: + - '*': + apis: [] + - team-a: {} +`; + const filePath = path.join(tmpDir, 'empty-override-filter.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadFilterConfig(filePath); + + expect(config!.apiSubFilters).toEqual({ '*': { operations: [] }, orders: {}, billing: {} }); + const operation = (api: string) => ({ type: ResourceType.ApiOperation, nameParts: [api, 'get'] }); + expect(shouldIncludeResource(operation('orders'), config)).toBe(true); + expect(shouldIncludeResource(operation('billing'), config)).toBe(true); + expect(shouldIncludeResource(operation('petstore'), config)).toBe(false); + const workspaceApi = { type: ResourceType.Api, nameParts: ['ws-orders'] }; + expect(shouldIncludeResource(workspaceApi, resolveWorkspaceFilter('team-a', config))).toBe(true); + expect(shouldIncludeResource(workspaceApi, resolveWorkspaceFilter('team-b', config))).toBe(false); + }); + + it('should ignore, with a warning, sub-filters keyed by an exclusion', async () => { + const warnSpy = vi.spyOn(logger, 'warn').mockImplementation(() => {}); + const content = ` +apis: + - '*' + - '!legacy': + operations: [] +workspaces: + - '!team-c': + apis: [] +`; + const filePath = path.join(tmpDir, 'exclusion-sub-filter.yaml'); + await fs.writeFile(filePath, content, 'utf-8'); + + const config = await loadFilterConfig(filePath); + + expect(config!.apis).toEqual(['*', '!legacy']); + expect(config!.apiSubFilters).toEqual({}); + expect(config!.workspaceSubFilters).toEqual({}); + expect(warnSpy).toHaveBeenCalledWith("Sub-filter for exclusion '!legacy' in 'apis' has no effect; ignoring."); + expect(warnSpy).toHaveBeenCalledWith("Sub-filter for exclusion '!team-c' in 'workspaces' has no effect; ignoring."); + warnSpy.mockRestore(); + }); + it('should load nested workspace filter entries', async () => { const content = ` workspaces: diff --git a/tests/unit/services/filter-service.test.ts b/tests/unit/services/filter-service.test.ts index 169b9277..0b58a08b 100644 --- a/tests/unit/services/filter-service.test.ts +++ b/tests/unit/services/filter-service.test.ts @@ -16,6 +16,7 @@ import { isWildcardPattern, wildcardToRegex, wildcardMatch, + findApiSubFilter, } from '../../../src/services/filter-service.js'; describe('filter-service', () => { @@ -47,6 +48,21 @@ describe('filter-service', () => { filter )).toBe(false); }); + + it('should use a workspace sub-filter keyed by a pattern', () => { + const filter: FilterConfig = { + workspaces: ['team-*'], + workspaceSubFilters: { + 'team-*': { namedValues: ['workspace-value'] }, + }, + }; + + expect(shouldReconcileResource(workspaceNamedValue, filter)).toBe(true); + expect(shouldReconcileResource( + { ...workspaceNamedValue, nameParts: ['other-value'] }, + filter + )).toBe(false); + }); }); describe('shouldIncludeResource', () => { @@ -345,6 +361,47 @@ describe('filter-service', () => { }); }); + describe('wildcard API sub-filters (#316) and tagDescriptions (#307)', () => { + const filter: FilterConfig = { + apis: ['*', 'orders'], + apiSubFilters: { + '*': { operations: [], schemas: [], tagDescriptions: [] }, + orders: { operations: ['get-order'] }, + }, + }; + const child = (type: ResourceType, api: string, name: string): ResourceDescriptor => ({ + type, + nameParts: [api, name], + }); + + it('should prefer a sub-filter keyed by the API name over a pattern, ignoring case, and find none without a match', () => { + expect(findApiSubFilter('ORDERS', filter.apiSubFilters)).toEqual({ operations: ['get-order'] }); + expect(findApiSubFilter('orders', { 'pet*': { schemas: [] } })).toBeUndefined(); + expect(findApiSubFilter('orders', undefined)).toBeUndefined(); + }); + + it('should exclude operations, schemas and tag descriptions from every API matched by the pattern', () => { + for (const type of [ResourceType.ApiOperation, ResourceType.ApiSchema, ResourceType.ApiTagDescription]) { + expect(shouldIncludeResource(child(type, 'petstore', 'x'), filter)).toBe(false); + } + expect(shouldIncludeResource(child(ResourceType.ApiDiagnostic, 'petstore', 'applicationinsights'), filter)).toBe(true); + }); + + it('should apply the named sub-filter to the named API only', () => { + expect(shouldIncludeResource(child(ResourceType.ApiOperation, 'orders', 'get-order'), filter)).toBe(true); + expect(shouldIncludeResource(child(ResourceType.ApiOperation, 'orders', 'delete-order'), filter)).toBe(false); + // The named sub-filter does not list schemas, so the API's schemas are all included. + expect(shouldIncludeResource(child(ResourceType.ApiSchema, 'orders', 'schema-1'), filter)).toBe(true); + }); + + it('should filter tag descriptions by name', () => { + const tagFilter: FilterConfig = { apis: ['orders'], apiSubFilters: { orders: { tagDescriptions: ['public'] } } }; + + expect(shouldIncludeResource(child(ResourceType.ApiTagDescription, 'orders', 'public'), tagFilter)).toBe(true); + expect(shouldIncludeResource(child(ResourceType.ApiTagDescription, 'orders', 'internal'), tagFilter)).toBe(false); + }); + }); + describe('extractRootApiName', () => { it('should return name as-is when no revision suffix', () => { expect(extractRootApiName('my-api')).toBe('my-api'); diff --git a/tests/unit/services/workspace-extractor.test.ts b/tests/unit/services/workspace-extractor.test.ts index e24f1b1f..6175de54 100644 --- a/tests/unit/services/workspace-extractor.test.ts +++ b/tests/unit/services/workspace-extractor.test.ts @@ -564,6 +564,26 @@ describe('workspace-extractor', () => { }); describe('resolveWorkspaceFilter', () => { + it('should apply a sub-filter keyed by a pattern to every workspace it matches', () => { + const filter: FilterConfig = { + workspaces: ['team-*'], + workspaceSubFilters: { + 'team-*': { apis: ['orders'], namedValues: [] }, + 'team-b': { apis: ['billing'] }, + '!team-c': { apis: [] }, + }, + }; + + expect(resolveWorkspaceFilter('team-a', filter)).toEqual( + expect.objectContaining({ apis: ['orders'], namedValues: [] }) + ); + // A sub-filter keyed by the workspace's name takes precedence over a pattern. + expect(resolveWorkspaceFilter('TEAM-B', filter)).toEqual(expect.objectContaining({ apis: ['billing'] })); + // Exclusion keys never apply. + expect(resolveWorkspaceFilter('team-c', filter)).toEqual(expect.objectContaining({ apis: ['orders'] })); + expect(resolveWorkspaceFilter('platform', filter)).toBeUndefined(); + }); + it('should return undefined when no filter provided', () => { expect(resolveWorkspaceFilter('ws-1')).toBeUndefined(); });