diff --git a/docs/guides/filtering-resources.md b/docs/guides/filtering-resources.md index 8a7676f9..4b08c976 100644 --- a/docs/guides/filtering-resources.md +++ b/docs/guides/filtering-resources.md @@ -249,6 +249,13 @@ apis: - If a sub-resource key is an **empty array** (`[]`), all sub-resources of that type are excluded - If a sub-resource key lists **names**, only those sub-resources are included +During extraction, the `operations` filter also applies to the API's OpenAPI specification +(`specification.yaml` or `specification.json`, including Swagger 2.0). Excluded operations +are removed from the specification, and paths with no remaining operations are removed. +Shared definitions and metadata on retained paths are preserved. Omitting `operations` +leaves the specification unchanged; `operations: []` removes all operations from it. +GraphQL, WSDL, and WADL specifications are not filtered. + ### Workspace sub-resource filters The configuration format supports specifying which workspace-scoped resources to extract: diff --git a/src/services/api-extractor.ts b/src/services/api-extractor.ts index 98b72d0b..2e477385 100644 --- a/src/services/api-extractor.ts +++ b/src/services/api-extractor.ts @@ -12,7 +12,8 @@ import { IArtifactStore } from '../clients/iartifact-store.js'; import { ApimServiceContext, ResourceDescriptor } from '../models/types.js'; import { ResourceType, RESOURCE_TYPE_METADATA } from '../models/resource-types.js'; import { FilterConfig } from '../models/config.js'; -import { shouldIncludeResource } from './filter-service.js'; +import * as yaml from 'js-yaml'; +import { extractRootApiName, shouldIncludeResource } from './filter-service.js'; import { extractResourceType, ExtractedResource } from './resource-extractor.js'; import { redactAndWarnPolicySecrets } from './secret-redactor.js'; import { logger } from '../lib/logger.js'; @@ -21,6 +22,22 @@ import { getNamePart } from '../lib/resource-path.js'; import { normalizeWsdl } from '../lib/wsdl-normalizer.js'; import { isWorkspaceScope, extractNameFromLink } from '../lib/workspace-link.js'; +/** + * `JSON.parse`'s reviver `context` argument and `JSON.rawJSON` are ES2024 + * runtime additions (supported by the project's Node.js target) not yet + * reflected in the configured TS `lib`. Augment the global `JSON` interface + * rather than casting through `unknown` at each call site. + */ +declare global { + interface JSON { + parse( + text: string, + reviver: (this: unknown, key: string, value: unknown, context: { source?: string }) => unknown + ): unknown; + rawJSON(text: string): unknown; + } +} + /** * Result of API-specific extraction for a single API. */ @@ -97,7 +114,7 @@ export async function extractApiResources( // Extract API specification (uses already-extracted schemas to detect // synthetic GraphQL without a second list call). const specificationResult = await extractApiSpecification( - client, store, context, apiDescriptor, apiJson, outputDir, result.schemas + client, store, context, apiDescriptor, apiJson, outputDir, result.schemas, filter ); result.specification = specificationResult.extracted; result.errorCount += specificationResult.errorCount; @@ -274,7 +291,8 @@ async function extractApiSpecification( apiDescriptor: ResourceDescriptor, apiJson: Record, outputDir: string, - extractedSchemas: ExtractedResource[] + extractedSchemas: ExtractedResource[], + filter?: FilterConfig ): Promise<{ extracted: boolean; errorCount: number }> { const properties = apiJson.properties as Record | undefined; const apiType = properties?.type as string | undefined; @@ -319,9 +337,12 @@ async function extractApiSpecification( // APIM's WSDL export can emit wsdl:part references qualified with the wrong // namespace prefix and multiple service ports, which its own importer then // rejects. Normalize so the extracted artifact round-trips through publish. - const content = spec.format === 'wsdl' + let content = spec.format === 'wsdl' ? normalizeWsdl(spec.content) : spec.content; + if (spec.format === 'yaml' || spec.format === 'json') { + content = filterOpenApiOperations(content, spec.format, apiDescriptor, filter); + } await store.writeContent( outputDir, @@ -340,6 +361,190 @@ async function extractApiSpecification( } } +/** + * Apply the same operation filter to the specification as to operation artifacts. + * Keep shared definitions and path metadata, but remove paths without selected methods. + * + * Parses and reserializes the document, taking care to avoid the precision/type + * loss that `JSON.parse`/js-yaml's defaults would otherwise introduce for scalars + * unrelated to the operations being removed (e.g. int64 examples round to the + * nearest double, bare YAML date-like strings are coerced to `Date`). This is not + * a full byte-for-byte preserving editor: YAML comments are not retained, and + * integers originally written in hex/octal/binary notation are re-emitted in + * decimal form, since the reserialization still rebuilds the whole document. + */ +function filterOpenApiOperations( + content: string, + format: 'yaml' | 'json', + apiDescriptor: ResourceDescriptor, + filter?: FilterConfig +): string { + const apiName = getNamePart(apiDescriptor.nameParts, 0); + const rootName = extractRootApiName(apiName).toLowerCase(); + const subFilterKey = Object.keys(filter?.apiSubFilters ?? {}).find( + key => key.toLowerCase() === rootName + ); + if (!subFilterKey || filter?.apiSubFilters?.[subFilterKey].operations === undefined) { + return content; + } + + const spec = parseOpenApiDocument(content, format); + const methods = new Set(['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace']); + let modified = false; + + for (const field of ['paths', 'x-ms-paths']) { + const paths = spec[field]; + if (!paths || typeof paths !== 'object' || Array.isArray(paths)) continue; + + for (const [path, value] of Object.entries(paths)) { + if (!path.startsWith('/') || !value || typeof value !== 'object' || Array.isArray(value)) continue; + const pathItem = value as Record; + let hasSelectedOperation = false; + + for (const method of Object.keys(pathItem)) { + if (!methods.has(method)) continue; + const operation = pathItem[method] as Record | null; + const operationId = operation?.operationId; + if (typeof operationId === 'string' && shouldIncludeResource({ + ...apiDescriptor, + type: ResourceType.ApiOperation, + nameParts: [apiName, operationId], + }, filter)) { + hasSelectedOperation = true; + } else { + delete pathItem[method]; + modified = true; + } + } + + if (!hasSelectedOperation) { + delete (paths as Record)[path]; + modified = true; + } + } + } + + if (!modified) return content; + return stringifyOpenApiDocument(spec, format); +} + +/** + * Parse a JSON/YAML OpenAPI document without losing precision on unrelated + * scalars. `JSON.parse`/`js-yaml` normally round very large integers (e.g. + * int64 examples) to the nearest representable double and, for YAML, coerce + * bare date-like scalars to `Date` objects. Both are avoided here so that + * only the operations targeted by the filter are actually changed. + */ +function parseOpenApiDocument(content: string, format: 'yaml' | 'json'): Record { + if (format === 'json') { + const reviver = (_key: string, value: unknown, context: { source?: string }): unknown => { + const source = context?.source; + if (typeof value === 'number' && source !== undefined) { + return JSON.rawJSON(source); + } + return value; + }; + return JSON.parse(content, reviver) as Record; + } + return yaml.load(content, { schema: PRECISE_YAML_SCHEMA }) as Record; +} + +/** + * Reserialize a document previously parsed by {@link parseOpenApiDocument}, + * emitting the preserved raw/precise scalars back in their original form. + */ +function stringifyOpenApiDocument(spec: Record, format: 'yaml' | 'json'): string { + return format === 'json' + ? JSON.stringify(spec, null, 2) + : yaml.dump(spec, { lineWidth: -1, schema: PRECISE_YAML_SCHEMA }); +} + +/** + * YAML `int` type that preserves full precision for integers outside the safe + * integer range (e.g. int64 examples) by representing them as `BigInt` + * instead of a lossy JS `number`. Built on the standard Core schema, which + * (unlike the Default schema) does not auto-convert bare date-like scalars to + * `Date` objects. + */ +const PRECISE_YAML_SCHEMA = yaml.CORE_SCHEMA.extend({ + implicit: [ + new yaml.Type('tag:yaml.org,2002:int', { + kind: 'scalar', + resolve: resolveYamlInteger, + construct: (data: string) => parsePreciseYamlInteger(data), + predicate: (data: unknown) => isPreciseYamlInteger(data), + // Always represented in decimal: the original literal style (e.g. hex, + // octal, binary) isn't retained on the plain number/BigInt values produced + // by `construct`, so a multi-style representer couldn't be selected from + // them. Non-decimal integers unrelated to the filtered operations are + // therefore re-emitted in decimal form; see the module-level docstring. + represent: (data: unknown) => castPreciseYamlValue(data).toString(10), + }), + ], +}); + +function castPreciseYamlValue(data: unknown): number | bigint { + return data as number | bigint; +} + +/** Ported from js-yaml's built-in int type resolver (not exported publicly). */ +function resolveYamlInteger(data: string | null): boolean { + if (data === null || data.length === 0) return false; + + let index = 0; + const max = data.length; + let ch = data[index]; + + if (ch === '-' || ch === '+') { + ch = data[++index]; + } + + if (ch === '0') { + if (index + 1 === max) return true; + ch = data[++index]; + + if (ch === 'b') return /^[01]+$/.test(data.slice(index + 1)); + if (ch === 'x') return /^[0-9a-fA-F]+$/.test(data.slice(index + 1)); + if (ch === 'o') return /^[0-7]+$/.test(data.slice(index + 1)); + } + + return /^[0-9]+$/.test(data.slice(index)); +} + +/** + * Parses a resolved YAML integer scalar, promoting to `BigInt` only when the + * value would otherwise lose precision as a JS `number`. + */ +function parsePreciseYamlInteger(data: string): number | bigint { + let value = data; + let sign = 1; + const ch0 = value[0]; + if (ch0 === '-' || ch0 === '+') { + if (ch0 === '-') sign = -1; + value = value.slice(1); + } + if (value === '0') return 0; + + let magnitude: number; + if (value[0] === '0' && value[1] === 'b') magnitude = parseInt(value.slice(2), 2); + else if (value[0] === '0' && value[1] === 'x') magnitude = parseInt(value.slice(2), 16); + else if (value[0] === '0' && value[1] === 'o') magnitude = parseInt(value.slice(2), 8); + else magnitude = parseInt(value, 10); + + const result = sign * magnitude; + if (Number.isSafeInteger(result)) return result; + + const preciseMagnitude = BigInt(value); + return sign === -1 ? -preciseMagnitude : preciseMagnitude; +} + +function isPreciseYamlInteger(data: unknown): boolean { + if (typeof data === 'bigint') return true; + return Object.prototype.toString.call(data) === '[object Number]' && + (data as number) % 1 === 0 && + !Object.is(data, -0); +} + /** * Returns true if any already-extracted ApiSchema resource has a contentType * indicating a GraphQL schema. Used to distinguish synthetic GraphQL APIs diff --git a/tests/unit/services/api-product-extractor.test.ts b/tests/unit/services/api-product-extractor.test.ts index 05263836..23017257 100644 --- a/tests/unit/services/api-product-extractor.test.ts +++ b/tests/unit/services/api-product-extractor.test.ts @@ -5,6 +5,7 @@ */ import { describe, it, expect, vi } from 'vitest'; +import * as yaml from 'js-yaml'; import { ResourceType } from '../../../src/models/resource-types.js'; import { ApimServiceContext, ResourceDescriptor } from '../../../src/models/types.js'; import { FilterConfig } from '../../../src/models/config.js'; @@ -80,6 +81,301 @@ describe('api-extractor', () => { expect(store.writeContent).toHaveBeenCalled(); }); + describe.each([ + { format: 'yaml', version: { openapi: '3.0.1' } }, + { format: 'json', version: { openapi: '3.0.1' } }, + { format: 'yaml', version: { swagger: '2.0' } }, + { format: 'json', version: { swagger: '2.0' } }, + ])('operation filtering in $format specifications ($version)', ({ format, version }) => { + const createOperation = { + operationId: 'create-resources', + responses: { '200': { description: 'Created' } }, + 'x-custom-operation': { preserved: true }, + }; + const testOperation = { + operationId: 'test', + responses: { '200': { description: 'OK' } }, + }; + const pathMetadata = { + parameters: [{ + name: 'id', in: 'query', required: false, + ...('swagger' in version ? { type: 'string' } : { schema: { type: 'string' } }), + }], + 'x-custom-path': { preserved: true }, + }; + const schemas = { Resource: { type: 'object', properties: { id: { type: 'string' } } } }; + const specification = { + ...version, + info: { title: 'Test API', version: '1.0' }, + ...('swagger' in version ? { definitions: schemas } : { components: { schemas } }), + paths: { + '/resources': { ...pathMetadata, post: createOperation, get: testOperation }, + '/test': { ...pathMetadata, get: testOperation }, + }, + 'x-custom-document': { preserved: true }, + }; + const content = format === 'json' + ? JSON.stringify(specification, null, 2) + : yaml.dump(specification); + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['api1'], + }; + + it.each([ + ['exact name', ['create-resources']], + ['case-insensitive name', ['CREATE-RESOURCES']], + ['wildcard', ['create-*']], + ['exclusion', ['*', '!test']], + ['pure exclusion', ['!test']], + ])('filters specification methods and operation artifacts by %s', async (_label, operations) => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + client._resources[ResourceType.ApiOperation] = [ + { name: 'create-resources', properties: { method: 'POST', urlTemplate: '/resources' } }, + { name: 'test', properties: { method: 'GET', urlTemplate: '/test' } }, + ]; + const store = createMockStore(); + const filter: FilterConfig = { + apis: ['api1'], + apiSubFilters: { api1: { operations } }, + }; + + const result = await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', filter + ); + + expect(result.specification).toBe(true); + expect(result.errorCount).toBe(0); + expect(result.operations.map(op => op.descriptor.nameParts[1])).toEqual(['create-resources']); + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + expect(written?.[4]).toBe(format); + const parsed = format === 'json' ? JSON.parse(written![2]) : yaml.load(written![2]); + expect(parsed).toEqual({ + ...specification, + paths: { '/resources': { ...pathMetadata, post: createOperation } }, + }); + }); + + it.each([ + ['empty list', []], + ['unmatched name', ['missing-operation']], + ])('removes all paths for an %s', async (_label, operations) => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + const store = createMockStore(); + + const result = await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apis: ['api1'], apiSubFilters: { api1: { operations } } } + ); + + expect(result.specification).toBe(true); + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + expect(yaml.load(written![2])).toEqual({ ...specification, paths: {} }); + }); + + it.each([ + undefined, + { apis: ['api1'] }, + { apiSubFilters: { api1: { diagnostics: [] } } }, + { apiSubFilters: { otherApi: { operations: [] } } }, + { apiSubFilters: { api1: { operations: ['*'] } } }, + ])('preserves the original content when operations are not filtered (%j)', async (filter) => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + const store = createMockStore(); + + await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', filter + ); + + expect(store.writeContent).toHaveBeenCalledWith( + '/output', apiDescriptor, content, 'specification', format + ); + }); + + it('applies case-insensitive API root filters to workspace revisions', async () => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + const store = createMockStore(); + const descriptor: ResourceDescriptor = { + ...apiDescriptor, nameParts: ['api1;rev=2'], workspace: 'team', + }; + + await extractApiResources( + client, store, testContext, descriptor, + { name: 'api1;rev=2', properties: {} }, '/output', + { apis: ['API1'], apiSubFilters: { API1: { operations: ['create-resources'] } } }, + 'team' + ); + + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + expect(written?.[1]).toEqual(descriptor); + expect(yaml.load(written![2])).toEqual({ + ...specification, + paths: { '/resources': { ...pathMetadata, post: createOperation } }, + }); + }); + + it('filters all HTTP methods and extended paths without removing shared metadata', async () => { + const paths = { + '/resources': { + ...pathMetadata, + post: createOperation, + get: testOperation, + put: testOperation, + delete: testOperation, + options: testOperation, + head: testOperation, + patch: testOperation, + trace: testOperation, + }, + '/unidentified': { get: { responses: { '200': { description: 'No operation ID' } } } }, + 'x-custom-paths': { preserved: true }, + }; + const extendedPaths = { + '/resources?type=new': { post: createOperation }, + '/test?type=internal': { get: testOperation }, + }; + const spec = { ...specification, paths, 'x-ms-paths': extendedPaths }; + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ + content: format === 'json' ? JSON.stringify(spec) : yaml.dump(spec), format, + }), + }); + const store = createMockStore(); + + await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apiSubFilters: { api1: { operations: ['create-resources'] } } } + ); + + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + expect(yaml.load(written![2])).toEqual({ + ...specification, + paths: { + '/resources': { ...pathMetadata, post: createOperation }, + 'x-custom-paths': { preserved: true }, + }, + 'x-ms-paths': { '/resources?type=new': { post: createOperation } }, + }); + }); + + it('preserves numeric literals and date-like strings unrelated to the removed operation', async () => { + const baseContent = format === 'json' + ? JSON.stringify(specification, null, 2) + : yaml.dump(specification); + // Embed raw literals directly rather than round-tripping them through a JS + // object, since the JS engine itself would already round these numeric + // values when parsing them in this test file. + const content = format === 'json' + ? baseContent.replace( + /^\{/, + '{\n "x-large-id": 9223372036854775807,\n' + + ' "x-large-decimal": 9223372036854775807.0,\n' + + ' "x-large-exponent": 1e400,\n' + + ' "x-release-date": "2023-01-01",' + ) + : `x-large-id: 9223372036854775807\nx-release-date: 2023-01-01\n${baseContent}`; + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + const store = createMockStore(); + + await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apiSubFilters: { api1: { operations: ['create-resources'] } } } + ); + + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + expect(written![2]).toContain( + format === 'json' ? '"x-large-id": 9223372036854775807' : 'x-large-id: 9223372036854775807' + ); + expect(written![2]).toContain( + format === 'json' ? '"x-release-date": "2023-01-01"' : 'x-release-date: 2023-01-01' + ); + if (format === 'json') { + expect(written![2]).toContain('"x-large-decimal": 9223372036854775807.0'); + expect(written![2]).toContain('"x-large-exponent": 1e400'); + } + }); + + it.skipIf(format !== 'yaml')('round-trips negative and out-of-range YAML integers via BigInt, decimalizing non-decimal styles', async () => { + const content = [ + 'x-negative-big-int: -9223372036854775808', + 'x-hex: 0x1A', + yaml.dump(specification), + ].join('\n'); + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content, format }), + }); + const store = createMockStore(); + + await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apiSubFilters: { api1: { operations: ['create-resources'] } } } + ); + + const written = store.writeContent.mock.calls.find(call => call[3] === 'specification'); + // Full precision preserved for the out-of-range negative integer... + expect(written![2]).toContain('x-negative-big-int: -9223372036854775808'); + // ...while a value in a non-decimal style is re-emitted in decimal, a + // documented limitation of reserializing the whole document. + expect(written![2]).toContain('x-hex: 26'); + }); + + it('reports malformed filtered specifications without writing unfiltered content', async () => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue({ content: '{ invalid', format }), + }); + const store = createMockStore(); + + const result = await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apiSubFilters: { api1: { operations: ['create-resources'] } } } + ); + + expect(result.specification).toBe(false); + expect(result.errorCount).toBe(1); + expect(store.writeContent).not.toHaveBeenCalled(); + }); + }); + + it.each([ + { format: 'graphql', content: 'type Query { hello: String }' }, + { format: 'wsdl', content: '' }, + { format: 'wadl', content: '' }, + ])('does not apply operation filters to $format specifications', async (spec) => { + const client = createMockClient({ + getApiSpecification: vi.fn().mockResolvedValue(spec), + }); + const store = createMockStore(); + const apiDescriptor: ResourceDescriptor = { type: ResourceType.Api, nameParts: ['api1'] }; + + const result = await extractApiResources( + client, store, testContext, apiDescriptor, + { name: 'api1', properties: {} }, '/output', + { apiSubFilters: { api1: { operations: [] } } } + ); + + expect(result.specification).toBe(true); + expect(store.writeContent).toHaveBeenCalledWith( + '/output', apiDescriptor, spec.content, 'specification', spec.format + ); + }); + it('should handle missing specification gracefully', async () => { const client = createMockClient({ getApiSpecification: vi.fn().mockResolvedValue(undefined),