Merge pull request #222 from Azure/petehauge-filter-exclusion-prefix

Support `!`-prefix exclusions in filter entries

Author: petehauge

docs/commands/extract.md

@@ -94,14 +94,17 @@ For local development, `az login` is the simplest option. For CI/CD pipelines, u
 
 By default, `apiops extract` exports **all** resources from the APIM instance (34 resource types including APIs, products, backends, named values, tags, policies, and more).
 
-To extract only specific resources, pass a YAML filter file with `--filter`. Filter entries support exact names and wildcard patterns (`*` for any characters, `?` for a single character):
+To extract only specific resources, pass a YAML filter file with `--filter`. Filter entries support exact names, wildcard patterns (`*` for any characters, `?` for a single character), and `!`-prefixed **exclusions** (e.g. `'!prod-legacy'` — see [`filtering-resources.md`](../guides/filtering-resources.md#excluding-resources-with-) for details):
+
+> **Always quote `!`-prefixed entries** — unquoted `- !prod-legacy` is parsed by YAML as a tag and fails to load.
 
 ```yaml
 # configuration.extractor.yaml
 apis:
   - echo-api
   - petstore-api
   - 'prod-*'             # Wildcard: all APIs starting with prod-
+  - '!prod-legacy-*'     # Exclusion: skip legacy prod APIs
 products:
   - starter
 backends:

docs/guides/filtering-resources.md

@@ -86,6 +86,7 @@ namedValues:
 - Names are matched case-insensitively against APIM resource names
 - Wildcard patterns are supported — `*` matches any characters, `?` matches a single character (see below)
 - Exact names and wildcard patterns can be mixed in the same array
+- Entries beginning with `!` are **exclusions** — see [Excluding resources with `!`](#excluding-resources-with-) below
 - An empty file extracts everything (same as no filter)
 - An empty array (`[]`) excludes ALL resources of that type
 
@@ -125,6 +126,54 @@ Wildcard matching is case-insensitive, just like exact matching. Special charact
 
 ---
 
+## Excluding resources with `!`
+
+Any filter entry whose first character is `!` is treated as an **exclusion**. Exclusions are applied *after* inclusions for the same list, so you can write patterns like "include everything matching this shape, except these specific ones."
+
+Semantics:
+
+- `!` must be the **first character** of the entry to count as negation. `foo!bar` is a literal name.
+- The rest of the entry is a normal filter value — exact name or wildcard pattern, matched case-insensitively.
+- A resource is included iff at least one inclusion matches it **and** no exclusion matches it.
+- A list containing only exclusions is treated as "include everything, then subtract" — equivalent to prepending an implicit `*`.
+- Exclusions respect the same semantics as inclusions: they match API root names (stripping revision suffixes), and excluding a parent (Api, Product, Gateway, Workspace) cascades to its children.
+
+> **Always quote `!`-prefixed entries in YAML.** An unquoted leading `!` (e.g. `- !prod-legacy-billing`) is parsed by YAML as a **tag** and produces an "unknown tag" error before the filter code ever sees the value. Wrap the entry in single or double quotes, exactly like the examples in this guide: `- '!prod-legacy-billing'`.
+
+### Examples
+
+```yaml
+# Include all prod-* APIs except one specific legacy API and any deprecated variants
+apis:
+  - 'prod-*'
+  - '!prod-legacy-billing'
+  - '!prod-*-deprecated'
+
+# Include every backend except the shared infra ones
+backends:
+  - '*'
+  - '!shared-monitoring'
+  - '!shared-*-infra'
+
+# Include every named value except Key Vault-backed ones (pure-exclusion list)
+namedValues:
+  - '!keyvault-*'
+```
+
+Exclusions work anywhere a string list is accepted, including sub-filter fields inside `apiSubFilters` and `workspaceSubFilters`:
+
+```yaml
+apis:
+  - 'my-api'
+apiSubFilters:
+  my-api:
+    operations:
+      - 'get-*'
+      - '!get-internal-*'   # keep all get-* operations except internal ones
+```
+
+---
+
 ## Nested Sub-Resource Filtering
 
 ### API sub-resource filters

src/services/filter-service.ts

@@ -238,6 +238,19 @@ export function wildcardMatch(pattern: string, text: string): boolean {
   return wildcardToRegex(pattern).test(text);
 }
 
+/**
+ * Check whether a single filter entry matches a resource name.
+ * Handles both exact (case-insensitive) and wildcard matching, and
+ * matches against the API root name (revision-suffix stripped) as well.
+ */
+function entryMatches(entry: string, lowerName: string, lowerRoot: string): boolean {
+  if (isWildcardPattern(entry)) {
+    return wildcardMatch(entry, lowerName) || wildcardMatch(entry, lowerRoot);
+  }
+  const lowerEntry = entry.toLowerCase();
+  return lowerName === lowerEntry || lowerRoot === lowerEntry;
+}
+
 /**
  * Match a resource name against a filter allowlist.
  *
@@ -246,6 +259,16 @@ export function wildcardMatch(pattern: string, text: string): boolean {
  * - non-empty array → case-insensitive exact match or wildcard pattern match
  *
  * Wildcard patterns use `*` (zero or more characters) and `?` (single character).
+ *
+ * Negation: entries beginning with `!` are treated as exclusions. Exclusions
+ * are evaluated after inclusions for the same list:
+ *   - If the list contains only exclusions, an implicit `*` include is assumed
+ *     ("include everything, then subtract").
+ *   - Otherwise a resource is included iff at least one inclusion matches
+ *     AND no exclusion matches.
+ * `!` must be the first character to be interpreted as negation; `foo!bar`
+ * is a literal name. `!` cannot appear in a valid APIM resource name, so a
+ * leading `!` is unambiguous.
  */
 function matchesFilter(name: string, allowlist: string[] | undefined): boolean {
   if (allowlist === undefined) {
@@ -256,17 +279,31 @@ function matchesFilter(name: string, allowlist: string[] | undefined): boolean {
     return false;
   }
 
+  const includes: string[] = [];
+  const excludes: string[] = [];
+  for (const entry of allowlist) {
+    if (entry.startsWith('!')) {
+      excludes.push(entry.slice(1));
+    } else {
+      includes.push(entry);
+    }
+  }
+
   const lowerName = name.toLowerCase();
   // For APIs, also match by root name (strip revision suffix)
   const lowerRoot = extractRootApiName(lowerName);
 
-  return allowlist.some((allowed) => {
-    if (isWildcardPattern(allowed)) {
-      return wildcardMatch(allowed, lowerName) || wildcardMatch(allowed, lowerRoot);
-    }
-    const lowerAllowed = allowed.toLowerCase();
-    return lowerName === lowerAllowed || lowerRoot === lowerAllowed;
-  });
+  // Pure-exclusion list is treated as "include-all, then subtract".
+  const included =
+    includes.length === 0
+      ? true
+      : includes.some((entry) => entryMatches(entry, lowerName, lowerRoot));
+
+  if (!included) {
+    return false;
+  }
+
+  return !excludes.some((entry) => entryMatches(entry, lowerName, lowerRoot));
 }
 
 /**

src/templates/configs/filter-config.yaml

@@ -116,3 +116,12 @@
 # - Use ? to match a single character: api-v? matches api-v1, api-v2
 # - Exact names and wildcard patterns can be mixed in the same list
 # - All matching is case-insensitive
+# - Prefix an entry with `!` to EXCLUDE it (e.g. '!prod-legacy-*'). A list
+#   containing only `!` entries means "include everything, then subtract."
+#   IMPORTANT: always QUOTE `!`-prefixed entries in YAML — an unquoted
+#   leading `!` is parsed as a YAML tag and fails to load.
+#   Example:
+#   apis:
+#     - 'prod-*'
+#     - '!prod-legacy-billing'
+#     - '!prod-*-deprecated'

src/templates/copilot/configure-filter-prompt.md

@@ -47,7 +47,7 @@ Walk through the resource types **one type at a time**. For each type, ask the u
 
 - **Extract ALL** — include every resource of this type. Leave this type **out** of the filter (APIOps extracts everything by default).
 - **Extract NONE** — exclude all resources of this type. Add the type with an empty array: `tags: []`.
-- **Extract SOME** — include only specific resources. The user provides which names (or wildcard patterns) to include. Matching is case-insensitive and supports `*` and `?` wildcards.
+- **Extract SOME** — include only specific resources. The user provides which names (or wildcard patterns) to include. Matching is case-insensitive and supports `*` and `?` wildcards. Entries can also be prefixed with `!` to **exclude** a name or pattern (e.g. `'!prod-legacy-*'`); a list containing only `!` entries means "include everything, then subtract." **Always quote `!`-prefixed entries in YAML** — an unquoted leading `!` is parsed as a YAML tag and fails to load.
 
 **Single-resource-type cadence for Step 1:**
 

tests/unit/services/filter-service.test.ts

@@ -583,4 +583,137 @@ describe('filter-service', () => {
       expect(shouldIncludeResource(excluded, filter)).toBe(false);
     });
   });
+
+  describe('negation (`!` prefix) exclusions', () => {
+    it('should treat a leading `!` as an exclusion in an otherwise-inclusive list', () => {
+      const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] };
+      const included: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-users'],
+      };
+      const excluded: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-legacy-billing'],
+      };
+      expect(shouldIncludeResource(included, filter)).toBe(true);
+      expect(shouldIncludeResource(excluded, filter)).toBe(false);
+    });
+
+    it('should support wildcard exclusions', () => {
+      const filter: FilterConfig = { apis: ['prod-*', '!prod-*-deprecated'] };
+      const included: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-users'],
+      };
+      const excluded: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-users-deprecated'],
+      };
+      expect(shouldIncludeResource(included, filter)).toBe(true);
+      expect(shouldIncludeResource(excluded, filter)).toBe(false);
+    });
+
+    it('should treat a pure-exclusion list as "include all, then subtract"', () => {
+      const filter: FilterConfig = { apis: ['!prod-legacy-billing'] };
+      const kept: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-users'],
+      };
+      const dropped: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-legacy-billing'],
+      };
+      expect(shouldIncludeResource(kept, filter)).toBe(true);
+      expect(shouldIncludeResource(dropped, filter)).toBe(false);
+    });
+
+    it('should treat a list with only wildcard exclusions as include-all-minus', () => {
+      const filter: FilterConfig = { namedValues: ['!keyvault-*'] };
+      const kept: ResourceDescriptor = {
+        type: ResourceType.NamedValue,
+        nameParts: ['api-token'],
+      };
+      const dropped: ResourceDescriptor = {
+        type: ResourceType.NamedValue,
+        nameParts: ['keyvault-secret'],
+      };
+      expect(shouldIncludeResource(kept, filter)).toBe(true);
+      expect(shouldIncludeResource(dropped, filter)).toBe(false);
+    });
+
+    it('should match exclusions case-insensitively', () => {
+      const filter: FilterConfig = { apis: ['*', '!Prod-Legacy-Billing'] };
+      const excluded: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-legacy-billing'],
+      };
+      expect(shouldIncludeResource(excluded, filter)).toBe(false);
+    });
+
+    it('should exclude API revisions when the root name matches an exclusion', () => {
+      const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] };
+      const revision: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['prod-legacy-billing;rev=2'],
+      };
+      expect(shouldIncludeResource(revision, filter)).toBe(false);
+    });
+
+    it('should cascade parent exclusions to child resources', () => {
+      const filter: FilterConfig = { apis: ['prod-*', '!prod-legacy-billing'] };
+      const policyOfExcludedApi: ResourceDescriptor = {
+        type: ResourceType.ApiPolicy,
+        nameParts: ['prod-legacy-billing'],
+      };
+      const policyOfIncludedApi: ResourceDescriptor = {
+        type: ResourceType.ApiPolicy,
+        nameParts: ['prod-users'],
+      };
+      const opOfExcludedApi: ResourceDescriptor = {
+        type: ResourceType.ApiOperation,
+        nameParts: ['prod-legacy-billing', 'get-invoices'],
+      };
+      expect(shouldIncludeResource(policyOfExcludedApi, filter)).toBe(false);
+      expect(shouldIncludeResource(policyOfIncludedApi, filter)).toBe(true);
+      expect(shouldIncludeResource(opOfExcludedApi, filter)).toBe(false);
+    });
+
+    it('should treat `!` only as negation when it is the first character', () => {
+      // "foo!bar" is a literal name, not an exclusion of "bar" that starts with "foo".
+      const filter: FilterConfig = { apis: ['foo!bar'] };
+      const literal: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['foo!bar'],
+      };
+      const other: ResourceDescriptor = {
+        type: ResourceType.Api,
+        nameParts: ['bar'],
+      };
+      // APIM resource names can't actually contain `!`, but the matcher must
+      // still treat non-leading `!` as a literal character rather than negation.
+      expect(shouldIncludeResource(literal, filter)).toBe(true);
+      expect(shouldIncludeResource(other, filter)).toBe(false);
+    });
+
+    it('should apply negation inside apiSubFilters', () => {
+      const filter: FilterConfig = {
+        apis: ['my-api'],
+        apiSubFilters: {
+          'my-api': {
+            operations: ['get-*', '!get-internal-*'],
+          },
+        },
+      };
+      const included: ResourceDescriptor = {
+        type: ResourceType.ApiOperation,
+        nameParts: ['my-api', 'get-users'],
+      };
+      const excluded: ResourceDescriptor = {
+        type: ResourceType.ApiOperation,
+        nameParts: ['my-api', 'get-internal-metrics'],
+      };
+      expect(shouldIncludeResource(included, filter)).toBe(true);
+      expect(shouldIncludeResource(excluded, filter)).toBe(false);
+    });
+  });
 });