fix: reconcile stale extraction artifacts safely

docs/commands/extract.md

@@ -66,6 +66,7 @@ apiops extract \
 | `--output <dir>` | string | `./apim-artifacts` | No | Output directory path |
 | `--filter <path>` | string | — | No | Filter configuration YAML file |
 | `--no-transitive` | boolean | false (transitive ON) | No | Disable transitive dependency inclusion |
+| `--remove-stale` | boolean | false | No | Remove stale managed artifacts in the selected filter and workspace scope after a fully successful extraction |
 
 ### Global flags
 

src/cli/extract-command.ts

@@ -8,26 +8,46 @@
  */
 
 import { Command } from 'commander';
-import { ExtractConfig } from '../models/config.js';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { ExtractConfig, FilterConfig } from '../models/config.js';
 import { ApimServiceContext } from '../models/types.js';
 import { runExtraction, ExtractionResult } from '../services/extract-service.js';
+import { shouldReconcileResource } from '../services/filter-service.js';
 import { loadFilterConfig } from '../lib/config-loader.js';
 import { logger, parseLogLevel } from '../lib/logger.js';
 import { ApimClient } from '../clients/apim-client.js';
 import { ArtifactStore } from '../clients/artifact-store.js';
+import { IArtifactStore } from '../clients/iartifact-store.js';
 import { getCloudConfig, buildArmBaseUrl } from '../lib/cloud-config.js';
+import { EXIT_FATAL, EXIT_SUCCESS } from '../lib/exit-codes.js';
 
 /**
  * Interface for extract command options (from CLI flags).
  */
-interface ExtractOptions {
+export interface ExtractOptions {
   resourceGroup: string;
   serviceName: string;
   output: string;
   filter?: string;
   transitive: boolean;
+  removeStale?: boolean;
 }
 
+export interface ExtractCommandDependencies {
+  runExtraction: typeof runExtraction;
+  createClient: (authScope: string) => ApimClient;
+  createStore: () => IArtifactStore;
+  exit: (code: number) => void;
+}
+
+const DEFAULT_DEPENDENCIES: ExtractCommandDependencies = {
+  runExtraction,
+  createClient: (authScope) => new ApimClient(authScope),
+  createStore: () => new ArtifactStore(),
+  exit: (code) => process.exit(code),
+};
+
 /**
  * Create and return the extract command for Commander.
  */
@@ -39,6 +59,7 @@ export function createExtractCommand(): Command {
     .option('--output <dir>', 'Output directory path', './apim-artifacts')
     .option('--filter <path>', 'Filter configuration YAML file')
     .option('--no-transitive', 'Disable transitive dependency inclusion')
+    .option('--remove-stale', 'Remove stale managed artifacts in the selected scope')
     .action(async (options: ExtractOptions, command: Command) => {
       const globalOpts = command.optsWithGlobals<{
         logLevel?: string;
@@ -57,21 +78,23 @@ export function createExtractCommand(): Command {
 /**
  * Execute the extract command.
  */
-async function executeExtract(
+export async function executeExtract(
   options: ExtractOptions,
   globalOpts: {
     logLevel?: string;
     subscriptionId?: string;
     cloud?: string;
     format?: string;
     apiVersion?: string;
-  }
+  },
+  dependencies: ExtractCommandDependencies = DEFAULT_DEPENDENCIES
 ): Promise<void> {
   const subscriptionId = globalOpts.subscriptionId ?? process.env.AZURE_SUBSCRIPTION_ID;
 
   if (!subscriptionId) {
     logger.error('Subscription ID required: use --subscription-id or set AZURE_SUBSCRIPTION_ID');
-    process.exit(2);
+    dependencies.exit(2);
+    return;
   }
 
   // Build service context
@@ -92,30 +115,49 @@ async function executeExtract(
   };
 
   // Load filter config if specified
-  let filterConfig;
+  let filterConfig: FilterConfig | undefined;
   if (options.filter) {
     filterConfig = await loadFilterConfig(options.filter);
     if (!filterConfig) {
       logger.error(`Filter file not found: ${options.filter}`);
-      process.exit(2);
+      dependencies.exit(2);
+      return;
     }
   }
 
-  // Build extract config
-  const extractConfig: ExtractConfig = {
-    service: context,
-    outputDir: options.output,
-    filter: filterConfig,
-    includeTransitive: options.transitive,
-    logLevel: parseLogLevel(globalOpts.logLevel ?? 'info'),
-  };
-
   // Create client and store
-  const client = new ApimClient(cloudConfig.authScope);
-  const store = new ArtifactStore();
-
-  // Run extraction
-  const result = await runExtraction(client, store, extractConfig);
+  const client = dependencies.createClient(cloudConfig.authScope);
+  const store = dependencies.createStore();
+
+  const outputDir = path.resolve(options.output);
+  const outputParent = path.dirname(outputDir);
+  const stagingPrefix = path.join(outputParent, `.${path.basename(outputDir)}.apiops-extract-`);
+  await fs.mkdir(outputParent, { recursive: true });
+  const stagingDir = await fs.mkdtemp(stagingPrefix);
+
+  let result: ExtractionResult;
+  try {
+    const extractConfig: ExtractConfig = {
+      service: context,
+      outputDir: stagingDir,
+      filter: filterConfig,
+      includeTransitive: options.transitive,
+      logLevel: parseLogLevel(globalOpts.logLevel ?? 'info'),
+    };
+
+    result = await dependencies.runExtraction(client, store, extractConfig);
+
+    if (result.exitCode !== EXIT_FATAL) {
+      await store.commitStagedExtraction(
+        stagingDir,
+        outputDir,
+        descriptor => shouldReconcileResource(descriptor, filterConfig),
+        shouldRemoveStaleArtifacts(result, options.removeStale ?? false)
+      );
+    }
+  } finally {
+    await fs.rm(stagingDir, { recursive: true, force: true });
+  }
 
   // Output results
   if (globalOpts.format === 'json') {
@@ -124,7 +166,14 @@ async function executeExtract(
     outputText(result);
   }
 
-  process.exit(result.exitCode);
+  dependencies.exit(result.exitCode);
+}
+
+export function shouldRemoveStaleArtifacts(
+  result: Pick<ExtractionResult, 'exitCode'>,
+  requested: boolean
+): boolean {
+  return requested && result.exitCode === EXIT_SUCCESS;
 }
 
 /**

src/clients/artifact-store.ts

@@ -17,10 +17,18 @@ import {
   buildSpecificationFilePath,
   buildAssociationFilePath,
   parseArtifactPath,
+  parseArtifactChangePath,
 } from '../lib/resource-path.js';
 import { logger } from '../lib/logger.js';
 
+interface CommitFileSystem {
+  rename: typeof fs.rename;
+  rm: typeof fs.rm;
+}
+
 export class ArtifactStore implements IArtifactStore {
+  constructor(private readonly commitFileSystem: CommitFileSystem = fs) {}
+
   async writeResource(
     baseDir: string,
     descriptor: ResourceDescriptor,
@@ -218,6 +226,73 @@ export class ArtifactStore implements IArtifactStore {
     }
   }
 
+  async commitStagedExtraction(
+    stagingDir: string,
+    baseDir: string,
+    isInScope: (descriptor: ResourceDescriptor) => boolean,
+    removeStale: boolean
+  ): Promise<void> {
+    const parentDir = path.dirname(baseDir);
+    const candidateDir = await fs.mkdtemp(path.join(parentDir, `.${path.basename(baseDir)}.candidate-`));
+    const backupDir = path.join(parentDir, `.${path.basename(baseDir)}.backup-${path.basename(candidateDir)}`);
+    const baseExists = await this.fileExists(baseDir);
+
+    try {
+      if (baseExists) {
+        await fs.cp(baseDir, candidateDir, { recursive: true, force: true });
+      }
+
+      if (removeStale && baseExists) {
+        const existingFiles = await this.listFiles(baseDir);
+        for (const existingFile of existingFiles) {
+          const descriptor = this.getOwningDescriptor(baseDir, existingFile);
+          if (!descriptor || !isInScope(descriptor)) {
+            continue;
+          }
+
+          const relativePath = path.relative(baseDir, existingFile);
+          const stagedFile = path.join(stagingDir, relativePath);
+          if (!(await this.fileExists(stagedFile))) {
+            await this.commitFileSystem.rm(path.join(candidateDir, relativePath), { force: true });
+          }
+        }
+      }
+
+      await fs.cp(stagingDir, candidateDir, { recursive: true, force: true });
+
+      if (baseExists) {
+        await this.commitFileSystem.rename(baseDir, backupDir);
+      }
+      try {
+        await this.commitFileSystem.rename(candidateDir, baseDir);
+      } catch (commitError) {
+        if (baseExists) {
+          try {
+            await this.commitFileSystem.rename(backupDir, baseDir);
+          } catch (restoreError) {
+            throw new AggregateError(
+              [commitError, restoreError],
+              `Failed to commit staged extraction and restore the original output. Backup remains at ${backupDir}`,
+              { cause: restoreError }
+            );
+          }
+        }
+        throw commitError;
+      }
+
+      if (baseExists) {
+        try {
+          await this.commitFileSystem.rm(backupDir, { recursive: true, force: true });
+        } catch (error) {
+          logger.warn(`Committed extraction but could not remove backup ${backupDir}: ${(error as Error).message}`);
+        }
+      }
+      logger.debug(`Committed staged extraction from ${stagingDir} to ${baseDir}`);
+    } finally {
+      await this.commitFileSystem.rm(candidateDir, { recursive: true, force: true });
+    }
+  }
+
   private async ensureDirectory(dirPath: string): Promise<void> {
     try {
       await fs.mkdir(dirPath, { recursive: true });
@@ -251,6 +326,51 @@ export class ArtifactStore implements IArtifactStore {
     }
   }
 
+  private async listFiles(directory: string): Promise<string[]> {
+    const files: string[] = [];
+    const entries = await fs.readdir(directory, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const entryPath = path.join(directory, entry.name);
+      if (entry.isDirectory()) {
+        files.push(...await this.listFiles(entryPath));
+      } else if (entry.isFile()) {
+        files.push(entryPath);
+      }
+    }
+
+    return files;
+  }
+
+  private getOwningDescriptor(baseDir: string, filePath: string): ResourceDescriptor | undefined {
+    const descriptor = parseArtifactChangePath(baseDir, filePath);
+    if (descriptor) {
+      return descriptor;
+    }
+
+    const fileName = path.basename(filePath);
+    if (!['apis.json', 'groups.json', 'tags.json'].includes(fileName)) {
+      return undefined;
+    }
+
+    const parentFile = fileName === 'apis.json' && path.basename(path.dirname(path.dirname(filePath))) === 'gateways'
+      ? 'gatewayInformation.json'
+      : 'productInformation.json';
+    return parseArtifactPath(baseDir, path.join(path.dirname(filePath), parentFile));
+  }
+
+  private async fileExists(filePath: string): Promise<boolean> {
+    try {
+      await fs.access(filePath);
+      return true;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+        return false;
+      }
+      throw error;
+    }
+  }
+
   private descriptorEquals(a: ResourceDescriptor, b: ResourceDescriptor): boolean {
     return (
       a.type === b.type &&

src/clients/iartifact-store.ts

@@ -88,4 +88,14 @@ export interface IArtifactStore {
     baseDir: string,
     descriptor: ResourceDescriptor
   ): Promise<void>;
+
+  /**
+   * Merge staged extraction output and optionally remove stale managed files.
+   */
+  commitStagedExtraction(
+    stagingDir: string,
+    baseDir: string,
+    isInScope: (descriptor: ResourceDescriptor) => boolean,
+    removeStale: boolean
+  ): Promise<void>;
 }