From f1aa98c606b9c93a9c636f2f19423b7f822fbef1 Mon Sep 17 00:00:00 2001 From: Aleksey Zheltov Date: Sun, 23 Aug 2026 19:57:28 +0000 Subject: [PATCH 1/5] Add .vscode/launch.json to gitignore (contains local subscription id) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a807d47b..ce68b716 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ Desktop.ini *~ .idea/ +# VS Code local debug config (may contain local subscription IDs / paths) +.vscode/launch.json + # Local testing output (use --output .local-extract for local runs) .local-extract*/ From f25335698c2ff12f735bc6fe51792dcc157bdb5d Mon Sep 17 00:00:00 2001 From: Aleksey Zheltov Date: Tue, 25 Aug 2026 16:31:36 +0000 Subject: [PATCH 2/5] fix(publisher,redactor): preserve schema $ref on reconcile PATCH; don't redact dynamic Authorization expressions (fixes #237) --- .../azure-devops/Check-ApiopsCliVersion.ps1 | 364 ++++++++ .../azure-devops/Delete-ALL-API-From-Root.ps1 | 735 ++++++++++++++++ .../azure-pipelines-delete-all-apis.yml | 261 ++++++ .../azure-pipelines-extract-onprem.yml | 760 +++++++++++++++++ ...re-pipelines-extract-serviceconnection.yml | 747 +++++++++++++++++ .../azure-pipelines-lint-apis.yml | 636 ++++++++++++++ .../azure-pipelines-publish-mi-env.yml | 351 ++++++++ .../azure-pipelines-publish-mi.yml | 297 +++++++ ...ipelines-publish-serviceconnection-env.yml | 306 +++++++ ...re-pipelines-publish-serviceconnection.yml | 292 +++++++ .../azure-pipelines_extract-mi.yml | 783 ++++++++++++++++++ pipelines/azure-devops/prod-overrides.yaml | 27 + src/services/api-publisher.ts | 44 + src/services/secret-redactor.ts | 8 +- tests/unit/services/api-publisher.test.ts | 94 ++- tests/unit/services/secret-redactor.test.ts | 19 + 16 files changed, 5715 insertions(+), 9 deletions(-) create mode 100644 pipelines/azure-devops/Check-ApiopsCliVersion.ps1 create mode 100644 pipelines/azure-devops/Delete-ALL-API-From-Root.ps1 create mode 100644 pipelines/azure-devops/azure-pipelines-delete-all-apis.yml create mode 100644 pipelines/azure-devops/azure-pipelines-extract-onprem.yml create mode 100644 pipelines/azure-devops/azure-pipelines-extract-serviceconnection.yml create mode 100644 pipelines/azure-devops/azure-pipelines-lint-apis.yml create mode 100644 pipelines/azure-devops/azure-pipelines-publish-mi-env.yml create mode 100644 pipelines/azure-devops/azure-pipelines-publish-mi.yml create mode 100644 pipelines/azure-devops/azure-pipelines-publish-serviceconnection-env.yml create mode 100644 pipelines/azure-devops/azure-pipelines-publish-serviceconnection.yml create mode 100644 pipelines/azure-devops/azure-pipelines_extract-mi.yml create mode 100644 pipelines/azure-devops/prod-overrides.yaml diff --git a/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 b/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 new file mode 100644 index 00000000..9dab8d57 --- /dev/null +++ b/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 @@ -0,0 +1,364 @@ +<# +.SYNOPSIS + Ensures the apiops CLI (@peterhauge/apiops-cli) is installed at the requested + version, installing or upgrading it via npm when necessary. + +.DESCRIPTION + Standalone version of the Azure DevOps pipeline step 'Check apiops-cli version'. + It verifies the Node.js toolchain, discovers any existing apiops install, + compares the installed version against the requested/registry version and + installs or upgrades as needed. + + On success it writes the resolved CLI path and version to the output, and (when + running inside Azure DevOps) also surfaces them as pipeline variables + APIOPS_PATH and APIOPS_VERSION. + +.PARAMETER PackageName + The npm package name. Defaults to '@peterhauge/apiops-cli'. + +.EXAMPLE + .\Check-ApiopsCliVersion.ps1 + +.EXAMPLE + .\Check-ApiopsCliVersion.ps1 -ApiopsVersion '0.2.1-alpha.0' + +.OUTPUTS + PSCustomObject with Path and Version properties. +#> +[CmdletBinding()] +param( + [string]$ApiopsVersion = 'latest', + [string]$PackageName = '@peterhauge/apiops-cli', + # npm registry URL used for the direct connectivity check (curl). + [string]$RegistryUrl = 'https://registry.npmjs.org', + # Deprecated / kept for backward compatibility with existing callers. Behaviour + # is now automatic: an unreachable registry is a WARNING when apiops is already + # installed locally, and a hard ERROR only when no local apiops version exists. + [switch]$AllowStaleOnRegistryFailure, + # Maximum time (seconds) to wait for 'npm install -g' before aborting, so a + # hung/interactive npm process cannot stall the run forever. + [int]$InstallTimeoutSeconds = 300, + # Maximum time (seconds) to wait for the registry version probe ('npm view'). + [int]$RegistryProbeTimeoutSeconds = 30 +) + +$ErrorActionPreference = 'Stop' +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +# Tracks whether the npm registry / apiops repo could not be reached. +$script:RegistryUnreachable = $false + +# Run an npm command via cmd.exe (so npm.cmd is used) with a hard timeout and a +# progress countdown. Returns an object with ExitCode, Output and TimedOut. +function Invoke-NpmCommand { + param( + [string[]]$NpmArgs, + [string]$Activity, + [int]$TimeoutSeconds + ) + # Prefer the Windows npm launcher (npm.cmd); Get-Command may return the + # extension-less 'npm' shell script, which Start-Process cannot execute. + $npmCmd = $null + $npmCmdInfo = Get-Command npm.cmd -ErrorAction SilentlyContinue + if (-not $npmCmdInfo) { $npmCmdInfo = Get-Command npm -ErrorAction SilentlyContinue } + if ($npmCmdInfo -and $npmCmdInfo.Source -like '*.cmd') { $npmCmd = $npmCmdInfo.Source } + if ($npmCmd) { $launcherArgs = @('/d','/c',"`"$npmCmd`"") + $NpmArgs } + else { $launcherArgs = @('/d','/c','npm') + $NpmArgs } + + $outFile = [System.IO.Path]::GetTempFileName() + $errFile = [System.IO.Path]::GetTempFileName() + $prevCI = $env:CI + $env:CI = '1' # non-interactive: npm never waits on a TTY + try { + $proc = Start-Process -FilePath 'cmd.exe' -ArgumentList $launcherArgs ` + -NoNewWindow -PassThru ` + -RedirectStandardOutput $outFile -RedirectStandardError $errFile + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $timedOut = $false + $reportEvery = 5 + $nextReport = Get-Date + while (-not $proc.HasExited) { + $now = Get-Date + if ($now -ge $deadline) { $timedOut = $true; break } + if ($now -ge $nextReport) { + $remaining = [int][math]::Ceiling(($deadline - $now).TotalSeconds) + Write-Host (" ... {0}, {1,4}s remaining before timeout" -f $Activity, $remaining) + $nextReport = $now.AddSeconds($reportEvery) + } + Start-Sleep -Milliseconds 500 + } + + $npmOutput = @() + $npmOutput += (Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) + $npmOutput += (Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue) + $text = ($npmOutput -join "`n") + + if ($timedOut) { + try { $proc.Kill($true) } catch { try { $proc.Kill() } catch {} } + return [PSCustomObject]@{ ExitCode = -1; Output = $text; TimedOut = $true } + } + return [PSCustomObject]@{ ExitCode = $proc.ExitCode; Output = $text; TimedOut = $false } + } finally { + $env:CI = $prevCI + Remove-Item -LiteralPath $outFile, $errFile -ErrorAction SilentlyContinue + } +} + +# Validate raw network connectivity to the npm registry using curl, which is a +# direct, unambiguous signal: any HTTP response (even 4xx/5xx) means reachable, +# while DNS / TLS / connection failures mean unreachable. This avoids inferring +# reachability from npm command output. +function Test-RegistryReachable { + param( + [string]$Url, + [int]$TimeoutSeconds + ) + Write-Host "Checking connectivity to npm registry '$Url' (timeout ${TimeoutSeconds}s)..." + + # Use the real curl binary (curl.exe). In Windows PowerShell 5.1 the bare + # name 'curl' is an alias for Invoke-WebRequest, so we must avoid the alias. + $curlCmd = Get-Command curl.exe -ErrorAction SilentlyContinue + if (-not $curlCmd) { $curlCmd = Get-Command curl -ErrorAction SilentlyContinue } + + if ($curlCmd -and $curlCmd.CommandType -ne 'Alias') { + # Run curl via Start-Process with redirected output files. Piping native + # stderr into the PowerShell pipeline (curl writes its progress meter / + # verbose log to stderr) raises a terminating NativeCommandError under + # $ErrorActionPreference='Stop'. Reachability is taken from the exit code: + # 0 = an HTTP response was received; non-zero = a failure that we further + # classify below (TLS/certificate errors still mean the host is reachable). + $outFile = [System.IO.Path]::GetTempFileName() + $errFile = [System.IO.Path]::GetTempFileName() + try { + $curlArgs = @('-s','-v','-o','NUL','--connect-timeout',"$TimeoutSeconds",'--max-time',"$TimeoutSeconds",$Url) + $proc = Start-Process -FilePath $curlCmd.Source -ArgumentList $curlArgs ` + -NoNewWindow -PassThru -Wait ` + -RedirectStandardOutput $outFile -RedirectStandardError $errFile + $code = $proc.ExitCode + $log = @() + $log += (Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue) + $log += (Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) + $log | Where-Object { $_ } | ForEach-Object { Write-Host " $_" } + if ($code -eq 0) { return $true } + # A TLS/certificate validation failure means we DID reach the host and + # completed (or nearly completed) a TLS handshake - the registry is + # reachable, only the certificate is untrusted. Because npm may trust a + # different CA store (or the cert is mid-rotation), this must not be + # treated as 'unreachable'. curl SSL/cert exit codes: + # 35 SSL connect error, 51 peer cert/fingerprint not OK, + # 58 local cert problem, 60 peer cert not authenticated by known CA, + # 66 SSL engine init failed, 77 CA cert file problem, 83 issuer check failed. + $sslCertExitCodes = @(35, 51, 58, 60, 66, 77, 83) + if ($sslCertExitCodes -contains $code) { + Write-Host "curl exited $code (TLS/certificate validation issue); npm registry '$Url' is reachable but its certificate could not be validated. Treating as reachable." + return $true + } + Write-Host "curl exited $code; npm registry '$Url' appears unreachable." + return $false + } finally { + Remove-Item -LiteralPath $outFile, $errFile -ErrorAction SilentlyContinue + } + } + + # Fallback when curl is unavailable: any HTTP response means reachable. + Write-Host 'curl not found; falling back to Invoke-WebRequest for the connectivity check.' + try { + $null = Invoke-WebRequest -Uri $Url -Method Head -TimeoutSec $TimeoutSeconds -UseBasicParsing + return $true + } catch { + if ($_.Exception.Response) { return $true } + # A certificate trust/validation failure still means the host was reached + # (the TLS handshake got far enough to receive a certificate). Reachability + # must not hinge on certificate trust, so treat these as reachable. + $msg = $_.Exception.Message + $inner = $_.Exception.InnerException + $combined = @($msg, ($inner.Message)) -join ' ' + if ($combined -match '(?i)certificate|trust relationship|SSL/TLS|secure channel|RemoteCertificate') { + Write-Host "Connectivity check hit a TLS/certificate validation issue ($msg); the registry is reachable but its certificate could not be validated. Treating as reachable." + return $true + } + Write-Host "Connectivity check failed: $msg" + return $false + } +} + +# Sanity check Node toolchain (CLI requires Node >= 22) +$nodeVersion = (& node --version) 2>$null +if (-not $nodeVersion) { throw 'Node.js is not installed on this machine. Install Node.js >= 22.' } +Write-Host "Node $nodeVersion / npm $(& npm --version)" + +$version = $ApiopsVersion +$pkgName = $PackageName +$pkg = "$pkgName@$version" + +function Find-Apiops { + $cmd = Get-Command apiops -ErrorAction SilentlyContinue + if ($cmd) { return $cmd.Source } + $candidates = @() + try { $candidates += (& npm prefix -g 2>$null) } catch {} + $candidates += @( + "$env:AppData\npm", + "$env:ProgramFiles\nodejs", + "${env:ProgramFiles(x86)}\nodejs" + ) + foreach ($dir in ($candidates | Where-Object { $_ })) { + foreach ($name in @('apiops.cmd','apiops.exe','apiops.ps1','apiops')) { + $p = Join-Path $dir $name + if (Test-Path $p) { return $p } + } + } + return $null +} + +function Get-ApiopsInstalledVersion([string]$exePath) { + if (-not $exePath) { return $null } + try { + $raw = & $exePath --version 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { return $null } + $m = [regex]::Match($raw, '\d+\.\d+\.\d+(?:[-+][\w\.]+)?') + if ($m.Success) { return $m.Value } + } catch {} + return $null +} + +function Get-NpmRegistryVersion([string]$pkgName, [string]$requested) { + # For pinned versions, the requested string IS the desired version (no registry call needed). + if ($requested -and $requested -ne 'latest') { return $requested } + # Connectivity is decided separately (Test-RegistryReachable); skip the version + # lookup entirely when the registry is already known to be unreachable. + if ($script:RegistryUnreachable) { return $null } + + Write-Host "Resolving latest '$pkgName' version from npm..." + $viewArgs = @('view',$pkgName,'version','--prefer-online','--no-progress', + '--fetch-retries=1','--fetch-retry-mintimeout=2000','--fetch-retry-maxtimeout=5000') + $res = Invoke-NpmCommand -NpmArgs $viewArgs -Activity 'resolving latest version' -TimeoutSeconds $RegistryProbeTimeoutSeconds + if ($res.Output) { ($res.Output -split "`n") | Where-Object { $_ } | ForEach-Object { Write-Host " $_" } } + + $v = ($res.Output -split "`n" | Where-Object { $_ -match '^\s*\d+\.\d+\.\d+' } | Select-Object -Last 1) + if ($v) { return $v.ToString().Trim() } + return $null +} + +function Install-Apiops([string]$pkg) { + Write-Host "Installing $pkg globally (timeout ${InstallTimeoutSeconds}s)..." + + # Run non-interactively: no progress spinner, no audit/fund network calls. + $npmArgs = @('install','-g',$pkg,'--no-progress','--no-audit','--no-fund','--loglevel=http') + $res = Invoke-NpmCommand -NpmArgs $npmArgs -Activity 'installing' -TimeoutSeconds $InstallTimeoutSeconds + if ($res.Output) { ($res.Output -split "`n") | Where-Object { $_ } | ForEach-Object { Write-Host $_ } } + + if ($res.TimedOut) { + # A timeout while fetching almost always means the repo is unreachable. + $script:RegistryUnreachable = $true + return 'Unreachable' + } + # On a real failure, classify with a direct curl connectivity check instead of + # parsing npm's output: an unreachable registry is non-critical, anything else + # is a genuine install error. + if ($null -ne $res.ExitCode -and $res.ExitCode -ne 0) { + if (-not (Test-RegistryReachable -Url $RegistryUrl -TimeoutSeconds $RegistryProbeTimeoutSeconds)) { + $script:RegistryUnreachable = $true + return 'Unreachable' + } + throw "npm install -g $pkg failed (exit $($res.ExitCode))." + } + + $npmPrefix = (& npm prefix -g).Trim() + if ($npmPrefix -and (Test-Path $npmPrefix) -and ($env:Path -notlike "*$npmPrefix*")) { + $env:Path = "$npmPrefix;$env:Path" + } + # Refresh PATH from registry (covers installs that updated it) + $env:Path = "$env:Path;" + + [System.Environment]::GetEnvironmentVariable('Path','Machine') + ';' + + [System.Environment]::GetEnvironmentVariable('Path','User') + + return 'Success' +} + +$apiopsPath = Find-Apiops +$installedVer = Get-ApiopsInstalledVersion $apiopsPath + +# Decide registry reachability up front via a direct curl connectivity check. +$registryReachable = Test-RegistryReachable -Url $RegistryUrl -TimeoutSeconds $RegistryProbeTimeoutSeconds +$script:RegistryUnreachable = -not $registryReachable + +$desiredVer = Get-NpmRegistryVersion $pkgName $version + +Write-Host ("Installed apiops version on machine: {0}" -f ($(if ($installedVer) { $installedVer } else { '' }))) +Write-Host ("Requested apiops version: {0}" -f $version) +if ($desiredVer) { Write-Host ("Resolved package version: {0}" -f $desiredVer) } + +# Always log apiops repository (npm registry) reachability so it's visible in logs. +if ($script:RegistryUnreachable) { + $msg = "apiops npm repository ($pkgName): UNREACHABLE." + if ($env:TF_BUILD) { Write-Host "##vso[task.logissue type=warning]$msg" } else { Write-Warning $msg } +} else { + Write-Host "apiops npm repository ($pkgName): REACHABLE." +} + +$needsInstall = $false +if ($script:RegistryUnreachable) { + # Network repo not reachable: rely on whatever is installed locally. + # Warning when a local version exists; hard error only when nothing is installed. + if ($apiopsPath -and $installedVer) { + Write-Warning "apiops repository is unreachable; continuing with the locally installed apiops version ($installedVer), which may be out of date." + } else { + throw "apiops repository is unreachable and no apiops CLI is installed locally. Cannot continue." + } +} elseif (-not $apiopsPath -or -not $installedVer) { + $needsInstall = $true + Write-Host 'apiops not found on machine; installation required.' +} elseif ($desiredVer -and ($installedVer -ne $desiredVer)) { + $needsInstall = $true + Write-Host "Installed version ($installedVer) differs from desired ($desiredVer); upgrading." +} else { + Write-Host 'apiops is up to date; skipping install.' +} + +if ($needsInstall) { + $installResult = Install-Apiops $pkg + + if ($installResult -eq 'Unreachable') { + # The npm registry / apiops repo could not be reached. This is a + # non-critical step when a usable apiops CLI is already installed: + # warn and continue with the existing version instead of failing. + if ($apiopsPath -and $installedVer) { + Write-Warning "Could not reach the npm registry/apiops repo to install '$pkg'. Continuing with the already-installed apiops version ($installedVer), which may be out of date." + } else { + throw "Could not reach the npm registry/apiops repo to install '$pkg', and no existing apiops CLI was found on this machine. Cannot continue." + } + } else { + $apiopsPath = Find-Apiops + if (-not $apiopsPath) { + $npmPrefix = (& npm prefix -g).Trim() + Write-Host "npm prefix -g => $npmPrefix" + if ($npmPrefix -and (Test-Path $npmPrefix)) { + Write-Host "Contents of npm global prefix:" + Get-ChildItem -LiteralPath $npmPrefix | Format-Table Name,Length -AutoSize | Out-String | Write-Host + } + throw 'apiops still not found after global install. Check that npm''s global prefix is on PATH.' + } + $installedVer = Get-ApiopsInstalledVersion $apiopsPath + } +} + +Write-Host "Using apiops CLI: $apiopsPath (version $installedVer)" + +# Expose results to the calling pipeline step (same runspace) so it can decide how +# to surface an online-repository outage (e.g. mark the step as a warning). +$global:ApiopsRepoReachable = -not [bool]$script:RegistryUnreachable +$global:ApiopsPath = $apiopsPath +$global:ApiopsVersion = $installedVer + +# If running inside Azure DevOps, surface the resolved CLI path + version to downstream steps. +if ($env:TF_BUILD) { + Write-Host "##vso[task.setvariable variable=APIOPS_PATH]$apiopsPath" + Write-Host "##vso[task.setvariable variable=APIOPS_VERSION]$installedVer" +} + +# Emit a result object for standalone callers. +[PSCustomObject]@{ + Path = $apiopsPath + Version = $installedVer +} diff --git a/pipelines/azure-devops/Delete-ALL-API-From-Root.ps1 b/pipelines/azure-devops/Delete-ALL-API-From-Root.ps1 new file mode 100644 index 00000000..fa6862bc --- /dev/null +++ b/pipelines/azure-devops/Delete-ALL-API-From-Root.ps1 @@ -0,0 +1,735 @@ +<# +.SYNOPSIS +Deletes ALL APIs, API Version Sets, Products, Subscriptions, Users, and API Tags +from an Azure API Management (APIM) instance (root or workspace scope where applicable). + +!! IRREVERSIBLE OPERATION !! + +Scope: +- Root: service-level resources (no workspace) +- Workspace: workspace-level resources (where supported by the resource type) + +.NOTES +- API version: 2024-05-01 +- Includes retry logic for DELETEs and polling of Azure-AsyncOperation (202 / async deletes). +#> + +[CmdletBinding()] +param( + # Target environment (supplied by the pipeline from the apim- variable group) + [Parameter(Mandatory = $true)] + [string] $SubscriptionId, + + [Parameter(Mandatory = $true)] + [string] $ResourceGroup, + + [Parameter(Mandatory = $true)] + [string] $ApimName, + + [string] $ApiVersion = "2024-05-01", + + # Scope configuration + # Root -> Deletes from service root (no workspace) + # Workspace -> Deletes from a specific workspace only + [ValidateSet("Root", "Workspace")] + [string] $ScopeType = "Root", + + # Used only if $ScopeType -eq "Workspace" + [string] $WorkspaceId = "Pilot-Workspace-For-Export", + + # Safety toggle: $true performs deletion, $false is a dry run + [bool] $PerformDeletion = $true, + + # Retry / async settings + [int] $MaxDeleteRetries = 5, + [int] $InitialRetryDelaySeconds = 2, + [int] $AsyncPollIntervalSeconds = 5, + [int] $AsyncPollTimeoutSeconds = 600 # 10 minutes max for async operations +) + +function Convert-SecureStringToPlain { + param( + [Parameter(Mandatory)] + [Security.SecureString] $SecureString + ) + $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($SecureString) + try { + [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) + } + finally { + if ($bstr -ne [IntPtr]::Zero) { + [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) + } + } +} + +function Get-AccessToken { + if (Get-Command Get-AzAccessToken -ErrorAction SilentlyContinue) { + try { + $tokenObj = Get-AzAccessToken -ResourceUrl "https://management.azure.com/" + if ($null -eq $tokenObj) { + Write-Warning "Get-AzAccessToken returned null." + } elseif ($tokenObj.Token) { + $rawToken = $tokenObj.Token + switch ($rawToken.GetType().Name) { + 'SecureString' { + if (Get-Command ConvertFrom-SecureString -ErrorAction SilentlyContinue | + Where-Object { $_.Parameters.ContainsKey('AsPlainText') }) { + try { + return (ConvertFrom-SecureString -SecureString $rawToken -AsPlainText) + } catch { + Write-Warning "ConvertFrom-SecureString -AsPlainText failed, falling back to Marshal." + } + } + return Convert-SecureStringToPlain -SecureString $rawToken + } + 'String' { + return $rawToken + } + default { + Write-Warning "Unexpected token type: $($rawToken.GetType().FullName). Attempting ToString()." + return "$rawToken" + } + } + } + } catch { + Write-Warning "Get-AzAccessToken failed: $($_.Exception.Message)" + } + } + + if (Get-Command az -ErrorAction SilentlyContinue) { + try { + $cliOut = az account get-access-token --resource https://management.azure.com/ --query accessToken -o tsv 2>$null + if ($cliOut) { return $cliOut } + } catch { + Write-Warning "Azure CLI access token fetch failed: $($_.Exception.Message)" + } + } + + throw "Unable to acquire Azure access token. Use Connect-AzAccount or az login." +} + +function Invoke-AzureRest { + param( + [Parameter(Mandatory)] [string] $Method, + [Parameter(Mandatory)] [string] $Uri, + [Parameter()] [object] $Body, + [Parameter()] [hashtable] $Headers + ) + $invokeParams = @{ + Method = $Method + Uri = $Uri + Headers = $Headers + ErrorAction = 'Stop' + } + if ($Body) { + $invokeParams.ContentType = "application/json" + $invokeParams.Body = ($Body | ConvertTo-Json -Depth 10) + } + Invoke-RestMethod @invokeParams +} + +function Get-BasePath { + param( + [Parameter(Mandatory)] [string] $SubscriptionId, + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $ApimName, + [Parameter(Mandatory)] [string] $ScopeType, + [Parameter()] [string] $WorkspaceId + ) + + $base = "https://management.azure.com/subscriptions/${SubscriptionId}/resourceGroups/${ResourceGroup}/providers/Microsoft.ApiManagement/service/${ApimName}" + + if ($ScopeType -eq "Workspace") { + if (-not $WorkspaceId) { + throw "ScopeType is 'Workspace' but WorkspaceId is not specified." + } + return "${base}/workspaces/${WorkspaceId}" + } + + return $base +} + +function Get-AllPaged { + param( + [Parameter(Mandatory)] [string] $BaseUri, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + + $headers = @{ Authorization = "Bearer ${AccessToken}" } + $uri = "${BaseUri}?api-version=${ApiVersion}" + + $all = @() + while ($uri) { + Write-Host "Fetching page: ${uri}" + $resp = Invoke-AzureRest -Method GET -Uri $uri -Headers $headers + if ($resp.value) { + $all += $resp.value + } + if ($resp.nextLink) { + $uri = $resp.nextLink + } else { + $uri = $null + } + } + return $all +} + +function Wait-ForAsyncOperation { + param( + [Parameter(Mandatory)] [string] $AsyncOperationUrl, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $PollIntervalSeconds, + [Parameter(Mandatory)] [int] $TimeoutSeconds + ) + + $headers = @{ Authorization = "Bearer ${AccessToken}" } + $start = Get-Date + + while ($true) { + $elapsed = (Get-Date) - $start + if ($elapsed.TotalSeconds -ge $TimeoutSeconds) { + throw "Async operation did not complete within ${TimeoutSeconds} seconds. Last polled URL: ${AsyncOperationUrl}" + } + + Write-Host "Polling async operation: ${AsyncOperationUrl}" + $resp = Invoke-AzureRest -Method GET -Uri $AsyncOperationUrl -Headers $headers + + $status = $resp.status + if (-not $status) { + Write-Warning "Async operation response has no 'status' field. Treating as success." + return + } + + Write-Host "Async status: ${status}" + switch ($status) { + "Succeeded" { return } + "Failed" { throw "Async operation failed. Response: $($resp | ConvertTo-Json -Depth 10)" } + default { + Start-Sleep -Seconds $PollIntervalSeconds + } + } + } +} + +function Delete-WithRetry { + param( + [Parameter(Mandatory)] [string] $DeleteUri, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds, + [Parameter(Mandatory)] [string] $ResourceDescription + ) + + $headers = @{ Authorization = "Bearer ${AccessToken}" } + $attempt = 0 + $delay = $InitialDelaySeconds + + while ($attempt -lt $MaxRetries) { + $attempt++ + try { + Write-Host "DELETE ${DeleteUri}" + $response = Invoke-WebRequest -Method DELETE -Uri $DeleteUri -Headers $headers -ErrorAction Stop + + $statusCode = [int]$response.StatusCode + $asyncLocation = $response.Headers["Location"] + $azureAsyncOperation = $response.Headers["Azure-AsyncOperation"] + + if ($statusCode -eq 202 -or $azureAsyncOperation -or $asyncLocation) { + $pollUrl = if ($azureAsyncOperation) { $azureAsyncOperation } elseif ($asyncLocation) { $asyncLocation } else { $null } + if ($pollUrl) { + Write-Host "Async delete initiated for ${ResourceDescription}. Polling: ${pollUrl}" + Wait-ForAsyncOperation -AsyncOperationUrl $pollUrl -AccessToken $AccessToken -PollIntervalSeconds $AsyncPollIntervalSeconds -TimeoutSeconds $AsyncPollTimeoutSeconds + } else { + Write-Warning "Async delete indicated but no Azure-AsyncOperation or Location header found. Assuming success." + } + } + + Write-Host "[SUCCESS] Deleted ${ResourceDescription}" + return $true + } catch { + $statusCode = $_.Exception.Response.StatusCode.value__ 2>$null + $message = $_.Exception.Message + Write-Warning "[Attempt ${attempt}/${MaxRetries}] Failed to delete '${ResourceDescription}' - StatusCode: ${statusCode} - ${message}" + + if ($statusCode -in 429,500,502,503,504) { + Write-Host "Transient error. Retrying in ${delay} second(s)..." + Start-Sleep -Seconds $delay + $delay = [int]([Math]::Min($delay * 2, 60)) + continue + } else { + Write-Error "Non-retriable status code (${statusCode}). Aborting delete for '${ResourceDescription}'." + return $false + } + } + } + + Write-Error "Max retries reached. Could not delete: ${ResourceDescription}" + return $false +} + +# ---------- GET helpers ---------- + +function Get-AllApis { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/apis" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllApiVersionSets { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/apiVersionSets" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllSubscriptions { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/subscriptions" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllProducts { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/products" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllUsers { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $SubscriptionId, + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $ApimName + ) + # Users are service-level, not workspace-level + $serviceBase = "https://management.azure.com/subscriptions/${SubscriptionId}/resourceGroups/${ResourceGroup}/providers/Microsoft.ApiManagement/service/${ApimName}" + $baseUri = "${serviceBase}/users" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllTags { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $SubscriptionId, + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $ApimName + ) + # Tags are also service-level (per your example), not workspace-specific + # GET /subscriptions/.../resourceGroups/.../providers/Microsoft.ApiManagement/service/{serviceName}/tags?api-version=... + $serviceBase = "https://management.azure.com/subscriptions/${SubscriptionId}/resourceGroups/${ResourceGroup}/providers/Microsoft.ApiManagement/service/${ApimName}" + $baseUri = "${serviceBase}/tags" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllPolicyFragments { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/policyFragments" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +function Get-AllNamedValues { + param( + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken + ) + $baseUri = "${BasePath}/namedValues" + Get-AllPaged -BaseUri $baseUri -ApiVersion $ApiVersion -AccessToken $AccessToken +} + +# ---------- DELETE helpers ---------- + +function Delete-ApiWithRetry { + param( + [Parameter(Mandatory)] [string] $ApiName, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/apis/${ApiName}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "API '${ApiName}'" +} + +function Delete-ApiVersionSetWithRetry { + param( + [Parameter(Mandatory)] [string] $VersionSetId, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/apiVersionSets/${VersionSetId}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "API Version Set '${VersionSetId}'" +} + +function Delete-SubscriptionWithRetry { + param( + [Parameter(Mandatory)] [string] $SubscriptionIdApim, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/subscriptions/${SubscriptionIdApim}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "Subscription '${SubscriptionIdApim}'" +} + +function Delete-ProductWithRetry { + param( + [Parameter(Mandatory)] [string] $ProductId, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/products/${ProductId}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "Product '${ProductId}'" +} + +function Delete-UserWithRetry { + param( + [Parameter(Mandatory)] [string] $UserId, + [Parameter(Mandatory)] [string] $SubscriptionId, + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $ApimName, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + + # DELETE https://management.azure.com/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ApiManagement/service/{serviceName}/users/{userId}?api-version=2024-05-01 + $serviceBase = "https://management.azure.com/subscriptions/${SubscriptionId}/resourceGroups/${ResourceGroup}/providers/Microsoft.ApiManagement/service/${ApimName}" + $deleteUri = "${serviceBase}/users/${UserId}?api-version=${ApiVersion}" + + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "User '${UserId}'" +} + +function Delete-TagWithRetry { + param( + [Parameter(Mandatory)] [string] $TagId, + [Parameter(Mandatory)] [string] $SubscriptionId, + [Parameter(Mandatory)] [string] $ResourceGroup, + [Parameter(Mandatory)] [string] $ApimName, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + + # As per your example: + # DELETE https://management.azure.com/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ApiManagement/service/{serviceName}/tags/{tagId}?api-version=2024-05-01 + $serviceBase = "https://management.azure.com/subscriptions/${SubscriptionId}/resourceGroups/${ResourceGroup}/providers/Microsoft.ApiManagement/service/${ApimName}" + $deleteUri = "${serviceBase}/tags/${TagId}?api-version=${ApiVersion}" + + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "Tag '${TagId}'" +} + +function Delete-PolicyFragmentWithRetry { + param( + [Parameter(Mandatory)] [string] $PolicyFragmentId, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/policyFragments/${PolicyFragmentId}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "Policy Fragment '${PolicyFragmentId}'" +} + +function Delete-NamedValueWithRetry { + param( + [Parameter(Mandatory)] [string] $NamedValueId, + [Parameter(Mandatory)] [string] $BasePath, + [Parameter(Mandatory)] [string] $ApiVersion, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [int] $MaxRetries, + [Parameter(Mandatory)] [int] $InitialDelaySeconds + ) + $deleteUri = "${BasePath}/namedValues/${NamedValueId}?api-version=${ApiVersion}" + Delete-WithRetry -DeleteUri $deleteUri -AccessToken $AccessToken -MaxRetries $MaxRetries -InitialDelaySeconds $InitialDelaySeconds -ResourceDescription "Named Value '${NamedValueId}'" +} + +# ==================== MAIN ==================== + +Write-Host "Starting deletion for APIM service '${ApimName}' in resource group '${ResourceGroup}' (Subscription: ${SubscriptionId})" -ForegroundColor Cyan +Write-Host "ScopeType: ${ScopeType}" -ForegroundColor Cyan +if ($ScopeType -eq "Workspace") { + Write-Host "WorkspaceId: ${WorkspaceId}" -ForegroundColor Cyan +} +if (-not $PerformDeletion) { + Write-Warning "PerformDeletion is FALSE. DRY RUN mode." +} + +$accessToken = Get-AccessToken +if (-not $accessToken -or [string]::IsNullOrWhiteSpace($accessToken)) { + throw "Access token acquisition succeeded but token string is empty." +} + +$basePath = Get-BasePath -SubscriptionId $SubscriptionId -ResourceGroup $ResourceGroup -ApimName $ApimName -ScopeType $ScopeType -WorkspaceId $WorkspaceId +$allResults = @() + +# ---- 1) APIs ---- +$apis = Get-AllApis -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $apis -or $apis.Count -eq 0) { + Write-Warning "No APIs were found." +} else { + Write-Host "Found $($apis.Count) API(s)." -ForegroundColor Green +} + +foreach ($api in $apis) { + $apiName = $api.name + Write-Host "Processing API: ${apiName}" + + if ($PerformDeletion) { + $deleted = Delete-ApiWithRetry -ApiName $apiName -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete API: ${apiName}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'API' + Name = $apiName + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 2) API Version Sets ---- +$versionSets = Get-AllApiVersionSets -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $versionSets -or $versionSets.Count -eq 0) { + Write-Warning "No API Version Sets were found." +} else { + Write-Host "Found $($versionSets.Count) API Version Set(s)." -ForegroundColor Green +} + +foreach ($vs in $versionSets) { + $vsId = $vs.name + Write-Host "Processing API Version Set: ${vsId}" + + if ($PerformDeletion) { + $deleted = Delete-ApiVersionSetWithRetry -VersionSetId $vsId -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete API Version Set: ${vsId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'ApiVersionSet' + Name = $vsId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 3) Products ---- +$products = Get-AllProducts -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $products -or $products.Count -eq 0) { + Write-Warning "No Products were found." +} else { + Write-Host "Found $($products.Count) Product(s)." -ForegroundColor Green +} + +foreach ($p in $products) { + $productId = $p.name + Write-Host "Processing Product: ${productId}" + + if ($PerformDeletion) { + $deleted = Delete-ProductWithRetry -ProductId $productId -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete Product: ${productId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'Product' + Name = $productId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 4) Subscriptions ---- +$subs = Get-AllSubscriptions -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $subs -or $subs.Count -eq 0) { + Write-Warning "No Subscriptions were found." +} else { + Write-Host "Found $($subs.Count) Subscription(s)." -ForegroundColor Green +} + +foreach ($sub in $subs) { + $sid = $sub.name + Write-Host "Processing Subscription: ${sid}" + + if ($PerformDeletion) { + $deleted = Delete-SubscriptionWithRetry -SubscriptionIdApim $sid -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete Subscription: ${sid}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'Subscription' + Name = $sid + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 5) Users (service-level) ---- +$users = Get-AllUsers -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -SubscriptionId $SubscriptionId -ResourceGroup $ResourceGroup -ApimName $ApimName +if (-not $users -or $users.Count -eq 0) { + Write-Warning "No Users were found." +} else { + Write-Host "Found $($users.Count) User(s)." -ForegroundColor Green +} + +foreach ($user in $users) { + $userId = $user.name + Write-Host "Processing User: ${userId}" + + if ($PerformDeletion) { + $deleted = Delete-UserWithRetry -UserId $userId -SubscriptionId $SubscriptionId -ResourceGroup $ResourceGroup -ApimName $ApimName -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete User: ${userId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'User' + Name = $userId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 6) Tags (service-level) ---- +$tags = Get-AllTags -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -SubscriptionId $SubscriptionId -ResourceGroup $ResourceGroup -ApimName $ApimName +if (-not $tags -or $tags.Count -eq 0) { + Write-Warning "No Tags were found." +} else { + Write-Host "Found $($tags.Count) Tag(s)." -ForegroundColor Green +} + +foreach ($tag in $tags) { + $tagId = $tag.name + Write-Host "Processing Tag: ${tagId}" + + if ($PerformDeletion) { + $deleted = Delete-TagWithRetry -TagId $tagId -SubscriptionId $SubscriptionId -ResourceGroup $ResourceGroup -ApimName $ApimName -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete Tag: ${tagId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'Tag' + Name = $tagId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 7) Policy Fragments ---- +# Deleted before Named Values because fragments can reference named values. +$policyFragments = Get-AllPolicyFragments -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $policyFragments -or $policyFragments.Count -eq 0) { + Write-Warning "No Policy Fragments were found." +} else { + Write-Host "Found $($policyFragments.Count) Policy Fragment(s)." -ForegroundColor Green +} + +foreach ($pf in $policyFragments) { + $pfId = $pf.name + Write-Host "Processing Policy Fragment: ${pfId}" + + if ($PerformDeletion) { + $deleted = Delete-PolicyFragmentWithRetry -PolicyFragmentId $pfId -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete Policy Fragment: ${pfId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'PolicyFragment' + Name = $pfId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- 8) Named Values ---- +$namedValues = Get-AllNamedValues -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken +if (-not $namedValues -or $namedValues.Count -eq 0) { + Write-Warning "No Named Values were found." +} else { + Write-Host "Found $($namedValues.Count) Named Value(s)." -ForegroundColor Green +} + +foreach ($nv in $namedValues) { + $nvId = $nv.name + Write-Host "Processing Named Value: ${nvId}" + + if ($PerformDeletion) { + $deleted = Delete-NamedValueWithRetry -NamedValueId $nvId -BasePath $basePath -ApiVersion $ApiVersion -AccessToken $accessToken -MaxRetries $MaxDeleteRetries -InitialDelaySeconds $InitialRetryDelaySeconds + } else { + Write-Host "[DRY RUN] Would delete Named Value: ${nvId}" + $deleted = $null + } + + $allResults += [pscustomobject]@{ + Type = 'NamedValue' + Name = $nvId + Deleted = $deleted + Timestamp = (Get-Date).ToString("u") + } +} + +# ---- Summary ---- +Write-Host "`nSummary:" -ForegroundColor Cyan +if ($allResults.Count -gt 0) { + $allResults | Format-Table -AutoSize +} else { + Write-Host "No resources processed." +} + +$failed = $allResults | Where-Object { $_.Deleted -eq $false } +if ($PerformDeletion -and $failed.Count -gt 0) { + Write-Warning "$($failed.Count) resource(s) failed to delete." +} elseif ($PerformDeletion) { + Write-Host "All deletions completed successfully (where applicable)." -ForegroundColor Green +} else { + Write-Host "Dry run complete. Set PerformDeletion = \$true to execute deletions." -ForegroundColor Yellow +} \ No newline at end of file diff --git a/pipelines/azure-devops/azure-pipelines-delete-all-apis.yml b/pipelines/azure-devops/azure-pipelines-delete-all-apis.yml new file mode 100644 index 00000000..3beeda8d --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-delete-all-apis.yml @@ -0,0 +1,261 @@ +# ===================================================================== +# APIM "Delete ALL resources" pipeline ***DESTRUCTIVE / IRREVERSIBLE*** +# +# Runs Delete-ALL-API-From-Root.ps1 against a chosen APIM instance to +# remove ALL APIs, API Version Sets, Products, Subscriptions, Users and +# Tags (Root or Workspace scope). +# +# Safety model (three independent gates): +# 1. Operator must type the exact APIM name AND type "DELETE" to confirm. +# 2. Stage 1 runs a DRY RUN first and prints everything that WOULD be +# deleted — nothing is removed in this stage. +# 3. Stage 2 (the real deletion) is a deployment job targeting the +# 'apim-delete-approval' Environment, which MUST be configured in +# Azure DevOps with a manual Approval check. The deletion does not +# start until an approver signs off. +# +# Auth: Managed Identity on a self-hosted agent (same model as Extract/Publish). +# ===================================================================== + +name: 'apim-delete-all-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand only +pr: none + +parameters: + - name: ENVIRONMENT + displayName: 'Environment to PURGE (loads variable group apim-; the target APIM is APIM_SERVICE_NAME from that group)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: confirmServiceName + displayName: 'Type the exact APIM service name of the selected environment to confirm' + type: string + + - name: confirmDelete + displayName: 'Type DELETE to confirm you want to remove ALL resources' + type: string + + - name: scopeType + displayName: 'Scope' + type: string + default: 'Root' + values: + - 'Root' + - 'Workspace' + + - name: workspaceId + displayName: 'Workspace id (only used when Scope = Workspace)' + type: string + default: 'none' + + - name: apiVersion + displayName: 'APIM REST API version' + type: string + default: '2024-05-01' + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - APIM resource group + # APIM_SERVICE_NAME - APIM service name to purge + # AZURE_SUBSCRIPTION_ID - target subscription id + # Optional (managed identity): + # IDENTITY_TYPE - 'system' (default) or 'user' + # USER_ASSIGNED_CLIENT_ID - client id of the user-assigned MI (IDENTITY_TYPE=user) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + +stages: +# --------------------------------------------------------------------- +# STAGE 1 — Validate input + DRY RUN (no deletion happens here) +# --------------------------------------------------------------------- +- stage: DryRun + displayName: 'Validate + Dry run (no deletion)' + jobs: + - job: dryrun + displayName: 'List resources that WOULD be deleted' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + steps: + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate confirmation inputs' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $service = '$(APIM_SERVICE_NAME)' + $confirmName = '${{ parameters.confirmServiceName }}' + $confirmWord = '${{ parameters.confirmDelete }}' + + if ([string]::IsNullOrWhiteSpace($service) -or $service -like '$(*') { + throw "APIM_SERVICE_NAME is not defined in the 'apim-${{ parameters.ENVIRONMENT }}' variable group." + } + if ($service -cne $confirmName) { + throw "Confirmation mismatch: 'confirmServiceName' ('$confirmName') does not exactly match the APIM service of environment '${{ parameters.ENVIRONMENT }}' ('$service')." + } + if ($confirmWord -cne 'DELETE') { + throw "You must type DELETE (in capitals) in the 'confirmDelete' parameter to proceed." + } + Write-Host "Confirmation accepted for APIM service '$service'." + + - task: PowerShell@2 + displayName: 'az login (managed identity)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable group; + # an unexpanded macro means the variable is not defined in the group. + $identityType = '$(IDENTITY_TYPE)' + if ($identityType -like '$(*') { $identityType = 'system' } + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + - task: PowerShell@2 + displayName: 'Verify access to APIM' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + - task: PowerShell@2 + displayName: 'DRY RUN: list resources to be deleted' + inputs: + targetType: 'filePath' + pwsh: false + filePath: '$(Build.SourcesDirectory)/Delete-ALL-API-From-Root.ps1' + arguments: > + -SubscriptionId '$(AZURE_SUBSCRIPTION_ID)' + -ResourceGroup '$(APIM_RESOURCE_GROUP)' + -ApimName '$(APIM_SERVICE_NAME)' + -ApiVersion '${{ parameters.apiVersion }}' + -ScopeType '${{ parameters.scopeType }}' + -WorkspaceId '${{ parameters.workspaceId }}' + -PerformDeletion $false + + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 + +# --------------------------------------------------------------------- +# STAGE 2 — Manual approval gate + REAL deletion +# The deployment job below targets the 'apim-delete-approval' +# Environment. Configure a manual Approval check on that Environment +# in Azure DevOps (Pipelines > Environments > apim-delete-approval > +# Approvals and checks) so a human must approve before deletion runs. +# --------------------------------------------------------------------- +- stage: Delete + displayName: 'Delete ALL resources (requires approval)' + dependsOn: DryRun + condition: succeeded() + jobs: + - deployment: delete + displayName: 'Run destructive deletion' + pool: + name: '$(AGENT_POOL)' + environment: 'apim-delete-approval' + timeoutInMinutes: 120 + strategy: + runOnce: + deploy: + steps: + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'az login (managed identity)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable group. + $identityType = '$(IDENTITY_TYPE)' + if ($identityType -like '$(*') { $identityType = 'system' } + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + - task: PowerShell@2 + displayName: 'DELETE ALL resources from $(APIM_SERVICE_NAME)' + inputs: + targetType: 'filePath' + pwsh: false + filePath: '$(Build.SourcesDirectory)/Delete-ALL-API-From-Root.ps1' + arguments: > + -SubscriptionId '$(AZURE_SUBSCRIPTION_ID)' + -ResourceGroup '$(APIM_RESOURCE_GROUP)' + -ApimName '$(APIM_SERVICE_NAME)' + -ApiVersion '${{ parameters.apiVersion }}' + -ScopeType '${{ parameters.scopeType }}' + -WorkspaceId '${{ parameters.workspaceId }}' + -PerformDeletion $true + + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 diff --git a/pipelines/azure-devops/azure-pipelines-extract-onprem.yml b/pipelines/azure-devops/azure-pipelines-extract-onprem.yml new file mode 100644 index 00000000..b8191b3a --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-extract-onprem.yml @@ -0,0 +1,760 @@ +# ===================================================================== +# APIM Extract pipeline — ON-PREMISES Azure DevOps Server variant. +# Difference vs. the cloud variant (azure-pipelines_Version5.yml): +# * Does NOT use PublishPipelineArtifact@1 (not supported on Azure +# DevOps Server). Extracted artifacts are committed straight to a +# new branch in this repo and consumed from there by the publish +# pipeline — no separate build/pipeline artifact is produced. +# Everything else (MI auth, apiops install/upgrade, branch push) is +# identical to the cloud version. +# ===================================================================== + +name: 'apim-extract-onprem-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: CONFIGURATION_YAML_PATH + displayName: 'Operation' + type: string + default: 'Extract All APIs' + values: + - 'Extract All APIs' + - 'Extract from filter file' + + - name: ENVIRONMENT + displayName: 'Target environment (loads variable group apim-)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts will be stored' + type: string + default: 'apim-artifacts' + + - name: branchPrefix + displayName: 'Prefix for the new branch created per extraction' + type: string + default: 'apim-extract' + + - name: gitUserName + displayName: 'Git author name for the commit' + type: string + default: 'APIM Extract Pipeline' + + - name: gitUserEmail + displayName: 'Git author email for the commit' + type: string + default: 'apim-extract@devops.local' + + # -------------------- Filter options -------------------- + - name: filterFile + displayName: 'Filter file path (used only when Operation = Extract from filter file; ignored for Extract All APIs)' + type: string + default: 'none' + + - name: noTransitive + displayName: 'Disable transitive dependency extraction (--no-transitive; only meaningful with a filter file)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - APIM resource group + # APIM_SERVICE_NAME - APIM service name + # AZURE_SUBSCRIPTION_ID - target subscription id + # Optional (managed identity): + # IDENTITY_TYPE - 'system' (default) or 'user' + # USER_ASSIGNED_CLIENT_ID - client id of the user-assigned MI (IDENTITY_TYPE=user) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + - name: ARTIFACT_PATH + value: '$(Build.ArtifactStagingDirectory)/apim-artifacts' + - name: EXTRACT_BRANCH + value: '${{ parameters.branchPrefix }}/$(APIM_SERVICE_NAME)-$(Build.BuildNumber)' + +jobs: +- job: extract + displayName: 'apiops extract [${{ parameters.ENVIRONMENT }}] (${{ parameters.CONFIGURATION_YAML_PATH }}) [on-prem]' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + variables: + - name: System.AccessToken + value: $(System.AccessToken) + steps: + + - checkout: self + persistCredentials: true + clean: true + fetchDepth: 0 + + # -------- Validate Node.js (24 LTS) on the agent -------- + - task: PowerShell@2 + displayName: 'Validate Node.js (24 LTS)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # ---- Connectivity probe to the Node.js distribution host ---- + $nodeDistUrl = 'https://nodejs.org/dist/index.json' + Write-Host "Checking connectivity to Node.js distribution '$nodeDistUrl'..." + $nodeHostReachable = $false + $curlCmd = Get-Command curl.exe -ErrorAction SilentlyContinue + if ($curlCmd -and $curlCmd.CommandType -ne 'Alias') { + $o = [System.IO.Path]::GetTempFileName() + $e = [System.IO.Path]::GetTempFileName() + try { + $p = Start-Process -FilePath $curlCmd.Source ` + -ArgumentList @('-s','-v','-o','NUL','--connect-timeout','15','--max-time','15',$nodeDistUrl) ` + -NoNewWindow -PassThru -Wait ` + -RedirectStandardOutput $o -RedirectStandardError $e + @(Get-Content $e -ErrorAction SilentlyContinue) + @(Get-Content $o -ErrorAction SilentlyContinue) | + Where-Object { $_ } | ForEach-Object { Write-Host " $_" } + $nodeHostReachable = ($p.ExitCode -eq 0) + } finally { Remove-Item $o, $e -ErrorAction SilentlyContinue } + } else { + Write-Host 'curl not found; skipping Node.js distribution connectivity probe.' + } + if ($nodeHostReachable) { + Write-Host 'Node.js distribution host: REACHABLE.' + } else { + Write-Host "##vso[task.logissue type=warning]Node.js distribution host appears UNREACHABLE; relying on the Node.js already installed on the agent." + } + + # ---- Validate the Node.js installed on the agent (no download) ---- + $nodeVerRaw = (& node --version) 2>$null + if (-not $nodeVerRaw) { + $hostState = if ($nodeHostReachable) { 'reachable' } else { 'unreachable' } + throw "Node.js is not installed on this agent. Install Node.js 24 LTS (the Node.js distribution host is $hostState)." + } + Write-Host "Detected Node.js $nodeVerRaw / npm $(& npm --version)." + $nodeVer = [version](($nodeVerRaw.TrimStart('v')) -replace '-.*$','') + if ($nodeVer.Major -lt 24) { + throw "Node.js $nodeVerRaw is installed but version >= 24 (v24 LTS) is required. Update Node.js on the agent." + } + Write-Host 'Node.js 24 LTS requirement satisfied.' + + # -------- Login with Managed Identity -------- + - task: PowerShell@2 + displayName: 'az login (managed identity)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable group; + # an unexpanded macro means the variable is not defined in the group. + $identityType = '$(IDENTITY_TYPE)' + if ($identityType -like '$(*') { $identityType = 'system' } + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + if (-not [string]::IsNullOrWhiteSpace($clientId)) { + Write-Host 'IDENTITY_TYPE=system: ignoring provided USER_ASSIGNED_CLIENT_ID.' + } + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + # -------- Sanity check that the MI can see the APIM -------- + - task: PowerShell@2 + displayName: 'Verify access to APIM' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops extract -------- + - task: PowerShell@2 + displayName: 'Run APIM Extract (${{ parameters.CONFIGURATION_YAML_PATH }})' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + New-Item -ItemType Directory -Force -Path "$(ARTIFACT_PATH)" | Out-Null + + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + if ('$(IDENTITY_TYPE)' -eq 'user' -and -not [string]::IsNullOrWhiteSpace($clientId)) { + $env:AZURE_CLIENT_ID = $clientId + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + # ---- Resolve filter based on selected Operation ---- + $operation = '${{ parameters.CONFIGURATION_YAML_PATH }}' + $filterFileRel = '${{ parameters.filterFile }}' + $filterArg = $null + + if ($operation -eq 'Extract from filter file') { + if ([string]::IsNullOrWhiteSpace($filterFileRel) -or $filterFileRel -eq 'none' -or $filterFileRel -eq '-') { + throw "Operation '$operation' requires the 'filterFile' parameter to be set (e.g. configuration.extractor.yaml)." + } + $filterPath = Join-Path '$(Build.SourcesDirectory)' $filterFileRel + if (-not (Test-Path $filterPath)) { + throw "Filter file '$filterPath' not found on the checked-out branch." + } + $filterArg = $filterPath + Write-Host "Operation: $operation" + Write-Host "Using filter file: $filterPath" + Write-Host '----- Filter file contents -----' + Get-Content -LiteralPath $filterPath | Out-String | Write-Host + Write-Host '--------------------------------' + } + else { + if (-not [string]::IsNullOrWhiteSpace($filterFileRel) -and $filterFileRel -ne 'none' -and $filterFileRel -ne '-') { + Write-Host "Operation '$operation' selected; ignoring filterFile parameter ('$filterFileRel')." + } + Write-Host "Operation: $operation (extracting ALL resources)" + } + + $cliArgs = @( + 'extract', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--output', "$(ARTIFACT_PATH)" + ) + if ($filterArg) { $cliArgs += @('--filter', $filterArg) } + if ('${{ parameters.noTransitive }}' -eq 'True') { + $cliArgs += '--no-transitive' + Write-Host 'Transitive dependency extraction DISABLED (--no-transitive).' + } + + Write-Host "Running: apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + if ($LASTEXITCODE -ne 0) { throw "apiops extract failed (exit $LASTEXITCODE)" } + + # ----------------- Spectral API linting (START) ----------------- + - task: PowerShell@2 + displayName: 'Install Spectral' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Pin a known-good version. Unpinned 'latest' has shipped regressions that crash + # with "Cannot read properties of null (reading 'enum')" on some specs. Change the + # version here if you need a different build. + $spectralPackage = '@stoplight/spectral-cli@6.11.1' + npm install -g $spectralPackage + if ($LASTEXITCODE -ne 0) { throw "npm install -g $spectralPackage failed (exit $LASTEXITCODE)" } + + # npm's global bin is often NOT on PATH for the agent service account, so + # resolve the spectral launcher explicitly and hand it to the next step. + $npmPrefix = (& npm prefix -g).Trim() + $spectral = $null + foreach ($name in @('spectral.cmd','spectral.exe','spectral')) { + $candidate = Join-Path $npmPrefix $name + if (Test-Path $candidate) { $spectral = $candidate; break } + } + if (-not $spectral) { + $cmd = Get-Command spectral -ErrorAction SilentlyContinue + if ($cmd) { $spectral = $cmd.Source } + } + if (-not $spectral) { throw "Spectral CLI not found after global install (npm prefix: $npmPrefix)." } + Write-Host "Spectral CLI: $spectral" + Write-Host "##vso[task.setvariable variable=SPECTRAL_PATH]$spectral" + + - task: PowerShell@2 + displayName: 'Resolve ruleset rule set (for passed-checks reporting)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Spectral only ever emits VIOLATIONS -- never the rules that passed. To report real + # passed checks we need the full ACTIVE rule set (the recommended spectral:oas rules + # that 'extends: spectral:oas' enables, plus the custom rules). We read it straight + # from the installed Spectral packages + the ruleset file, so the denominator is + # accurate. If anything here fails, the report falls back to the rules observed in + # this run, so reporting never breaks. + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + $rulesetLocal = '$(Build.ArtifactStagingDirectory)/spectral-ruleset.yaml' + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $scriptFile = '$(Build.ArtifactStagingDirectory)/extract-rules.cjs' + Remove-Item -LiteralPath $rulesFile -ErrorAction SilentlyContinue + + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $ruleset -OutFile $rulesetLocal -UseBasicParsing + } catch { + Write-Host "##vso[task.logissue type=warning]Could not download ruleset for rule extraction: $($_.Exception.Message)" + } + + $gRoot = (& npm root -g).Trim() + + # Node script: union of (recommended OpenAPI rules from @stoplight/spectral-rulesets) + # and (custom rules parsed from the ruleset YAML, honouring false/off disables). + $js = @( + 'const fs = require("fs");', + 'const path = require("path");', + 'const { createRequire } = require("module");', + 'try {', + ' const gRoot = process.env.SPECTRAL_GLOBAL_MODULES;', + ' const req = createRequire(path.join(gRoot, "@stoplight", "spectral-cli", "package.json"));', + ' const set = new Set();', + ' try {', + ' const { oas } = req("@stoplight/spectral-rulesets");', + ' for (const [k, v] of Object.entries(oas.rules)) { if (v && v.recommended !== false) set.add(k); }', + ' } catch (e) { process.stderr.write("OAS_RULES_ERROR: " + e + "\n"); }', + ' try {', + ' const yaml = req("@stoplight/yaml");', + ' const doc = yaml.parse(fs.readFileSync(process.env.SPECTRAL_RULESET_FILE, "utf8"));', + ' if (doc && doc.rules) {', + ' for (const [k, v] of Object.entries(doc.rules)) {', + ' if (v === false || v === "off" || v === 0) { set.delete(k); continue; }', + ' set.add(k);', + ' }', + ' }', + ' } catch (e) { process.stderr.write("CUSTOM_RULES_ERROR: " + e + "\n"); }', + ' process.stdout.write(Array.from(set).join("\n"));', + '} catch (e) {', + ' process.stderr.write("RULE_EXTRACT_ERROR: " + (e && e.stack ? e.stack : String(e)));', + ' process.exit(3);', + '}' + ) + Set-Content -LiteralPath $scriptFile -Value $js -Encoding utf8 + + $env:SPECTRAL_GLOBAL_MODULES = $gRoot + $env:SPECTRAL_RULESET_FILE = $rulesetLocal + $out = & node $scriptFile 2>&1 + $code = $LASTEXITCODE + # stdout = rule names (one per line); stderr diagnostics carry known prefixes. + $ruleNames = @($out | Where-Object { $_ -and ($_ -notmatch '^(OAS_RULES_ERROR|CUSTOM_RULES_ERROR|RULE_EXTRACT_ERROR)') }) + if ($code -eq 0 -and $ruleNames.Count -gt 0) { + $ruleNames | Set-Content -LiteralPath $rulesFile -Encoding utf8 + Write-Host "Resolved active rule set: $($ruleNames.Count) rules." + } else { + Write-Host "##vso[task.logissue type=warning]Could not resolve full rule set (node exit $code); passed-checks will fall back to the rules observed in this run." + Write-Host ($out -join "`n") + } + exit 0 + + - task: PowerShell@2 + displayName: 'Run Spectral Linting' + continueOnError: true + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $spectral = '$(SPECTRAL_PATH)' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + $specRoot = '$(ARTIFACT_PATH)/apis' + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + + Remove-Item -LiteralPath $json, $crashLog -ErrorAction SilentlyContinue + + # Lint ONLY the per-API OpenAPI specification file: apis//specification.*. + # Match at ONE level under 'apis' (NOT -Recurse) so nested specification.* files under + # operations/, schemas/, etc. are excluded -- we want exactly one spec per API. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + Write-Host "Spectral CLI: $spectral" + Write-Host "Spec root: $specRoot" + Write-Host "Ruleset: $ruleset" + Write-Host "Spec files: $($specFiles.Count)" + + # Lint each API spec individually so one malformed spec that crashes Spectral/Nimma + # ("Cannot read properties of null (reading 'enum')") is isolated to that file instead + # of aborting the whole run; every other spec still gets reported. + $all = New-Object System.Collections.Generic.List[object] + $worst = 0 + foreach ($f in $specFiles) { + Write-Host "----- Linting: $f" + $per = [System.IO.Path]::GetTempFileName() + & $spectral lint --format stylish --format json --output.json $per --fail-severity warn $f -r $ruleset + $code = $LASTEXITCODE + if ($code -ge 2) { + Write-Host "##vso[task.logissue type=warning]Spectral CRASHED on: $f (exit $code)" + Add-Content -LiteralPath $crashLog -Value $f + if ($code -gt $worst) { $worst = $code } + } elseif ($code -eq 1 -and $worst -lt 1) { + $worst = 1 + } + if (Test-Path -LiteralPath $per) { + $raw = Get-Content -LiteralPath $per -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + try { + $parsed = ConvertFrom-Json -InputObject $raw + # PS 5.1 may surface the JSON array as a single nested object; flatten one level + # so EACH finding is added individually (not the whole array as one element). + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { [void]$all.Add($sub) } + } else { + [void]$all.Add($item) + } + } + } catch { + Write-Host "##vso[task.logissue type=warning]Could not parse Spectral JSON for: $f" + } + } + Remove-Item -LiteralPath $per -ErrorAction SilentlyContinue + } + } + + # Merge all per-file findings into the single JSON the report builder consumes. + if ($all.Count -gt 0) { + ($all | ConvertTo-Json -Depth 50) | Set-Content -LiteralPath $json -Encoding utf8 + } else { + '[]' | Set-Content -LiteralPath $json -Encoding utf8 + } + + $crashed = if (Test-Path -LiteralPath $crashLog) { @(Get-Content -LiteralPath $crashLog).Count } else { 0 } + Write-Host "Linted $($specFiles.Count) spec file(s); $($all.Count) finding(s); $crashed crashed." + # 0 = all clean, 1 = findings, 2+ = at least one spec crashed Spectral. + Write-Host "##vso[task.setvariable variable=SPECTRAL_EXIT]$worst" + # Findings/crashes are surfaced via the published test results, not via this task's + # exit code, so end cleanly to avoid a misleading red '##[error]'. + $global:LASTEXITCODE = 0 + exit 0 + + - task: PowerShell@2 + displayName: 'Build severity-labelled lint report' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $junit = '$(Build.ArtifactStagingDirectory)/spectral-result.xml' + $specRoot = '$(ARTIFACT_PATH)/apis' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + # Exit code from the 'Run Spectral Linting' step: 0 clean, 1 findings, 2+ Spectral error. + $spectralExit = '$(SPECTRAL_EXIT)' + + # Spec files that crashed Spectral (one absolute path per line) -> flagged as failures. + $crashedSet = @{} + if (Test-Path -LiteralPath $crashLog) { + foreach ($line in (Get-Content -LiteralPath $crashLog)) { + $t = $line.Trim() + if ($t) { $crashedSet[($t -replace '\\','/').ToLowerInvariant()] = $t } + } + } + + # Spectral severity codes -> labels used as a prefix in each test name. + $sevName = @{ 0 = 'ERROR'; 1 = 'WARN'; 2 = 'INFO'; 3 = 'HINT' } + + $results = @() + if (Test-Path -LiteralPath $json) { + $raw = Get-Content -LiteralPath $json -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + $parsed = ConvertFrom-Json -InputObject $raw + # Windows PowerShell 5.1 can surface the JSON array as a single nested object; + # flatten one level so each element is an individual Spectral result. + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { $results += $sub } + } else { + $results += $item + } + } + } + } + + # Enumerate the SAME per-API specs the lint step used (apis//specification.*), + # one level under 'apis' (NOT -Recurse), so report counts match the lint scope exactly. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + $doc = New-Object System.Xml.XmlDocument + [void]$doc.AppendChild($doc.CreateXmlDeclaration('1.0','utf-8',$null)) + $root = $doc.AppendChild($doc.CreateElement('testsuites')) + + function Add-LintSuite { + param($Doc, $Root, $SevName, [string]$Source, $Items, $AllCodes) + $Items = @($Items) + $AllCodes = @($AllCodes) + $ts = $Doc.CreateElement('testsuite') + $ts.SetAttribute('name', $Source) + $ts.SetAttribute('errors','0') + + # Codes that FIRED on this spec become failures; every other rule in the + # run-wide rule set is reported as a real PASSED check for this spec. + $firedCodes = @{} + foreach ($r in $Items) { $c = [string]$r.code; if ($c) { $firedCodes[$c] = $true } } + $passedCodes = @($AllCodes | Where-Object { $_ -and -not $firedCodes.ContainsKey($_) }) + + $failCount = $Items.Count + $passCount = $passedCodes.Count + if ($failCount -eq 0 -and $passCount -eq 0) { + # Nothing fired anywhere in the run -> a single generic passing check. + $ts.SetAttribute('tests','1') + $ts.SetAttribute('failures','0') + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name','No lint issues found') + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + [void]$Root.AppendChild($ts) + return + } + + $ts.SetAttribute('tests',[string]($failCount + $passCount)) + $ts.SetAttribute('failures',[string]$failCount) + + # Derive the API folder name from the spec path so passed checks can be linked + # back to a specific API (a passed rule applies to the whole spec, not one path). + $apiName = '' + if ($Source) { + $apiName = (($Source -replace '\\','/') -split '/apis/')[-1].Split('/')[0] + } + + # One PASSED testcase per rule that did NOT fire on this spec. + foreach ($pc in $passedCodes) { + $tc = $Doc.CreateElement('testcase') + $passName = "[PASS] $pc" + if ($apiName) { $passName += " ($apiName)" } + $tc.SetAttribute('name',$passName) + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + } + foreach ($r in $Items) { + $sevRaw = $r.severity + if ($sevRaw -is [System.Array]) { $sevRaw = $sevRaw[0] } + $sev = if ($null -ne $sevRaw) { [int]$sevRaw } else { 1 } + $label = if ($SevName.ContainsKey($sev)) { $SevName[$sev] } else { "SEV$sev" } + $pathStr = if ($r.path) { ($r.path -join '/') } else { '' } + $line = 0; $col = 0 + if ($r.range -and $r.range.start) { + $line = [int]$r.range.start.line + 1 + $col = [int]$r.range.start.character + 1 + } + $code = [string]$r.code + $name = "[$label] $code" + if ($pathStr) { $name += " ($pathStr)" } + + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name',$name) + $tc.SetAttribute('classname',$Source) + + # Every finding stays a Failed outcome; severity is conveyed via name + failure type. + $fail = $Doc.CreateElement('failure') + $fail.SetAttribute('message',[string]$r.message) + $fail.SetAttribute('type',$label) + $detail = "line $line, col $col, $($r.message) ($code) at path #/$pathStr" + [void]$fail.AppendChild($Doc.CreateCDataSection($detail)) + [void]$tc.AppendChild($fail) + + # Attach the offending spec file to this result (ADO Server 2022.2+/cloud). + if ($Source -and (Test-Path -LiteralPath $Source)) { + $so = $Doc.CreateElement('system-out') + $so.InnerText = "[[ATTACHMENT|$Source]]" + [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + } + [void]$Root.AppendChild($ts) + } + + # Index findings by normalized source path so each spec file can be matched + # to its findings regardless of slash/case differences. + $findingsByNorm = @{} + foreach ($r in $results) { + $src = [string]$r.source + $norm = ($src -replace '\\','/').ToLowerInvariant() + if (-not $findingsByNorm.ContainsKey($norm)) { + $findingsByNorm[$norm] = [pscustomobject]@{ Source = $src; Items = @() } + } + $findingsByNorm[$norm].Items += $r + } + + # Denominator for passed-checks = the full ACTIVE rule set resolved from the installed + # Spectral packages + ruleset (written by the 'Resolve ruleset rule set' step). Falls + # back to the rules observed in this run if resolution was unavailable. + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $firedCodes = @($results | ForEach-Object { [string]$_.code } | Where-Object { $_ } | Select-Object -Unique) + $ruleUniverse = @() + if (Test-Path -LiteralPath $rulesFile) { + $ruleUniverse = @(Get-Content -LiteralPath $rulesFile | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } + # Union so any rule that fired but is missing from the resolved set still appears. + $allCodes = @($ruleUniverse + $firedCodes | Where-Object { $_ } | Select-Object -Unique) + + if ($specFiles.Count -eq 0 -and $results.Count -eq 0 -and $crashedSet.Count -eq 0) { + Add-LintSuite $doc $root $sevName 'API lint' @() @() + } else { + $seen = @{} + foreach ($file in $specFiles) { + $norm = ($file -replace '\\','/').ToLowerInvariant() + $seen[$norm] = $true + if ($crashedSet.ContainsKey($norm)) { + # This spec crashed Spectral -> explicit failure (NOT counted as passed). + $ts = $doc.CreateElement('testsuite') + $ts.SetAttribute('name',$file) + $ts.SetAttribute('tests','1'); $ts.SetAttribute('failures','1'); $ts.SetAttribute('errors','0') + $tc = $doc.CreateElement('testcase') + $tc.SetAttribute('name','Spectral crashed while linting this spec') + $tc.SetAttribute('classname',$file) + $fail = $doc.CreateElement('failure') + $fail.SetAttribute('message','Spectral threw an error on this specification (unsupported or malformed construct).') + $fail.SetAttribute('type','ERROR') + [void]$fail.AppendChild($doc.CreateCDataSection("Spectral failed on $file. This spec was NOT linted; fix the spec or exclude it. See the 'Run Spectral Linting' log.")) + [void]$tc.AppendChild($fail) + if (Test-Path -LiteralPath $file) { + $so = $doc.CreateElement('system-out'); $so.InnerText = "[[ATTACHMENT|$file]]"; [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + [void]$root.AppendChild($ts) + } elseif ($findingsByNorm.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } else { + Add-LintSuite $doc $root $sevName $file @() $allCodes + } + } + # Findings whose source was not matched to an enumerated spec file. + foreach ($norm in $findingsByNorm.Keys) { + if (-not $seen.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } + } + } + + $doc.Save($junit) + $filesWithIssues = $findingsByNorm.Count + $crashedFiles = $crashedSet.Count + $cleanFiles = [Math]::Max(0, $specFiles.Count - $filesWithIssues - $crashedFiles) + Write-Host "Wrote JUnit report: $($results.Count) finding(s) across $filesWithIssues file(s); $crashedFiles crashed; $cleanFiles passed -> $junit" + + - task: PublishTestResults@2 + displayName: 'Publish Spectral lint results' + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/spectral-result.xml' + searchFolder: '$(Build.ArtifactStagingDirectory)' + testRunTitle: 'API lint results $(Build.SourceBranchName)' + failTaskOnFailedTests: false + # ----------------- Spectral API linting (END) ----------------- + + # -------- Commit extracted artifacts to a NEW branch in this repo -------- + - task: PowerShell@2 + displayName: 'Push artifacts to new repo branch' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + $branch = '$(EXTRACT_BRANCH)' + $targetFolder = '${{ parameters.targetRepoFolder }}' + $artifactPath = '$(ARTIFACT_PATH)' + $repoFolderAbs = Join-Path '$(Build.SourcesDirectory)' $targetFolder + + Write-Host "Branch: $branch" + Write-Host "Target folder: $repoFolderAbs" + + git config user.email '${{ parameters.gitUserEmail }}' + git config user.name '${{ parameters.gitUserName }}' + + if (Test-Path $repoFolderAbs) { Remove-Item -Recurse -Force $repoFolderAbs } + New-Item -ItemType Directory -Force -Path $repoFolderAbs | Out-Null + Copy-Item -Path (Join-Path $artifactPath '*') -Destination $repoFolderAbs -Recurse -Force + + git checkout -b $branch + + git add -- $targetFolder + $pending = git status --porcelain + if ([string]::IsNullOrWhiteSpace($pending)) { + Write-Host 'No changes detected against the base branch; nothing to commit.' + } else { + git commit -m "APIM extract: $(APIM_SERVICE_NAME) (build $(Build.BuildNumber))" + if ($LASTEXITCODE -ne 0) { throw "git commit failed (exit $LASTEXITCODE)" } + } + + git push --set-upstream origin $branch + if ($LASTEXITCODE -ne 0) { throw "git push failed (exit $LASTEXITCODE)" } + + Write-Host "##vso[task.setvariable variable=extractBranch;isOutput=true]$branch" + Write-Host "Pushed branch '$branch' with artifacts under '$targetFolder/'." + name: pushBranch + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + # -------- Always logout -------- + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 diff --git a/pipelines/azure-devops/azure-pipelines-extract-serviceconnection.yml b/pipelines/azure-devops/azure-pipelines-extract-serviceconnection.yml new file mode 100644 index 00000000..5500c343 --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-extract-serviceconnection.yml @@ -0,0 +1,747 @@ +# ===================================================================== +# APIM Extract pipeline — authenticates via an Azure DevOps SERVICE +# CONNECTION (no Managed Identity required). +# +# Use this variant when: +# * The self-hosted agent does NOT have a Managed Identity (typical +# for on-prem Azure DevOps Server agents running on plain VMs). +# * You already manage Azure auth through a Service Connection +# (workload-identity federation, service principal + secret, or +# service principal + certificate). +# +# Differences vs. azure-pipelines_Version5.yml: +# * `az login` step is replaced by AzureCLI@2 (handles auth itself). +# * Service connection name comes from the per-environment variable +# group (AZURE_SERVICE_CONNECTION). +# * Removes identityType / userAssignedClientId (irrelevant for SC). +# Everything else (apiops install/upgrade, filter, branch push) is the +# same. +# ===================================================================== + +name: 'apim-extract-sc-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: CONFIGURATION_YAML_PATH + displayName: 'Operation' + type: string + default: 'Extract All APIs' + values: + - 'Extract All APIs' + - 'Extract from filter file' + + - name: ENVIRONMENT + displayName: 'Target environment (loads variable group apim-)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts will be stored' + type: string + default: 'apim-artifacts' + + - name: branchPrefix + displayName: 'Prefix for the new branch created per extraction' + type: string + default: 'apim-extract' + + - name: gitUserName + displayName: 'Git author name for the commit' + type: string + default: 'APIM Extract Pipeline' + + - name: gitUserEmail + displayName: 'Git author email for the commit' + type: string + default: 'apim-extract@devops.local' + + # -------------------- Filter options -------------------- + - name: filterFile + displayName: 'Filter file path (used only when Operation = Extract from filter file; ignored for Extract All APIs)' + type: string + default: 'none' + + - name: noTransitive + displayName: 'Disable transitive dependency extraction (--no-transitive; only meaningful with a filter file)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - APIM resource group + # APIM_SERVICE_NAME - APIM service name + # AZURE_SERVICE_CONNECTION - ARM service connection used for Azure / APIM auth + # (the pipeline must be authorized to use it) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + - name: ARTIFACT_PATH + value: '$(Build.ArtifactStagingDirectory)/apim-artifacts' + - name: EXTRACT_BRANCH + value: '${{ parameters.branchPrefix }}/$(APIM_SERVICE_NAME)-$(Build.BuildNumber)' + +jobs: +- job: extract + displayName: 'apiops extract via Service Connection [${{ parameters.ENVIRONMENT }}] (${{ parameters.CONFIGURATION_YAML_PATH }})' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + variables: + - name: System.AccessToken + value: $(System.AccessToken) + steps: + + # Checkout with credentials persisted so we can push a new branch later. + - checkout: self + persistCredentials: true + clean: true + fetchDepth: 0 + + # -------- Validate Node.js (24 LTS) on the agent -------- + - task: PowerShell@2 + displayName: 'Validate Node.js (24 LTS)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # ---- Connectivity probe to the Node.js distribution host (informative) ---- + $nodeDistUrl = 'https://nodejs.org/dist/index.json' + Write-Host "Checking connectivity to Node.js distribution '$nodeDistUrl'..." + $nodeHostReachable = $false + $curlCmd = Get-Command curl.exe -ErrorAction SilentlyContinue + if ($curlCmd -and $curlCmd.CommandType -ne 'Alias') { + $o = [System.IO.Path]::GetTempFileName() + $e = [System.IO.Path]::GetTempFileName() + try { + $p = Start-Process -FilePath $curlCmd.Source ` + -ArgumentList @('-s','-v','-o','NUL','--connect-timeout','15','--max-time','15',$nodeDistUrl) ` + -NoNewWindow -PassThru -Wait ` + -RedirectStandardOutput $o -RedirectStandardError $e + @(Get-Content $e -ErrorAction SilentlyContinue) + @(Get-Content $o -ErrorAction SilentlyContinue) | + Where-Object { $_ } | ForEach-Object { Write-Host " $_" } + $nodeHostReachable = ($p.ExitCode -eq 0) + } finally { Remove-Item $o, $e -ErrorAction SilentlyContinue } + } else { + Write-Host 'curl not found; skipping Node.js distribution connectivity probe.' + } + if ($nodeHostReachable) { + Write-Host 'Node.js distribution host: REACHABLE.' + } else { + Write-Host "##vso[task.logissue type=warning]Node.js distribution host appears UNREACHABLE; relying on the Node.js already installed on the agent." + } + + # ---- Validate the Node.js installed on the agent (no download) ---- + $nodeVerRaw = (& node --version) 2>$null + if (-not $nodeVerRaw) { + $hostState = if ($nodeHostReachable) { 'reachable' } else { 'unreachable' } + throw "Node.js is not installed on this agent. Install Node.js 24 LTS (the Node.js distribution host is $hostState)." + } + Write-Host "Detected Node.js $nodeVerRaw / npm $(& npm --version)." + $nodeVer = [version](($nodeVerRaw.TrimStart('v')) -replace '-.*$','') + if ($nodeVer.Major -lt 24) { + throw "Node.js $nodeVerRaw is installed but version >= 24 (v24 LTS) is required. Update Node.js on the agent." + } + Write-Host 'Node.js 24 LTS requirement satisfied.' + + # -------- Sanity check that the SC can see the APIM -------- + - task: AzureCLI@2 + displayName: 'Verify access to APIM (service connection)' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' # Windows PowerShell 5.1 + scriptLocation: 'inlineScript' + inlineScript: | + $ErrorActionPreference = 'Stop' + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # Surface the subscription ID resolved from the service connection + $subId = (az account show --query id -o tsv).Trim() + Write-Host "##vso[task.setvariable variable=AZURE_SUBSCRIPTION_ID]$subId" + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + # Auth is not required to resolve/install the CLI, so this runs as a plain + # PowerShell step (not via the service connection). + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops extract (Service Connection injects creds for CLI) -------- + - task: AzureCLI@2 + displayName: 'Run APIM Extract (${{ parameters.CONFIGURATION_YAML_PATH }})' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' + scriptLocation: 'inlineScript' + addSpnToEnvironment: true + inlineScript: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + New-Item -ItemType Directory -Force -Path "$(ARTIFACT_PATH)" | Out-Null + + # ---- Hand SC credentials to DefaultAzureCredential (used by apiops) ---- + # AzureCLI@2 with addSpnToEnvironment exposes these PowerShell variables + # for the script: $env:servicePrincipalId, $env:servicePrincipalKey, + # $env:tenantId, and $env:idToken (for federated SCs). + if ($env:servicePrincipalId) { + $env:AZURE_CLIENT_ID = $env:servicePrincipalId + $env:AZURE_TENANT_ID = $env:tenantId + if ($env:servicePrincipalKey) { + # Service principal + secret SCs + $env:AZURE_CLIENT_SECRET = $env:servicePrincipalKey + Write-Host 'Service connection type: service principal + secret' + } elseif ($env:idToken) { + # Workload-identity federation SCs + $env:AZURE_FEDERATED_TOKEN = $env:idToken + Write-Host 'Service connection type: workload identity federation' + } else { + Write-Host 'Service connection has no secret / id token exposed; relying on ambient az CLI session.' + } + } else { + Write-Host 'addSpnToEnvironment did not expose SPN vars; relying on ambient az CLI session.' + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + # ---- Resolve filter based on selected Operation ---- + $operation = '${{ parameters.CONFIGURATION_YAML_PATH }}' + $filterFileRel = '${{ parameters.filterFile }}' + $filterArg = $null + + if ($operation -eq 'Extract from filter file') { + if ([string]::IsNullOrWhiteSpace($filterFileRel) -or $filterFileRel -eq 'none' -or $filterFileRel -eq '-') { + throw "Operation '$operation' requires the 'filterFile' parameter to be set (e.g. configuration.extractor.yaml)." + } + $filterPath = Join-Path '$(Build.SourcesDirectory)' $filterFileRel + if (-not (Test-Path $filterPath)) { + throw "Filter file '$filterPath' not found on the checked-out branch." + } + $filterArg = $filterPath + Write-Host "Operation: $operation" + Write-Host "Using filter file: $filterPath" + Write-Host '----- Filter file contents -----' + Get-Content -LiteralPath $filterPath | Out-String | Write-Host + Write-Host '--------------------------------' + } + else { + if (-not [string]::IsNullOrWhiteSpace($filterFileRel) -and $filterFileRel -ne 'none' -and $filterFileRel -ne '-') { + Write-Host "Operation '$operation' selected; ignoring filterFile parameter ('$filterFileRel')." + } + Write-Host "Operation: $operation (extracting ALL resources)" + } + + $cliArgs = @( + 'extract', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--output', "$(ARTIFACT_PATH)" + ) + if ($filterArg) { $cliArgs += @('--filter', $filterArg) } + if ('${{ parameters.noTransitive }}' -eq 'True') { + $cliArgs += '--no-transitive' + Write-Host 'Transitive dependency extraction DISABLED (--no-transitive).' + } + + Write-Host "Running: apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + if ($LASTEXITCODE -ne 0) { throw "apiops extract failed (exit $LASTEXITCODE)" } + + # ----------------- Spectral API linting (START) ----------------- + - task: PowerShell@2 + displayName: 'Install Spectral' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Pin a known-good version. Unpinned 'latest' has shipped regressions that crash + # with "Cannot read properties of null (reading 'enum')" on some specs. Change the + # version here if you need a different build. + $spectralPackage = '@stoplight/spectral-cli@6.11.1' + npm install -g $spectralPackage + if ($LASTEXITCODE -ne 0) { throw "npm install -g $spectralPackage failed (exit $LASTEXITCODE)" } + + # npm's global bin is often NOT on PATH for the agent service account, so + # resolve the spectral launcher explicitly and hand it to the next step. + $npmPrefix = (& npm prefix -g).Trim() + $spectral = $null + foreach ($name in @('spectral.cmd','spectral.exe','spectral')) { + $candidate = Join-Path $npmPrefix $name + if (Test-Path $candidate) { $spectral = $candidate; break } + } + if (-not $spectral) { + $cmd = Get-Command spectral -ErrorAction SilentlyContinue + if ($cmd) { $spectral = $cmd.Source } + } + if (-not $spectral) { throw "Spectral CLI not found after global install (npm prefix: $npmPrefix)." } + Write-Host "Spectral CLI: $spectral" + Write-Host "##vso[task.setvariable variable=SPECTRAL_PATH]$spectral" + + - task: PowerShell@2 + displayName: 'Resolve ruleset rule set (for passed-checks reporting)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Spectral only ever emits VIOLATIONS -- never the rules that passed. To report real + # passed checks we need the full ACTIVE rule set (the recommended spectral:oas rules + # that 'extends: spectral:oas' enables, plus the custom rules). We read it straight + # from the installed Spectral packages + the ruleset file, so the denominator is + # accurate. If anything here fails, the report falls back to the rules observed in + # this run, so reporting never breaks. + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + $rulesetLocal = '$(Build.ArtifactStagingDirectory)/spectral-ruleset.yaml' + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $scriptFile = '$(Build.ArtifactStagingDirectory)/extract-rules.cjs' + Remove-Item -LiteralPath $rulesFile -ErrorAction SilentlyContinue + + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $ruleset -OutFile $rulesetLocal -UseBasicParsing + } catch { + Write-Host "##vso[task.logissue type=warning]Could not download ruleset for rule extraction: $($_.Exception.Message)" + } + + $gRoot = (& npm root -g).Trim() + + # Node script: union of (recommended OpenAPI rules from @stoplight/spectral-rulesets) + # and (custom rules parsed from the ruleset YAML, honouring false/off disables). + $js = @( + 'const fs = require("fs");', + 'const path = require("path");', + 'const { createRequire } = require("module");', + 'try {', + ' const gRoot = process.env.SPECTRAL_GLOBAL_MODULES;', + ' const req = createRequire(path.join(gRoot, "@stoplight", "spectral-cli", "package.json"));', + ' const set = new Set();', + ' try {', + ' const { oas } = req("@stoplight/spectral-rulesets");', + ' for (const [k, v] of Object.entries(oas.rules)) { if (v && v.recommended !== false) set.add(k); }', + ' } catch (e) { process.stderr.write("OAS_RULES_ERROR: " + e + "\n"); }', + ' try {', + ' const yaml = req("@stoplight/yaml");', + ' const doc = yaml.parse(fs.readFileSync(process.env.SPECTRAL_RULESET_FILE, "utf8"));', + ' if (doc && doc.rules) {', + ' for (const [k, v] of Object.entries(doc.rules)) {', + ' if (v === false || v === "off" || v === 0) { set.delete(k); continue; }', + ' set.add(k);', + ' }', + ' }', + ' } catch (e) { process.stderr.write("CUSTOM_RULES_ERROR: " + e + "\n"); }', + ' process.stdout.write(Array.from(set).join("\n"));', + '} catch (e) {', + ' process.stderr.write("RULE_EXTRACT_ERROR: " + (e && e.stack ? e.stack : String(e)));', + ' process.exit(3);', + '}' + ) + Set-Content -LiteralPath $scriptFile -Value $js -Encoding utf8 + + $env:SPECTRAL_GLOBAL_MODULES = $gRoot + $env:SPECTRAL_RULESET_FILE = $rulesetLocal + $out = & node $scriptFile 2>&1 + $code = $LASTEXITCODE + # stdout = rule names (one per line); stderr diagnostics carry known prefixes. + $ruleNames = @($out | Where-Object { $_ -and ($_ -notmatch '^(OAS_RULES_ERROR|CUSTOM_RULES_ERROR|RULE_EXTRACT_ERROR)') }) + if ($code -eq 0 -and $ruleNames.Count -gt 0) { + $ruleNames | Set-Content -LiteralPath $rulesFile -Encoding utf8 + Write-Host "Resolved active rule set: $($ruleNames.Count) rules." + } else { + Write-Host "##vso[task.logissue type=warning]Could not resolve full rule set (node exit $code); passed-checks will fall back to the rules observed in this run." + Write-Host ($out -join "`n") + } + exit 0 + + - task: PowerShell@2 + displayName: 'Run Spectral Linting' + continueOnError: true + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $spectral = '$(SPECTRAL_PATH)' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + $specRoot = '$(ARTIFACT_PATH)/apis' + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + + Remove-Item -LiteralPath $json, $crashLog -ErrorAction SilentlyContinue + + # Lint ONLY the per-API OpenAPI specification file: apis//specification.*. + # Match at ONE level under 'apis' (NOT -Recurse) so nested specification.* files under + # operations/, schemas/, etc. are excluded -- we want exactly one spec per API. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + Write-Host "Spectral CLI: $spectral" + Write-Host "Spec root: $specRoot" + Write-Host "Ruleset: $ruleset" + Write-Host "Spec files: $($specFiles.Count)" + + # Lint each API spec individually so one malformed spec that crashes Spectral/Nimma + # ("Cannot read properties of null (reading 'enum')") is isolated to that file instead + # of aborting the whole run; every other spec still gets reported. + $all = New-Object System.Collections.Generic.List[object] + $worst = 0 + foreach ($f in $specFiles) { + Write-Host "----- Linting: $f" + $per = [System.IO.Path]::GetTempFileName() + & $spectral lint --format stylish --format json --output.json $per --fail-severity warn $f -r $ruleset + $code = $LASTEXITCODE + if ($code -ge 2) { + Write-Host "##vso[task.logissue type=warning]Spectral CRASHED on: $f (exit $code)" + Add-Content -LiteralPath $crashLog -Value $f + if ($code -gt $worst) { $worst = $code } + } elseif ($code -eq 1 -and $worst -lt 1) { + $worst = 1 + } + if (Test-Path -LiteralPath $per) { + $raw = Get-Content -LiteralPath $per -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + try { + $parsed = ConvertFrom-Json -InputObject $raw + # PS 5.1 may surface the JSON array as a single nested object; flatten one level + # so EACH finding is added individually (not the whole array as one element). + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { [void]$all.Add($sub) } + } else { + [void]$all.Add($item) + } + } + } catch { + Write-Host "##vso[task.logissue type=warning]Could not parse Spectral JSON for: $f" + } + } + Remove-Item -LiteralPath $per -ErrorAction SilentlyContinue + } + } + + # Merge all per-file findings into the single JSON the report builder consumes. + if ($all.Count -gt 0) { + ($all | ConvertTo-Json -Depth 50) | Set-Content -LiteralPath $json -Encoding utf8 + } else { + '[]' | Set-Content -LiteralPath $json -Encoding utf8 + } + + $crashed = if (Test-Path -LiteralPath $crashLog) { @(Get-Content -LiteralPath $crashLog).Count } else { 0 } + Write-Host "Linted $($specFiles.Count) spec file(s); $($all.Count) finding(s); $crashed crashed." + # 0 = all clean, 1 = findings, 2+ = at least one spec crashed Spectral. + Write-Host "##vso[task.setvariable variable=SPECTRAL_EXIT]$worst" + # Findings/crashes are surfaced via the published test results, not via this task's + # exit code, so end cleanly to avoid a misleading red '##[error]'. + $global:LASTEXITCODE = 0 + exit 0 + + - task: PowerShell@2 + displayName: 'Build severity-labelled lint report' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $junit = '$(Build.ArtifactStagingDirectory)/spectral-result.xml' + $specRoot = '$(ARTIFACT_PATH)/apis' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + # Exit code from the 'Run Spectral Linting' step: 0 clean, 1 findings, 2+ Spectral error. + $spectralExit = '$(SPECTRAL_EXIT)' + + # Spec files that crashed Spectral (one absolute path per line) -> flagged as failures. + $crashedSet = @{} + if (Test-Path -LiteralPath $crashLog) { + foreach ($line in (Get-Content -LiteralPath $crashLog)) { + $t = $line.Trim() + if ($t) { $crashedSet[($t -replace '\\','/').ToLowerInvariant()] = $t } + } + } + + # Spectral severity codes -> labels used as a prefix in each test name. + $sevName = @{ 0 = 'ERROR'; 1 = 'WARN'; 2 = 'INFO'; 3 = 'HINT' } + + $results = @() + if (Test-Path -LiteralPath $json) { + $raw = Get-Content -LiteralPath $json -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + $parsed = ConvertFrom-Json -InputObject $raw + # Windows PowerShell 5.1 can surface the JSON array as a single nested object; + # flatten one level so each element is an individual Spectral result. + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { $results += $sub } + } else { + $results += $item + } + } + } + } + + # Enumerate the SAME per-API specs the lint step used (apis//specification.*), + # one level under 'apis' (NOT -Recurse), so report counts match the lint scope exactly. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + $doc = New-Object System.Xml.XmlDocument + [void]$doc.AppendChild($doc.CreateXmlDeclaration('1.0','utf-8',$null)) + $root = $doc.AppendChild($doc.CreateElement('testsuites')) + + function Add-LintSuite { + param($Doc, $Root, $SevName, [string]$Source, $Items, $AllCodes) + $Items = @($Items) + $AllCodes = @($AllCodes) + $ts = $Doc.CreateElement('testsuite') + $ts.SetAttribute('name', $Source) + $ts.SetAttribute('errors','0') + + # Codes that FIRED on this spec become failures; every other rule in the + # run-wide rule set is reported as a real PASSED check for this spec. + $firedCodes = @{} + foreach ($r in $Items) { $c = [string]$r.code; if ($c) { $firedCodes[$c] = $true } } + $passedCodes = @($AllCodes | Where-Object { $_ -and -not $firedCodes.ContainsKey($_) }) + + $failCount = $Items.Count + $passCount = $passedCodes.Count + if ($failCount -eq 0 -and $passCount -eq 0) { + # Nothing fired anywhere in the run -> a single generic passing check. + $ts.SetAttribute('tests','1') + $ts.SetAttribute('failures','0') + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name','No lint issues found') + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + [void]$Root.AppendChild($ts) + return + } + + $ts.SetAttribute('tests',[string]($failCount + $passCount)) + $ts.SetAttribute('failures',[string]$failCount) + + # Derive the API folder name from the spec path so passed checks can be linked + # back to a specific API (a passed rule applies to the whole spec, not one path). + $apiName = '' + if ($Source) { + $apiName = (($Source -replace '\\','/') -split '/apis/')[-1].Split('/')[0] + } + + # One PASSED testcase per rule that did NOT fire on this spec. + foreach ($pc in $passedCodes) { + $tc = $Doc.CreateElement('testcase') + $passName = "[PASS] $pc" + if ($apiName) { $passName += " ($apiName)" } + $tc.SetAttribute('name',$passName) + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + } + foreach ($r in $Items) { + $sevRaw = $r.severity + if ($sevRaw -is [System.Array]) { $sevRaw = $sevRaw[0] } + $sev = if ($null -ne $sevRaw) { [int]$sevRaw } else { 1 } + $label = if ($SevName.ContainsKey($sev)) { $SevName[$sev] } else { "SEV$sev" } + $pathStr = if ($r.path) { ($r.path -join '/') } else { '' } + $line = 0; $col = 0 + if ($r.range -and $r.range.start) { + $line = [int]$r.range.start.line + 1 + $col = [int]$r.range.start.character + 1 + } + $code = [string]$r.code + $name = "[$label] $code" + if ($pathStr) { $name += " ($pathStr)" } + + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name',$name) + $tc.SetAttribute('classname',$Source) + + # Every finding stays a Failed outcome; severity is conveyed via name + failure type. + $fail = $Doc.CreateElement('failure') + $fail.SetAttribute('message',[string]$r.message) + $fail.SetAttribute('type',$label) + $detail = "line $line, col $col, $($r.message) ($code) at path #/$pathStr" + [void]$fail.AppendChild($Doc.CreateCDataSection($detail)) + [void]$tc.AppendChild($fail) + + # Attach the offending spec file to this result (ADO Server 2022.2+/cloud). + if ($Source -and (Test-Path -LiteralPath $Source)) { + $so = $Doc.CreateElement('system-out') + $so.InnerText = "[[ATTACHMENT|$Source]]" + [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + } + [void]$Root.AppendChild($ts) + } + + # Index findings by normalized source path so each spec file can be matched + # to its findings regardless of slash/case differences. + $findingsByNorm = @{} + foreach ($r in $results) { + $src = [string]$r.source + $norm = ($src -replace '\\','/').ToLowerInvariant() + if (-not $findingsByNorm.ContainsKey($norm)) { + $findingsByNorm[$norm] = [pscustomobject]@{ Source = $src; Items = @() } + } + $findingsByNorm[$norm].Items += $r + } + + # Denominator for passed-checks = the full ACTIVE rule set resolved from the installed + # Spectral packages + ruleset (written by the 'Resolve ruleset rule set' step). Falls + # back to the rules observed in this run if resolution was unavailable. + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $firedCodes = @($results | ForEach-Object { [string]$_.code } | Where-Object { $_ } | Select-Object -Unique) + $ruleUniverse = @() + if (Test-Path -LiteralPath $rulesFile) { + $ruleUniverse = @(Get-Content -LiteralPath $rulesFile | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } + # Union so any rule that fired but is missing from the resolved set still appears. + $allCodes = @($ruleUniverse + $firedCodes | Where-Object { $_ } | Select-Object -Unique) + + if ($specFiles.Count -eq 0 -and $results.Count -eq 0 -and $crashedSet.Count -eq 0) { + Add-LintSuite $doc $root $sevName 'API lint' @() @() + } else { + $seen = @{} + foreach ($file in $specFiles) { + $norm = ($file -replace '\\','/').ToLowerInvariant() + $seen[$norm] = $true + if ($crashedSet.ContainsKey($norm)) { + # This spec crashed Spectral -> explicit failure (NOT counted as passed). + $ts = $doc.CreateElement('testsuite') + $ts.SetAttribute('name',$file) + $ts.SetAttribute('tests','1'); $ts.SetAttribute('failures','1'); $ts.SetAttribute('errors','0') + $tc = $doc.CreateElement('testcase') + $tc.SetAttribute('name','Spectral crashed while linting this spec') + $tc.SetAttribute('classname',$file) + $fail = $doc.CreateElement('failure') + $fail.SetAttribute('message','Spectral threw an error on this specification (unsupported or malformed construct).') + $fail.SetAttribute('type','ERROR') + [void]$fail.AppendChild($doc.CreateCDataSection("Spectral failed on $file. This spec was NOT linted; fix the spec or exclude it. See the 'Run Spectral Linting' log.")) + [void]$tc.AppendChild($fail) + if (Test-Path -LiteralPath $file) { + $so = $doc.CreateElement('system-out'); $so.InnerText = "[[ATTACHMENT|$file]]"; [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + [void]$root.AppendChild($ts) + } elseif ($findingsByNorm.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } else { + Add-LintSuite $doc $root $sevName $file @() $allCodes + } + } + # Findings whose source was not matched to an enumerated spec file. + foreach ($norm in $findingsByNorm.Keys) { + if (-not $seen.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } + } + } + + $doc.Save($junit) + $filesWithIssues = $findingsByNorm.Count + $crashedFiles = $crashedSet.Count + $cleanFiles = [Math]::Max(0, $specFiles.Count - $filesWithIssues - $crashedFiles) + Write-Host "Wrote JUnit report: $($results.Count) finding(s) across $filesWithIssues file(s); $crashedFiles crashed; $cleanFiles passed -> $junit" + + - task: PublishTestResults@2 + displayName: 'Publish Spectral lint results' + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/spectral-result.xml' + searchFolder: '$(Build.ArtifactStagingDirectory)' + testRunTitle: 'API lint results $(Build.SourceBranchName)' + failTaskOnFailedTests: false + # ----------------- Spectral API linting (END) ----------------- + + # -------- Commit extracted artifacts to a NEW branch in this repo -------- + - task: PowerShell@2 + displayName: 'Push artifacts to new repo branch' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + $branch = '$(EXTRACT_BRANCH)' + $targetFolder = '${{ parameters.targetRepoFolder }}' + $artifactPath = '$(ARTIFACT_PATH)' + $repoFolderAbs = Join-Path '$(Build.SourcesDirectory)' $targetFolder + + Write-Host "Branch: $branch" + Write-Host "Target folder: $repoFolderAbs" + + git config user.email '${{ parameters.gitUserEmail }}' + git config user.name '${{ parameters.gitUserName }}' + + if (Test-Path $repoFolderAbs) { Remove-Item -Recurse -Force $repoFolderAbs } + New-Item -ItemType Directory -Force -Path $repoFolderAbs | Out-Null + Copy-Item -Path (Join-Path $artifactPath '*') -Destination $repoFolderAbs -Recurse -Force + + git checkout -b $branch + + git add -- $targetFolder + $pending = git status --porcelain + if ([string]::IsNullOrWhiteSpace($pending)) { + Write-Host 'No changes detected against the base branch; nothing to commit.' + } else { + git commit -m "APIM extract: $(APIM_SERVICE_NAME) (build $(Build.BuildNumber))" + if ($LASTEXITCODE -ne 0) { throw "git commit failed (exit $LASTEXITCODE)" } + } + + git push --set-upstream origin $branch + if ($LASTEXITCODE -ne 0) { throw "git push failed (exit $LASTEXITCODE)" } + + Write-Host "##vso[task.setvariable variable=extractBranch;isOutput=true]$branch" + Write-Host "Pushed branch '$branch' with artifacts under '$targetFolder/'." + name: pushBranch + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/pipelines/azure-devops/azure-pipelines-lint-apis.yml b/pipelines/azure-devops/azure-pipelines-lint-apis.yml new file mode 100644 index 00000000..24ca7dff --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-lint-apis.yml @@ -0,0 +1,636 @@ +# ===================================================================== +# APIM Spectral Lint pipeline — STANDALONE API linting. +# +# Lints the OpenAPI specifications already committed to this repo under +# /apis//specification.*, using Spectral and a +# shared ruleset. Unlike the Extract pipeline (which lints freshly +# extracted artifacts before pushing them), this pipeline needs NO +# Azure authentication and does NOT talk to APIM — it only reads files +# from the checked-out branch. Use it to: +# * Gate pull requests that touch API specs (enable the `pr` trigger +# and/or a branch-policy build validation on this pipeline). +# * Run linting on demand against any branch. +# +# Output: +# * A JUnit report published to the run's Tests tab. Each rule that +# fired is a Failed test (severity in the name); every other active +# rule is a Passed test per spec. Specs that crash Spectral are +# surfaced as explicit failures. +# * The raw Spectral JSON + ruleset are published as a build artifact. +# +# By default findings do NOT fail the pipeline (they are reported via +# the Tests tab). Set `failOnLintIssues = true` to fail the run when any +# finding at or above `failSeverity` is present. +# ===================================================================== + +name: 'apim-lint-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none # set to a branch list (or configure a build-validation policy) to gate PRs + +parameters: + - name: targetRepoFolder + displayName: 'Folder (inside repo) where API artifacts live' + type: string + default: 'apim-artifacts' + + - name: ENVIRONMENT + displayName: 'Environment (loads variable group apim- for the agent pool)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: rulesetUrl + displayName: 'Spectral ruleset (URL or repo-relative path)' + type: string + default: 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + + - name: spectralVersion + displayName: 'Spectral CLI npm version' + type: string + default: '6.11.1' + + - name: failSeverity + displayName: 'Minimum severity Spectral treats as a finding' + type: string + default: 'warn' + values: + - 'error' + - 'warn' + - 'info' + - 'hint' + + - name: failOnLintIssues + displayName: 'Fail the pipeline when findings are present (otherwise report only)' + type: boolean + default: false + +variables: + # AGENT_POOL comes from the 'apim-' variable group (Pipelines > Library). + - group: 'apim-${{ parameters.ENVIRONMENT }}' + - name: SPEC_ROOT + value: '$(Build.SourcesDirectory)/${{ parameters.targetRepoFolder }}/apis' + +jobs: +- job: lint + displayName: 'Spectral lint APIs in ${{ parameters.targetRepoFolder }}' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 30 + steps: + + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate selected branch contains API artifacts' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + Write-Host "Ref: $(Build.SourceBranch)" + Write-Host "Commit: $(Build.SourceVersion)" + + $specRoot = '$(SPEC_ROOT)' + if (-not (Test-Path -LiteralPath $specRoot)) { + throw "Expected API folder '$specRoot' not found on this branch. Pick a branch that contains '${{ parameters.targetRepoFolder }}/apis'." + } + Write-Host "API root: $specRoot" + + # -------- Validate Node.js is available on the agent -------- + - task: PowerShell@2 + displayName: 'Validate Node.js' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $node = Get-Command node -ErrorAction SilentlyContinue + if (-not $node) { throw 'Node.js was not found on the agent. Spectral requires Node.js (install an LTS release on the agent).' } + Write-Host "Node.js: $(node --version) at $($node.Source)" + Write-Host "npm: $(npm --version)" + + # ----------------- Spectral API linting (START) ----------------- + - task: PowerShell@2 + displayName: 'Install Spectral' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Pin a known-good version. Unpinned 'latest' has shipped regressions that crash + # with "Cannot read properties of null (reading 'enum')" on some specs. Change the + # version via the 'spectralVersion' parameter if you need a different build. + $spectralPackage = '@stoplight/spectral-cli@${{ parameters.spectralVersion }}' + npm install -g $spectralPackage + if ($LASTEXITCODE -ne 0) { throw "npm install -g $spectralPackage failed (exit $LASTEXITCODE)" } + + # npm's global bin is often NOT on PATH for the agent service account, so + # resolve the spectral launcher explicitly and hand it to the next step. + $npmPrefix = (& npm prefix -g).Trim() + $spectral = $null + foreach ($name in @('spectral.cmd','spectral.exe','spectral')) { + $candidate = Join-Path $npmPrefix $name + if (Test-Path $candidate) { $spectral = $candidate; break } + } + if (-not $spectral) { + $cmd = Get-Command spectral -ErrorAction SilentlyContinue + if ($cmd) { $spectral = $cmd.Source } + } + if (-not $spectral) { throw "Spectral CLI not found after global install (npm prefix: $npmPrefix)." } + Write-Host "Spectral CLI: $spectral" + Write-Host "##vso[task.setvariable variable=SPECTRAL_PATH]$spectral" + + - task: PowerShell@2 + displayName: 'Resolve ruleset (for passed-checks reporting)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Spectral only ever emits VIOLATIONS -- never the rules that passed. To report real + # passed checks we need the full ACTIVE rule set (the recommended spectral:oas rules + # that 'extends: spectral:oas' enables, plus the custom rules). We read it straight + # from the installed Spectral packages + the ruleset file, so the denominator is + # accurate. If anything here fails, the report falls back to the rules observed in + # this run, so reporting never breaks. + $ruleset = '${{ parameters.rulesetUrl }}' + $rulesetLocal = '$(Build.ArtifactStagingDirectory)/spectral-ruleset.yaml' + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $scriptFile = '$(Build.ArtifactStagingDirectory)/extract-rules.cjs' + Remove-Item -LiteralPath $rulesFile -ErrorAction SilentlyContinue + + # Ruleset may be an http(s) URL or a repo-relative path; normalise to a local file + # so the rule-extraction script and the lint step both consume the same content. + if ($ruleset -match '^(?i)https?://') { + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $ruleset -OutFile $rulesetLocal -UseBasicParsing + } catch { + Write-Host "##vso[task.logissue type=warning]Could not download ruleset for rule extraction: $($_.Exception.Message)" + } + } else { + $localRuleset = Join-Path '$(Build.SourcesDirectory)' $ruleset + if (Test-Path -LiteralPath $localRuleset) { + Copy-Item -LiteralPath $localRuleset -Destination $rulesetLocal -Force + } else { + Write-Host "##vso[task.logissue type=warning]Ruleset path '$localRuleset' not found; passed-checks reporting will fall back to observed rules." + } + } + + $gRoot = (& npm root -g).Trim() + + # Node script: union of (recommended OpenAPI rules from @stoplight/spectral-rulesets) + # and (custom rules parsed from the ruleset YAML, honouring false/off disables). + $js = @( + 'const fs = require("fs");', + 'const path = require("path");', + 'const { createRequire } = require("module");', + 'try {', + ' const gRoot = process.env.SPECTRAL_GLOBAL_MODULES;', + ' const req = createRequire(path.join(gRoot, "@stoplight", "spectral-cli", "package.json"));', + ' const set = new Set();', + ' try {', + ' const { oas } = req("@stoplight/spectral-rulesets");', + ' for (const [k, v] of Object.entries(oas.rules)) { if (v && v.recommended !== false) set.add(k); }', + ' } catch (e) { process.stderr.write("OAS_RULES_ERROR: " + e + "\n"); }', + ' try {', + ' const yaml = req("@stoplight/yaml");', + ' const doc = yaml.parse(fs.readFileSync(process.env.SPECTRAL_RULESET_FILE, "utf8"));', + ' if (doc && doc.rules) {', + ' for (const [k, v] of Object.entries(doc.rules)) {', + ' if (v === false || v === "off" || v === 0) { set.delete(k); continue; }', + ' set.add(k);', + ' }', + ' }', + ' } catch (e) { process.stderr.write("CUSTOM_RULES_ERROR: " + e + "\n"); }', + ' process.stdout.write(Array.from(set).join("\n"));', + '} catch (e) {', + ' process.stderr.write("RULE_EXTRACT_ERROR: " + (e && e.stack ? e.stack : String(e)));', + ' process.exit(3);', + '}' + ) + Set-Content -LiteralPath $scriptFile -Value $js -Encoding utf8 + + $env:SPECTRAL_GLOBAL_MODULES = $gRoot + $env:SPECTRAL_RULESET_FILE = $rulesetLocal + # Node prints deprecation warnings (e.g. punycode DEP0040) to stderr. In Windows + # PowerShell 5.1 ANY native stderr is turned into a NativeCommandError that terminates + # under $ErrorActionPreference='Stop' -- even when redirected with 2> to a file. Run + # node via Start-Process so stdout/stderr go straight to files and never touch + # PowerShell's error stream; success is decided solely by the process exit code. + $outFile = '$(Build.ArtifactStagingDirectory)/extract-rules.out' + $errFile = '$(Build.ArtifactStagingDirectory)/extract-rules.err' + $proc = Start-Process -FilePath 'node' -ArgumentList "`"$scriptFile`"" -NoNewWindow -Wait -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile + $code = $proc.ExitCode + $out = if (Test-Path -LiteralPath $outFile) { Get-Content -LiteralPath $outFile } else { @() } + if ((Test-Path -LiteralPath $errFile) -and (Get-Item -LiteralPath $errFile).Length -gt 0) { + Write-Host "node stderr:`n$(Get-Content -LiteralPath $errFile -Raw)" + } + # stdout = rule names (one per line); stderr diagnostics carry known prefixes. + $ruleNames = @($out | Where-Object { $_ -and ($_ -notmatch '^(OAS_RULES_ERROR|CUSTOM_RULES_ERROR|RULE_EXTRACT_ERROR)') }) + if ($code -eq 0 -and $ruleNames.Count -gt 0) { + $ruleNames | Set-Content -LiteralPath $rulesFile -Encoding utf8 + Write-Host "Resolved active rule set: $($ruleNames.Count) rules." + } else { + Write-Host "##vso[task.logissue type=warning]Could not resolve full rule set (node exit $code); passed-checks will fall back to the rules observed in this run." + Write-Host ($out -join "`n") + } + exit 0 + + - task: PowerShell@2 + displayName: 'Run Spectral Linting' + continueOnError: true + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $spectral = '$(SPECTRAL_PATH)' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + $specRoot = '$(SPEC_ROOT)' + # Prefer the locally-resolved ruleset (URL or repo path already normalised); fall + # back to the original value so the step still runs if resolution was skipped. + $ruleset = '$(Build.ArtifactStagingDirectory)/spectral-ruleset.yaml' + if (-not (Test-Path -LiteralPath $ruleset)) { $ruleset = '${{ parameters.rulesetUrl }}' } + + Remove-Item -LiteralPath $json, $crashLog -ErrorAction SilentlyContinue + + # Lint ONLY the per-API OpenAPI specification file: apis//specification.*. + # Match at ONE level under 'apis' (NOT -Recurse) so nested specification.* files under + # operations/, schemas/, etc. are excluded -- we want exactly one spec per API. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + Write-Host "Spectral CLI: $spectral" + Write-Host "Spec root: $specRoot" + Write-Host "Ruleset: $ruleset" + Write-Host "Fail severity: ${{ parameters.failSeverity }}" + Write-Host "Spec files: $($specFiles.Count)" + + # Lint each API spec individually so one malformed spec that crashes Spectral/Nimma + # ("Cannot read properties of null (reading 'enum')") is isolated to that file instead + # of aborting the whole run; every other spec still gets reported. + # Per-file wall-clock time is captured so the JUnit report carries real durations + # (otherwise the Tests tab shows a misleading "0s Run duration"). + $all = New-Object System.Collections.Generic.List[object] + $worst = 0 + $durations = @{} + foreach ($f in $specFiles) { + Write-Host "----- Linting: $f" + $per = [System.IO.Path]::GetTempFileName() + $sw = [System.Diagnostics.Stopwatch]::StartNew() + & $spectral lint --format stylish --format json --output.json $per --fail-severity ${{ parameters.failSeverity }} $f -r $ruleset + $code = $LASTEXITCODE + $sw.Stop() + $durations[$f] = [Math]::Round($sw.Elapsed.TotalSeconds, 3) + if ($code -ge 2) { + Write-Host "##vso[task.logissue type=warning]Spectral CRASHED on: $f (exit $code)" + Add-Content -LiteralPath $crashLog -Value $f + if ($code -gt $worst) { $worst = $code } + } elseif ($code -eq 1 -and $worst -lt 1) { + $worst = 1 + } + if (Test-Path -LiteralPath $per) { + $raw = Get-Content -LiteralPath $per -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + try { + $parsed = ConvertFrom-Json -InputObject $raw + # PS 5.1 may surface the JSON array as a single nested object; flatten one level + # so EACH finding is added individually (not the whole array as one element). + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { [void]$all.Add($sub) } + } else { + [void]$all.Add($item) + } + } + } catch { + Write-Host "##vso[task.logissue type=warning]Could not parse Spectral JSON for: $f" + } + } + Remove-Item -LiteralPath $per -ErrorAction SilentlyContinue + } + } + + # Merge all per-file findings into the single JSON the report builder consumes. + if ($all.Count -gt 0) { + ($all | ConvertTo-Json -Depth 50) | Set-Content -LiteralPath $json -Encoding utf8 + } else { + '[]' | Set-Content -LiteralPath $json -Encoding utf8 + } + + # Persist per-file lint durations (seconds) so the report builder can stamp the + # JUnit report with real times instead of leaving the run at 0s. + $durFile = '$(Build.ArtifactStagingDirectory)/spectral-durations.json' + if ($durations.Count -gt 0) { + ($durations | ConvertTo-Json -Depth 3) | Set-Content -LiteralPath $durFile -Encoding utf8 + } else { + '{}' | Set-Content -LiteralPath $durFile -Encoding utf8 + } + + $crashed = if (Test-Path -LiteralPath $crashLog) { @(Get-Content -LiteralPath $crashLog).Count } else { 0 } + Write-Host "Linted $($specFiles.Count) spec file(s); $($all.Count) finding(s); $crashed crashed." + # 0 = all clean, 1 = findings, 2+ = at least one spec crashed Spectral. + Write-Host "##vso[task.setvariable variable=SPECTRAL_EXIT]$worst" + # Findings/crashes are surfaced via the published test results, not via this task's + # exit code, so end cleanly to avoid a misleading red '##[error]'. + $global:LASTEXITCODE = 0 + exit 0 + + - task: PowerShell@2 + displayName: 'Build severity-labelled lint report' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $junit = '$(Build.ArtifactStagingDirectory)/spectral-result.xml' + $specRoot = '$(SPEC_ROOT)' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + # Exit code from the 'Run Spectral Linting' step: 0 clean, 1 findings, 2+ Spectral error. + $spectralExit = '$(SPECTRAL_EXIT)' + + # Per-file lint durations (seconds) captured by the lint step, keyed by normalized + # spec path, so each JUnit testsuite/testcase carries a real time (avoids "0s"). + $durByNorm = @{} + $durFile = '$(Build.ArtifactStagingDirectory)/spectral-durations.json' + if (Test-Path -LiteralPath $durFile) { + try { + $dobj = Get-Content -LiteralPath $durFile -Raw | ConvertFrom-Json + if ($dobj) { + foreach ($p in $dobj.PSObject.Properties) { + $norm = ($p.Name -replace '\\','/').ToLowerInvariant() + $durByNorm[$norm] = [double]$p.Value + } + } + } catch { } + } + + # Spec files that crashed Spectral (one absolute path per line) -> flagged as failures. + $crashedSet = @{} + if (Test-Path -LiteralPath $crashLog) { + foreach ($line in (Get-Content -LiteralPath $crashLog)) { + $t = $line.Trim() + if ($t) { $crashedSet[($t -replace '\\','/').ToLowerInvariant()] = $t } + } + } + + # Spectral severity codes -> labels used as a prefix in each test name. + $sevName = @{ 0 = 'ERROR'; 1 = 'WARN'; 2 = 'INFO'; 3 = 'HINT' } + + $results = @() + if (Test-Path -LiteralPath $json) { + $raw = Get-Content -LiteralPath $json -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + $parsed = ConvertFrom-Json -InputObject $raw + # Windows PowerShell 5.1 can surface the JSON array as a single nested object; + # flatten one level so each element is an individual Spectral result. + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { $results += $sub } + } else { + $results += $item + } + } + } + } + + # Enumerate the SAME per-API specs the lint step used (apis//specification.*), + # one level under 'apis' (NOT -Recurse), so report counts match the lint scope exactly. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + $doc = New-Object System.Xml.XmlDocument + [void]$doc.AppendChild($doc.CreateXmlDeclaration('1.0','utf-8',$null)) + $root = $doc.AppendChild($doc.CreateElement('testsuites')) + + function Add-LintSuite { + param($Doc, $Root, $SevName, [string]$Source, $Items, $AllCodes, [double]$DurationSeconds = 0) + $Items = @($Items) + $AllCodes = @($AllCodes) + $ts = $Doc.CreateElement('testsuite') + $ts.SetAttribute('name', $Source) + $ts.SetAttribute('errors','0') + # Wall-clock lint time for this spec (whole seconds; no milliseconds). + $ts.SetAttribute('time', [string]([int][Math]::Round($DurationSeconds, 0))) + + # Codes that FIRED on this spec become failures; every other rule in the + # run-wide rule set is reported as a real PASSED check for this spec. + $firedCodes = @{} + foreach ($r in $Items) { $c = [string]$r.code; if ($c) { $firedCodes[$c] = $true } } + $passedCodes = @($AllCodes | Where-Object { $_ -and -not $firedCodes.ContainsKey($_) }) + + $failCount = $Items.Count + $passCount = $passedCodes.Count + if ($failCount -eq 0 -and $passCount -eq 0) { + # Nothing fired anywhere in the run -> a single generic passing check. + $ts.SetAttribute('tests','1') + $ts.SetAttribute('failures','0') + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name','No lint issues found') + $tc.SetAttribute('classname',$Source) + $tc.SetAttribute('time', [string]([int][Math]::Round($DurationSeconds, 0))) + [void]$ts.AppendChild($tc) + [void]$Root.AppendChild($ts) + return + } + + $ts.SetAttribute('tests',[string]($failCount + $passCount)) + $ts.SetAttribute('failures',[string]$failCount) + # Report whole-second durations only (no milliseconds). Put the spec's full lint + # time on the first testcase and 0 on the rest, so the run total (sum of testcase + # times) stays an integer number of seconds that ADO renders as "Xs" / "Xm Ys". + $durWhole = [int][Math]::Round($DurationSeconds, 0) + $durAssigned = $false + + # Derive the API folder name from the spec path so passed checks can be linked + # back to a specific API (a passed rule applies to the whole spec, not one path). + $apiName = '' + if ($Source) { + $apiName = (($Source -replace '\\','/') -split '/apis/')[-1].Split('/')[0] + } + + # One PASSED testcase per rule that did NOT fire on this spec. + foreach ($pc in $passedCodes) { + $tc = $Doc.CreateElement('testcase') + $passName = "[PASS] $pc" + if ($apiName) { $passName += " ($apiName)" } + $tc.SetAttribute('name',$passName) + $tc.SetAttribute('classname',$Source) + $tcTime = if (-not $durAssigned) { $durAssigned = $true; $durWhole } else { 0 } + $tc.SetAttribute('time', [string]$tcTime) + [void]$ts.AppendChild($tc) + } + foreach ($r in $Items) { + $sevRaw = $r.severity + if ($sevRaw -is [System.Array]) { $sevRaw = $sevRaw[0] } + $sev = if ($null -ne $sevRaw) { [int]$sevRaw } else { 1 } + $label = if ($SevName.ContainsKey($sev)) { $SevName[$sev] } else { "SEV$sev" } + $pathStr = if ($r.path) { ($r.path -join '/') } else { '' } + $line = 0; $col = 0 + if ($r.range -and $r.range.start) { + $line = [int]$r.range.start.line + 1 + $col = [int]$r.range.start.character + 1 + } + $code = [string]$r.code + $name = "[$label] $code" + if ($pathStr) { $name += " ($pathStr)" } + + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name',$name) + $tc.SetAttribute('classname',$Source) + $tcTime = if (-not $durAssigned) { $durAssigned = $true; $durWhole } else { 0 } + $tc.SetAttribute('time', [string]$tcTime) + + # Every finding stays a Failed outcome; severity is conveyed via name + failure type. + $fail = $Doc.CreateElement('failure') + $fail.SetAttribute('message',[string]$r.message) + $fail.SetAttribute('type',$label) + $detail = "line $line, col $col, $($r.message) ($code) at path #/$pathStr" + [void]$fail.AppendChild($Doc.CreateCDataSection($detail)) + [void]$tc.AppendChild($fail) + + # Attach the offending spec file to this result (ADO Server 2022.2+/cloud). + if ($Source -and (Test-Path -LiteralPath $Source)) { + $so = $Doc.CreateElement('system-out') + $so.InnerText = "[[ATTACHMENT|$Source]]" + [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + } + [void]$Root.AppendChild($ts) + } + + # Index findings by normalized source path so each spec file can be matched + # to its findings regardless of slash/case differences. + $findingsByNorm = @{} + foreach ($r in $results) { + $src = [string]$r.source + $norm = ($src -replace '\\','/').ToLowerInvariant() + if (-not $findingsByNorm.ContainsKey($norm)) { + $findingsByNorm[$norm] = [pscustomobject]@{ Source = $src; Items = @() } + } + $findingsByNorm[$norm].Items += $r + } + + # Denominator for passed-checks = the full ACTIVE rule set resolved from the installed + # Spectral packages + ruleset (written by the 'Resolve ruleset' step). Falls back to + # the rules observed in this run if resolution was unavailable. + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $firedCodes = @($results | ForEach-Object { [string]$_.code } | Where-Object { $_ } | Select-Object -Unique) + $ruleUniverse = @() + if (Test-Path -LiteralPath $rulesFile) { + $ruleUniverse = @(Get-Content -LiteralPath $rulesFile | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } + # Union so any rule that fired but is missing from the resolved set still appears. + $allCodes = @($ruleUniverse + $firedCodes | Where-Object { $_ } | Select-Object -Unique) + + if ($specFiles.Count -eq 0 -and $results.Count -eq 0 -and $crashedSet.Count -eq 0) { + Add-LintSuite $doc $root $sevName 'API lint' @() @() 0 + } else { + $seen = @{} + foreach ($file in $specFiles) { + $norm = ($file -replace '\\','/').ToLowerInvariant() + $seen[$norm] = $true + $fileDur = if ($durByNorm.ContainsKey($norm)) { [double]$durByNorm[$norm] } else { 0 } + if ($crashedSet.ContainsKey($norm)) { + # This spec crashed Spectral -> explicit failure (NOT counted as passed). + $ts = $doc.CreateElement('testsuite') + $ts.SetAttribute('name',$file) + $ts.SetAttribute('tests','1'); $ts.SetAttribute('failures','1'); $ts.SetAttribute('errors','0') + $ts.SetAttribute('time', [string]([int][Math]::Round($fileDur, 0))) + $tc = $doc.CreateElement('testcase') + $tc.SetAttribute('name','Spectral crashed while linting this spec') + $tc.SetAttribute('classname',$file) + $tc.SetAttribute('time', [string]([int][Math]::Round($fileDur, 0))) + $fail = $doc.CreateElement('failure') + $fail.SetAttribute('message','Spectral threw an error on this specification (unsupported or malformed construct).') + $fail.SetAttribute('type','ERROR') + [void]$fail.AppendChild($doc.CreateCDataSection("Spectral failed on $file. This spec was NOT linted; fix the spec or exclude it. See the 'Run Spectral Linting' log.")) + [void]$tc.AppendChild($fail) + if (Test-Path -LiteralPath $file) { + $so = $doc.CreateElement('system-out'); $so.InnerText = "[[ATTACHMENT|$file]]"; [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + [void]$root.AppendChild($ts) + } elseif ($findingsByNorm.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes $fileDur + } else { + Add-LintSuite $doc $root $sevName $file @() $allCodes $fileDur + } + } + # Findings whose source was not matched to an enumerated spec file. + foreach ($norm in $findingsByNorm.Keys) { + if (-not $seen.ContainsKey($norm)) { + $unDur = if ($durByNorm.ContainsKey($norm)) { [double]$durByNorm[$norm] } else { 0 } + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes $unDur + } + } + } + + # Stamp the run-level total as whole seconds (sum of the per-file rounded seconds), + # so the Tests tab shows minutes/seconds without milliseconds. + $totalDur = 0 + foreach ($v in $durByNorm.Values) { $totalDur += [int][Math]::Round([double]$v, 0) } + $root.SetAttribute('time', [string]$totalDur) + + $doc.Save($junit) + $filesWithIssues = $findingsByNorm.Count + $crashedFiles = $crashedSet.Count + $cleanFiles = [Math]::Max(0, $specFiles.Count - $filesWithIssues - $crashedFiles) + Write-Host "Wrote JUnit report: $($results.Count) finding(s) across $filesWithIssues file(s); $crashedFiles crashed; $cleanFiles passed -> $junit" + + - task: PublishTestResults@2 + displayName: 'Publish Spectral lint results' + condition: always() + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/spectral-result.xml' + searchFolder: '$(Build.ArtifactStagingDirectory)' + testRunTitle: 'API lint results $(Build.SourceBranchName)' + failTaskOnFailedTests: false + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Spectral raw output' + condition: always() + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'spectral-lint' + publishLocation: 'Container' + + # Optional gate: fail the run when findings/crashes are present. Runs LAST so the + # report + artifacts are always published first. Controlled by 'failOnLintIssues'. + - task: PowerShell@2 + displayName: 'Fail pipeline on lint findings' + condition: and(succeeded(), eq('${{ parameters.failOnLintIssues }}', true)) + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # SPECTRAL_EXIT: 0 clean, 1 findings at/above failSeverity, 2+ Spectral crashed a spec. + $exit = [int]('0' + '$(SPECTRAL_EXIT)'.Trim()) + if ($exit -ge 1) { + throw "Spectral reported lint findings or crashes (severity threshold '${{ parameters.failSeverity }}', code $exit). See the Tests tab for details." + } + Write-Host 'No lint findings at or above the configured severity.' + # ----------------- Spectral API linting (END) ----------------- diff --git a/pipelines/azure-devops/azure-pipelines-publish-mi-env.yml b/pipelines/azure-devops/azure-pipelines-publish-mi-env.yml new file mode 100644 index 00000000..587d3b02 --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-publish-mi-env.yml @@ -0,0 +1,351 @@ +# ===================================================================== +# APIM Publish pipeline — ENVIRONMENT-AWARE variant of +# azure-pipelines-publish-mi.yml. +# +# Environment. That gives you: +# * Approval gate(s) before the job is dispatched to the agent +# (configure on the environment, not in YAML). +# * Deployment history per environment (branch, commit, outcome, +# who ran it, who approved it). +# * Other checks: branch control, business hours, exclusive lock, +# required template, ServiceNow ticket, etc. +# +# Prerequisite (one-time setup, per environment you intend to use): +# Azure DevOps Server: Pipelines > Environments > New environment +# - Name: apim- (e.g. apim-dev, apim-prod) — must match the +# ENVIRONMENT parameter value prefixed with 'apim-'. +# - Resource: None (this is a virtual environment; the actual +# agent comes from the Self-hosted pool, not from the environment). +# Then add approvals/checks under Approvals and checks. +# +# Auth: Managed Identity on a self-hosted agent (same model as Extract). +# ===================================================================== + +name: 'apim-publish-env-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts live on the selected branch' + type: string + default: 'apim-artifacts' + + - name: ENVIRONMENT + displayName: 'Destination environment (loads variable group apim- and targets the apim- Azure DevOps Environment)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: publishMode + displayName: 'Publish mode' + type: string + default: 'publish-all-artifacts-in-repo' + values: + - 'publish-all-artifacts-in-repo' + - 'publish-artifacts-in-last-commit' + + - name: overridesFile + displayName: 'Overrides file (relative to repo root, e.g. configuration.prod.yaml). Replaces env-specific values (namedValues, backends.url, apis.serviceUrl, loggers.resourceId, diagnostics.loggerId) at publish time. Use "none" to skip.' + type: string + default: 'none' + + - name: deleteUnmatched + displayName: 'Delete resources in APIM that are not in the source artifacts (--delete-unmatched). Cannot be combined with publish-artifacts-in-last-commit.' + type: boolean + default: false + + - name: dryRun + displayName: 'Dry-run (do not apply changes; prints planned create/update/delete actions)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - destination APIM resource group + # APIM_SERVICE_NAME - destination APIM service name + # AZURE_SUBSCRIPTION_ID - destination subscription id + # Optional (managed identity): + # IDENTITY_TYPE - 'system' (default) or 'user' + # USER_ASSIGNED_CLIENT_ID - client id of the user-assigned MI (IDENTITY_TYPE=user) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + +jobs: +- deployment: publish + displayName: 'apiops publish [${{ parameters.ENVIRONMENT }}]' + # Bind to an Azure DevOps Environment. Approvals/checks configured on + # the environment (Pipelines > Environments > > Approvals and + # checks) gate this job. Every run is recorded in the environment's + # deployment history with branch, commit, and outcome. + environment: 'apim-${{ parameters.ENVIRONMENT }}' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + strategy: + runOnce: + deploy: + steps: + + # Deployment jobs do NOT auto-checkout the repo; must opt in + # explicitly. Checks out the branch selected in the Run pipeline + # dialog (Build.SourceBranch). + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate selected branch contains artifacts' + # Pass parameter values as environment variables rather than + # injecting them into the PowerShell source. Values are read at + # runtime via $env:, so a quote or brace in any value cannot + # corrupt the script and break parsing. + env: + P_ENVIRONMENT: apim-${{ parameters.ENVIRONMENT }} + P_TARGET_FOLDER: ${{ parameters.targetRepoFolder }} + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + # Build.SourceBranch is the full ref (e.g. refs/heads/apim-extract/apim-dev-...) + # Build.SourceBranchName is just the leaf segment. + $ref = '$(Build.SourceBranch)' + $shortRef = '$(Build.SourceBranchName)' + Write-Host "Running against ref: $ref" + Write-Host "Short branch name: $shortRef" + Write-Host "Commit: $(Build.SourceVersion)" + Write-Host "Environment: $env:P_ENVIRONMENT" + + $folder = Join-Path '$(Build.SourcesDirectory)' $env:P_TARGET_FOLDER + if (-not (Test-Path $folder)) { + throw "Expected artifacts folder '$folder' not found on branch '$ref'. Pick a branch produced by the Extract pipeline." + } + Write-Host "Artifacts folder: $folder" + + # -------- Login with Managed Identity -------- + - task: PowerShell@2 + displayName: 'az login (managed identity)' + env: + P_USER_CLIENT_ID: $(USER_ASSIGNED_CLIENT_ID) + P_IDENTITY_TYPE: $(IDENTITY_TYPE) + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable + # group; an unexpanded macro means the variable is not defined. + $identityType = $env:P_IDENTITY_TYPE + if (-not $identityType -or $identityType -like '$(*') { $identityType = 'system' } + $clientId = $env:P_USER_CLIENT_ID + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + # -------- Sanity check destination APIM -------- + - task: PowerShell@2 + displayName: 'Verify access to destination APIM' + env: + P_RESOURCE_GROUP: $(APIM_RESOURCE_GROUP) + P_SERVICE_NAME: $(APIM_SERVICE_NAME) + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + az apim show ` + --resource-group $env:P_RESOURCE_GROUP ` + --name $env:P_SERVICE_NAME ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops publish -------- + - task: PowerShell@2 + displayName: 'Run APIM Publish' + # Parameters are passed as environment variables (read at runtime + # via $env:) instead of being injected into the PowerShell source. + # This makes the script immune to values that contain quotes or + # braces, which would otherwise corrupt parsing. + env: + P_USER_CLIENT_ID: $(USER_ASSIGNED_CLIENT_ID) + P_IDENTITY_TYPE: $(IDENTITY_TYPE) + P_APIOPS_VERSION: ${{ parameters.apiopsVersion }} + P_TARGET_FOLDER: ${{ parameters.targetRepoFolder }} + P_RESOURCE_GROUP: $(APIM_RESOURCE_GROUP) + P_SERVICE_NAME: $(APIM_SERVICE_NAME) + P_PUBLISH_MODE: ${{ parameters.publishMode }} + P_DELETE_UNMATCHED: ${{ parameters.deleteUnmatched }} + P_DRY_RUN: ${{ parameters.dryRun }} + P_OVERRIDES_FILE: ${{ parameters.overridesFile }} + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + # ---- Auth via DefaultAzureCredential (Managed Identity on the agent) ---- + $identityType = $env:P_IDENTITY_TYPE + if (-not $identityType -or $identityType -like '$(*') { $identityType = 'system' } + $clientId = $env:P_USER_CLIENT_ID + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + if ($identityType -eq 'user' -and -not [string]::IsNullOrWhiteSpace($clientId)) { + $env:AZURE_CLIENT_ID = $clientId + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + $inputFolder = Join-Path '$(Build.SourcesDirectory)' $env:P_TARGET_FOLDER + Write-Host "Publishing from: $inputFolder" + + $publishMode = $env:P_PUBLISH_MODE + $deleteUnmatched = ($env:P_DELETE_UNMATCHED -eq 'True') + + # Per apiops docs: --delete-unmatched requires a full repo scan, + # so it cannot be combined with incremental (--commit-id) publish. + if ($deleteUnmatched -and $publishMode -eq 'publish-artifacts-in-last-commit') { + throw "'deleteUnmatched' cannot be combined with publishMode 'publish-artifacts-in-last-commit'. Use 'publish-all-artifacts-in-repo' instead." + } + + $cliArgs = @( + 'publish', + '--resource-group', $env:P_RESOURCE_GROUP, + '--service-name', $env:P_SERVICE_NAME, + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--source', "$inputFolder" + ) + + # Incremental publish: limit to resources changed in the triggering commit + if ($publishMode -eq 'publish-artifacts-in-last-commit') { + $cliArgs += @('--commit-id', '$(Build.SourceVersion)') + Write-Host "Incremental publish for commit $(Build.SourceVersion)" + } else { + Write-Host 'Full publish of all artifacts in repo' + } + + # Optional overrides file (relative to repo root) + $overrides = $env:P_OVERRIDES_FILE + if (-not [string]::IsNullOrWhiteSpace($overrides) -and $overrides -ne 'none' -and $overrides -ne '-') { + $overridesPath = Join-Path '$(Build.SourcesDirectory)' $overrides + if (-not (Test-Path $overridesPath)) { + throw "Overrides file '$overridesPath' not found on the selected branch." + } + if (-not ($overridesPath -match '\.ya?ml$')) { + throw "Overrides file '$overridesPath' does not have a .yaml/.yml extension. Provide a valid YAML (.yaml/.yml) overrides file." + } + $cliArgs += @('--overrides', $overridesPath) + Write-Host "Using overrides file: $overridesPath" + Write-Host '----- Overrides file contents -----' + Get-Content -LiteralPath $overridesPath | Out-String | Write-Host + Write-Host '-----------------------------------' + } else { + Write-Host 'No overrides file specified; publishing artifact values as-is.' + } + + if ($deleteUnmatched) { + $cliArgs += '--delete-unmatched' + Write-Host '##vso[task.logissue type=warning]--delete-unmatched is ENABLED. Resources present in APIM but absent from the source folder will be DELETED.' + } + + if ($env:P_DRY_RUN -eq 'True') { + $cliArgs += '--dry-run' + Write-Host 'Dry-run mode: no changes will be applied to Azure.' + } + + Write-Host "apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + $exit = $LASTEXITCODE + # IMPORTANT: reset $LASTEXITCODE immediately. Otherwise the + # PowerShell host exits with apiops's exit code (1) and the + # PowerShell@2 task is marked Failed regardless of any + # task.complete logging command we emit below. + $global:LASTEXITCODE = 0 + # apiops-cli exit codes (see docs/reference/exit-codes.md): + # 0 = full success + # 1 = partial failure (some resources failed, others succeeded) + # 2 = fatal (auth / config / network / all resources failed) + # Treat partial failure as a WARNING so the pipeline run is flagged + # (SucceededWithIssues) but downstream steps still execute. Fatal + # errors keep failing the job. + switch ($exit) { + 0 { + Write-Host 'apiops publish succeeded (exit 0).' + exit 0 + } + 1 { + Write-Host "##vso[task.logissue type=warning]apiops publish completed with PARTIAL failures (exit 1) - some resources failed to publish. See output above." + Write-Host "##vso[task.complete result=SucceededWithIssues;]" + # exit 0 so the PowerShell@2 task itself reports success; + # the task.complete directive above downgrades it to SucceededWithIssues. + exit 0 + } + 2 { throw "apiops publish FATAL error (exit 2) - publish could not proceed." } + default { throw "apiops publish failed (exit $exit)." } + } + + # -------- Always logout -------- + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 diff --git a/pipelines/azure-devops/azure-pipelines-publish-mi.yml b/pipelines/azure-devops/azure-pipelines-publish-mi.yml new file mode 100644 index 00000000..da1633c6 --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-publish-mi.yml @@ -0,0 +1,297 @@ +# ===================================================================== +# APIM Publish pipeline — applies extracted artifacts to a destination APIM. +# Reads the artifacts from the branch the pipeline is QUEUED ON +# (use the built-in "Branch/tag" dropdown in the Run pipeline dialog; +# pick one of the apim-extract/* branches produced by the Extract pipeline). +# Artifacts are expected under the configured `targetRepoFolder`. +# +# Auth: Managed Identity on a self-hosted agent (same model as Extract). +# ===================================================================== + +name: 'apim-publish-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts live on the selected branch' + type: string + default: 'apim-artifacts' + + - name: ENVIRONMENT + displayName: 'Destination environment (loads variable group apim-)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: publishMode + displayName: 'Publish mode' + type: string + default: 'publish-all-artifacts-in-repo' + values: + - 'publish-all-artifacts-in-repo' + - 'publish-artifacts-in-last-commit' + + - name: overridesFile + displayName: 'Overrides file (relative to repo root, e.g. configuration.prod.yaml). Replaces env-specific values (namedValues, backends.url, apis.serviceUrl, loggers.resourceId, diagnostics.loggerId) at publish time. Use "none" to skip.' + type: string + default: 'none' + + - name: deleteUnmatched + displayName: 'Delete resources in APIM that are not in the source artifacts (--delete-unmatched). Cannot be combined with publish-artifacts-in-last-commit.' + type: boolean + default: false + + - name: dryRun + displayName: 'Dry-run (do not apply changes; prints planned create/update/delete actions)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - destination APIM resource group + # APIM_SERVICE_NAME - destination APIM service name + # AZURE_SUBSCRIPTION_ID - destination subscription id + # Optional (managed identity): + # IDENTITY_TYPE - 'system' (default) or 'user' + # USER_ASSIGNED_CLIENT_ID - client id of the user-assigned MI (IDENTITY_TYPE=user) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + +jobs: +- job: publish + displayName: 'apiops publish [${{ parameters.ENVIRONMENT }}]' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + steps: + + # Check out the branch selected in the Run pipeline dialog (Build.SourceBranch). + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate selected branch contains artifacts' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + # Build.SourceBranch is the full ref (e.g. refs/heads/apim-extract/apim-dev-...) + # Build.SourceBranchName is just the leaf segment. + $ref = '$(Build.SourceBranch)' + $shortRef = '$(Build.SourceBranchName)' + Write-Host "Running against ref: $ref" + Write-Host "Short branch name: $shortRef" + Write-Host "Commit: $(Build.SourceVersion)" + + $folder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + if (-not (Test-Path $folder)) { + throw "Expected artifacts folder '$folder' not found on branch '$ref'. Pick a branch produced by the Extract pipeline." + } + Write-Host "Artifacts folder: $folder" + + # -------- Login with Managed Identity -------- + - task: PowerShell@2 + displayName: 'az login (managed identity)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable group; + # an unexpanded macro means the variable is not defined in the group. + $identityType = '$(IDENTITY_TYPE)' + if ($identityType -like '$(*') { $identityType = 'system' } + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + # -------- Sanity check destination APIM -------- + - task: PowerShell@2 + displayName: 'Verify access to destination APIM' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops publish -------- + - task: PowerShell@2 + displayName: 'Run APIM Publish' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + # ---- Auth via DefaultAzureCredential (Managed Identity on the agent) ---- + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + if ('$(IDENTITY_TYPE)' -eq 'user' -and -not [string]::IsNullOrWhiteSpace($clientId)) { + $env:AZURE_CLIENT_ID = $clientId + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + $inputFolder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + Write-Host "Publishing from: $inputFolder" + + $publishMode = '${{ parameters.publishMode }}' + $deleteUnmatched = ('${{ parameters.deleteUnmatched }}' -eq 'True') + + # Per apiops docs: --delete-unmatched requires a full repo scan, + # so it cannot be combined with incremental (--commit-id) publish. + if ($deleteUnmatched -and $publishMode -eq 'publish-artifacts-in-last-commit') { + throw "'deleteUnmatched' cannot be combined with publishMode 'publish-artifacts-in-last-commit'. Use 'publish-all-artifacts-in-repo' instead." + } + + $cliArgs = @( + 'publish', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--source', "$inputFolder" + ) + + # Incremental publish: limit to resources changed in the triggering commit + if ($publishMode -eq 'publish-artifacts-in-last-commit') { + $cliArgs += @('--commit-id', '$(Build.SourceVersion)') + Write-Host "Incremental publish for commit $(Build.SourceVersion)" + } else { + Write-Host 'Full publish of all artifacts in repo' + } + + # Optional overrides file (relative to repo root) + $overrides = '${{ parameters.overridesFile }}' + if (-not [string]::IsNullOrWhiteSpace($overrides) -and $overrides -ne 'none' -and $overrides -ne '-') { + $overridesPath = Join-Path '$(Build.SourcesDirectory)' $overrides + if (-not (Test-Path $overridesPath)) { + throw "Overrides file '$overridesPath' not found on the selected branch." + } + if (-not ($overridesPath -match '\.ya?ml$')) { + throw "Overrides file '$overridesPath' does not have a .yaml/.yml extension. Provide a valid YAML (.yaml/.yml) overrides file." + } + $cliArgs += @('--overrides', $overridesPath) + Write-Host "Using overrides file: $overridesPath" + Write-Host '----- Overrides file contents -----' + Get-Content -LiteralPath $overridesPath | Out-String | Write-Host + Write-Host '-----------------------------------' + } else { + Write-Host 'No overrides file specified; publishing artifact values as-is.' + } + + if ($deleteUnmatched) { + $cliArgs += '--delete-unmatched' + Write-Host '##vso[task.logissue type=warning]--delete-unmatched is ENABLED. Resources present in APIM but absent from the source folder will be DELETED.' + } + + if ('${{ parameters.dryRun }}' -eq 'True') { + $cliArgs += '--dry-run' + Write-Host 'Dry-run mode: no changes will be applied to Azure.' + } + + Write-Host "apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + $exit = $LASTEXITCODE + # IMPORTANT: reset $LASTEXITCODE immediately. Otherwise the + # PowerShell host exits with apiops's exit code (1) and the + # PowerShell@2 task is marked Failed regardless of any + # task.complete logging command we emit below. + $global:LASTEXITCODE = 0 + # apiops-cli exit codes (see docs/reference/exit-codes.md): + # 0 = full success + # 1 = partial failure (some resources failed, others succeeded) + # 2 = fatal (auth / config / network / all resources failed) + # Treat partial failure as a WARNING so the pipeline run is flagged + # (SucceededWithIssues) but downstream steps still execute. Fatal + # errors keep failing the job. + switch ($exit) { + 0 { + Write-Host 'apiops publish succeeded (exit 0).' + exit 0 + } + 1 { + Write-Host "##vso[task.logissue type=warning]apiops publish completed with PARTIAL failures (exit 1) - some resources failed to publish. See output above." + Write-Host "##vso[task.complete result=SucceededWithIssues;]" + # exit 0 so the PowerShell@2 task itself reports success; + # the task.complete directive above downgrades it to SucceededWithIssues. + exit 0 + } + 2 { throw "apiops publish FATAL error (exit 2) - publish could not proceed." } + default { throw "apiops publish failed (exit $exit)." } + } + + # -------- Always logout -------- + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 diff --git a/pipelines/azure-devops/azure-pipelines-publish-serviceconnection-env.yml b/pipelines/azure-devops/azure-pipelines-publish-serviceconnection-env.yml new file mode 100644 index 00000000..b7f62bc0 --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-publish-serviceconnection-env.yml @@ -0,0 +1,306 @@ +# ===================================================================== +# APIM Publish pipeline — ENVIRONMENT-AWARE variant of +# azure-pipelines-publish-serviceconnection.yml. +# +# Identical behaviour to the base pipeline (artifact validation, +# AzureCLI@2 service connection auth, apiops install, publish, +# partial-failure-as-warning), but the job is a DEPLOYMENT job bound +# to an Azure DevOps Environment. That gives you: +# * Approval gate(s) before the job is dispatched to the agent +# (configure on the environment, not in YAML). +# * Deployment history per environment (branch, commit, outcome, +# who ran it, who approved it). +# * Other checks: branch control, business hours, exclusive lock, +# required template, ServiceNow ticket, etc. +# +# Prerequisite (one-time setup, per environment you intend to use): +# Azure DevOps Server: Pipelines > Environments > New environment +# - Name: apim- (e.g. apim-dev, apim-prod) — must match the +# ENVIRONMENT parameter value prefixed with 'apim-'. +# - Resource: None (this is a virtual environment; the actual +# agent comes from the Self-hosted pool, not from the environment). +# Then add approvals/checks under Approvals and checks. +# +# Use this variant when: +# * The self-hosted agent does NOT have a Managed Identity (typical +# for on-prem Azure DevOps Server agents running on plain VMs). +# * You already manage Azure auth through a Service Connection. +# ===================================================================== + +name: 'apim-publish-sc-env-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts live on the selected branch' + type: string + default: 'apim-artifacts' + + - name: ENVIRONMENT + displayName: 'Destination environment (loads variable group apim- and targets the apim- Azure DevOps Environment)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: publishMode + displayName: 'Publish mode' + type: string + default: 'publish-all-artifacts-in-repo' + values: + - 'publish-all-artifacts-in-repo' + - 'publish-artifacts-in-last-commit' + + - name: overridesFile + displayName: 'Overrides file (relative to repo root, e.g. configuration.prod.yaml). Replaces env-specific values (namedValues, backends.url, apis.serviceUrl, loggers.resourceId, diagnostics.loggerId) at publish time. Use "none" to skip.' + type: string + default: 'none' + + - name: deleteUnmatched + displayName: 'Delete resources in APIM that are not in the source artifacts (--delete-unmatched). Cannot be combined with publish-artifacts-in-last-commit.' + type: boolean + default: false + + - name: dryRun + displayName: 'Dry-run (do not apply changes; prints planned create/update/delete actions)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - destination APIM resource group + # APIM_SERVICE_NAME - destination APIM service name + # AZURE_SERVICE_CONNECTION - ARM service connection used for Azure / APIM auth + # (the pipeline must be authorized to use it) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + +jobs: +- deployment: publish + displayName: 'apiops publish via Service Connection [${{ parameters.ENVIRONMENT }}]' + # Bind to an Azure DevOps Environment. Approvals/checks configured on + # the environment (Pipelines > Environments > > Approvals and + # checks) gate this job. Every run is recorded in the environment's + # deployment history with branch, commit, and outcome. + environment: 'apim-${{ parameters.ENVIRONMENT }}' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + strategy: + runOnce: + deploy: + steps: + + # Deployment jobs do NOT auto-checkout the repo; must opt in + # explicitly. Checks out the branch selected in the Run pipeline + # dialog (Build.SourceBranch). + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate selected branch contains artifacts' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + # Build.SourceBranch is the full ref (e.g. refs/heads/apim-extract/apim-dev-...) + # Build.SourceBranchName is just the leaf segment. + $ref = '$(Build.SourceBranch)' + $shortRef = '$(Build.SourceBranchName)' + Write-Host "Running against ref: $ref" + Write-Host "Short branch name: $shortRef" + Write-Host "Commit: $(Build.SourceVersion)" + Write-Host "Environment: apim-${{ parameters.ENVIRONMENT }}" + + $folder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + if (-not (Test-Path $folder)) { + throw "Expected artifacts folder '$folder' not found on branch '$ref'. Pick a branch produced by the Extract pipeline." + } + Write-Host "Artifacts folder: $folder" + + # -------- Sanity check that the SC can see the destination APIM -------- + - task: AzureCLI@2 + displayName: 'Verify access to destination APIM (service connection)' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' # Windows PowerShell 5.1 + scriptLocation: 'inlineScript' + inlineScript: | + $ErrorActionPreference = 'Stop' + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # Surface the subscription ID resolved from the service connection + $subId = (az account show --query id -o tsv).Trim() + Write-Host "##vso[task.setvariable variable=AZURE_SUBSCRIPTION_ID]$subId" + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + # Auth is not required to resolve/install the CLI, so this runs as a plain + # PowerShell step (not via the service connection). + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops publish (Service Connection injects creds for CLI) -------- + - task: AzureCLI@2 + displayName: 'Run APIM Publish' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' + scriptLocation: 'inlineScript' + addSpnToEnvironment: true + workingDirectory: '$(Build.SourcesDirectory)' + inlineScript: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + # ---- Hand SC credentials to DefaultAzureCredential (used by apiops) ---- + # AzureCLI@2 with addSpnToEnvironment exposes these PowerShell variables + # for the script: $env:servicePrincipalId, $env:servicePrincipalKey, + # $env:tenantId, and $env:idToken (for federated SCs). + if ($env:servicePrincipalId) { + $env:AZURE_CLIENT_ID = $env:servicePrincipalId + $env:AZURE_TENANT_ID = $env:tenantId + if ($env:servicePrincipalKey) { + # Service principal + secret SCs + $env:AZURE_CLIENT_SECRET = $env:servicePrincipalKey + Write-Host 'Service connection type: service principal + secret' + } elseif ($env:idToken) { + # Workload-identity federation SCs + $env:AZURE_FEDERATED_TOKEN = $env:idToken + Write-Host 'Service connection type: workload identity federation' + } else { + Write-Host 'Service connection has no secret / id token exposed; relying on ambient az CLI session.' + } + } else { + Write-Host 'addSpnToEnvironment did not expose SPN vars; relying on ambient az CLI session.' + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + $inputFolder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + Write-Host "Publishing from: $inputFolder" + + $publishMode = '${{ parameters.publishMode }}' + $deleteUnmatched = ('${{ parameters.deleteUnmatched }}' -eq 'True') + + # Per apiops docs: --delete-unmatched requires a full repo scan, + # so it cannot be combined with incremental (--commit-id) publish. + if ($deleteUnmatched -and $publishMode -eq 'publish-artifacts-in-last-commit') { + throw "'deleteUnmatched' cannot be combined with publishMode 'publish-artifacts-in-last-commit'. Use 'publish-all-artifacts-in-repo' instead." + } + + $cliArgs = @( + 'publish', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--source', "$inputFolder" + ) + + # Incremental publish: limit to resources changed in the triggering commit + if ($publishMode -eq 'publish-artifacts-in-last-commit') { + $cliArgs += @('--commit-id', '$(Build.SourceVersion)') + Write-Host "Incremental publish for commit $(Build.SourceVersion)" + } else { + Write-Host 'Full publish of all artifacts in repo' + } + + # Optional overrides file (relative to repo root) + $overrides = '${{ parameters.overridesFile }}' + if (-not [string]::IsNullOrWhiteSpace($overrides) -and $overrides -ne 'none' -and $overrides -ne '-') { + $overridesPath = Join-Path '$(Build.SourcesDirectory)' $overrides + if (-not (Test-Path $overridesPath)) { + throw "Overrides file '$overridesPath' not found on the selected branch." + } + if (-not ($overridesPath -match '\.ya?ml$')) { + throw "Overrides file '$overridesPath' does not have a .yaml/.yml extension. Provide a valid YAML (.yaml/.yml) overrides file." + } + $cliArgs += @('--overrides', $overridesPath) + Write-Host "Using overrides file: $overridesPath" + Write-Host '----- Overrides file contents -----' + Get-Content -LiteralPath $overridesPath | Out-String | Write-Host + Write-Host '-----------------------------------' + } else { + Write-Host 'No overrides file specified; publishing artifact values as-is.' + } + + if ($deleteUnmatched) { + $cliArgs += '--delete-unmatched' + Write-Host '##vso[task.logissue type=warning]--delete-unmatched is ENABLED. Resources present in APIM but absent from the source folder will be DELETED.' + } + + if ('${{ parameters.dryRun }}' -eq 'True') { + $cliArgs += '--dry-run' + Write-Host 'Dry-run mode: no changes will be applied to Azure.' + } + + Write-Host "apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + $exit = $LASTEXITCODE + # IMPORTANT: reset $LASTEXITCODE immediately. Otherwise the + # PowerShell host exits with apiops's exit code (1) and the + # PowerShell@2 / AzureCLI@2 task is marked Failed regardless of + # any task.complete logging command we emit below. + $global:LASTEXITCODE = 0 + # apiops-cli exit codes (see docs/reference/exit-codes.md): + # 0 = full success + # 1 = partial failure (some resources failed, others succeeded) + # 2 = fatal (auth / config / network / all resources failed) + # Treat partial failure as a WARNING so the pipeline run is flagged + # (SucceededWithIssues) but downstream steps still execute. Fatal + # errors keep failing the job. + switch ($exit) { + 0 { + Write-Host 'apiops publish succeeded (exit 0).' + exit 0 + } + 1 { + Write-Host "##vso[task.logissue type=warning]apiops publish completed with PARTIAL failures (exit 1) - some resources failed to publish. See output above." + Write-Host "##vso[task.complete result=SucceededWithIssues;]" + # exit 0 so the task itself reports success; the + # task.complete directive above downgrades it to SucceededWithIssues. + exit 0 + } + 2 { throw "apiops publish FATAL error (exit 2) - publish could not proceed." } + default { throw "apiops publish failed (exit $exit)." } + } diff --git a/pipelines/azure-devops/azure-pipelines-publish-serviceconnection.yml b/pipelines/azure-devops/azure-pipelines-publish-serviceconnection.yml new file mode 100644 index 00000000..51384b74 --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines-publish-serviceconnection.yml @@ -0,0 +1,292 @@ +# ===================================================================== +# APIM Publish pipeline — authenticates via an Azure DevOps SERVICE +# CONNECTION (no Managed Identity required). +# +# Applies extracted artifacts to a destination APIM. Reads artifacts +# from the branch the pipeline is QUEUED ON (use the built-in +# "Branch/tag" dropdown in the Run pipeline dialog; pick one of the +# apim-extract/* branches produced by the Extract pipeline). +# Artifacts are expected under the configured `targetRepoFolder`. +# +# Use this variant when: +# * The self-hosted agent does NOT have a Managed Identity (typical +# for on-prem Azure DevOps Server agents running on plain VMs). +# * You already manage Azure auth through a Service Connection +# (workload-identity federation, service principal + secret, or +# service principal + certificate). +# +# Differences vs. azure-pipelines-publish-mi.yml: +# * `az login` step is replaced by AzureCLI@2 (handles auth itself). +# * Service connection name comes from the per-environment variable +# group (AZURE_SERVICE_CONNECTION). +# * Removes identityType / userAssignedClientId (irrelevant for SC). +# Everything else (artifact validation, apiops install/upgrade, +# publish modes, overrides, delete-unmatched, dry-run) is the same. +# ===================================================================== + +name: 'apim-publish-sc-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts live on the selected branch' + type: string + default: 'apim-artifacts' + + - name: ENVIRONMENT + displayName: 'Destination environment (loads variable group apim-)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: publishMode + displayName: 'Publish mode' + type: string + default: 'publish-all-artifacts-in-repo' + values: + - 'publish-all-artifacts-in-repo' + - 'publish-artifacts-in-last-commit' + + - name: overridesFile + displayName: 'Overrides file (relative to repo root, e.g. configuration.prod.yaml). Replaces env-specific values (namedValues, backends.url, apis.serviceUrl, loggers.resourceId, diagnostics.loggerId) at publish time. Use "none" to skip.' + type: string + default: 'none' + + - name: deleteUnmatched + displayName: 'Delete resources in APIM that are not in the source artifacts (--delete-unmatched). Cannot be combined with publish-artifacts-in-last-commit.' + type: boolean + default: false + + - name: dryRun + displayName: 'Dry-run (do not apply changes; prints planned create/update/delete actions)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - destination APIM resource group + # APIM_SERVICE_NAME - destination APIM service name + # AZURE_SERVICE_CONNECTION - ARM service connection used for Azure / APIM auth + # (the pipeline must be authorized to use it) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + +jobs: +- job: publish + displayName: 'apiops publish via Service Connection [${{ parameters.ENVIRONMENT }}]' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + steps: + + # Check out the branch selected in the Run pipeline dialog (Build.SourceBranch). + - checkout: self + clean: true + fetchDepth: 1 + + - task: PowerShell@2 + displayName: 'Validate selected branch contains artifacts' + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + # Build.SourceBranch is the full ref (e.g. refs/heads/apim-extract/apim-dev-...) + # Build.SourceBranchName is just the leaf segment. + $ref = '$(Build.SourceBranch)' + $shortRef = '$(Build.SourceBranchName)' + Write-Host "Running against ref: $ref" + Write-Host "Short branch name: $shortRef" + Write-Host "Commit: $(Build.SourceVersion)" + + $folder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + if (-not (Test-Path $folder)) { + throw "Expected artifacts folder '$folder' not found on branch '$ref'. Pick a branch produced by the Extract pipeline." + } + Write-Host "Artifacts folder: $folder" + + # -------- Sanity check that the SC can see the destination APIM -------- + - task: AzureCLI@2 + displayName: 'Verify access to destination APIM (service connection)' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' # Windows PowerShell 5.1 + scriptLocation: 'inlineScript' + inlineScript: | + $ErrorActionPreference = 'Stop' + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # Surface the subscription ID resolved from the service connection + $subId = (az account show --query id -o tsv).Trim() + Write-Host "##vso[task.setvariable variable=AZURE_SUBSCRIPTION_ID]$subId" + + # -------- Check / install apiops CLI (separate step for easy version tracking) -------- + # Auth is not required to resolve/install the CLI, so this runs as a plain + # PowerShell step (not via the service connection). + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops publish (Service Connection injects creds for CLI) -------- + - task: AzureCLI@2 + displayName: 'Run APIM Publish' + inputs: + azureSubscription: '$(AZURE_SERVICE_CONNECTION)' + scriptType: 'ps' + scriptLocation: 'inlineScript' + addSpnToEnvironment: true + workingDirectory: '$(Build.SourcesDirectory)' + inlineScript: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + # ---- Hand SC credentials to DefaultAzureCredential (used by apiops) ---- + # AzureCLI@2 with addSpnToEnvironment exposes these PowerShell variables + # for the script: $env:servicePrincipalId, $env:servicePrincipalKey, + # $env:tenantId, and $env:idToken (for federated SCs). + if ($env:servicePrincipalId) { + $env:AZURE_CLIENT_ID = $env:servicePrincipalId + $env:AZURE_TENANT_ID = $env:tenantId + if ($env:servicePrincipalKey) { + # Service principal + secret SCs + $env:AZURE_CLIENT_SECRET = $env:servicePrincipalKey + Write-Host 'Service connection type: service principal + secret' + } elseif ($env:idToken) { + # Workload-identity federation SCs + $env:AZURE_FEDERATED_TOKEN = $env:idToken + Write-Host 'Service connection type: workload identity federation' + } else { + Write-Host 'Service connection has no secret / id token exposed; relying on ambient az CLI session.' + } + } else { + Write-Host 'addSpnToEnvironment did not expose SPN vars; relying on ambient az CLI session.' + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + $inputFolder = Join-Path '$(Build.SourcesDirectory)' '${{ parameters.targetRepoFolder }}' + Write-Host "Publishing from: $inputFolder" + + $publishMode = '${{ parameters.publishMode }}' + $deleteUnmatched = ('${{ parameters.deleteUnmatched }}' -eq 'True') + + # Per apiops docs: --delete-unmatched requires a full repo scan, + # so it cannot be combined with incremental (--commit-id) publish. + if ($deleteUnmatched -and $publishMode -eq 'publish-artifacts-in-last-commit') { + throw "'deleteUnmatched' cannot be combined with publishMode 'publish-artifacts-in-last-commit'. Use 'publish-all-artifacts-in-repo' instead." + } + + $cliArgs = @( + 'publish', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--source', "$inputFolder" + ) + + # Incremental publish: limit to resources changed in the triggering commit + if ($publishMode -eq 'publish-artifacts-in-last-commit') { + $cliArgs += @('--commit-id', '$(Build.SourceVersion)') + Write-Host "Incremental publish for commit $(Build.SourceVersion)" + } else { + Write-Host 'Full publish of all artifacts in repo' + } + + # Optional overrides file (relative to repo root) + $overrides = '${{ parameters.overridesFile }}' + if (-not [string]::IsNullOrWhiteSpace($overrides) -and $overrides -ne 'none' -and $overrides -ne '-') { + $overridesPath = Join-Path '$(Build.SourcesDirectory)' $overrides + if (-not (Test-Path $overridesPath)) { + throw "Overrides file '$overridesPath' not found on the selected branch." + } + if (-not ($overridesPath -match '\.ya?ml$')) { + throw "Overrides file '$overridesPath' does not have a .yaml/.yml extension. Provide a valid YAML (.yaml/.yml) overrides file." + } + $cliArgs += @('--overrides', $overridesPath) + Write-Host "Using overrides file: $overridesPath" + Write-Host '----- Overrides file contents -----' + Get-Content -LiteralPath $overridesPath | Out-String | Write-Host + Write-Host '-----------------------------------' + } else { + Write-Host 'No overrides file specified; publishing artifact values as-is.' + } + + if ($deleteUnmatched) { + $cliArgs += '--delete-unmatched' + Write-Host '##vso[task.logissue type=warning]--delete-unmatched is ENABLED. Resources present in APIM but absent from the source folder will be DELETED.' + } + + if ('${{ parameters.dryRun }}' -eq 'True') { + $cliArgs += '--dry-run' + Write-Host 'Dry-run mode: no changes will be applied to Azure.' + } + + Write-Host "apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + $exit = $LASTEXITCODE + # IMPORTANT: reset $LASTEXITCODE immediately. Otherwise the + # PowerShell host exits with apiops's exit code (1) and the + # PowerShell@2 / AzureCLI@2 task is marked Failed regardless of + # any task.complete logging command we emit below. + $global:LASTEXITCODE = 0 + # apiops-cli exit codes (see docs/reference/exit-codes.md): + # 0 = full success + # 1 = partial failure (some resources failed, others succeeded) + # 2 = fatal (auth / config / network / all resources failed) + # Treat partial failure as a WARNING so the pipeline run is flagged + # (SucceededWithIssues) but downstream steps still execute. Fatal + # errors keep failing the job. + switch ($exit) { + 0 { + Write-Host 'apiops publish succeeded (exit 0).' + exit 0 + } + 1 { + Write-Host "##vso[task.logissue type=warning]apiops publish completed with PARTIAL failures (exit 1) - some resources failed to publish. See output above." + Write-Host "##vso[task.complete result=SucceededWithIssues;]" + # exit 0 so the task itself reports success; the + # task.complete directive above downgrades it to SucceededWithIssues. + exit 0 + } + 2 { throw "apiops publish FATAL error (exit 2) - publish could not proceed." } + default { throw "apiops publish failed (exit $exit)." } + } diff --git a/pipelines/azure-devops/azure-pipelines_extract-mi.yml b/pipelines/azure-devops/azure-pipelines_extract-mi.yml new file mode 100644 index 00000000..aab48c2d --- /dev/null +++ b/pipelines/azure-devops/azure-pipelines_extract-mi.yml @@ -0,0 +1,783 @@ +# ===================================================================== +# APIM Extract pipeline — authenticates with Managed Identity (no SC) +# Requires: self-hosted agent on Azure VM/VMSS/AKS/Container App or +# Azure Arc-enabled server with the MI granted RBAC on APIM. +# ===================================================================== + +name: 'apim-extract-$(Date:yyyyMMdd)-$(Rev:r)' + +trigger: none # run on demand +pr: none + +parameters: + - name: CONFIGURATION_YAML_PATH + displayName: 'Operation' + type: string + default: 'Extract All APIs' + values: + - 'Extract All APIs' + - 'Extract from filter file' + + - name: ENVIRONMENT + displayName: 'Target environment (loads variable group apim-)' + type: string + default: 'dev' + values: + - 'dev' + - 'prod' + + - name: apiopsVersion + displayName: 'apiops-cli npm version (installed if missing)' + type: string + default: 'latest' + + - name: publishToRepo + displayName: 'Commit extracted artifacts to repo branch' + type: boolean + default: true + + - name: targetRepoFolder + displayName: 'Folder (inside repo) where artifacts will be stored' + type: string + default: 'apim-artifacts' + + - name: branchPrefix + displayName: 'Prefix for the new branch created per extraction' + type: string + default: 'apim-extract' + + - name: gitUserName + displayName: 'Git author name for the commit' + type: string + default: 'APIM Extract Pipeline' + + - name: gitUserEmail + displayName: 'Git author email for the commit' + type: string + default: 'apim-extract@devops.local' + + # -------------------- Filter options -------------------- + # NOTE: Azure DevOps cannot hide/disable parameters at runtime based on + # another parameter's value. `filterFile` is REQUIRED only when + # Operation = 'Extract from filter file'; it is IGNORED otherwise. + - name: filterFile + displayName: 'Filter file path (used only when Operation = Extract from filter file; ignored for Extract All APIs)' + type: string + default: 'none' + + - name: noTransitive + displayName: 'Disable transitive dependency extraction (--no-transitive; only meaningful with a filter file)' + type: boolean + default: false + +variables: + # Environment-specific settings come from the 'apim-' variable group + # (Pipelines > Library). Required variables: + # AGENT_POOL - self-hosted agent pool name + # APIM_RESOURCE_GROUP - APIM resource group + # APIM_SERVICE_NAME - APIM service name + # AZURE_SUBSCRIPTION_ID - target subscription id + # Optional (managed identity): + # IDENTITY_TYPE - 'system' (default) or 'user' + # USER_ASSIGNED_CLIENT_ID - client id of the user-assigned MI (IDENTITY_TYPE=user) + - group: 'apim-${{ parameters.ENVIRONMENT }}' + - name: ARTIFACT_NAME + value: 'apim-artifacts' + - name: ARTIFACT_PATH + value: '$(Build.ArtifactStagingDirectory)/apim-artifacts' + # Branch name is computed once and surfaced as an output for downstream pipelines + - name: EXTRACT_BRANCH + value: '${{ parameters.branchPrefix }}/$(APIM_SERVICE_NAME)-$(Build.BuildNumber)' + +jobs: +- job: extract + displayName: 'apiops extract [${{ parameters.ENVIRONMENT }}] (${{ parameters.CONFIGURATION_YAML_PATH }})' + pool: + name: '$(AGENT_POOL)' + timeoutInMinutes: 60 + # Allow the job to read System.AccessToken so we can push the branch back to the repo + variables: + - name: System.AccessToken + value: $(System.AccessToken) + steps: + + # Checkout with credentials persisted so we can push a new branch later. + - checkout: self + persistCredentials: true + clean: true + fetchDepth: 0 + + # -------- Validate Node.js (24 LTS) on the agent -------- + - task: PowerShell@2 + displayName: 'Validate Node.js (24 LTS)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + + # ---- Connectivity probe to the Node.js distribution host (informative) ---- + $nodeDistUrl = 'https://nodejs.org/dist/index.json' + Write-Host "Checking connectivity to Node.js distribution '$nodeDistUrl'..." + $nodeHostReachable = $false + $curlCmd = Get-Command curl.exe -ErrorAction SilentlyContinue + if ($curlCmd -and $curlCmd.CommandType -ne 'Alias') { + $o = [System.IO.Path]::GetTempFileName() + $e = [System.IO.Path]::GetTempFileName() + try { + $p = Start-Process -FilePath $curlCmd.Source ` + -ArgumentList @('-s','-v','-o','NUL','--connect-timeout','15','--max-time','15',$nodeDistUrl) ` + -NoNewWindow -PassThru -Wait ` + -RedirectStandardOutput $o -RedirectStandardError $e + @(Get-Content $e -ErrorAction SilentlyContinue) + @(Get-Content $o -ErrorAction SilentlyContinue) | + Where-Object { $_ } | ForEach-Object { Write-Host " $_" } + $nodeHostReachable = ($p.ExitCode -eq 0) + } finally { Remove-Item $o, $e -ErrorAction SilentlyContinue } + } else { + Write-Host 'curl not found; skipping Node.js distribution connectivity probe.' + } + if ($nodeHostReachable) { + Write-Host 'Node.js distribution host: REACHABLE.' + } else { + Write-Host "##vso[task.logissue type=warning]Node.js distribution host appears UNREACHABLE; relying on the Node.js already installed on the agent." + } + + # ---- Validate the Node.js installed on the agent (no download) ---- + $nodeVerRaw = (& node --version) 2>$null + if (-not $nodeVerRaw) { + $hostState = if ($nodeHostReachable) { 'reachable' } else { 'unreachable' } + throw "Node.js is not installed on this agent. Install Node.js 24 LTS (the Node.js distribution host is $hostState)." + } + Write-Host "Detected Node.js $nodeVerRaw / npm $(& npm --version)." + $nodeVer = [version](($nodeVerRaw.TrimStart('v')) -replace '-.*$','') + if ($nodeVer.Major -lt 24) { + throw "Node.js $nodeVerRaw is installed but version >= 24 (v24 LTS) is required. Update Node.js on the agent." + } + Write-Host 'Node.js 24 LTS requirement satisfied.' + + # -------- Login with Managed Identity -------- + - task: PowerShell@2 + displayName: 'az login (managed identity)' + inputs: + targetType: 'inline' + pwsh: false # use Windows PowerShell 5.1 that ships with the OS + script: | + $ErrorActionPreference = 'Stop' + + # IDENTITY_TYPE / USER_ASSIGNED_CLIENT_ID come from the variable group; + # an unexpanded macro means the variable is not defined in the group. + $identityType = '$(IDENTITY_TYPE)' + if ($identityType -like '$(*') { $identityType = 'system' } + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + + if ($identityType -eq 'user') { + if ([string]::IsNullOrWhiteSpace($clientId)) { + Write-Host "##vso[task.logissue type=error]USER_ASSIGNED_CLIENT_ID is required when IDENTITY_TYPE=user" + exit 1 + } + Write-Host 'Logging in with USER-assigned managed identity...' + az login --identity --client-id "$clientId" 1>$null + } else { + if (-not [string]::IsNullOrWhiteSpace($clientId)) { + Write-Host 'IDENTITY_TYPE=system: ignoring provided USER_ASSIGNED_CLIENT_ID.' + } + Write-Host 'Logging in with SYSTEM-assigned managed identity...' + az login --identity 1>$null + } + if ($LASTEXITCODE -ne 0) { throw "az login failed (exit $LASTEXITCODE)" } + + az account set --subscription "$(AZURE_SUBSCRIPTION_ID)" + if ($LASTEXITCODE -ne 0) { throw "az account set failed (exit $LASTEXITCODE)" } + az account show --query '{sub:name, tenant:tenantId, user:user.name}' -o table + + # -------- Sanity check that the MI can see the APIM -------- + - task: PowerShell@2 + displayName: 'Verify access to APIM' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + az apim show ` + --resource-group '$(APIM_RESOURCE_GROUP)' ` + --name '$(APIM_SERVICE_NAME)' ` + --query '{name:name, sku:sku.name, location:location}' -o table + if ($LASTEXITCODE -ne 0) { throw "az apim show failed (exit $LASTEXITCODE)" } + + # -------- Check / install apiops CLI (delegated to shared script) -------- + - task: PowerShell@2 + displayName: 'Check apiops-cli version' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Delegates Node/apiops version check + install to the shared repo script. + # It sets APIOPS_PATH / APIOPS_VERSION for downstream steps and treats an + # unreachable npm registry as a warning when apiops is already installed. + & "$(Build.SourcesDirectory)/Check-ApiopsCliVersion.ps1" ` + -ApiopsVersion '${{ parameters.apiopsVersion }}' ` + -AllowStaleOnRegistryFailure + + # If the online apiops repository was unreachable, finish this step as a + # warning (orange) without failing the pipeline; the locally installed + # apiops version is used for the run. + if ($global:ApiopsRepoReachable -eq $false) { + Write-Host "##vso[task.logissue type=warning]apiops online repository was unreachable; using locally installed apiops version $global:ApiopsVersion." + Write-Host "##vso[task.complete result=SucceededWithIssues;]apiops online repository unreachable" + } + + # -------- Run apiops extract -------- + - task: PowerShell@2 + displayName: 'Run APIM Extract (${{ parameters.CONFIGURATION_YAML_PATH }})' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + + New-Item -ItemType Directory -Force -Path "$(ARTIFACT_PATH)" | Out-Null + + # ---- Auth via DefaultAzureCredential (Managed Identity on the agent) ---- + $clientId = '$(USER_ASSIGNED_CLIENT_ID)' + if ($clientId -eq 'none' -or $clientId -eq '-' -or $clientId -like '$(*') { $clientId = '' } + if ('$(IDENTITY_TYPE)' -eq 'user' -and -not [string]::IsNullOrWhiteSpace($clientId)) { + # Tells ManagedIdentityCredential which user-assigned identity to use + $env:AZURE_CLIENT_ID = $clientId + } + $env:AZURE_SUBSCRIPTION_ID = '$(AZURE_SUBSCRIPTION_ID)' + + # ---- apiops CLI resolved/installed by the 'Check apiops-cli version' step ---- + $apiopsPath = '$(APIOPS_PATH)' + Write-Host "Using apiops CLI: $apiopsPath (version $(APIOPS_VERSION))" + + # ---- Resolve filter based on selected Operation ---- + $operation = '${{ parameters.CONFIGURATION_YAML_PATH }}' + $filterFileRel = '${{ parameters.filterFile }}' + $filterArg = $null + + if ($operation -eq 'Extract from filter file') { + if ([string]::IsNullOrWhiteSpace($filterFileRel) -or $filterFileRel -eq 'none' -or $filterFileRel -eq '-') { + throw "Operation '$operation' requires the 'filterFile' parameter to be set (e.g. configuration.extractor.yaml)." + } + $filterPath = Join-Path '$(Build.SourcesDirectory)' $filterFileRel + if (-not (Test-Path $filterPath)) { + throw "Filter file '$filterPath' not found on the checked-out branch." + } + $filterArg = $filterPath + Write-Host "Operation: $operation" + Write-Host "Using filter file: $filterPath" + Write-Host '----- Filter file contents -----' + Get-Content -LiteralPath $filterPath | Out-String | Write-Host + Write-Host '--------------------------------' + } + else { + if (-not [string]::IsNullOrWhiteSpace($filterFileRel) -and $filterFileRel -ne 'none' -and $filterFileRel -ne '-') { + Write-Host "Operation '$operation' selected; ignoring filterFile parameter ('$filterFileRel')." + } + Write-Host "Operation: $operation (extracting ALL resources)" + } + + # ---- Build CLI arg list (splatting is safer than backtick line continuations) ---- + $cliArgs = @( + 'extract', + '--resource-group', '$(APIM_RESOURCE_GROUP)', + '--service-name', '$(APIM_SERVICE_NAME)', + '--subscription-id', '$(AZURE_SUBSCRIPTION_ID)', + '--output', "$(ARTIFACT_PATH)" + ) + if ($filterArg) { $cliArgs += @('--filter', $filterArg) } + if ('${{ parameters.noTransitive }}' -eq 'True') { + $cliArgs += '--no-transitive' + Write-Host 'Transitive dependency extraction DISABLED (--no-transitive).' + } + + Write-Host "Running: apiops $($cliArgs -join ' ')" + & $apiopsPath @cliArgs + if ($LASTEXITCODE -ne 0) { throw "apiops extract failed (exit $LASTEXITCODE)" } + + # ----------------- Spectral API linting (START) ----------------- + - task: PowerShell@2 + displayName: 'Install Spectral' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Pin a known-good version. Unpinned 'latest' has shipped regressions that crash + # with "Cannot read properties of null (reading 'enum')" on some specs. Change the + # version here if you need a different build. + $spectralPackage = '@stoplight/spectral-cli@6.11.1' + npm install -g $spectralPackage + if ($LASTEXITCODE -ne 0) { throw "npm install -g $spectralPackage failed (exit $LASTEXITCODE)" } + + # npm's global bin is often NOT on PATH for the agent service account, so + # resolve the spectral launcher explicitly and hand it to the next step. + $npmPrefix = (& npm prefix -g).Trim() + $spectral = $null + foreach ($name in @('spectral.cmd','spectral.exe','spectral')) { + $candidate = Join-Path $npmPrefix $name + if (Test-Path $candidate) { $spectral = $candidate; break } + } + if (-not $spectral) { + $cmd = Get-Command spectral -ErrorAction SilentlyContinue + if ($cmd) { $spectral = $cmd.Source } + } + if (-not $spectral) { throw "Spectral CLI not found after global install (npm prefix: $npmPrefix)." } + Write-Host "Spectral CLI: $spectral" + Write-Host "##vso[task.setvariable variable=SPECTRAL_PATH]$spectral" + + - task: PowerShell@2 + displayName: 'Resolve ruleset rule set (for passed-checks reporting)' + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + # Spectral only ever emits VIOLATIONS -- never the rules that passed. To report real + # passed checks we need the full ACTIVE rule set (the recommended spectral:oas rules + # that 'extends: spectral:oas' enables, plus the custom rules). We read it straight + # from the installed Spectral packages + the ruleset file, so the denominator is + # accurate. If anything here fails, the report falls back to the rules observed in + # this run, so reporting never breaks. + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + $rulesetLocal = '$(Build.ArtifactStagingDirectory)/spectral-ruleset.yaml' + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $scriptFile = '$(Build.ArtifactStagingDirectory)/extract-rules.cjs' + Remove-Item -LiteralPath $rulesFile -ErrorAction SilentlyContinue + + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri $ruleset -OutFile $rulesetLocal -UseBasicParsing + } catch { + Write-Host "##vso[task.logissue type=warning]Could not download ruleset for rule extraction: $($_.Exception.Message)" + } + + $gRoot = (& npm root -g).Trim() + + # Node script: union of (recommended OpenAPI rules from @stoplight/spectral-rulesets) + # and (custom rules parsed from the ruleset YAML, honouring false/off disables). + $js = @( + 'const fs = require("fs");', + 'const path = require("path");', + 'const { createRequire } = require("module");', + 'try {', + ' const gRoot = process.env.SPECTRAL_GLOBAL_MODULES;', + ' const req = createRequire(path.join(gRoot, "@stoplight", "spectral-cli", "package.json"));', + ' const set = new Set();', + ' try {', + ' const { oas } = req("@stoplight/spectral-rulesets");', + ' for (const [k, v] of Object.entries(oas.rules)) { if (v && v.recommended !== false) set.add(k); }', + ' } catch (e) { process.stderr.write("OAS_RULES_ERROR: " + e + "\n"); }', + ' try {', + ' const yaml = req("@stoplight/yaml");', + ' const doc = yaml.parse(fs.readFileSync(process.env.SPECTRAL_RULESET_FILE, "utf8"));', + ' if (doc && doc.rules) {', + ' for (const [k, v] of Object.entries(doc.rules)) {', + ' if (v === false || v === "off" || v === 0) { set.delete(k); continue; }', + ' set.add(k);', + ' }', + ' }', + ' } catch (e) { process.stderr.write("CUSTOM_RULES_ERROR: " + e + "\n"); }', + ' process.stdout.write(Array.from(set).join("\n"));', + '} catch (e) {', + ' process.stderr.write("RULE_EXTRACT_ERROR: " + (e && e.stack ? e.stack : String(e)));', + ' process.exit(3);', + '}' + ) + Set-Content -LiteralPath $scriptFile -Value $js -Encoding utf8 + + $env:SPECTRAL_GLOBAL_MODULES = $gRoot + $env:SPECTRAL_RULESET_FILE = $rulesetLocal + $out = & node $scriptFile 2>&1 + $code = $LASTEXITCODE + # stdout = rule names (one per line); stderr diagnostics carry known prefixes. + $ruleNames = @($out | Where-Object { $_ -and ($_ -notmatch '^(OAS_RULES_ERROR|CUSTOM_RULES_ERROR|RULE_EXTRACT_ERROR)') }) + if ($code -eq 0 -and $ruleNames.Count -gt 0) { + $ruleNames | Set-Content -LiteralPath $rulesFile -Encoding utf8 + Write-Host "Resolved active rule set: $($ruleNames.Count) rules." + } else { + Write-Host "##vso[task.logissue type=warning]Could not resolve full rule set (node exit $code); passed-checks will fall back to the rules observed in this run." + Write-Host ($out -join "`n") + } + exit 0 + + - task: PowerShell@2 + displayName: 'Run Spectral Linting' + continueOnError: true + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $spectral = '$(SPECTRAL_PATH)' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + $specRoot = '$(ARTIFACT_PATH)/apis' + $ruleset = 'https://raw.githubusercontent.com/connectedcircuits/devops-api-linter/main/rules.yaml' + + Remove-Item -LiteralPath $json, $crashLog -ErrorAction SilentlyContinue + + # Lint ONLY the per-API OpenAPI specification file: apis//specification.*. + # Match at ONE level under 'apis' (NOT -Recurse) so nested specification.* files under + # operations/, schemas/, etc. are excluded -- we want exactly one spec per API. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + Write-Host "Spectral CLI: $spectral" + Write-Host "Spec root: $specRoot" + Write-Host "Ruleset: $ruleset" + Write-Host "Spec files: $($specFiles.Count)" + + # Lint each API spec individually so one malformed spec that crashes Spectral/Nimma + # ("Cannot read properties of null (reading 'enum')") is isolated to that file instead + # of aborting the whole run; every other spec still gets reported. + $all = New-Object System.Collections.Generic.List[object] + $worst = 0 + foreach ($f in $specFiles) { + Write-Host "----- Linting: $f" + $per = [System.IO.Path]::GetTempFileName() + & $spectral lint --format stylish --format json --output.json $per --fail-severity warn $f -r $ruleset + $code = $LASTEXITCODE + if ($code -ge 2) { + Write-Host "##vso[task.logissue type=warning]Spectral CRASHED on: $f (exit $code)" + Add-Content -LiteralPath $crashLog -Value $f + if ($code -gt $worst) { $worst = $code } + } elseif ($code -eq 1 -and $worst -lt 1) { + $worst = 1 + } + if (Test-Path -LiteralPath $per) { + $raw = Get-Content -LiteralPath $per -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + try { + $parsed = ConvertFrom-Json -InputObject $raw + # PS 5.1 may surface the JSON array as a single nested object; flatten one level + # so EACH finding is added individually (not the whole array as one element). + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { [void]$all.Add($sub) } + } else { + [void]$all.Add($item) + } + } + } catch { + Write-Host "##vso[task.logissue type=warning]Could not parse Spectral JSON for: $f" + } + } + Remove-Item -LiteralPath $per -ErrorAction SilentlyContinue + } + } + + # Merge all per-file findings into the single JSON the report builder consumes. + if ($all.Count -gt 0) { + ($all | ConvertTo-Json -Depth 50) | Set-Content -LiteralPath $json -Encoding utf8 + } else { + '[]' | Set-Content -LiteralPath $json -Encoding utf8 + } + + $crashed = if (Test-Path -LiteralPath $crashLog) { @(Get-Content -LiteralPath $crashLog).Count } else { 0 } + Write-Host "Linted $($specFiles.Count) spec file(s); $($all.Count) finding(s); $crashed crashed." + # 0 = all clean, 1 = findings, 2+ = at least one spec crashed Spectral. + Write-Host "##vso[task.setvariable variable=SPECTRAL_EXIT]$worst" + # Findings/crashes are surfaced via the published test results, not via this task's + # exit code, so end cleanly to avoid a misleading red '##[error]'. + $global:LASTEXITCODE = 0 + exit 0 + + - task: PowerShell@2 + displayName: 'Build severity-labelled lint report' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + $ErrorActionPreference = 'Stop' + $json = '$(Build.ArtifactStagingDirectory)/spectral-result.json' + $junit = '$(Build.ArtifactStagingDirectory)/spectral-result.xml' + $specRoot = '$(ARTIFACT_PATH)/apis' + $crashLog = '$(Build.ArtifactStagingDirectory)/spectral-crashes.txt' + # Exit code from the 'Run Spectral Linting' step: 0 clean, 1 findings, 2+ Spectral error. + $spectralExit = '$(SPECTRAL_EXIT)' + + # Spec files that crashed Spectral (one absolute path per line) -> flagged as failures. + $crashedSet = @{} + if (Test-Path -LiteralPath $crashLog) { + foreach ($line in (Get-Content -LiteralPath $crashLog)) { + $t = $line.Trim() + if ($t) { $crashedSet[($t -replace '\\','/').ToLowerInvariant()] = $t } + } + } + + # Spectral severity codes -> labels used as a prefix in each test name. + $sevName = @{ 0 = 'ERROR'; 1 = 'WARN'; 2 = 'INFO'; 3 = 'HINT' } + + $results = @() + if (Test-Path -LiteralPath $json) { + $raw = Get-Content -LiteralPath $json -Raw + if (-not [string]::IsNullOrWhiteSpace($raw)) { + $parsed = ConvertFrom-Json -InputObject $raw + # Windows PowerShell 5.1 can surface the JSON array as a single nested object; + # flatten one level so each element is an individual Spectral result. + foreach ($item in @($parsed)) { + if ($item -is [System.Collections.IEnumerable] -and $item -isnot [string]) { + foreach ($sub in $item) { $results += $sub } + } else { + $results += $item + } + } + } + } + + # Enumerate the SAME per-API specs the lint step used (apis//specification.*), + # one level under 'apis' (NOT -Recurse), so report counts match the lint scope exactly. + $specNames = @('specification.json','specification.yaml','specification.yml') + $specFiles = @() + if (Test-Path -LiteralPath $specRoot) { + $specFiles = Get-ChildItem -LiteralPath $specRoot -Directory -ErrorAction SilentlyContinue | ForEach-Object { + Get-ChildItem -LiteralPath $_.FullName -File -ErrorAction SilentlyContinue | Where-Object { $specNames -contains $_.Name } + } | Select-Object -ExpandProperty FullName + } + + $doc = New-Object System.Xml.XmlDocument + [void]$doc.AppendChild($doc.CreateXmlDeclaration('1.0','utf-8',$null)) + $root = $doc.AppendChild($doc.CreateElement('testsuites')) + + function Add-LintSuite { + param($Doc, $Root, $SevName, [string]$Source, $Items, $AllCodes) + $Items = @($Items) + $AllCodes = @($AllCodes) + $ts = $Doc.CreateElement('testsuite') + $ts.SetAttribute('name', $Source) + $ts.SetAttribute('errors','0') + + # Codes that FIRED on this spec become failures; every other rule in the + # run-wide rule set is reported as a real PASSED check for this spec. + $firedCodes = @{} + foreach ($r in $Items) { $c = [string]$r.code; if ($c) { $firedCodes[$c] = $true } } + $passedCodes = @($AllCodes | Where-Object { $_ -and -not $firedCodes.ContainsKey($_) }) + + $failCount = $Items.Count + $passCount = $passedCodes.Count + if ($failCount -eq 0 -and $passCount -eq 0) { + # Nothing fired anywhere in the run -> a single generic passing check. + $ts.SetAttribute('tests','1') + $ts.SetAttribute('failures','0') + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name','No lint issues found') + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + [void]$Root.AppendChild($ts) + return + } + + $ts.SetAttribute('tests',[string]($failCount + $passCount)) + $ts.SetAttribute('failures',[string]$failCount) + + # Derive the API folder name from the spec path so passed checks can be linked + # back to a specific API (a passed rule applies to the whole spec, not one path). + $apiName = '' + if ($Source) { + $apiName = (($Source -replace '\\','/') -split '/apis/')[-1].Split('/')[0] + } + + # One PASSED testcase per rule that did NOT fire on this spec. + foreach ($pc in $passedCodes) { + $tc = $Doc.CreateElement('testcase') + $passName = "[PASS] $pc" + if ($apiName) { $passName += " ($apiName)" } + $tc.SetAttribute('name',$passName) + $tc.SetAttribute('classname',$Source) + [void]$ts.AppendChild($tc) + } + foreach ($r in $Items) { + $sevRaw = $r.severity + if ($sevRaw -is [System.Array]) { $sevRaw = $sevRaw[0] } + $sev = if ($null -ne $sevRaw) { [int]$sevRaw } else { 1 } + $label = if ($SevName.ContainsKey($sev)) { $SevName[$sev] } else { "SEV$sev" } + $pathStr = if ($r.path) { ($r.path -join '/') } else { '' } + $line = 0; $col = 0 + if ($r.range -and $r.range.start) { + $line = [int]$r.range.start.line + 1 + $col = [int]$r.range.start.character + 1 + } + $code = [string]$r.code + $name = "[$label] $code" + if ($pathStr) { $name += " ($pathStr)" } + + $tc = $Doc.CreateElement('testcase') + $tc.SetAttribute('name',$name) + $tc.SetAttribute('classname',$Source) + + # Every finding stays a Failed outcome; severity is conveyed via name + failure type. + $fail = $Doc.CreateElement('failure') + $fail.SetAttribute('message',[string]$r.message) + $fail.SetAttribute('type',$label) + $detail = "line $line, col $col, $($r.message) ($code) at path #/$pathStr" + [void]$fail.AppendChild($Doc.CreateCDataSection($detail)) + [void]$tc.AppendChild($fail) + + # Attach the offending spec file to this result (ADO Server 2022.2+/cloud). + if ($Source -and (Test-Path -LiteralPath $Source)) { + $so = $Doc.CreateElement('system-out') + $so.InnerText = "[[ATTACHMENT|$Source]]" + [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + } + [void]$Root.AppendChild($ts) + } + + # Index findings by normalized source path so each spec file can be matched + # to its findings regardless of slash/case differences. + $findingsByNorm = @{} + foreach ($r in $results) { + $src = [string]$r.source + $norm = ($src -replace '\\','/').ToLowerInvariant() + if (-not $findingsByNorm.ContainsKey($norm)) { + $findingsByNorm[$norm] = [pscustomobject]@{ Source = $src; Items = @() } + } + $findingsByNorm[$norm].Items += $r + } + + # Denominator for passed-checks = the full ACTIVE rule set resolved from the installed + # Spectral packages + ruleset (written by the 'Resolve ruleset rule set' step). Falls + # back to the rules observed in this run if resolution was unavailable. + $rulesFile = '$(Build.ArtifactStagingDirectory)/spectral-rules.txt' + $firedCodes = @($results | ForEach-Object { [string]$_.code } | Where-Object { $_ } | Select-Object -Unique) + $ruleUniverse = @() + if (Test-Path -LiteralPath $rulesFile) { + $ruleUniverse = @(Get-Content -LiteralPath $rulesFile | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } + # Union so any rule that fired but is missing from the resolved set still appears. + $allCodes = @($ruleUniverse + $firedCodes | Where-Object { $_ } | Select-Object -Unique) + + if ($specFiles.Count -eq 0 -and $results.Count -eq 0 -and $crashedSet.Count -eq 0) { + Add-LintSuite $doc $root $sevName 'API lint' @() @() + } else { + $seen = @{} + foreach ($file in $specFiles) { + $norm = ($file -replace '\\','/').ToLowerInvariant() + $seen[$norm] = $true + if ($crashedSet.ContainsKey($norm)) { + # This spec crashed Spectral -> explicit failure (NOT counted as passed). + $ts = $doc.CreateElement('testsuite') + $ts.SetAttribute('name',$file) + $ts.SetAttribute('tests','1'); $ts.SetAttribute('failures','1'); $ts.SetAttribute('errors','0') + $tc = $doc.CreateElement('testcase') + $tc.SetAttribute('name','Spectral crashed while linting this spec') + $tc.SetAttribute('classname',$file) + $fail = $doc.CreateElement('failure') + $fail.SetAttribute('message','Spectral threw an error on this specification (unsupported or malformed construct).') + $fail.SetAttribute('type','ERROR') + [void]$fail.AppendChild($doc.CreateCDataSection("Spectral failed on $file. This spec was NOT linted; fix the spec or exclude it. See the 'Run Spectral Linting' log.")) + [void]$tc.AppendChild($fail) + if (Test-Path -LiteralPath $file) { + $so = $doc.CreateElement('system-out'); $so.InnerText = "[[ATTACHMENT|$file]]"; [void]$tc.AppendChild($so) + } + [void]$ts.AppendChild($tc) + [void]$root.AppendChild($ts) + } elseif ($findingsByNorm.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } else { + Add-LintSuite $doc $root $sevName $file @() $allCodes + } + } + # Findings whose source was not matched to an enumerated spec file. + foreach ($norm in $findingsByNorm.Keys) { + if (-not $seen.ContainsKey($norm)) { + Add-LintSuite $doc $root $sevName $findingsByNorm[$norm].Source $findingsByNorm[$norm].Items $allCodes + } + } + } + + $doc.Save($junit) + $filesWithIssues = $findingsByNorm.Count + $crashedFiles = $crashedSet.Count + $cleanFiles = [Math]::Max(0, $specFiles.Count - $filesWithIssues - $crashedFiles) + Write-Host "Wrote JUnit report: $($results.Count) finding(s) across $filesWithIssues file(s); $crashedFiles crashed; $cleanFiles passed -> $junit" + + - task: PublishTestResults@2 + displayName: 'Publish Spectral lint results' + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/spectral-result.xml' + searchFolder: '$(Build.ArtifactStagingDirectory)' + testRunTitle: 'API lint results $(Build.SourceBranchName)' + failTaskOnFailedTests: false + # ----------------- Spectral API linting (END) ----------------- + + # -------- Publish extracted artifacts (pipeline artifact, for download/debug) -------- + - task: PublishPipelineArtifact@1 + displayName: 'Publish extracted artifacts' + inputs: + targetPath: '$(ARTIFACT_PATH)' + artifact: '$(ARTIFACT_NAME)' + publishLocation: 'pipeline' + + # -------- Commit extracted artifacts to a NEW branch in this repo -------- + - task: PowerShell@2 + displayName: 'Push artifacts to new repo branch' + condition: and(succeeded(), eq('${{ parameters.publishToRepo }}', true)) + inputs: + targetType: 'inline' + pwsh: false + workingDirectory: '$(Build.SourcesDirectory)' + script: | + $ErrorActionPreference = 'Stop' + + $branch = '$(EXTRACT_BRANCH)' + $targetFolder = '${{ parameters.targetRepoFolder }}' + $artifactPath = '$(ARTIFACT_PATH)' + $repoFolderAbs = Join-Path '$(Build.SourcesDirectory)' $targetFolder + + Write-Host "Branch: $branch" + Write-Host "Target folder: $repoFolderAbs" + + git config user.email '${{ parameters.gitUserEmail }}' + git config user.name '${{ parameters.gitUserName }}' + + # Create (or reset) the destination folder with the freshly extracted artifacts + if (Test-Path $repoFolderAbs) { Remove-Item -Recurse -Force $repoFolderAbs } + New-Item -ItemType Directory -Force -Path $repoFolderAbs | Out-Null + Copy-Item -Path (Join-Path $artifactPath '*') -Destination $repoFolderAbs -Recurse -Force + + # Create a brand-new branch from the current commit + git checkout -b $branch + + git add -- $targetFolder + $pending = git status --porcelain + if ([string]::IsNullOrWhiteSpace($pending)) { + Write-Host 'No changes detected against the base branch; nothing to commit.' + } else { + git commit -m "APIM extract: $(APIM_SERVICE_NAME) (build $(Build.BuildNumber))" + if ($LASTEXITCODE -ne 0) { throw "git commit failed (exit $LASTEXITCODE)" } + } + + git push --set-upstream origin $branch + if ($LASTEXITCODE -ne 0) { throw "git push failed (exit $LASTEXITCODE)" } + + # Expose the branch name as a pipeline output for the publisher pipeline + Write-Host "##vso[task.setvariable variable=extractBranch;isOutput=true]$branch" + Write-Host "Pushed branch '$branch' with artifacts under '$targetFolder/'." + name: pushBranch + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + # -------- Always logout -------- + - task: PowerShell@2 + displayName: 'az logout' + condition: always() + inputs: + targetType: 'inline' + pwsh: false + script: | + try { az logout } catch {} + try { az cache purge } catch {} + exit 0 \ No newline at end of file diff --git a/pipelines/azure-devops/prod-overrides.yaml b/pipelines/azure-devops/prod-overrides.yaml new file mode 100644 index 00000000..b995f103 --- /dev/null +++ b/pipelines/azure-devops/prod-overrides.yaml @@ -0,0 +1,27 @@ +diagnostics: + - name: applicationinsights + properties: + loggerId: "/subscriptions/8ed73338-8d94-4cdf-a286-604fd0acf798/resourceGroups/UK-Infrastructure/providers/Microsoft.ApiManagement/service/prod-apim-uk-01/loggers/apim-uk-appinsights" + +loggers: + - name: apim-uk-appinsights + properties: + resourceId: "/subscriptions/8ed73338-8d94-4cdf-a286-604fd0acf798/resourceGroups/UK-Infrastructure/providers/microsoft.insights/components/APIM-UK-Prod-AppInsights" + +apis: + - name: swagger-petstore + properties: + serviceUrl: 'https://prod.petstore.swagger.io/v2' + authenticationSettings: + oAuth2: null + openid: null + oAuth2AuthenticationSettings: [] + openidAuthenticationSettings: [] + + - name: webapitest + properties: + authenticationSettings: + oAuth2: null + openid: null + oAuth2AuthenticationSettings: [] + openidAuthenticationSettings: [] \ No newline at end of file diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index cbb9c248..d81d2ba1 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -450,6 +450,16 @@ async function reconcileOperationsAfterSpecImport( } } + // The spec import already bound request/responses representations to the + // schemas it created. PATCH replaces those arrays wholesale, so re-sending + // them without schemaId/typeName (the source IDs don't exist on the target) + // would wipe the binding the importer just created. Drop them and reconcile + // only importer-agnostic metadata. + if (hasSchemaBoundRepresentations(patchProps)) { + delete patchProps.request; + delete patchProps.responses; + } + // Strip source schema refs; APIM rebinds on import and drops stale IDs. stripRepresentationSchemaRefs(patchProps); @@ -475,6 +485,40 @@ async function reconcileOperationsAfterSpecImport( } } +/** + * Check whether any request/responses representation carries a schema binding + * (schemaId or typeName). Such operations are fully owned by the spec import + * and must not have request/responses re-sent in the reconcile PATCH. + */ +function hasSchemaBoundRepresentations(patchProps: Record): boolean { + const hasRef = (items: unknown): boolean => + Array.isArray(items) && + items.some( + (item) => + item !== null && + typeof item === 'object' && + (Object.hasOwn(item, 'schemaId') || Object.hasOwn(item, 'typeName')) && + ((item as Record).schemaId != null || + (item as Record).typeName != null) + ); + + const request = patchProps.request; + if (request && typeof request === 'object' && hasRef((request as Record).representations)) { + return true; + } + + const responses = patchProps.responses; + if (Array.isArray(responses)) { + for (const response of responses) { + if (response && typeof response === 'object' && hasRef((response as Record).representations)) { + return true; + } + } + } + + return false; +} + /** * Strip source schema refs from operation parameters and representations * before PATCH. APIM assigns new schema IDs on spec import, so stale IDs diff --git a/src/services/secret-redactor.ts b/src/services/secret-redactor.ts index 12e4eb7c..468ccec7 100644 --- a/src/services/secret-redactor.ts +++ b/src/services/secret-redactor.ts @@ -84,12 +84,18 @@ function isApimNamedValueReference(value: string): boolean { return NAMED_VALUE_REFERENCE_PATTERN.test(value); } +// Policy expressions (@(...) or @{...}) compute values at runtime from context +// and are not literal secrets, so they must be preserved as-is. +function isPolicyExpression(value: string): boolean { + return value.startsWith('@(') || value.startsWith('@{'); +} + function shouldRedactLiteral(value: string): boolean { const trimmed = value.trim(); if (!trimmed || trimmed === REDACTION_MARKER) { return false; } - return !isApimNamedValueReference(trimmed); + return !isApimNamedValueReference(trimmed) && !isPolicyExpression(trimmed); } function redactAuthorizationHeaderValue( diff --git a/tests/unit/services/api-publisher.test.ts b/tests/unit/services/api-publisher.test.ts index ea9d4df6..c847bd11 100644 --- a/tests/unit/services/api-publisher.test.ts +++ b/tests/unit/services/api-publisher.test.ts @@ -1067,6 +1067,10 @@ describe('api-publisher', () => { }); it('should reconcile operations via PATCH even in incremental mode (commitId set)', async () => { + mockRunParallel.mockImplementation(async (tasks: Array<() => Promise>) => { + for (const task of tasks) await task(); + }); + const client = createMockClient(); const children = [ { type: ResourceType.ApiPolicy, nameParts: ['petstore', 'policy-1'] }, @@ -1085,8 +1089,11 @@ describe('api-publisher', () => { return { name: 'create-item', properties: { + displayName: 'Create item', request: { - representations: [{ contentType: 'application/json', schemaId: 'my-schema' }], + representations: [ + { contentType: 'application/json', schemaId: 'my-schema', example: { id: 1 } }, + ], }, }, }; @@ -1117,6 +1124,14 @@ describe('api-publisher', () => { return sum + tasks.length; }, 0); expect(totalTasks).toBe(2); + + // Schema-bound request must NOT be re-sent — the spec import owns it. + // Only importer-agnostic metadata is reconciled. + expect(client.patchResource).toHaveBeenCalledWith( + testContext, + expect.objectContaining({ type: ResourceType.ApiOperation, nameParts: ['petstore', 'create-item'] }), + { properties: { displayName: 'Create item' } } + ); }); it('should skip operation republish in incremental mode when operation description is null', async () => { @@ -1174,7 +1189,11 @@ describe('api-publisher', () => { expect(totalTasks).toBe(1); }); - it('should re-publish operations with schema references in response representations', async () => { + it('should not re-send schema-bound response representations in reconcile PATCH', async () => { + mockRunParallel.mockImplementation(async (tasks: Array<() => Promise>) => { + for (const task of tasks) await task(); + }); + const client = createMockClient(); const children = [ { type: ResourceType.ApiOperation, nameParts: ['petstore', 'get-item'] }, @@ -1212,12 +1231,71 @@ describe('api-publisher', () => { await publishApi(client, store, testContext, apiDescriptor, testConfig); - // get-item reconcile task (1 total, no initial child publish tasks) - const totalTasks = mockRunParallel.mock.calls.reduce((sum, call) => { - const tasks = call[0] as unknown[]; - return sum + tasks.length; - }, 0); - expect(totalTasks).toBe(1); + // The operation only has schema-bound responses, which the spec import + // owns entirely — nothing is left to reconcile, so no PATCH is sent. + expect(client.patchResource).not.toHaveBeenCalled(); + }); + + it('should still strip schema refs when representations have no schema binding elsewhere', async () => { + mockRunParallel.mockImplementation(async (tasks: Array<() => Promise>) => { + for (const task of tasks) await task(); + }); + + const client = createMockClient(); + const children = [ + { type: ResourceType.ApiOperation, nameParts: ['petstore', 'get-item'] }, + ]; + const store = createMockStore(children); + + store.readResource.mockImplementation(async (_dir: string, descriptor: ResourceDescriptor) => { + if (descriptor.type === ResourceType.Api) { + return { name: 'petstore', properties: { path: 'petstore' } }; + } + if (descriptor.type === ResourceType.ApiOperation) { + return { + name: 'get-item', + properties: { + templateParameters: [{ name: 'id', type: 'string', schemaId: 'stale-schema' }], + responses: [ + { + statusCode: 200, + representations: [{ contentType: 'application/json', example: { ok: true } }], + }, + ], + }, + }; + } + return null; + }); + store.readContent.mockResolvedValue({ + content: 'openapi: "3.0.0"', + format: 'yaml', + }); + + const apiDescriptor: ResourceDescriptor = { + type: ResourceType.Api, + nameParts: ['petstore'], + }; + + await publishApi(client, store, testContext, apiDescriptor, testConfig); + + // No representation carries a schema binding, so responses are still sent + // (preserving examples) and stale refs elsewhere are stripped. + expect(client.patchResource).toHaveBeenCalledWith( + testContext, + expect.objectContaining({ type: ResourceType.ApiOperation, nameParts: ['petstore', 'get-item'] }), + { + properties: { + templateParameters: [{ name: 'id', type: 'string' }], + responses: [ + { + statusCode: 200, + representations: [{ contentType: 'application/json', example: { ok: true } }], + }, + ], + }, + } + ); }); it('should PATCH operations with only allow-listed properties after spec import', async () => { diff --git a/tests/unit/services/secret-redactor.test.ts b/tests/unit/services/secret-redactor.test.ts index 303cbdcf..82c6422e 100644 --- a/tests/unit/services/secret-redactor.test.ts +++ b/tests/unit/services/secret-redactor.test.ts @@ -157,5 +157,24 @@ describe('secret-redactor', () => { expect(redactedContent).toBe(policyXml); expect(findings).toEqual([]); }); + + it('should not redact policy expressions', () => { + const policyXml = ` + + + + @("Bearer " + (string)context.Variables["workloadBearer"]) + + Bearer @(context.Variables["token"]) + @{ return context.Variables.GetValueOrDefault<string>("key"); } + @(context.Request.Headers.GetValueOrDefault("x-sig")) + + `; + + const { redactedContent, findings } = redactPolicySecrets(policyXml); + + expect(redactedContent).toBe(policyXml); + expect(findings).toEqual([]); + }); }); }); From f43efdec2ebcb1ca95769d3af5e4d11310569d67 Mon Sep 17 00:00:00 2001 From: Aleksey Zheltov Date: Tue, 25 Aug 2026 17:33:24 +0000 Subject: [PATCH 3/5] fix(lint): resolve no-unsafe-argument errors in api-publisher --- src/services/api-publisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/api-publisher.ts b/src/services/api-publisher.ts index d81d2ba1..b252a6b1 100644 --- a/src/services/api-publisher.ts +++ b/src/services/api-publisher.ts @@ -494,7 +494,7 @@ function hasSchemaBoundRepresentations(patchProps: Record): boo const hasRef = (items: unknown): boolean => Array.isArray(items) && items.some( - (item) => + (item: unknown) => item !== null && typeof item === 'object' && (Object.hasOwn(item, 'schemaId') || Object.hasOwn(item, 'typeName')) && From debfe4e5b847083f9b557338edff52a912688c53 Mon Sep 17 00:00:00 2001 From: Aleksey Zheltov <71097129+Alexey-Zheltov@users.noreply.github.com> Date: Tue, 25 Aug 2026 19:01:57 +0000 Subject: [PATCH 4/5] Delete pipelines/azure-devops/prod-overrides.yaml --- pipelines/azure-devops/prod-overrides.yaml | 27 ---------------------- 1 file changed, 27 deletions(-) delete mode 100644 pipelines/azure-devops/prod-overrides.yaml diff --git a/pipelines/azure-devops/prod-overrides.yaml b/pipelines/azure-devops/prod-overrides.yaml deleted file mode 100644 index b995f103..00000000 --- a/pipelines/azure-devops/prod-overrides.yaml +++ /dev/null @@ -1,27 +0,0 @@ -diagnostics: - - name: applicationinsights - properties: - loggerId: "/subscriptions/8ed73338-8d94-4cdf-a286-604fd0acf798/resourceGroups/UK-Infrastructure/providers/Microsoft.ApiManagement/service/prod-apim-uk-01/loggers/apim-uk-appinsights" - -loggers: - - name: apim-uk-appinsights - properties: - resourceId: "/subscriptions/8ed73338-8d94-4cdf-a286-604fd0acf798/resourceGroups/UK-Infrastructure/providers/microsoft.insights/components/APIM-UK-Prod-AppInsights" - -apis: - - name: swagger-petstore - properties: - serviceUrl: 'https://prod.petstore.swagger.io/v2' - authenticationSettings: - oAuth2: null - openid: null - oAuth2AuthenticationSettings: [] - openidAuthenticationSettings: [] - - - name: webapitest - properties: - authenticationSettings: - oAuth2: null - openid: null - oAuth2AuthenticationSettings: [] - openidAuthenticationSettings: [] \ No newline at end of file From 58b55f4ea35fde56742c5d3fdf308d6b8c79e375 Mon Sep 17 00:00:00 2001 From: Alexey Zheltov Date: Wed, 26 Aug 2026 12:38:38 +0000 Subject: [PATCH 5/5] rename npm package to @azure-tools/apiops-cli, replace all references to @peterhauge/apiops-cli with @azure-tools/apiops-cli across package.json, init templates, generated pipelines, tests, docs, walkthroughs, and the Azure DevOps version-check script. Tarball filename patterns updated to azure-tools-apiops-cli-.tgz and package-lock.json regenerated for the new package name. --- .copilot/skills/release-apiops-version/SKILL.md | 2 +- .github/ISSUE_TEMPLATE/bug-report.yml | 2 +- .squad/agents/nodejsdev/charter.md | 2 +- .squad/agents/nodejsdev/history.md | 4 ++-- .squad/agents/testengineer/history.md | 2 +- .squad/decisions.md | 4 ++-- CHANGELOG.md | 2 +- README.md | 2 +- docs/README.md | 2 +- docs/ci-cd/azure-devops.md | 2 +- docs/commands/init.md | 2 +- docs/getting-started.md | 2 +- docs/guides/migration-from-v1.md | 6 +++--- .../air-gapped-azure-devops-local-registry.md | 8 ++++---- .../air-gapped-azure-devops-offline-tarball.md | 16 ++++++++-------- .../air-gapped-github-actions-local-registry.md | 2 +- .../air-gapped-github-actions-offline-tarball.md | 14 +++++++------- package-lock.json | 4 ++-- package.json | 2 +- .../azure-devops/Check-ApiopsCliVersion.ps1 | 6 +++--- src/cli/init-command.ts | 2 +- src/templates/azure-devops/extract-pipeline.ts | 4 ++-- src/templates/configs/package-json.ts | 2 +- tests/unit/services/init-service.test.ts | 4 ++-- .../azure-devops/extract-pipeline.test.ts | 2 +- .../unit/templates/configs/package-json.test.ts | 4 ++-- 26 files changed, 52 insertions(+), 52 deletions(-) diff --git a/.copilot/skills/release-apiops-version/SKILL.md b/.copilot/skills/release-apiops-version/SKILL.md index 52113afd..01d59a11 100644 --- a/.copilot/skills/release-apiops-version/SKILL.md +++ b/.copilot/skills/release-apiops-version/SKILL.md @@ -1,7 +1,7 @@ # Skill: Release apiops-cli Version **Confidence:** high -**Scope:** Any agent (or human) cutting a new release of `@peterhauge/apiops-cli` +**Scope:** Any agent (or human) cutting a new release of `@azure-tools/apiops-cli` ## What diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 4db234c1..a7ab7ade 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -31,7 +31,7 @@ body: id: version attributes: label: apiops CLI version - description: "Run `apiops --version` (or `npm list -g @peterhauge/apiops-cli`) to find the installed version." + description: "Run `apiops --version` (or `npm list -g @azure-tools/apiops-cli`) to find the installed version." validations: required: true diff --git a/.squad/agents/nodejsdev/charter.md b/.squad/agents/nodejsdev/charter.md index 90437abc..f27a9918 100644 --- a/.squad/agents/nodejsdev/charter.md +++ b/.squad/agents/nodejsdev/charter.md @@ -41,7 +41,7 @@ These are the concrete conventions I enforce in this project. - CLI commands wire Commander options to service functions — thin layer, no business logic in commands #### Dual-Mode Package Consumption (Decision: 2026-04-29) -- **Public npm mode** (default): `--cli-package` omitted → generates `package.json` referencing `"@peterhauge/apiops-cli": "latest"` from npm +- **Public npm mode** (default): `--cli-package` omitted → generates `package.json` referencing `"@azure-tools/apiops-cli": "latest"` from npm - **Local tarball mode**: `--cli-package ` → copies tarball to `.apiops/` directory, generates `package.json` with `"apiops": "file:.apiops/{tarball}"` - Both modes must work — backward compatibility is non-negotiable diff --git a/.squad/agents/nodejsdev/history.md b/.squad/agents/nodejsdev/history.md index 3da74d13..9e11a1c3 100644 --- a/.squad/agents/nodejsdev/history.md +++ b/.squad/agents/nodejsdev/history.md @@ -122,7 +122,7 @@ program.version(packageJson.version); ### 2026-04-29: Dual-Mode Init — Public npm vs Local Tarball -**Problem:** After publishing `@peterhauge/apiops-cli` to npm, `apiops init` still required `--cli-package ` pointing to a local .tgz tarball, making the workflow cumbersome for users who just want to use the public package. +**Problem:** After publishing `@azure-tools/apiops-cli` to npm, `apiops init` still required `--cli-package ` pointing to a local .tgz tarball, making the workflow cumbersome for users who just want to use the public package. **Solution:** Made `--cli-package` optional and implemented two modes: @@ -133,7 +133,7 @@ program.version(packageJson.version); 2. **Public npm mode** (when `--cli-package` NOT provided): - No tarball copy, no `.apiops/` directory - - Generates package.json with `"@peterhauge/apiops-cli": "latest"` + - Generates package.json with `"@azure-tools/apiops-cli": "latest"` - Use case: Standard consumption after publishing to npm **Implementation Details:** diff --git a/.squad/agents/testengineer/history.md b/.squad/agents/testengineer/history.md index 330302fe..d69e99cb 100644 --- a/.squad/agents/testengineer/history.md +++ b/.squad/agents/testengineer/history.md @@ -130,7 +130,7 @@ - Updated local mode tests to use `{ mode: 'local', tarballRelPath: '...' }` - Added 6 new tests for npm mode covering: - Valid JSON generation - - `@peterhauge/apiops-cli` dependency with `latest` version + - `@azure-tools/apiops-cli` dependency with `latest` version - No `apiops` dependency (should be undefined) - Standard package.json properties (private, name, version) - Newline termination diff --git a/.squad/decisions.md b/.squad/decisions.md index c34657a1..4da01ed3 100644 --- a/.squad/decisions.md +++ b/.squad/decisions.md @@ -128,8 +128,8 @@ ### 2026-04-29T14:30:00Z: apiops init Dual-Mode Package Consumption **By:** NodeJsDev **Status:** Implemented -**What:** Made `--cli-package` optional in `apiops init`. The command now supports two package consumption modes: (1) **Public npm mode** (default, when `--cli-package` NOT provided): generates package.json with `"@peterhauge/apiops-cli": "latest"`, no local tarball copy, no `.apiops/` directory created, standard consumption pattern after npm publish. (2) **Local tarball mode** (when `--cli-package ` provided): copies tarball to `.apiops/` directory, generates package.json with `"apiops": "file:.apiops/{tarball}"`, preserves existing behavior for local development/testing. -**Why:** After publishing to npm as `@peterhauge/apiops-cli`, requiring users to download the package and run `apiops init --cli-package ./tarball.tgz` added unnecessary friction. Most users want to reference the public package directly. The change is backward compatible — existing workflows with `--cli-package` continue to work unchanged. Improves user experience with simpler onboarding. +**What:** Made `--cli-package` optional in `apiops init`. The command now supports two package consumption modes: (1) **Public npm mode** (default, when `--cli-package` NOT provided): generates package.json with `"@azure-tools/apiops-cli": "latest"`, no local tarball copy, no `.apiops/` directory created, standard consumption pattern after npm publish. (2) **Local tarball mode** (when `--cli-package ` provided): copies tarball to `.apiops/` directory, generates package.json with `"apiops": "file:.apiops/{tarball}"`, preserves existing behavior for local development/testing. +**Why:** After publishing to npm as `@azure-tools/apiops-cli`, requiring users to download the package and run `apiops init --cli-package ./tarball.tgz` added unnecessary friction. Most users want to reference the public package directly. The change is backward compatible — existing workflows with `--cli-package` continue to work unchanged. Improves user experience with simpler onboarding. ### 2026-04-21T19:35:00Z: SOAP/WADL spec extraction prefers link format with inline XML fallback **By:** ApimExpert (via Squad session with a user) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c827d4..ae060060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,7 +140,7 @@ This project uses [Semantic Versioning](https://semver.org/) with alpha pre-rele ### Features - **Azure DevOps `init`** — interactive Copilot prompt with managed identity / WIF support ([#31](https://github.com/Azure/apiops-cli/pull/31)) -- **Public npm registry support** — install directly from `@peterhauge/apiops-cli` on npmjs.com ([#28](https://github.com/Azure/apiops-cli/pull/28)) +- **Public npm registry support** — install directly from `@azure-tools/apiops-cli` on npmjs.com ([#28](https://github.com/Azure/apiops-cli/pull/28)) ### Bug Fixes diff --git a/README.md b/README.md index aec7ad71..d40cc3bc 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ **Prerequisites:** An Azure subscription with an existing APIM resource, and Node.js ≥ 22. ```bash -npm install -g @peterhauge/apiops-cli +npm install -g @azure-tools/apiops-cli ``` ## Authentication diff --git a/docs/README.md b/docs/README.md index 90900afd..5301afd8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,7 +29,7 @@ flowchart LR ## Install ```bash -npm install -g @peterhauge/apiops-cli +npm install -g @azure-tools/apiops-cli ``` Requires Node.js 22 or later. diff --git a/docs/ci-cd/azure-devops.md b/docs/ci-cd/azure-devops.md index b1374146..99f4aac6 100644 --- a/docs/ci-cd/azure-devops.md +++ b/docs/ci-cd/azure-devops.md @@ -291,7 +291,7 @@ In your `package.json`, pin to a specific version: ```json { "dependencies": { - "@peterhauge/apiops-cli": "1.2.3" + "@azure-tools/apiops-cli": "1.2.3" } } ``` diff --git a/docs/commands/init.md b/docs/commands/init.md index 3c963ae3..c66b242b 100644 --- a/docs/commands/init.md +++ b/docs/commands/init.md @@ -106,7 +106,7 @@ In interactive mode (the default when running in a terminal), `apiops init` prom ## Package consumption modes -By default, generated pipeline files reference the published npm package `@peterhauge/apiops-cli`. This is the standard consumption pattern — no local files are needed. +By default, generated pipeline files reference the published npm package `@azure-tools/apiops-cli`. This is the standard consumption pattern — no local files are needed. If you pass `--cli-package `, the tarball is copied into a `.apiops/` directory and the generated `package.json` references it as a local file dependency. This mode is useful for local development and testing before the package is published to npm. diff --git a/docs/getting-started.md b/docs/getting-started.md index 8390c317..beccdc0b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ Extract your Azure API Management configuration, version it in git, and publish ## Install ```bash -npm install -g @peterhauge/apiops-cli +npm install -g @azure-tools/apiops-cli ``` Verify the installation: diff --git a/docs/guides/migration-from-v1.md b/docs/guides/migration-from-v1.md index b37f7fb5..72ad6d1c 100644 --- a/docs/guides/migration-from-v1.md +++ b/docs/guides/migration-from-v1.md @@ -21,7 +21,7 @@ apiops-cli is a single Node.js CLI that covers the full workflow with less setup |---------|-------------------|-----------------| | **Runtime** | .NET SDK or Docker | Node.js 22+ | | **CLI** | Separate Extractor/Publisher binaries | Single `apiops` CLI | -| **Install** | Docker pull or .NET tool install | `npm install -g @peterhauge/apiops-cli` | +| **Install** | Docker pull or .NET tool install | `npm install -g @azure-tools/apiops-cli` | | **Configuration** | `configuration.extractor.yaml` + `configuration.publisher.yaml` | Single filter YAML + override YAML | | **Authentication** | Azure service connections / env vars | `DefaultAzureCredential` (Azure CLI, OIDC, service principal, managed identity) | | **Scaffolding** | Manual pipeline setup | `apiops init` generates pipelines, config, directory structure | @@ -44,7 +44,7 @@ apiops-cli supports all APIOps Toolkit resource types plus: `GlobalSchema`, `Pol ### 1. Install apiops-cli ```bash -npm install -g @peterhauge/apiops-cli +npm install -g @azure-tools/apiops-cli ``` Verify: @@ -274,7 +274,7 @@ apiops extract --cloud usgov ... | Issue | Cause | Fix | |-------|-------|-----| -| `apiops: command not found` | CLI not installed globally | Run `npm install -g @peterhauge/apiops-cli` | +| `apiops: command not found` | CLI not installed globally | Run `npm install -g @azure-tools/apiops-cli` | | Artifacts not recognized | Unexpected directory structure | Verify your artifacts follow the standard layout (`apis/{name}/apiInformation.json`, etc.) | | Authentication fails in pipeline | APIOps Toolkit used service connection env vars; apiops-cli uses `DefaultAzureCredential` | See [Authentication Guide](./authentication.md). For GitHub Actions, use `azure/login` with OIDC. For Azure DevOps, use `AzureCLI@2` task. | | Override values not applied | Wrong override file format or path | Check YAML structure matches apiops-cli format. Pass with `--overrides `. | diff --git a/docs/walkthrough/air-gapped-azure-devops-local-registry.md b/docs/walkthrough/air-gapped-azure-devops-local-registry.md index c61ade03..5ba5f115 100644 --- a/docs/walkthrough/air-gapped-azure-devops-local-registry.md +++ b/docs/walkthrough/air-gapped-azure-devops-local-registry.md @@ -94,7 +94,7 @@ az devops invoke \ ### 1.2 Configure an upstream source -**[Configure an upstream source](https://learn.microsoft.com/en-us/azure/devops/artifacts/how-to/set-up-upstream-sources?view=azure-devops)** pointing to `https://registry.npmjs.org`. The upstream is only used during controlled sync windows; once `@peterhauge/apiops-cli` and its dependencies are cached, the feed serves them locally. +**[Configure an upstream source](https://learn.microsoft.com/en-us/azure/devops/artifacts/how-to/set-up-upstream-sources?view=azure-devops)** pointing to `https://registry.npmjs.org`. The upstream is only used during controlled sync windows; once `@azure-tools/apiops-cli` and its dependencies are cached, the feed serves them locally. ```bash cat > feed-upstream.json <<'JSON' @@ -121,10 +121,10 @@ az devops invoke \ ``` ### 1.3 Populate the feed -**[Populate the feed](https://learn.microsoft.com/en-us/azure/devops/artifacts/npm/npmrc?view=azure-devops)** from a connected workstation by running `npm install @peterhauge/apiops-cli` against the feed registry URL. This pulls the package and its transitive dependencies into the feed cache. +**[Populate the feed](https://learn.microsoft.com/en-us/azure/devops/artifacts/npm/npmrc?view=azure-devops)** from a connected workstation by running `npm install @azure-tools/apiops-cli` against the feed registry URL. This pulls the package and its transitive dependencies into the feed cache. ```bash -npm install @peterhauge/apiops-cli \ +npm install @azure-tools/apiops-cli \ --registry "$FEED_REGISTRY" \ --//pkgs.dev.azure.com/${ORG}/${PROJECT}/_packaging/${FEED}/npm/registry/:_authToken="$(az account get-access-token --resource https://app.vssps.visualstudio.com --query accessToken -o tsv)" ``` @@ -272,7 +272,7 @@ Sync the feed during a connectivity window to pull the new version, then update ```bash # Update package.json to the latest CLI version available in the feed -npm install @peterhauge/apiops-cli --registry "$FEED_REGISTRY" --//pkgs.dev.azure.com/${ORG}/${PROJECT}/_packaging/${FEED}/npm/registry/:_authToken="$(az account get-access-token --resource https://app.vssps.visualstudio.com --query accessToken -o tsv)" +npm install @azure-tools/apiops-cli --registry "$FEED_REGISTRY" --//pkgs.dev.azure.com/${ORG}/${PROJECT}/_packaging/${FEED}/npm/registry/:_authToken="$(az account get-access-token --resource https://app.vssps.visualstudio.com --query accessToken -o tsv)" # Rebuild lock file from package.json npm install diff --git a/docs/walkthrough/air-gapped-azure-devops-offline-tarball.md b/docs/walkthrough/air-gapped-azure-devops-offline-tarball.md index 763001c9..e9e48881 100644 --- a/docs/walkthrough/air-gapped-azure-devops-offline-tarball.md +++ b/docs/walkthrough/air-gapped-azure-devops-offline-tarball.md @@ -53,10 +53,10 @@ flowchart LR On the connected workstation: ```bash -npm pack @peterhauge/apiops-cli +npm pack @azure-tools/apiops-cli ``` -This produces `peterhauge-apiops-cli-.tgz` in the current directory. +This produces `azure-tools-apiops-cli-.tgz` in the current directory. --- @@ -70,7 +70,7 @@ Pass `--cli-package` so the generated `package.json` references the local tarbal apiops init \ --ci azure-devops \ --environments dev,prod \ - --cli-package /peterhauge-apiops-cli-.tgz + --cli-package /azure-tools-apiops-cli-.tgz ``` This command generates: @@ -111,7 +111,7 @@ For the offline-tarball workflow, commit the files that make the pipeline fully | File Name | Description | |-----------|-------------| -| `.apiops/peterhauge-apiops-cli-.tgz` | CLI package consumed by the pipelines. | +| `.apiops/azure-tools-apiops-cli-.tgz` | CLI package consumed by the pipelines. | | `package.json` | Contains the `file:` dependency pointing to the tarball. | | `package-lock.json` | Required for deterministic offline installs with `npm ci --offline`. | | `.azdo/pipelines/run-apiops-extractor.yml` | Azure DevOps extract pipeline definition. | @@ -120,7 +120,7 @@ For the offline-tarball workflow, commit the files that make the pipeline fully ```bash git add \ - .apiops/peterhauge-apiops-cli-*.tgz \ + .apiops/azure-tools-apiops-cli-*.tgz \ package.json \ package-lock.json \ .azdo/pipelines/run-apiops-extractor.yml \ @@ -179,8 +179,8 @@ Trigger the extract pipeline manually from **Pipelines → Run pipeline** and ve ## Upgrading the CLI Version -1. On a connected workstation, run `npm pack @peterhauge/apiops-cli` for the new version -2. Replace `.apiops/peterhauge-apiops-cli-*.tgz` with the new tarball and update the `file:` path in `package.json` +1. On a connected workstation, run `npm pack @azure-tools/apiops-cli` for the new version +2. Replace `.apiops/azure-tools-apiops-cli-*.tgz` with the new tarball and update the `file:` path in `package.json` 3. Regenerate `package-lock.json` ```bash npm install @@ -197,7 +197,7 @@ Trigger the extract pipeline manually from **Pipelines → Run pipeline** and ve 5. Commit the tarball and updated lock file ```bash git add \ - .apiops/peterhauge-apiops-cli-*.tgz \ + .apiops/azure-tools-apiops-cli-*.tgz \ package.json \ package-lock.json \ git commit -m "chore: commit updated offline-tarball apiops bootstrap files" diff --git a/docs/walkthrough/air-gapped-github-actions-local-registry.md b/docs/walkthrough/air-gapped-github-actions-local-registry.md index 68ccc6ce..c906e688 100644 --- a/docs/walkthrough/air-gapped-github-actions-local-registry.md +++ b/docs/walkthrough/air-gapped-github-actions-local-registry.md @@ -52,7 +52,7 @@ flowchart LR Set up the [GitHub Packages](https://docs.github.com/en/enterprise-server@latest/admin/packages/getting-started-with-github-packages-for-your-enterprise) npm registry on your GHES instance so it serves packages to your air-gapped runners without requiring internet access at install time. 1. **[Enable GitHub Packages on GHES](https://docs.github.com/en/enterprise-server@latest/admin/packages/getting-started-with-github-packages-for-your-enterprise)** — turn on the Packages service for your enterprise and configure the storage backend. The npm registry endpoint is `https://npm./`. -2. **Populate the registry** from a connected workstation by running `npm install @peterhauge/apiops-cli` against the GHES npm registry URL. This pulls the package and its transitive dependencies into the registry cache. +2. **Populate the registry** from a connected workstation by running `npm install @azure-tools/apiops-cli` against the GHES npm registry URL. This pulls the package and its transitive dependencies into the registry cache. 3. **Add a project `.npmrc`** that points `registry=` at your GHES npm endpoint and sets `//npm./:_authToken=${NODE_AUTH_TOKEN}` so authentication is read from an environment variable injected at workflow runtime. Commit `.npmrc` so workflows and developers resolve against the local registry. > **Tip:** Follow [Authenticating to GitHub Packages](https://docs.github.com/en/enterprise-server@latest/packages/working-with-a-github-packages-registry/working-with-the-npm-registry#authenticating-to-github-packages) for the exact `.npmrc` format your GHES version expects. diff --git a/docs/walkthrough/air-gapped-github-actions-offline-tarball.md b/docs/walkthrough/air-gapped-github-actions-offline-tarball.md index a70fbca3..90856b24 100644 --- a/docs/walkthrough/air-gapped-github-actions-offline-tarball.md +++ b/docs/walkthrough/air-gapped-github-actions-offline-tarball.md @@ -53,17 +53,17 @@ flowchart LR On the connected workstation: ```bash -npm pack @peterhauge/apiops-cli +npm pack @azure-tools/apiops-cli ``` -This produces `peterhauge-apiops-cli-.tgz` in the current directory. +This produces `azure-tools-apiops-cli-.tgz` in the current directory. Commit the tarball into your repository (e.g., under `.apiops/`) so the workflow can reference it by path: ```bash mkdir -p .apiops -mv peterhauge-apiops-cli-*.tgz .apiops/ -git add .apiops/peterhauge-apiops-cli-*.tgz +mv azure-tools-apiops-cli-*.tgz .apiops/ +git add .apiops/azure-tools-apiops-cli-*.tgz ``` --- @@ -76,7 +76,7 @@ Pass `--cli-package` so the generated `package.json` references the local tarbal apiops init \ --ci github-actions \ --environments dev,prod \ - --cli-package ./.apiops/peterhauge-apiops-cli-.tgz \ + --cli-package ./.apiops/azure-tools-apiops-cli-.tgz \ --non-interactive ``` @@ -215,8 +215,8 @@ Trigger the extract workflow manually from **Actions → Run workflow** and veri ## Upgrading the CLI Version -1. On a connected workstation, run `npm pack @peterhauge/apiops-cli` for the new version -2. Replace `.apiops/peterhauge-apiops-cli-*.tgz` with the new tarball and update the `file:` path in `package.json` +1. On a connected workstation, run `npm pack @azure-tools/apiops-cli` for the new version +2. Replace `.apiops/azure-tools-apiops-cli-*.tgz` with the new tarball and update the `file:` path in `package.json` 3. Regenerate `package-lock.json` (`npm install`) 4. Re-populate and re-transfer the npm cache (`npm ci` on the workstation, then copy `~/.npm/_cacache/`) 5. Commit the tarball and updated lock file diff --git a/package-lock.json b/package-lock.json index 2fa603f3..c9c06628 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@peterhauge/apiops-cli", + "name": "@azure-tools/apiops-cli", "version": "0.4.0-alpha.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@peterhauge/apiops-cli", + "name": "@azure-tools/apiops-cli", "version": "0.4.0-alpha.2", "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index f385abb6..62e97995 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "@peterhauge/apiops-cli", + "name": "@azure-tools/apiops-cli", "version": "0.4.0-alpha.2", "schemaVersion": "1", "description": "CLI tool for Azure API Management configuration-as-code", diff --git a/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 b/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 index 9dab8d57..91fac6bc 100644 --- a/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 +++ b/pipelines/azure-devops/Check-ApiopsCliVersion.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Ensures the apiops CLI (@peterhauge/apiops-cli) is installed at the requested + Ensures the apiops CLI (@azure-tools/apiops-cli) is installed at the requested version, installing or upgrading it via npm when necessary. .DESCRIPTION @@ -14,7 +14,7 @@ APIOPS_PATH and APIOPS_VERSION. .PARAMETER PackageName - The npm package name. Defaults to '@peterhauge/apiops-cli'. + The npm package name. Defaults to '@azure-tools/apiops-cli'. .EXAMPLE .\Check-ApiopsCliVersion.ps1 @@ -28,7 +28,7 @@ [CmdletBinding()] param( [string]$ApiopsVersion = 'latest', - [string]$PackageName = '@peterhauge/apiops-cli', + [string]$PackageName = '@azure-tools/apiops-cli', # npm registry URL used for the direct connectivity check (curl). [string]$RegistryUrl = 'https://registry.npmjs.org', # Deprecated / kept for backward compatibility with existing callers. Behaviour diff --git a/src/cli/init-command.ts b/src/cli/init-command.ts index ea040523..9cf31fd4 100644 --- a/src/cli/init-command.ts +++ b/src/cli/init-command.ts @@ -33,7 +33,7 @@ export function createInitCommand(): Command { .option('--non-interactive', 'Skip interactive prompts (requires --ci)', false) .option('--artifact-dir ', 'Artifact directory path', './apim-artifacts') .option('--environments ', 'Comma-separated environment names', 'dev,prod') - .option('--cli-package ', 'Path to apiops npm tarball (from npm pack). If not provided, uses @peterhauge/apiops-cli from npm registry') + .option('--cli-package ', 'Path to apiops npm tarball (from npm pack). If not provided, uses @azure-tools/apiops-cli from npm registry') .option('--force', 'Overwrite existing files without prompting', false) .action(async (options: InitOptions) => { try { diff --git a/src/templates/azure-devops/extract-pipeline.ts b/src/templates/azure-devops/extract-pipeline.ts index 6ae64781..46737120 100644 --- a/src/templates/azure-devops/extract-pipeline.ts +++ b/src/templates/azure-devops/extract-pipeline.ts @@ -97,7 +97,7 @@ steps: scriptType: 'bash' scriptLocation: 'inlineScript' inlineScript: | - npx @peterhauge/apiops-cli extract \\ + npx @azure-tools/apiops-cli extract \\ --resource-group "$(APIM_RESOURCE_GROUP)" \\ --service-name "$(APIM_SERVICE_NAME)" \\ --output ${config.artifactDir} \\ @@ -111,7 +111,7 @@ steps: scriptType: 'bash' scriptLocation: 'inlineScript' inlineScript: | - npx @peterhauge/apiops-cli extract \\ + npx @azure-tools/apiops-cli extract \\ --resource-group "$(APIM_RESOURCE_GROUP)" \\ --service-name "$(APIM_SERVICE_NAME)" \\ --output ${config.artifactDir} \\ diff --git a/src/templates/configs/package-json.ts b/src/templates/configs/package-json.ts index a596523e..e6adc927 100644 --- a/src/templates/configs/package-json.ts +++ b/src/templates/configs/package-json.ts @@ -24,7 +24,7 @@ export function generatePackageJson(config: PackageJsonConfig): string { (pkg.dependencies as Record).apiops = `file:${posixPath}`; } else { // Public npm registry mode - (pkg.dependencies as Record)['@peterhauge/apiops-cli'] = 'latest'; + (pkg.dependencies as Record)['@azure-tools/apiops-cli'] = 'latest'; } return JSON.stringify(pkg, null, 2) + '\n'; diff --git a/tests/unit/services/init-service.test.ts b/tests/unit/services/init-service.test.ts index 37ea7545..29a50e14 100644 --- a/tests/unit/services/init-service.test.ts +++ b/tests/unit/services/init-service.test.ts @@ -507,7 +507,7 @@ describe('init-service', () => { expect(pkgCalls).toHaveLength(1); const content = pkgCalls[0][1] as string; const pkg = JSON.parse(content); - expect(pkg.dependencies['@peterhauge/apiops-cli']).toBe('latest'); + expect(pkg.dependencies['@azure-tools/apiops-cli']).toBe('latest'); expect(pkg.dependencies.apiops).toBeUndefined(); }); @@ -659,7 +659,7 @@ describe('init-service', () => { const content = pkgCalls[0][1] as string; const pkg = JSON.parse(content); expect(pkg.dependencies.lodash).toBe('^4.17.21'); - expect(pkg.dependencies['@peterhauge/apiops-cli']).toBe('latest'); + expect(pkg.dependencies['@azure-tools/apiops-cli']).toBe('latest'); }); }); diff --git a/tests/unit/templates/azure-devops/extract-pipeline.test.ts b/tests/unit/templates/azure-devops/extract-pipeline.test.ts index f511448d..100ad106 100644 --- a/tests/unit/templates/azure-devops/extract-pipeline.test.ts +++ b/tests/unit/templates/azure-devops/extract-pipeline.test.ts @@ -123,7 +123,7 @@ describe('azure-devops/extract-pipeline', () => { it('should use npm ci to install dependencies (uses tgz from package.json)', () => { const pipeline = generateExtractPipeline(defaultConfig); expect(pipeline).toContain('npm ci'); - expect(pipeline).toContain('npx @peterhauge/apiops-cli extract'); + expect(pipeline).toContain('npx @azure-tools/apiops-cli extract'); }); }); }); diff --git a/tests/unit/templates/configs/package-json.test.ts b/tests/unit/templates/configs/package-json.test.ts index 3441dab2..8b5f3122 100644 --- a/tests/unit/templates/configs/package-json.test.ts +++ b/tests/unit/templates/configs/package-json.test.ts @@ -53,10 +53,10 @@ describe('configs/package-json', () => { expect(() => JSON.parse(content)).not.toThrow(); }); - it('should include @peterhauge/apiops-cli dependency', () => { + it('should include @azure-tools/apiops-cli dependency', () => { const content = generatePackageJson({ mode: 'npm' }); const pkg = JSON.parse(content); - expect(pkg.dependencies['@peterhauge/apiops-cli']).toBe('latest'); + expect(pkg.dependencies['@azure-tools/apiops-cli']).toBe('latest'); }); it('should NOT include apiops dependency', () => {