feat(scheduler): surface failed and at-risk scheduled tasks

backend/Config/CIPPTimers.json

@@ -284,5 +284,13 @@
     "RunOnProcessor": true,
     "TZOffset": true,
     "PreferredProcessor": "standards"
+  },
+  {
+    "Id": "0b31a4b7-4104-479e-a358-a57675f80bd5",
+    "Command": "Start-CIPPTaskPreflightCheck",
+    "Description": "Flag planned scheduled tasks whose prerequisites are no longer available",
+    "Cron": "0 7 */6 * * *",
+    "Priority": 25,
+    "RunOnProcessor": true
   }
 ]

backend/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1

@@ -39,12 +39,12 @@ function Push-ExecScheduledCommand {
     $CurrentTask = Get-AzDataTableEntity @Table -Filter "PartitionKey eq '$($task.PartitionKey)' and RowKey eq '$($task.RowKey)'"
     if (!$CurrentTask) {
         Write-Information "The task $($task.Name) for tenant $($task.Tenant) does not exist in the ScheduledTasks table. Exiting."
-        Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+        Set-CippScheduledTaskContext -TaskId ''
         return
     }
     if ($CurrentTask.TaskState -eq 'Completed' -and !$IsMultiTenantTask) {
         Write-Information "The task $($task.Name) for tenant $($task.Tenant) is already completed. Skipping execution."
-        Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+        Set-CippScheduledTaskContext -TaskId ''
         return
     }
     # Task should be 'Pending' (queued by orchestrator) or 'Running' (retry/recovery)
@@ -68,7 +68,7 @@ function Push-ExecScheduledCommand {
         # If executed within last 15 minutes, skip (likely a duplicate pickup)
         if ($timeSinceExecution -lt 900) {
             Write-Information "One-time task $($task.Name) for tenant $Tenant was recently executed ($timeSinceExecution seconds ago). Skipping to prevent duplicate execution."
-            Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+            Set-CippScheduledTaskContext -TaskId ''
             return
         }
     }
@@ -114,7 +114,7 @@ function Push-ExecScheduledCommand {
                     TaskState     = 'Planned'
                     ScheduledTime = [string]$nextRunUnixTime
                 }
-                Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+                Set-CippScheduledTaskContext -TaskId ''
                 return
             }
         }
@@ -142,11 +142,16 @@ function Push-ExecScheduledCommand {
                 RowKey       = $task.RowKey
                 Results      = "$Results"
                 TaskState    = $State
+                HasErrors    = $true
+                ErrorSummary = "$Results"
+                AtRisk       = $false
+                AtRiskReason = ''
+                Acknowledged = $false
             }
         }
 
         Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Failed to execute task $($task.Name): The command $($Item.Command) does not exist." -sev Error
-        Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+        Set-CippScheduledTaskContext -TaskId ''
         return
     }
 
@@ -160,9 +165,14 @@ function Push-ExecScheduledCommand {
                 RowKey       = $task.RowKey
                 Results      = "$Results"
                 TaskState    = $State
+                HasErrors    = $true
+                ErrorSummary = "$Results"
+                AtRisk       = $false
+                AtRiskReason = ''
+                Acknowledged = $false
             }
         }
-        Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+        Set-CippScheduledTaskContext -TaskId ''
         return
     }
     if ($Item.Command -in (Get-CIPPSchedulerBlockedCommands)) {
@@ -175,9 +185,14 @@ function Push-ExecScheduledCommand {
                 RowKey       = $task.RowKey
                 Results      = "$Results"
                 TaskState    = $State
+                HasErrors    = $true
+                ErrorSummary = "$Results"
+                AtRisk       = $false
+                AtRiskReason = ''
+                Acknowledged = $false
             }
         }
-        Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+        Set-CippScheduledTaskContext -TaskId ''
         return
     }
 
@@ -265,7 +280,16 @@ function Push-ExecScheduledCommand {
                 Write-Information "Starting task: $($Item.Command) for tenant: $Tenant with parameters: $($commandParameters | ConvertTo-Json -Depth 10)"
                 $results = & $Item.Command @commandParameters
             } catch {
-                $results = "Task Failed: $($_.Exception.Message)"
+                # Commands like New-CIPPUserTask throw a hashtable carrying their partial results
+                # (throw @{'Results' = ...}). A thrown hashtable's Exception.Message is the literal
+                # type name, so unwrap it - otherwise the stored result reads
+                # "Task Failed: System.Collections.Hashtable" instead of what actually went wrong.
+                $ThrownObject = $_.TargetObject
+                $results = if ($ThrownObject -is [System.Collections.IDictionary] -and $ThrownObject.Results) {
+                    "Task Failed: $(@($ThrownObject.Results) -join ' | ')"
+                } else {
+                    "Task Failed: $($_.Exception.Message)"
+                }
                 $State = 'Failed'
             }
             Write-Information 'Ran the command. Processing results'
@@ -354,9 +378,15 @@ function Push-ExecScheduledCommand {
                 Results       = "$errorMessage"
                 ScheduledTime = "$nextRunUnixTime"
                 TaskState     = $State
+                HasErrors     = $true
+                ErrorSummary  = "$errorMessage"
+                AtRisk        = $false
+                AtRiskReason  = ''
+                Acknowledged  = $false
             }
         }
         Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Failed to execute task $($task.Name): $errorMessage" -sev Error -LogData (Get-CippExceptionData -Exception $_.Exception)
+        $TaskFailureLogged = $true
     }
 
     # For orchestrator-based commands, skip post-execution alerts as they will be handled by the orchestrator's post-execution function
@@ -374,6 +404,21 @@ function Push-ExecScheduledCommand {
         Send-CIPPScheduledTaskAlert @AlertParams
     }
 
+    # A task can finish without throwing while a step inside it failed - the clearest case being a user
+    # that is created successfully but never licensed because the tenant has no licences left. Those steps
+    # log at Error severity, so the errors collected against this task's context are what tell the two
+    # apart. Without this the row reads Completed and the failure only exists as text inside Results.
+    $TaskErrors = @(Get-CippScheduledTaskError)
+    $TaskHasErrors = $TaskErrors.Count -gt 0
+    $TaskErrorSummary = if ($TaskHasErrors) { $TaskErrors -join ' | ' } else { '' }
+    if ($TaskErrorSummary.Length -gt 4000) {
+        $TaskErrorSummary = $TaskErrorSummary.Substring(0, 4000)
+    }
+
+    # Every terminal write below also clears AtRisk: the preflight flag is a prediction about a run
+    # that has now happened, and nothing else ever re-examines a task that has left the Planned
+    # state - without this, a flagged task that ran would sit in the at-risk view forever.
+
     try {
         # For orchestrator-based commands, skip task state update as it will be handled by post-execution
         if ($Item.Command -in $OrchestratorBasedCommands) {
@@ -386,6 +431,11 @@ function Push-ExecScheduledCommand {
                         RowKey       = $task.RowKey
                         Results      = "$results"
                         TaskState    = 'Failed'
+                        HasErrors    = $true
+                        ErrorSummary = "$results"
+                        AtRisk       = $false
+                        AtRiskReason = ''
+                        Acknowledged = $false
                     }
                 } else {
                     # Update task state to 'Processing' to indicate orchestration is in progress
@@ -402,12 +452,20 @@ function Push-ExecScheduledCommand {
             # The PostExecution function will aggregate all results and update the parent task
             Write-Information "Multi-tenant execution for tenant $Tenant - parent task state will be updated by PostExecution"
         } elseif ($task.Recurrence -eq '0' -or [string]::IsNullOrEmpty($task.Recurrence) -or $Trigger.ExecutionMode.value -eq 'once' -or $Trigger.ExecutionMode -eq 'once') {
-            Write-Information 'Recurrence empty or 0. Task is not recurring. Setting task state to completed.'
+            # $State is set to 'Failed' when the command threw. Before, it was only honoured for
+            # orchestrator-based commands, so every other failing task was written as Completed.
+            $FinalState = if ($State -eq 'Failed') { 'Failed' } else { 'Completed' }
+            Write-Information "Recurrence empty or 0. Task is not recurring. Setting task state to $FinalState."
             Update-AzDataTableEntity -Force @Table -Entity @{
                 PartitionKey = $task.PartitionKey
                 RowKey       = $task.RowKey
                 Results      = "$StoredResults"
-                TaskState    = 'Completed'
+                TaskState    = $FinalState
+                HasErrors    = $TaskHasErrors
+                ErrorSummary = $TaskErrorSummary
+                AtRisk       = $false
+                AtRiskReason = ''
+                Acknowledged = $false
             }
         } else {
             #if recurrence is just a number, add it in days.
@@ -429,22 +487,41 @@ function Push-ExecScheduledCommand {
             }
 
             $nextRunUnixTime = [int64]$task.ScheduledTime + [int64]$secondsToAdd
+            # A recurring task that failed still has to be rescheduled, so it goes to 'Failed - Planned'
+            # rather than 'Failed' - the orchestrator picks that state back up on the next cycle.
+            $FinalState = if ($State -eq 'Failed') { 'Failed - Planned' } else { 'Planned' }
             Write-Information "The job is recurring. It was scheduled for $($task.ScheduledTime). The next runtime should be $nextRunUnixTime"
             Update-AzDataTableEntity -Force @Table -Entity @{
                 PartitionKey  = $task.PartitionKey
                 RowKey        = $task.RowKey
                 Results       = "$StoredResults"
-                TaskState     = 'Planned'
+                TaskState     = $FinalState
                 ScheduledTime = "$nextRunUnixTime"
+                HasErrors     = $TaskHasErrors
+                ErrorSummary  = $TaskErrorSummary
+                AtRisk        = $false
+                AtRiskReason  = ''
+                Acknowledged  = $false
             }
         }
     } catch {
         Write-Warning "Failed to update task state: $($_.Exception.Message)"
         Write-Information $_.InvocationInfo.PositionMessage
     }
-    if ($TaskType -ne 'Alert') {
-        Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Successfully executed task: $($task.Name)" -sev Info
+    if ($TaskType -ne 'Alert' -and -not $TaskFailureLogged) {
+        if ($State -eq 'Failed') {
+            # $StoredResults, not $results: by this point a thrown command's message has been wrapped
+            # into a hashtable by the result processing above, which interpolates as
+            # "System.Collections.Hashtable" and loses the error entirely.
+            Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Failed to execute task $($task.Name): $StoredResults" -sev Error
+        } elseif ($TaskHasErrors) {
+            # The task itself ran, but a step inside it failed. Logged as an error so it can raise a
+            # notification rather than being reported as a clean success.
+            Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Executed task $($task.Name) with errors: $TaskErrorSummary" -sev Error
+        } else {
+            Write-LogMessage -API 'Scheduler_UserTasks' -tenant $Tenant -tenantid $TenantInfo.customerId -message "Successfully executed task: $($task.Name)" -sev Info
+        }
     }
-    Remove-Variable -Name ScheduledTaskId -Scope Script -ErrorAction SilentlyContinue
+    Set-CippScheduledTaskContext -TaskId ''
     return 'Task Completed Successfully.'
 }

backend/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Push-ScheduledTaskPostExecution.ps1

@@ -65,6 +65,14 @@ function Push-ScheduledTaskPostExecution {
     # Prepare aggregated results message
     $AggregatedMessage = "Multi-tenant task completed: $SuccessCount successful, $FailureCount failed (Total: $TotalTenants tenants)"
 
+    # The task state below only goes to Failed when *every* tenant failed, so a partial failure - 3 of
+    # 50 tenants - otherwise reports as a clean Completed and is invisible. Flag any failure so those
+    # tasks still surface for review.
+    $HasFailures = $FailureCount -gt 0
+    $FailureSummary = if ($HasFailures) {
+        "$FailureCount of $TotalTenants tenant(s) failed. Per-tenant detail is available in the More Info pane."
+    } else { '' }
+
     # Calculate next run time for recurring tasks
     if ($IsRecurring -and !$IsTriggerOnce) {
         # Convert recurrence to seconds
@@ -95,6 +103,9 @@ function Push-ScheduledTaskPostExecution {
                 Results       = $AggregatedMessage
                 TaskState     = if ($FailureCount -gt 0 -and $FailureCount -eq $TotalTenants) { 'Failed - Planned' } else { 'Planned' }
                 ScheduledTime = "$nextRunUnixTime"
+                HasErrors     = $HasFailures
+                ErrorSummary  = $FailureSummary
+                Acknowledged  = $false
             }
         } else {
             # Invalid recurrence, mark as completed
@@ -104,6 +115,9 @@ function Push-ScheduledTaskPostExecution {
                 RowKey       = $ParentTask.RowKey
                 Results      = "$AggregatedMessage - Warning: Invalid recurrence, task will not repeat"
                 TaskState    = 'Completed'
+                HasErrors    = $HasFailures
+                ErrorSummary = $FailureSummary
+                Acknowledged = $false
             }
         }
     } else {
@@ -114,6 +128,9 @@ function Push-ScheduledTaskPostExecution {
             RowKey       = $ParentTask.RowKey
             Results      = $AggregatedMessage
             TaskState    = if ($FailureCount -gt 0 -and $FailureCount -eq $TotalTenants) { 'Failed' } else { 'Completed' }
+            HasErrors    = $HasFailures
+            ErrorSummary = $FailureSummary
+            Acknowledged = $false
         }
     }