Implement AutoScheduler engine, MDE alert detection, job timeout, MDE onboarding, and domain credentials

src/Api/ShieldChecker.BackendApi/Program.cs

@@ -78,6 +78,11 @@
 // ── Audit Service ─────────────────────────────────────────────────────────────
 builder.Services.AddScoped<ShieldChecker.BackendApi.Services.AuditService>();
 
+// ── Background Services ──────────────────────────────────────────────────────
+builder.Services.AddHostedService<ShieldChecker.BackendApi.Services.AutoSchedulerService>();
+builder.Services.AddHostedService<ShieldChecker.BackendApi.Services.MdeAlertDetectionService>();
+builder.Services.AddHostedService<ShieldChecker.BackendApi.Services.JobTimeoutService>();
+
 var app = builder.Build();
 
 // Initialise / validate the database on startup

src/Api/ShieldChecker.BackendApi/Services/AutoSchedulerService.cs

@@ -0,0 +1,279 @@
+using Microsoft.EntityFrameworkCore;
+using ShieldChecker.DataAccess;
+using ShieldChecker.DataAccess.Models;
+
+namespace ShieldChecker.BackendApi.Services
+{
+    /// <summary>
+    /// Background service that evaluates AutoSchedule entries every 60 seconds and
+    /// creates queued TestJob records for schedules whose NextExecution has passed.
+    /// Uses a SchedulerMutex row to prevent concurrent schedule evaluation across replicas.
+    /// </summary>
+    public sealed class AutoSchedulerService : BackgroundService
+    {
+        private readonly IServiceScopeFactory _scopeFactory;
+        private readonly ILogger<AutoSchedulerService> _logger;
+
+        /// <summary>Maximum age of a mutex before it is considered stale and can be reclaimed.</summary>
+        private static readonly TimeSpan MutexTimeout = TimeSpan.FromMinutes(5);
+
+        public AutoSchedulerService(IServiceScopeFactory scopeFactory, ILogger<AutoSchedulerService> logger)
+        {
+            _scopeFactory = scopeFactory;
+            _logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("AutoSchedulerService started.");
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                try
+                {
+                    await EvaluateSchedulesAsync(stoppingToken);
+                }
+                catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+                {
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "AutoSchedulerService encountered an error during schedule evaluation.");
+                }
+
+                await Task.Delay(TimeSpan.FromSeconds(60), stoppingToken);
+            }
+
+            _logger.LogInformation("AutoSchedulerService stopped.");
+        }
+
+        internal async Task EvaluateSchedulesAsync(CancellationToken ct)
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var context = scope.ServiceProvider.GetRequiredService<ShieldCheckerContext>();
+
+            // Try to acquire the scheduler mutex
+            if (!await TryAcquireMutexAsync(context, ct))
+            {
+                _logger.LogTrace("AutoScheduler mutex is held by another instance. Skipping this cycle.");
+                return;
+            }
+
+            try
+            {
+                var now = DateTime.UtcNow;
+                var dueSchedules = await context.AutoSchedule
+                    .Include(s => s.TestDefinitions)
+                    .Where(s => s.Enabled && s.NextExecution <= now)
+                    .ToListAsync(ct);
+
+                if (dueSchedules.Count == 0)
+                {
+                    _logger.LogTrace("No due schedules found.");
+                    return;
+                }
+
+                _logger.LogInformation("Found {Count} due schedule(s) to process.", dueSchedules.Count);
+
+                foreach (var schedule in dueSchedules)
+                {
+                    await ProcessScheduleAsync(context, schedule, now, ct);
+                }
+            }
+            finally
+            {
+                await ReleaseMutexAsync(context, ct);
+            }
+        }
+
+        private async Task ProcessScheduleAsync(ShieldCheckerContext context, AutoSchedule schedule, DateTime now, CancellationToken ct)
+        {
+            _logger.LogInformation("Processing schedule '{Name}' (ID: {Id}).", schedule.Name, schedule.ID);
+
+            // Determine eligible test definitions
+            var candidates = schedule.TestDefinitions
+                .Where(t => t.Enabled == true)
+                .ToList();
+
+            // If no specific tests are selected, select all enabled tests matching filters
+            if (candidates.Count == 0)
+            {
+                var query = context.UseCaseTests.Where(t => t.Enabled == true);
+
+                if (schedule.FilterOperatingSystem.HasValue)
+                {
+                    query = query.Where(t => t.OperatingSystem == schedule.FilterOperatingSystem.Value);
+                }
+
+                candidates = await query.ToListAsync(ct);
+            }
+            else if (schedule.FilterOperatingSystem.HasValue)
+            {
+                candidates = candidates
+                    .Where(t => t.OperatingSystem == schedule.FilterOperatingSystem.Value)
+                    .ToList();
+            }
+
+            // Apply execution filter
+            if (schedule.FilterExecution.HasValue && schedule.FilterExecution.Value != FilterExecution.None)
+            {
+                candidates = await ApplyExecutionFilterAsync(context, candidates, schedule.FilterExecution.Value, now, ct);
+            }
+
+            // Apply random count filter
+            if (schedule.FilterRandomCount.HasValue && schedule.FilterRandomCount.Value > 0 && candidates.Count > schedule.FilterRandomCount.Value)
+            {
+                candidates = candidates
+                    .OrderBy(_ => Guid.NewGuid())
+                    .Take(schedule.FilterRandomCount.Value)
+                    .ToList();
+            }
+
+            // Create queued jobs
+            int jobsCreated = 0;
+            foreach (var test in candidates)
+            {
+                var job = new TestJob
+                {
+                    UseCaseID = test.ID,
+                    Created = now,
+                    Modified = now,
+                    Status = JobStatus.Queued,
+                    Result = JobResult.Undetermined,
+                    SchedulerLog = $"Auto-scheduled by '{schedule.Name}' (ID: {schedule.ID})"
+                };
+                context.TestJobs.Add(job);
+                jobsCreated++;
+            }
+
+            // Advance NextExecution
+            schedule.NextExecution = CalculateNextExecution(schedule.NextExecution, schedule.Type);
+
+            await context.SaveChangesAsync(ct);
+
+            _logger.LogInformation(
+                "Schedule '{Name}' (ID: {Id}): created {Count} job(s). Next execution: {Next}.",
+                schedule.Name, schedule.ID, jobsCreated, schedule.NextExecution);
+        }
+
+        private static async Task<List<TestDefinition>> ApplyExecutionFilterAsync(
+            ShieldCheckerContext context,
+            List<TestDefinition> candidates,
+            FilterExecution filter,
+            DateTime now,
+            CancellationToken ct)
+        {
+            var cutoff = filter switch
+            {
+                FilterExecution.OnlyWhenNoSuccessJobInPastWeek => now.AddDays(-7),
+                FilterExecution.OnlyWhenNoSuccessJobInPastMonth => now.AddDays(-30),
+                FilterExecution.OnlyWhenNoJobInPastWeek => now.AddDays(-7),
+                FilterExecution.OnlyWhenNoJobInPastMonth => now.AddDays(-30),
+                _ => now
+            };
+
+            bool successOnly = filter == FilterExecution.OnlyWhenNoSuccessJobInPastWeek
+                            || filter == FilterExecution.OnlyWhenNoSuccessJobInPastMonth;
+
+            var candidateIds = candidates.Select(c => c.ID).ToList();
+
+            // Find test IDs that already have matching jobs in the time window
+            IQueryable<TestJob> jobQuery = context.TestJobs
+                .Where(j => candidateIds.Contains(j.UseCaseID) && j.Created >= cutoff);
+
+            if (successOnly)
+            {
+                jobQuery = jobQuery.Where(j => j.Result == JobResult.Success);
+            }
+
+            var testIdsWithJobs = await jobQuery
+                .Select(j => j.UseCaseID)
+                .Distinct()
+                .ToListAsync(ct);
+
+            return candidates.Where(c => !testIdsWithJobs.Contains(c.ID)).ToList();
+        }
+
+        internal static DateTime CalculateNextExecution(DateTime current, AutoScheduleType type)
+        {
+            return type switch
+            {
+                AutoScheduleType.Daily => current.AddDays(1),
+                AutoScheduleType.Weekly => current.AddDays(7),
+                AutoScheduleType.Monthly => current.AddMonths(1),
+                AutoScheduleType.Quarterly => current.AddMonths(3),
+                _ => current.AddDays(1)
+            };
+        }
+
+        private async Task<bool> TryAcquireMutexAsync(ShieldCheckerContext context, CancellationToken ct)
+        {
+            var mutex = await context.SchedulerMutex
+                .FirstOrDefaultAsync(m => m.SchedulerType == SchedulerType.Worker, ct);
+
+            var now = DateTime.UtcNow;
+            string owner = Environment.MachineName;
+
+            if (mutex == null)
+            {
+                context.SchedulerMutex.Add(new SchedulerMutex
+                {
+                    Owner = owner,
+                    SchedulerType = SchedulerType.Worker,
+                    Start = now
+                });
+                try
+                {
+                    await context.SaveChangesAsync(ct);
+                    return true;
+                }
+                catch (DbUpdateException)
+                {
+                    // Another instance beat us to it
+                    return false;
+                }
+            }
+
+            // If the mutex is stale (owner crashed), reclaim it
+            if (now - mutex.Start > MutexTimeout)
+            {
+                _logger.LogWarning("Reclaiming stale AutoScheduler mutex from '{Owner}' (started {Start}).",
+                    mutex.Owner, mutex.Start);
+                mutex.Owner = owner;
+                mutex.Start = now;
+                try
+                {
+                    await context.SaveChangesAsync(ct);
+                    return true;
+                }
+                catch (DbUpdateConcurrencyException)
+                {
+                    return false;
+                }
+            }
+
+            // Mutex is held by another instance and not stale
+            return false;
+        }
+
+        private async Task ReleaseMutexAsync(ShieldCheckerContext context, CancellationToken ct)
+        {
+            var mutex = await context.SchedulerMutex
+                .FirstOrDefaultAsync(m => m.SchedulerType == SchedulerType.Worker, ct);
+
+            if (mutex != null)
+            {
+                context.SchedulerMutex.Remove(mutex);
+                try
+                {
+                    await context.SaveChangesAsync(ct);
+                }
+                catch (DbUpdateException ex)
+                {
+                    _logger.LogWarning(ex, "Failed to release AutoScheduler mutex.");
+                }
+            }
+        }
+    }
+}

src/Api/ShieldChecker.BackendApi/Services/JobTimeoutService.cs

@@ -0,0 +1,97 @@
+using Microsoft.EntityFrameworkCore;
+using ShieldChecker.DataAccess;
+using ShieldChecker.DataAccess.Models;
+
+namespace ShieldChecker.BackendApi.Services
+{
+    /// <summary>
+    /// Background service that periodically scans for jobs exceeding the configured
+    /// <see cref="Settings.JobTimeout"/> and marks them as Error with an Undetermined result.
+    /// </summary>
+    public sealed class JobTimeoutService : BackgroundService
+    {
+        private readonly IServiceScopeFactory _scopeFactory;
+        private readonly ILogger<JobTimeoutService> _logger;
+
+        public JobTimeoutService(IServiceScopeFactory scopeFactory, ILogger<JobTimeoutService> logger)
+        {
+            _scopeFactory = scopeFactory;
+            _logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("JobTimeoutService started.");
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                try
+                {
+                    await EnforceTimeoutsAsync(stoppingToken);
+                }
+                catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+                {
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "JobTimeoutService encountered an error.");
+                }
+
+                await Task.Delay(TimeSpan.FromMinutes(2), stoppingToken);
+            }
+
+            _logger.LogInformation("JobTimeoutService stopped.");
+        }
+
+        internal async Task EnforceTimeoutsAsync(CancellationToken ct)
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var context = scope.ServiceProvider.GetRequiredService<ShieldCheckerContext>();
+
+            var settings = await context.Settings.FirstOrDefaultAsync(ct);
+            if (settings == null || settings.JobTimeout <= 0)
+            {
+                _logger.LogTrace("Job timeout not configured or is zero. Skipping.");
+                return;
+            }
+
+            var now = DateTime.UtcNow;
+            var cutoff = now.AddMinutes(-settings.JobTimeout);
+
+            // Find jobs that are in active processing states and have exceeded the timeout
+            var timedOutJobs = await context.TestJobs
+                .Where(j =>
+                    (j.Status == JobStatus.WaitingForMDE
+                     || j.Status == JobStatus.WaitingForDetection
+                     || j.Status == JobStatus.Queued)
+                    && j.Created < cutoff)
+                .ToListAsync(ct);
+
+            if (timedOutJobs.Count == 0)
+            {
+                _logger.LogTrace("No timed-out jobs found.");
+                return;
+            }
+
+            _logger.LogInformation("Found {Count} timed-out job(s) (timeout: {Timeout} min).",
+                timedOutJobs.Count, settings.JobTimeout);
+
+            foreach (var job in timedOutJobs)
+            {
+                job.Status = JobStatus.Error;
+                job.Result = JobResult.Undetermined;
+                job.Modified = now;
+
+                string timestamp = now.ToString("yyyy-MM-dd HH:mm:ss");
+                string logEntry = $"[{timestamp}] Job timed out after {settings.JobTimeout} minutes.{Environment.NewLine}";
+                job.SchedulerLog = string.IsNullOrEmpty(job.SchedulerLog) ? logEntry : job.SchedulerLog + logEntry;
+
+                _logger.LogWarning("Job {JobId} timed out (created: {Created}, timeout: {Timeout} min).",
+                    job.ID, job.Created, settings.JobTimeout);
+            }
+
+            await context.SaveChangesAsync(ct);
+        }
+    }
+}