[BRE-1058] fix alpine race condition (#6156)

* alpine race condition during shutdown fix

* change catch to only be for relevant task cancelled, added a debug log

* test commit for build and test

* remove testing comment

src/Core/HostedServices/ApplicationCacheHostedService.cs

@@ -67,19 +67,25 @@ public class ApplicationCacheHostedService : IHostedService, IDisposable
 
     public virtual async Task StopAsync(CancellationToken cancellationToken)
     {
+        // Step 1: Signal ExecuteAsync to stop gracefully
+        _cts?.Cancel();
+
+        // Step 2: Wait for ExecuteAsync to finish cleanly
+        if (_executingTask != null)
+        {
+            await _executingTask;
+        }
+
+        // Step 3: Now safely dispose resources (ExecuteAsync is done)
         await _subscriptionReceiver.CloseAsync(cancellationToken);
         await _serviceBusClient.DisposeAsync();
-        _cts?.Cancel();
+
+        // Step 4: Clean up subscription
         try
         {
             await _serviceBusAdministrationClient.DeleteSubscriptionAsync(_topicName, _subName, cancellationToken);
         }
         catch { }
-
-        if (_executingTask != null)
-        {
-            await _executingTask;
-        }
     }
 
     public virtual void Dispose()
@@ -87,15 +93,39 @@ public class ApplicationCacheHostedService : IHostedService, IDisposable
 
     private async Task ExecuteAsync(CancellationToken cancellationToken)
     {
-        await foreach (var message in _subscriptionReceiver.ReceiveMessagesAsync(cancellationToken))
+        while (!cancellationToken.IsCancellationRequested)
         {
             try
             {
-                await ProcessMessageAsync(message, cancellationToken);
+                var messages = await _subscriptionReceiver.ReceiveMessagesAsync(
+                    maxMessages: 1,
+                    maxWaitTime: TimeSpan.FromSeconds(30),
+                    cancellationToken);
+
+                if (messages?.Any() == true)
+                {
+                    foreach (var message in messages)
+                    {
+                        try
+                        {
+                            await ProcessMessageAsync(message, cancellationToken);
+                        }
+                        catch (Exception e)
+                        {
+                            _logger.LogError(e, "Error processing messages in ApplicationCacheHostedService");
+                        }
+                    }
+                }
+            }
+            catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("ServiceBus receiver disposed during Alpine container shutdown");
+                break;
             }
-            catch (Exception e)
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
-                _logger.LogError(e, "Error processing messages in ApplicationCacheHostedService");
+                _logger.LogDebug("ServiceBus operation cancelled during Alpine container shutdown");
+                break;
             }
         }
     }

src/EventsProcessor/AzureQueueHostedService.cs

@@ -86,11 +86,24 @@ public class AzureQueueHostedService : IHostedService, IDisposable
                     await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                 }
             }
+            catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                break;
+            }
             catch (Exception ex)
             {
                 _logger.LogError(ex, "Error occurred processing message block.");
 
-                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                try
+                {
+                    await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+                }
+                catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+                {
+                    _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                    break;
+                }
             }
         }
 

src/Notifications/AzureQueueHostedService.cs

@@ -87,6 +87,11 @@ public class AzureQueueHostedService : IHostedService, IDisposable
                     await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                 }
             }
+            catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                _logger.LogDebug("Task.Delay cancelled during Alpine container shutdown");
+                break;
+            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error processing messages.");