Adds LinearBackoffClientConnectionRetryFilter in the default client…

… services (#8793)

* Adds `DefaultClientConnectRetryFilter` and registers it in `DefaultClientServices`

* Renamed the default filter to indicate that is its linear backoff

---------

Co-authored-by: Ledjon Behluli <[email protected]>

src/Orleans.Core/Core/DefaultClientServices.cs

@@ -86,6 +86,7 @@ public static void AddDefaultServices(IServiceCollection services)
             services.TryAddFromExisting<IInternalClusterClient, ClusterClient>();
             services.AddFromExisting<IHostedService, ClusterClient>();
             services.AddTransient<IOptions<MessagingOptions>>(static sp => sp.GetRequiredService<IOptions<ClientMessagingOptions>>());
+            services.TryAddSingleton<IClientConnectionRetryFilter, LinearBackoffClientConnectionRetryFilter>();
 
             services.AddSingleton<IConfigureOptions<GrainTypeOptions>, DefaultGrainTypeOptionsProvider>();
 

src/Orleans.Core/Core/IClientConnectionRetryFilter.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using Orleans.Runtime;
 
 namespace Orleans
 {
@@ -17,4 +18,30 @@ public interface IClientConnectionRetryFilter
         /// <returns><see langword="true"/> if connection should be re-attempted, <see langword="false"/> if attempts to connect to the cluster should be aborted.</returns>
         Task<bool> ShouldRetryConnectionAttempt(Exception exception, CancellationToken cancellationToken);
     }
+
+    internal sealed class LinearBackoffClientConnectionRetryFilter : IClientConnectionRetryFilter
+    {
+        private int _retryCount = 0;
+
+        private const int MaxRetry = 5;
+        private const int Delay = 1_500;
+
+        public async Task<bool> ShouldRetryConnectionAttempt(
+            Exception exception,
+            CancellationToken cancellationToken)
+        {
+            if (_retryCount >= MaxRetry)
+            {
+                return false;
+            }
+
+            if (!cancellationToken.IsCancellationRequested && exception is SiloUnavailableException)
+            {
+                await Task.Delay(++_retryCount * Delay, cancellationToken);
+                return true;
+            }
+
+            return false;
+        }
+    }
 }