From 9cbb184ab9a0a7d3df4f03f06412caa62b00f680 Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Tue, 18 Feb 2025 13:52:47 +0000 Subject: [PATCH 1/6] log on retry and max retry --- .../templates/ApiClient.mustache | 9 ++++-- .../templates/test-ApiClientTests.mustache | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/resources/sdk/pureclouddotnet/templates/ApiClient.mustache b/resources/sdk/pureclouddotnet/templates/ApiClient.mustache index a3317fb35..40d500882 100644 --- a/resources/sdk/pureclouddotnet/templates/ApiClient.mustache +++ b/resources/sdk/pureclouddotnet/templates/ApiClient.mustache @@ -364,7 +364,7 @@ namespace {{packageName}}.Client } } - if ((int)response.StatusCode < 200 || (int)response.StatusCode >= 300) + if ((int)response.StatusCode < 200 || (int)response.StatusCode >= 300) { Configuration.Logger.Error(method.ToString(), url, postBody, response.Content, (int)response.StatusCode, headerParams, response.Headers? .GroupBy(header => header?.Name) .Select(header => new @@ -373,7 +373,10 @@ namespace {{packageName}}.Client Value = header.Select(x => x?.Value)?.ToList() }).ToDictionary(header => header?.Name?.ToString(), header => String.Join(", ", header?.Value?.ToArray())) ?? new Dictionary()); - + if ((int)response.StatusCode == 429) { + Console.WriteLine("Max number of retries exceeded"); + } + } return (Object) response; } @@ -1118,6 +1121,7 @@ namespace {{packageName}}.Client //If status code is 50x then wait for every 3 Sec and retry until 5 minutes then after wait for every 9 Sec and retry until next 5 minutes afterwards wait for every 27 Sec and retry. retryCount++; + Console.WriteLine(retryCount); return waitBeforeRetry(getWaitTimeExp(Math.Min(3, Math.Floor(stopwatch.ElapsedMilliseconds / backoffIntervalMs * 1.0) + 1))); } stopwatch.Stop(); @@ -1128,6 +1132,7 @@ namespace {{packageName}}.Client { try { + Console.WriteLine("Rate limit encountered. Retry the request in [{0}] seconds", retryAfterMs/1000); Thread.Sleep((int) retryAfterMs); } catch (ThreadInterruptedException) diff --git a/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache b/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache index 0d164f0d3..6f7fb256c 100644 --- a/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache +++ b/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache @@ -109,6 +109,36 @@ public class ApiClientTests stopwatch.Stop(); } + [Test] + public void InvokeTestWith_429_And_Max_Retries_Exceeded() + { + var retryConfig = new ApiClient.RetryConfiguration + { + MaxRetryTimeSec = 6, + RetryAfterDefaultMs = 100, + RetryMax = 0 + }; + + var mockHttp = new MockHttpMessageHandler(); + mockHttp.When("*").Respond((req) => + { + var response = new HttpResponseMessage((System.Net.HttpStatusCode)429); + response.Headers.Add("Retry-After", "3"); + return Task.FromResult(response); + }); + + var apiClient = new ApiClient(new {{packageName}}.Client.Configuration()); + apiClient.RetryConfig = retryConfig; + apiClient.ClientOptions.HttpMessageHandler = mockHttp; + + stopwatch = Stopwatch.StartNew(); + RestResponse user = (RestResponse)apiClient.CallApi(path, method, queryParams, postBody, headerParams, formParams, fileParams, pathParams, contentType); + + Assert.IsTrue(stopwatch.ElapsedMilliseconds >= 3000 && stopwatch.ElapsedMilliseconds < 3100, "Since maxRetryTime is not provided it will not retry even if the status code is 429"); + Assert.AreEqual(429, (int)user.StatusCode); + stopwatch.Stop(); + } + [Test] public void InvokeTestWith_502() { From 6519f105169517ab10f7de0c3e2c0f9e8b2f1291 Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Tue, 18 Feb 2025 14:27:07 +0000 Subject: [PATCH 2/6] update assert message --- .../sdk/pureclouddotnet/templates/test-ApiClientTests.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache b/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache index 6f7fb256c..aa6155ddd 100644 --- a/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache +++ b/resources/sdk/pureclouddotnet/templates/test-ApiClientTests.mustache @@ -134,7 +134,7 @@ public class ApiClientTests stopwatch = Stopwatch.StartNew(); RestResponse user = (RestResponse)apiClient.CallApi(path, method, queryParams, postBody, headerParams, formParams, fileParams, pathParams, contentType); - Assert.IsTrue(stopwatch.ElapsedMilliseconds >= 3000 && stopwatch.ElapsedMilliseconds < 3100, "Since maxRetryTime is not provided it will not retry even if the status code is 429"); + Assert.IsTrue(stopwatch.ElapsedMilliseconds >= 3000 && stopwatch.ElapsedMilliseconds < 3100, "Since RetryMax is 0, after one retry exception is thrown"); Assert.AreEqual(429, (int)user.StatusCode); stopwatch.Stop(); } From c49b12bf29b7aacc215cb65bdc77af4cd6447d4f Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Mon, 24 Feb 2025 09:56:37 +0000 Subject: [PATCH 3/6] retry logging for Go SDK --- .../purecloudgo/templates/apiclient.mustache | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/resources/sdk/purecloudgo/templates/apiclient.mustache b/resources/sdk/purecloudgo/templates/apiclient.mustache index fec33db39..221e5c50b 100644 --- a/resources/sdk/purecloudgo/templates/apiclient.mustache +++ b/resources/sdk/purecloudgo/templates/apiclient.mustache @@ -6,6 +6,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "log" "io/ioutil" "net/http" "net/url" @@ -48,6 +49,7 @@ func NewAPIClient(c *Configuration) APIClient { client := retryablehttp.NewClient() client.Logger = nil client.HTTPClient.Timeout = timeout + client.Backoff = customBackoff return APIClient{ client: *client, @@ -219,8 +221,13 @@ func (c *APIClient) CallAPI(path string, method string, } } - - c.client.CheckRetry = DefaultRetryPolicy + c.client.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + retry, err := DefaultRetryPolicy(ctx, resp, err) + if !retry && resp.StatusCode == http.StatusTooManyRequests { + log.Println("Max Retries Exceeded") + } + return retry, err + } if c.configuration.ProxyConfiguration != nil { var proxyUrl *url.URL @@ -472,3 +479,11 @@ func getFieldName(t reflect.Type, field string) string { func toTime(o interface{}) *time.Time { return o.(*time.Time) } + +func customBackoff(min, max time.Duration, attemptNumber int, resp *http.Response) time.Duration { + sleep := retryablehttp.DefaultBackoff(min, max, attemptNumber, resp) + if resp != nil && resp.StatusCode == http.StatusTooManyRequests { + log.Printf("Rate limit encountered. Retry the request in [%.0f] seconds\n", sleep.Seconds()) + } + return sleep +} \ No newline at end of file From f8c8324e4fd89726fcafef901cb7aa5bb4dc4fcb Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Wed, 26 Feb 2025 09:56:30 +0000 Subject: [PATCH 4/6] python: added retry logging --- .../purecloudpython/templates/rest.mustache | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/resources/sdk/purecloudpython/templates/rest.mustache b/resources/sdk/purecloudpython/templates/rest.mustache index 060af3271..0c0ee599b 100644 --- a/resources/sdk/purecloudpython/templates/rest.mustache +++ b/resources/sdk/purecloudpython/templates/rest.mustache @@ -98,7 +98,7 @@ class RESTClientObject(object): proxy_username = Configuration().proxy_username proxy_password = Configuration().proxy_password - retries = urllib3.util.Retry() + retries = CustomRetry() retries.allowed_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'TRACE'} # https pool manager if proxy: @@ -270,3 +270,19 @@ class ApiException(Exception): error_message += "HTTP response body: {0}\n".format(self.body) return error_message + +class CustomRetry(urllib3.util.Retry): + def __init__(self, total=0, connect=None, read=None, redirect=None, status=None, other=None, backoff_factor=0): + super().__init__(total=total, connect=connect, read=read, redirect=redirect, status=status, other=other, backoff_factor=backoff_factor) + + def sleep(self, response=None): + sleep_duration = self.get_backoff_time() + print(f"Rate limit encountered. Retry the request in [{sleep_duration}] seconds") + super().sleep(response) + + def increment(self, method=None, url=None, repsonse=None, error=None, _pool=None, _stacktrace=None): + retry = super.increment(method, url, repsonse, error, _pool, _stacktrace) + if retry.is_exhausted(): + print(f"Max number of retries exceeded: {e}") + + return retry \ No newline at end of file From 03768a99e6eef28e113760265d05bfc8fc4eb362 Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Mon, 24 Feb 2025 12:09:43 +0000 Subject: [PATCH 5/6] webmessage: added retry logging on max retry --- resources/sdk/webmessagingjava/templates/ApiClient.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/sdk/webmessagingjava/templates/ApiClient.mustache b/resources/sdk/webmessagingjava/templates/ApiClient.mustache index fa6904303..5d3b417a8 100644 --- a/resources/sdk/webmessagingjava/templates/ApiClient.mustache +++ b/resources/sdk/webmessagingjava/templates/ApiClient.mustache @@ -493,7 +493,7 @@ public class ApiClient implements AutoCloseable { return new ApiResponseWrapper<>(statusCode, reasonPhrase, headers, body, entity); } else { - String message = "error"; + String message = (statusCode == 429) ? "Max number of retries exceeded" : "error"; String body = response.readBody(); throw new WebMessagingException(statusCode, message, headers, body); } From 0f9d76376f83a2c32e43ca5e4034f38c4d3423d3 Mon Sep 17 00:00:00 2001 From: fergalmcl Date: Wed, 26 Feb 2025 12:52:36 +0000 Subject: [PATCH 6/6] dotnet: remove debug print --- resources/sdk/pureclouddotnet/templates/ApiClient.mustache | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/sdk/pureclouddotnet/templates/ApiClient.mustache b/resources/sdk/pureclouddotnet/templates/ApiClient.mustache index 40d500882..5fb983a99 100644 --- a/resources/sdk/pureclouddotnet/templates/ApiClient.mustache +++ b/resources/sdk/pureclouddotnet/templates/ApiClient.mustache @@ -1121,7 +1121,6 @@ namespace {{packageName}}.Client //If status code is 50x then wait for every 3 Sec and retry until 5 minutes then after wait for every 9 Sec and retry until next 5 minutes afterwards wait for every 27 Sec and retry. retryCount++; - Console.WriteLine(retryCount); return waitBeforeRetry(getWaitTimeExp(Math.Min(3, Math.Floor(stopwatch.ElapsedMilliseconds / backoffIntervalMs * 1.0) + 1))); } stopwatch.Stop();