OCPBUGS-54628: Add retry method if we hit jira rate limit (#1313)

* Add retry method if we hit jira rate limit

Jira implemented rate limit due to which tests
which should be skipped due to known issues do not get
skipped.

This PR implements retry with exponential backoff
to retry the request if we hit http status code 429(too many requests)

Signed-off-by: Niranjan M.R <[email protected]>

* use Retry-After header first before trying exponential backoff

Signed-off-by: Niranjan M.R <[email protected]>

* use http api constants for status code instead of numbers

Signed-off-by: Niranjan M.R <[email protected]>

* log before we wait for retry time to expire

log information about the time need to wait
before retrying to fetch bug status

Signed-off-by: Niranjan M.R <[email protected]>

---------

Signed-off-by: Niranjan M.R <[email protected]>
Co-authored-by: Niranjan M.R <[email protected]>

Author: mrniranjan

test/e2e/performanceprofile/functests/utils/jira/status.go

@@ -2,12 +2,21 @@ package jira
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/pkg/errors"
+	"math/rand"
 	"net/http"
 	"net/url"
+	"strconv"
+	"time"
+
+	testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
 )
 
-const JIRA_BASE_URL = "https://issues.redhat.com"
+const (
+	JIRA_BASE_URL = "https://issues.redhat.com"
+	MAX_RETRIES   = 3
+)
 
 type JiraIssueStatusResponseFieldsStatus = struct {
 	Name string `json:"name"`
@@ -26,28 +35,75 @@ type JiraIssueStatusResponse = struct {
 func RetrieveJiraStatus(key string) (*JiraIssueStatusResponse, error) {
 	local_params := url.Values{}
 	local_params.Set("fields", "summary,status")
-
 	fullUrl := JIRA_BASE_URL + "/rest/api/2/issue/" + key + "?" + local_params.Encode()
-	req, err := http.NewRequest("GET", fullUrl, nil)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create request for Jira status of %s", key)
+	jiraResponse := JiraIssueStatusResponse{}
+	for retryCount := 0; retryCount <= MAX_RETRIES; retryCount++ {
+		req, err := http.NewRequest("GET", fullUrl, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create request for Jira status of %s", key)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create client to get jira status of %s", key)
+		}
+		defer resp.Body.Close()
+		switch resp.StatusCode {
+		case http.StatusOK:
+			decoder := json.NewDecoder(resp.Body)
+			err = decoder.Decode(&jiraResponse)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to decode jira status of %s", key)
+			}
+		case http.StatusTooManyRequests:
+			if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" {
+				retryTime, err := convertRetryAfterTime(retryAfter)
+				testlog.Infof("Using Retry-After header to retry after %s seconds", retryTime.String())
+				if err != nil {
+					return nil, err
+				}
+				time.Sleep(retryTime)
+			} else {
+				testlog.Info("No Retry-After header found using exponential backoff method to retry")
+				delay := retryWithBackOff(retryCount)
+				testlog.Infof("Wait for %s seconds before retry to fetch bug status", delay.String())
+				time.Sleep(delay)
+			}
+		default:
+			return nil, errors.Errorf("failed to get jira status of %s: %d %s", key, resp.StatusCode, resp.Status)
+		}
 	}
-	ret, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to create client to get jira status of %s", key)
-	}
-	defer ret.Body.Close()
+	return &jiraResponse, nil
+}
 
-	if ret.StatusCode != http.StatusOK {
-		return nil, errors.Errorf("failed to get jira status of %s: %d %s", key, ret.StatusCode, ret.Status)
-	}
+// retryWithBackOff use exponential backoff retries when we hit rate limit
+func retryWithBackOff(attempts int) time.Duration {
+	// we have a limit of 5 req/sec
+	initialDelay := 200 * time.Millisecond
+	maxDelay := 5 * time.Second
+	delay := initialDelay * (1 << attempts)
+
+	// add jitter
+	jitter := time.Duration(rand.Int63n(int64(delay / 2)))
+	delay += jitter
 
-	response := JiraIssueStatusResponse{}
-	decoder := json.NewDecoder(ret.Body)
-	err = decoder.Decode(&response)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to decode jira status of %s", key)
+	if delay > maxDelay {
+		delay = maxDelay
 	}
+	return delay
+}
 
-	return &response, nil
+// convertRetryAfterTime converts time in Retry-After in time.Duration
+// the format can be in the form of date or seconds
+func convertRetryAfterTime(retryAfter string) (time.Duration, error) {
+	if seconds, err := strconv.Atoi(retryAfter); err == nil {
+		return time.Duration(seconds) * time.Second, nil
+	}
+	if t, err := http.ParseTime(retryAfter); err != nil {
+		wait := time.Until(t)
+		if wait > 0 {
+			return wait, nil
+		}
+		return 0, nil
+	}
+	return 0, fmt.Errorf("Invalid Retry-After time value %s", retryAfter)
 }