Retry Pattern in Microservices

Order Service                         Payment Service
     |---- attempt 1: charge(order-1001) --->|  (timeout)
     |                                        x
     |   wait ~1 s
     |---- attempt 2: charge(order-1001) --->|
     |<--------------- 200 OK ---------------|  (charged once)
     |
  create order, reply "Order placed"

package com.example.resilience;

import java.time.Duration;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Predicate;
import java.util.function.Supplier;

public class RetryExecutor {

    private final int maxAttempts;
    private final Duration initialDelay;
    private final double multiplier;
    private final Duration maxDelay;
    private final Predicate<Exception> isRetryable;

    public RetryExecutor(int maxAttempts, Duration initialDelay, double multiplier,
                         Duration maxDelay, Predicate<Exception> isRetryable) {
        this.maxAttempts = maxAttempts;
        this.initialDelay = initialDelay;
        this.multiplier = multiplier;
        this.maxDelay = maxDelay;
        this.isRetryable = isRetryable;
    }

    public <T> T execute(Supplier<T> operation) {
        for (int attempt = 1; ; attempt++) {
            try {
                return operation.get();
            } catch (RuntimeException ex) {
                // Give up on permanent errors or after the last attempt
                if (!isRetryable.test(ex) || attempt >= maxAttempts) {
                    throw ex;
                }
                Duration wait = delayBefore(attempt + 1);
                System.out.printf("Attempt %d failed (%s). Retrying in %d ms%n",
                        attempt, ex.getClass().getSimpleName(), wait.toMillis());
                sleep(wait);
            }
        }
    }

    // Exponential backoff with jitter: a random wait between half and the full backoff value
    private Duration delayBefore(int nextAttempt) {
        double backoff = initialDelay.toMillis() * Math.pow(multiplier, nextAttempt - 2);
        long capped = (long) Math.min(backoff, maxDelay.toMillis());
        long withJitter = ThreadLocalRandom.current().nextLong(capped / 2, capped + 1);
        return Duration.ofMillis(withJitter);
    }

    private static void sleep(Duration duration) {
        try {
            Thread.sleep(duration.toMillis());
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IllegalStateException("Retry interrupted", e);
        }
    }
}

RetryExecutor retry = new RetryExecutor(
        3,                          // at most 3 attempts in total
        Duration.ofSeconds(1),      // first wait about 1 second
        2.0,                        // then about 2 seconds
        Duration.ofSeconds(10),     // never wait longer than 10 seconds
        ex -> ex instanceof ResourceAccessException                    // timeouts, connection errors
           || ex instanceof HttpServerErrorException.ServiceUnavailable);  // 503

PaymentResult result = retry.execute(
        () -> paymentClient.charge(order.id(), order.total()));

resilience4j:
  retry:
    instances:
      payment:
        max-attempts: 3                    # includes the first call
        wait-duration: 1s                  # first wait
        enable-exponential-backoff: true
        exponential-backoff-multiplier: 2  # 1s, then 2s
        retry-exceptions:
          - org.springframework.web.client.ResourceAccessException
          - org.springframework.web.client.HttpServerErrorException
        ignore-exceptions:
          - org.springframework.web.client.HttpClientErrorException

package com.example.order.client;

import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestClient;

import io.github.resilience4j.retry.annotation.Retry;

@Component
public class PaymentClient {

    private final RestClient restClient;

    public PaymentClient(RestClient.Builder builder) {
        this.restClient = builder.baseUrl("http://payment-service").build();
    }

    @Retry(name = "payment", fallbackMethod = "paymentPending")
    public PaymentResult charge(ChargeRequest request) {
        return restClient.post()
                .uri("/payments")
                // The same key on every attempt: the payment service charges only once
                .header("Idempotency-Key", request.orderId())
                .body(request)
                .retrieve()
                .body(PaymentResult.class);
    }

    // Called after all attempts have failed
    private PaymentResult paymentPending(ChargeRequest request, Exception ex) {
        return PaymentResult.pending(request.orderId(),
                "Payment is being confirmed. We will email you shortly.");
    }
}

package com.example.order.config;

import java.time.Duration;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import io.github.resilience4j.common.retry.configuration.RetryConfigCustomizer;
import io.github.resilience4j.core.IntervalFunction;

@Configuration
public class RetryConfiguration {

    @Bean
    public RetryConfigCustomizer paymentRetryJitter() {
        return RetryConfigCustomizer.of("payment", builder -> builder
                .intervalFunction(IntervalFunction.ofExponentialRandomBackoff(
                        Duration.ofSeconds(1),   // initial wait
                        2.0,                     // multiplier
                        0.5)));                  // +/- 50% randomness
    }
}

package com.example.order.client;

import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Component;
import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestClient;

@Component
public class InventoryClient {

    private final RestClient restClient;

    public InventoryClient(RestClient.Builder builder) {
        this.restClient = builder.baseUrl("http://inventory-service").build();
    }

    @Retryable(retryFor = ResourceAccessException.class,
               maxAttempts = 3,
               backoff = @Backoff(delay = 500, multiplier = 2, random = true))
    public StockStatus checkStock(String productId) {
        return restClient.get()
                .uri("/stock/{id}", productId)
                .retrieve()
                .body(StockStatus.class);
    }

    // Runs when all attempts have failed
    @Recover
    public StockStatus stockUnknown(ResourceAccessException ex, String productId) {
        return StockStatus.unknown(productId);
    }
}

@Service
public class PaymentService {

    private final PaymentRepository payments;
    private final CardProcessor cardProcessor;

    public PaymentService(PaymentRepository payments, CardProcessor cardProcessor) {
        this.payments = payments;
        this.cardProcessor = cardProcessor;
    }

    @Transactional
    public PaymentResult charge(String idempotencyKey, ChargeRequest request) {
        // Already processed? Return the stored result instead of charging again
        Optional<Payment> existing = payments.findByIdempotencyKey(idempotencyKey);
        if (existing.isPresent()) {
            return PaymentResult.from(existing.get());
        }

        String transactionId = cardProcessor.charge(request.cardToken(), request.amount());
        Payment payment = payments.save(new Payment(idempotencyKey, transactionId, request.amount()));
        return PaymentResult.from(payment);
    }
}

@Retry(name = "payment")
@CircuitBreaker(name = "payment", fallbackMethod = "paymentPending")
@TimeLimiter(name = "payment")
public CompletableFuture<PaymentResult> charge(ChargeRequest request) {
    return CompletableFuture.supplyAsync(() -> paymentClient.charge(request));
}

Client  -> retries 3 times
  API Gateway  -> retries 3 times each
    Order Service  -> retries 3 times each
      Payment Service receives up to 3 x 3 x 3 = 27 requests
      for ONE user click - while it is already struggling.

package com.example.shipping.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
import org.springframework.kafka.listener.DefaultErrorHandler;
import org.springframework.kafka.support.ExponentialBackOffWithMaxRetries;

@Configuration
public class KafkaErrorHandlingConfig {

    @Bean
    public DefaultErrorHandler errorHandler(KafkaTemplate<Object, Object> template) {
        // After the retries fail, publish the message to "<topic>-dlt"
        DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(template);

        ExponentialBackOffWithMaxRetries backOff = new ExponentialBackOffWithMaxRetries(3);
        backOff.setInitialInterval(1_000);   // 1 s
        backOff.setMultiplier(2.0);          // 2 s, 4 s
        backOff.setMaxInterval(10_000);

        DefaultErrorHandler handler = new DefaultErrorHandler(recoverer, backOff);
        // Bad data will never succeed: send it to the dead-letter topic immediately
        handler.addNotRetryableExceptions(IllegalArgumentException.class);
        return handler;
    }
}

@Configuration
public class RetryLoggingConfig {

    private static final Logger log = LoggerFactory.getLogger(RetryLoggingConfig.class);

    public RetryLoggingConfig(RetryRegistry retryRegistry) {
        retryRegistry.retry("payment").getEventPublisher()
                .onRetry(event -> log.warn("Retry {} for '{}' after {}",
                        event.getNumberOfRetryAttempts(),
                        event.getName(),
                        event.getLastThrowable().toString()))
                .onError(event -> log.error("All retries failed for '{}'", event.getName()));
    }
}