Health Check API in Microservices: Tell the Platform When Your Service Can Work

import com.sun.net.httpserver.HttpServer;
import java.io.File;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BooleanSupplier;

public class HealthEndpointDemo {

    static final AtomicBoolean databaseReachable = new AtomicBoolean(true); // pretend connection pool state

    public static void main(String[] args) throws Exception {
        Map<String, BooleanSupplier> checks = new LinkedHashMap<>();
        checks.put("database", databaseReachable::get);
        checks.put("diskSpace", () -> new File(".").getUsableSpace() > 10L * 1024 * 1024); // more than 10 MB free

        HttpServer server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); // port 0 = any free port
        server.createContext("/health", exchange -> {
            StringBuilder components = new StringBuilder();
            boolean allUp = true;
            for (var check : checks.entrySet()) {
                boolean up = check.getValue().getAsBoolean();
                allUp &= up;
                if (!components.isEmpty()) components.append(',');
                components.append("\"%s\":\"%s\"".formatted(check.getKey(), up ? "UP" : "DOWN"));
            }
            String body = "{\"status\":\"%s\",\"components\":{%s}}".formatted(allUp ? "UP" : "DOWN", components);
            byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
            exchange.getResponseHeaders().set("Content-Type", "application/json");
            exchange.sendResponseHeaders(allUp ? 200 : 503, bytes.length); // the status code is what probes read
            exchange.getResponseBody().write(bytes);
            exchange.close();
        });
        server.start();

        HttpClient client = HttpClient.newHttpClient();
        URI uri = URI.create("http://localhost:" + server.getAddress().getPort() + "/health");
        try {
            probe(client, uri, "Normal operation");
            databaseReachable.set(false); // simulate the database going away
            probe(client, uri, "Database unreachable");
        } finally {
            server.stop(0);
        }
    }

    static void probe(HttpClient client, URI uri, String label) throws Exception {
        HttpResponse<String> response = client.send(HttpRequest.newBuilder(uri).GET().build(),
                HttpResponse.BodyHandlers.ofString());
        System.out.println(label + ":");
        System.out.println("  HTTP " + response.statusCode() + "  " + response.body());
    }
}

Normal operation:
  HTTP 200  {"status":"UP","components":{"database":"UP","diskSpace":"UP"}}
Database unreachable:
  HTTP 503  {"status":"DOWN","components":{"database":"DOWN","diskSpace":"UP"}}

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

public class ProbeRestartDemo {

    static final int FAILURE_THRESHOLD = 3; // Kubernetes default for both probes; one tick = one probe period

    static class Instance {
        final String name;
        int failedLiveness, failedReadiness, restarts;
        boolean inLoadBalancer = true;
        Instance(String name) { this.name = name; }
    }

    // livenessChecksDatabase = the mistake: the liveness check also pings the shared database
    static void simulate(String title, boolean livenessChecksDatabase) {
        System.out.println(title);
        List<Instance> instances = new ArrayList<>(List.of(new Instance("pod-1"), new Instance("pod-2"), new Instance("pod-3")));
        for (int tick = 1; tick <= 9; tick++) {
            boolean databaseUp = tick == 1 || tick == 9; // the database is down during ticks 2 to 8
            Map<String, List<String>> events = new LinkedHashMap<>(); // what happened -> which pods
            for (Instance pod : instances) {
                boolean live = !livenessChecksDatabase || databaseUp; // the process itself is always healthy here
                boolean ready = databaseUp;                            // it can't serve orders without the database
                pod.failedLiveness = live ? 0 : pod.failedLiveness + 1;
                if (pod.failedLiveness == FAILURE_THRESHOLD) {        // the orchestrator kills and restarts it
                    pod.restarts++;
                    pod.failedLiveness = 0;
                    events.computeIfAbsent("RESTARTED", k -> new ArrayList<>()).add(pod.name);
                }
                pod.failedReadiness = ready ? 0 : pod.failedReadiness + 1;
                boolean shouldServe = pod.inLoadBalancer ? pod.failedReadiness < FAILURE_THRESHOLD : ready; // one success brings it back
                if (pod.inLoadBalancer != shouldServe) {
                    pod.inLoadBalancer = shouldServe;
                    events.computeIfAbsent(ready ? "back in load balancer" : "removed from load balancer", k -> new ArrayList<>()).add(pod.name);
                }
            }
            StringBuilder line = new StringBuilder();
            events.forEach((what, pods) -> line.append(what).append(": ").append(String.join(" ", pods)).append("  "));
            System.out.printf("  tick %d  db=%-4s  %s%n", tick, databaseUp ? "UP" : "DOWN", line.isEmpty() ? "-" : line.toString().strip());
        }
        int total = instances.stream().mapToInt(p -> p.restarts).sum();
        System.out.println("  restarts: " + total);
    }

    public static void main(String[] args) {
        simulate("Liveness checks the database (wrong):", true);
        simulate("Liveness checks only the process, readiness checks the database:", false);
    }
}

Liveness checks the database (wrong):
  tick 1  db=UP    -
  tick 2  db=DOWN  -
  tick 3  db=DOWN  -
  tick 4  db=DOWN  RESTARTED: pod-1 pod-2 pod-3  removed from load balancer: pod-1 pod-2 pod-3
  tick 5  db=DOWN  -
  tick 6  db=DOWN  -
  tick 7  db=DOWN  RESTARTED: pod-1 pod-2 pod-3
  tick 8  db=DOWN  -
  tick 9  db=UP    back in load balancer: pod-1 pod-2 pod-3
  restarts: 6
Liveness checks only the process, readiness checks the database:
  tick 1  db=UP    -
  tick 2  db=DOWN  -
  tick 3  db=DOWN  -
  tick 4  db=DOWN  removed from load balancer: pod-1 pod-2 pod-3
  tick 5  db=DOWN  -
  tick 6  db=DOWN  -
  tick 7  db=DOWN  -
  tick 8  db=DOWN  -
  tick 9  db=UP    back in load balancer: pod-1 pod-2 pod-3
  restarts: 0

import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;

public class HealthCheckBudgetDemo {

    record Check(String name, long delayMillis) {}
    record Report(String overall, Map<String, String> components) {}

    static final AtomicInteger realChecks = new AtomicInteger(); // how often we really hit the dependencies

    // Runs all checks in parallel; each one gets at most timeoutMillis
    static Report runChecks(List<Check> checks, long timeoutMillis, ExecutorService pool) {
        realChecks.incrementAndGet();
        Map<String, CompletableFuture<String>> futures = new LinkedHashMap<>();
        for (Check c : checks) {
            futures.put(c.name(), CompletableFuture.supplyAsync(() -> {
                sleep(c.delayMillis()); // stands in for a query or a ping
                return "UP";
            }, pool).orTimeout(timeoutMillis, TimeUnit.MILLISECONDS)
              .exceptionally(e -> e instanceof TimeoutException
                      ? "DOWN (timed out after " + timeoutMillis + " ms)" : "DOWN"));
        }
        Map<String, String> result = new LinkedHashMap<>();
        futures.forEach((name, f) -> result.put(name, f.join()));
        boolean allUp = result.values().stream().allMatch("UP"::equals);
        return new Report(allUp ? "UP" : "DOWN", result);
    }

    // Caches the last report so frequent probes don't hammer the database
    static class CachedHealth {
        private final long ttlMillis;
        private Report last;
        private long lastAt = Long.MIN_VALUE;
        CachedHealth(long ttlMillis) { this.ttlMillis = ttlMillis; }

        Report get(long nowMillis, java.util.function.Supplier<Report> compute) {
            if (last == null || nowMillis - lastAt >= ttlMillis) {
                last = compute.get();
                lastAt = nowMillis;
            }
            return last;
        }
    }

    static void sleep(long millis) {
        try { Thread.sleep(millis); } catch (InterruptedException e) { Thread.currentThread().interrupt(); }
    }

    public static void main(String[] args) {
        List<Check> checks = List.of(new Check("database", 40), new Check("searchIndex", 3_000)); // search is hanging
        try (ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor()) {
            long start = System.nanoTime();
            Report report = runChecks(checks, 300, pool);
            long tookMillis = (System.nanoTime() - start) / 1_000_000;
            System.out.println("Overall: " + report.overall());
            report.components().forEach((k, v) -> System.out.println("  " + k + ": " + v));
            System.out.println("Answered within a 1 s probe timeout: " + (tookMillis < 1_000));

            CachedHealth cache = new CachedHealth(5_000);
            realChecks.set(0);
            long[] probeTimes = {0, 1_000, 2_000, 4_999, 5_000, 9_000}; // simulated clock, in milliseconds
            for (long t : probeTimes) {
                cache.get(t, () -> runChecks(List.of(new Check("database", 5)), 300, pool));
            }
            System.out.println(probeTimes.length + " probes, database really checked " + realChecks.get() + " times");
        }
    }
}

Overall: DOWN
  database: UP
  searchIndex: DOWN (timed out after 300 ms)
Answered within a 1 s probe timeout: true
6 probes, database really checked 2 times