외부 서비스 장애로부터 살아남기

Create Room Request
    |
    v
Room created (Redis)
    |
    v
QR Code image generated (ZXing)
    |
    v
Upload to Oracle Object Storage    <-- external service call
    |
    v
Generate Public URL
    |
    v
Broadcast QR Code URL via WebSocket

@Override
public String upload(@NonNull String contents, byte[] data) {
    try {
        return uploadQrCodeToObjectStorage(contents, data);
    } catch (Exception e) {
        throw new StorageServiceException(
            RoomErrorCode.QR_CODE_UPLOAD_FAILED,
            RoomErrorCode.QR_CODE_UPLOAD_FAILED.getMessage(), e);
    }
}

+-------------------------------------------------------------------+
|               Failure Type vs Behavior                            |
+-------------------------------------------------------------------+
|                                                                   |
|  Failure Type         Before              After                   |
|  -----------------------------------------------------------------|
|                                                                   |
|  Transient            Fail immediately    Retry and recover       |
|  (network blip,       -> no QR code       -> QR code delivered    |
|   timeout)                                                        |
|                                                                   |
|  -----------------------------------------------------------------|
|                                                                   |
|  Persistent           Retry every time    Fail fast               |
|  (Oracle Cloud        -> wasted resource  -> protect resources    |
|   outage)             -> thread pool busy -> fast response        |
|                                                                   |
+-------------------------------------------------------------------+

resilience4j:
  retry:
    retry-aspect-order: 2
    instances:
      oracleStorage:
        maxAttempts: 2
        waitDuration: 500ms
        retryExceptions:
          - java.io.IOException
          - com.oracle.bmc.model.BmcException
          - java.lang.RuntimeException

@Test
@DisplayName("retryExceptions에 없는 예외는 리트라이하지 않는다")
void retryExceptions에_없는_예외는_리트라이_안함() {
    when(objectStorage.putObject(any(PutObjectRequest.class)))
            .thenThrow(new IllegalArgumentException("Invalid argument"));

    Supplier<String> decoratedSupplier = Retry.decorateSupplier(
            retry,
            () -> storageService.upload("test", "data".getBytes())
    );

    assertThatThrownBy(decoratedSupplier::get)
            .isInstanceOf(IllegalArgumentException.class);

    // 1번만 호출됨 (리트라이 안 함)
    verify(objectStorage, times(1)).putObject(any(PutObjectRequest.class));
}

resilience4j:
  circuitbreaker:
    circuit-breaker-aspect-order: 1
    instances:
      oracleStorage:
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 10
        failureRateThreshold: 50
        waitDurationInOpenState: 30s
        permittedNumberOfCallsInHalfOpenState: 3
        automaticTransitionFromOpenToHalfOpenEnabled: true
        recordExceptions:
          - java.io.IOException
          - com.oracle.bmc.model.BmcException
          - java.lang.RuntimeException

                    failure rate < 50%
              +------------------------+
              |                        |
              v                        |
         +---------+             +-----------+
  ------>| CLOSED  |------------>|   OPEN    |
         +---------+ failure    +-----------+
              ^       rate >= 50%      |
              |                        | after 30s
              |                        v
              |                 +------------+
              +-----------------|  HALF_OPEN |
                success rate    +------------+
                 sufficient      3 trial calls

circuit-breaker-aspect-order: 1
retry-aspect-order: 2

Request --> CircuitBreaker (order=1) --> Retry (order=2) --> actual method

+-------------------------------------------------------------------+
|                   Aspect Order Comparison                         |
+-------------------------------------------------------------------+
|                                                                   |
|  Order            Circuit OPEN         Circuit CLOSED             |
|  -----------------------------------------------------------------|
|                                                                   |
|  CB -> Retry      Block immediately,   Retry first, then record  |
|  (correct)        no retry attempt     final result to CB         |
|                                                                   |
|  -----------------------------------------------------------------|
|                                                                   |
|  Retry -> CB      CallNotPermitted     Each attempt recorded     |
|  (wrong)          thrown twice,         to CB separately,         |
|                   pointless retry      failure count inflated    |
|                                                                   |
+-------------------------------------------------------------------+

@Override
@CircuitBreaker(name = "oracleStorage", fallbackMethod = "uploadFallback")
@Retry(name = "oracleStorage")
public String upload(@NonNull String contents, byte[] data) {
    if (data.length == 0) {
        throw new StorageServiceException(
            RoomErrorCode.QR_CODE_UPLOAD_FAILED, "QR 이미지 바이트가 비어 있습니다.");
    }
    return doUpload(contents, data);
}

private String uploadFallback(String contents, byte[] data, Exception e) {
    meterRegistry.counter("oracle.objectstorage.qr.upload.failed",
            "error", e.getClass().getSimpleName()).increment();

    if (e instanceof CallNotPermittedException) {
        log.warn("서킷 브레이커 OPEN 상태 - Oracle Storage 호출 차단됨: contents={}", contents);
        throw new StorageServiceException(RoomErrorCode.QR_CODE_UPLOAD_FAILED,
                "스토리지 서비스가 일시적으로 사용 불가능합니다. 잠시 후 다시 시도해주세요.");
    }

    log.error("Oracle Object Storage QR 코드 업로드 실패: contents={}, error={}",
            contents, e.getMessage(), e);
    throw new StorageServiceException(RoomErrorCode.QR_CODE_UPLOAD_FAILED,
            "QR 코드 업로드에 실패했습니다. 잠시 후 다시 시도해주세요.", e);
}

RoomService.createRoom()
  |
  v
QrCodeService.generateQrCodeAsync()  (@Async)
  |
  v
OracleObjectStorageService.upload()
  |
  v
[CircuitBreaker: oracleStorage]
  |
  +-- OPEN
  |     |
  |     v
  |   uploadFallback()
  |     -> "service temporarily unavailable" exception
  |     -> QrCodeService broadcasts ERROR status
  |
  +-- CLOSED / HALF_OPEN
        |
        v
      [Retry: oracleStorage]
        |
        +-- 1st attempt SUCCESS
        |     -> return QR code URL
        |     -> QrCodeService broadcasts SUCCESS
        |
        +-- 1st attempt FAIL --> wait 500ms --> 2nd attempt
              |
              +-- SUCCESS: return URL
              |
              +-- FAIL: uploadFallback()
                    -> record failure metric
                    -> record failure to CircuitBreaker
                    -> QrCodeService broadcasts ERROR

@TestConfiguration
static class TestConfig {
    @Bean
    @Primary
    public OracleObjectStorageService oracleObjectStorageService(
            ObjectStorage objectStorage,
            OracleObjectStorageProperties oracleProperties,
            QrProperties qrProperties,
            MeterRegistry meterRegistry
    ) {
        return new OracleObjectStorageService(
            objectStorage, oracleProperties, qrProperties, meterRegistry);
    }
}

@Test
@DisplayName("재시도 횟수를 초과하는 지속적인 실패 발생 시, fallback이 StorageServiceException을 던진다")
void 지속적_실패_시_StorageServiceException_발생() {
    when(objectStorage.putObject(any(PutObjectRequest.class)))
            .thenThrow(new BmcException(500, "ServiceCode", "Persistent failure", "requestId"));

    assertThatThrownBy(() ->
        oracleObjectStorageService.upload("test-contents", "test-data".getBytes()))
            .isInstanceOf(StorageServiceException.class)
            .hasMessageContaining("QR 코드 업로드에 실패했습니다");

    // maxAttempts(2)만큼 호출됨
    verify(objectStorage, times(2)).putObject(any(PutObjectRequest.class));
}

management:
  endpoints:
    web:
      exposure:
        include: "health,info,metrics,prometheus,circuitbreakers,retries"