fix: classify upstream failures and prevent empty billed responses

This commit is contained in:
wlx committed 2026-09-28 04:55:00 +08:00
1 parent b455d8d79f
commit 9acc69d4c7
28 files changed
+1449 -131

No files matched your search

@@ -20,6 +20,32 @@ import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
class GatewayDeploymentTest {
@Test
void capacitySnapshotReadsBothLiveAndWaitingBucketsAndDoesNotInventZeroOnFailure() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
doReturn(List.of(2L, 3L, 0L, 1L)).when(redis).execute(any(RedisScript.class), anyList());
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 4, "moyi:test:");
var snapshot = guard.capacitySnapshot(List.of("model-a", "model-b"));
assertTrue(snapshot.available());
assertEquals(new GatewayGuard.Capacity(2, 3), snapshot.groups().get("model-a"));
assertEquals(new GatewayGuard.Capacity(0, 1), snapshot.groups().get("model-b"));
verify(redis).execute(any(RedisScript.class), eq(List.of(
"moyi:test:gateway:slots:model:model-a", "moyi:test:gateway:waiting:model:model-a",
"moyi:test:gateway:slots:model:model-b", "moyi:test:gateway:waiting:model:model-b")));
doThrow(new org.springframework.dao.DataAccessResourceFailureException("offline")).when(redis).execute(any(RedisScript.class), anyList());
assertFalse(guard.capacitySnapshot(List.of("model-a")).available());
assertTrue(guard.capacitySnapshot(List.of("model-a")).groups().isEmpty());
}
@Test
void queueTimeoutCleansItsWaitingEntry() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
doReturn(0L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class));
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 1, 1, 15, 10, "moyi:test:");
assertThrows(GatewayLimitException.class, () -> guard.acquire(Map.of("id", "model-a"), UUID.randomUUID(), 15));
verify(redis).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:waiting:model:model-a")), any(Object[].class));
}
@Test
void redisRateKeysAreEnvironmentScoped() {
StringRedisTemplate redis = mock(StringRedisTemplate.class);
@@ -108,6 +134,65 @@ class GatewayDeploymentTest {
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock","messages",List.of(Map.of("role","user","content","test"))),null,new MockHttpServletResponse()).getStatusCode().value());
}
@Test
void syncResponsePreservesReasoningAndFinishReasonWhenContentIsEmpty() {
CatalogService catalog = mock(CatalogService.class);
GatewayGuard guard = mock(GatewayGuard.class);
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
ChatBillingService billing = mock(ChatBillingService.class);
String userId = UUID.randomUUID().toString();
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
Map<String, Object> model = Map.of("id", "reasoning-route", "providerId", "provider-relay", "upstreamModel", "reasoning-model", "mode", "sync");
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("reasoning-route")).thenReturn(model);
when(catalog.keyCanInvoke(key, "reasoning-route")).thenReturn(true);
when(provider.isConfigured("provider-relay")).thenReturn(true);
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
"id", "upstream-response",
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", "", "reasoning_content", "private reasoning"), "finish_reason", "length")),
"usage", Map.of("prompt_tokens", 2, "completion_tokens", 16, "total_tokens", 18)
));
when(provider.chat(eq("provider-relay"), eq("reasoning-model"), anyMap())).thenReturn(upstreamResponse);
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
when(billing.settle(any(), anyMap(), anyInt(), isNull())).thenReturn(java.math.BigDecimal.ZERO);
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
var response = controller.chatCompletions("Bearer test-key", Map.of("model", "reasoning-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse());
assertEquals(200, response.getStatusCode().value());
Map<?, ?> choice = (Map<?, ?>)((List<?>)response.getBody().get("choices")).getFirst();
Map<?, ?> message = (Map<?, ?>)choice.get("message");
assertEquals("", message.get("content"));
assertEquals("private reasoning", message.get("reasoning_content"));
assertEquals("length", choice.get("finish_reason"));
}
@Test
void trulyEmptySyncResponseIsRejectedBeforeSettlement() {
CatalogService catalog = mock(CatalogService.class);
GatewayGuard guard = mock(GatewayGuard.class);
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
ChatBillingService billing = mock(ChatBillingService.class);
String userId = UUID.randomUUID().toString();
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
Map<String, Object> model = Map.of("id", "empty-route", "providerId", "provider-relay", "upstreamModel", "empty-model", "mode", "sync");
when(catalog.authenticateKey("test-key")).thenReturn(key);
when(catalog.activeModel("empty-route")).thenReturn(model);
when(catalog.keyCanInvoke(key, "empty-route")).thenReturn(true);
when(provider.isConfigured("provider-relay")).thenReturn(true);
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
"id", "upstream-empty",
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", ""), "finish_reason", "stop"))
));
when(provider.chat(eq("provider-relay"), eq("empty-model"), anyMap())).thenReturn(upstreamResponse);
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "empty-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse()));
assertEquals("UPSTREAM_RESPONSE_INVALID", error.getCode());
verify(billing, never()).settle(any(), anyMap(), anyInt(), any());
verify(billing).fail(any(), eq("UPSTREAM_RESPONSE_INVALID"), anyString(), anyInt());
}
@Test
void unimplementedMediaProviderDoesNotCreateFakeTask() {
JdbcTemplate jdbc = mock(JdbcTemplate.class);