fix: classify upstream failures and prevent empty billed responses
This commit is contained in:
1 parent
b455d8d79f
commit
9acc69d4c7
28 files changed
+1449
-131
No files matched your search
+85
@@ -20,6 +20,32 @@ import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class GatewayDeploymentTest {
|
||||
@Test
|
||||
void capacitySnapshotReadsBothLiveAndWaitingBucketsAndDoesNotInventZeroOnFailure() {
|
||||
StringRedisTemplate redis = mock(StringRedisTemplate.class);
|
||||
doReturn(List.of(2L, 3L, 0L, 1L)).when(redis).execute(any(RedisScript.class), anyList());
|
||||
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 4, "moyi:test:");
|
||||
var snapshot = guard.capacitySnapshot(List.of("model-a", "model-b"));
|
||||
assertTrue(snapshot.available());
|
||||
assertEquals(new GatewayGuard.Capacity(2, 3), snapshot.groups().get("model-a"));
|
||||
assertEquals(new GatewayGuard.Capacity(0, 1), snapshot.groups().get("model-b"));
|
||||
verify(redis).execute(any(RedisScript.class), eq(List.of(
|
||||
"moyi:test:gateway:slots:model:model-a", "moyi:test:gateway:waiting:model:model-a",
|
||||
"moyi:test:gateway:slots:model:model-b", "moyi:test:gateway:waiting:model:model-b")));
|
||||
doThrow(new org.springframework.dao.DataAccessResourceFailureException("offline")).when(redis).execute(any(RedisScript.class), anyList());
|
||||
assertFalse(guard.capacitySnapshot(List.of("model-a")).available());
|
||||
assertTrue(guard.capacitySnapshot(List.of("model-a")).groups().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void queueTimeoutCleansItsWaitingEntry() {
|
||||
StringRedisTemplate redis = mock(StringRedisTemplate.class);
|
||||
doReturn(0L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class));
|
||||
GatewayGuard guard = new GatewayGuard(redis, 60, 300, 1, 1, 15, 10, "moyi:test:");
|
||||
assertThrows(GatewayLimitException.class, () -> guard.acquire(Map.of("id", "model-a"), UUID.randomUUID(), 15));
|
||||
verify(redis).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:waiting:model:model-a")), any(Object[].class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void redisRateKeysAreEnvironmentScoped() {
|
||||
StringRedisTemplate redis = mock(StringRedisTemplate.class);
|
||||
@@ -108,6 +134,65 @@ class GatewayDeploymentTest {
|
||||
assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock","messages",List.of(Map.of("role","user","content","test"))),null,new MockHttpServletResponse()).getStatusCode().value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void syncResponsePreservesReasoningAndFinishReasonWhenContentIsEmpty() {
|
||||
CatalogService catalog = mock(CatalogService.class);
|
||||
GatewayGuard guard = mock(GatewayGuard.class);
|
||||
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
|
||||
ChatBillingService billing = mock(ChatBillingService.class);
|
||||
String userId = UUID.randomUUID().toString();
|
||||
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
|
||||
Map<String, Object> model = Map.of("id", "reasoning-route", "providerId", "provider-relay", "upstreamModel", "reasoning-model", "mode", "sync");
|
||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||
when(catalog.activeModel("reasoning-route")).thenReturn(model);
|
||||
when(catalog.keyCanInvoke(key, "reasoning-route")).thenReturn(true);
|
||||
when(provider.isConfigured("provider-relay")).thenReturn(true);
|
||||
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
||||
"id", "upstream-response",
|
||||
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", "", "reasoning_content", "private reasoning"), "finish_reason", "length")),
|
||||
"usage", Map.of("prompt_tokens", 2, "completion_tokens", 16, "total_tokens", 18)
|
||||
));
|
||||
when(provider.chat(eq("provider-relay"), eq("reasoning-model"), anyMap())).thenReturn(upstreamResponse);
|
||||
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
||||
when(billing.settle(any(), anyMap(), anyInt(), isNull())).thenReturn(java.math.BigDecimal.ZERO);
|
||||
|
||||
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
|
||||
var response = controller.chatCompletions("Bearer test-key", Map.of("model", "reasoning-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse());
|
||||
assertEquals(200, response.getStatusCode().value());
|
||||
Map<?, ?> choice = (Map<?, ?>)((List<?>)response.getBody().get("choices")).getFirst();
|
||||
Map<?, ?> message = (Map<?, ?>)choice.get("message");
|
||||
assertEquals("", message.get("content"));
|
||||
assertEquals("private reasoning", message.get("reasoning_content"));
|
||||
assertEquals("length", choice.get("finish_reason"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void trulyEmptySyncResponseIsRejectedBeforeSettlement() {
|
||||
CatalogService catalog = mock(CatalogService.class);
|
||||
GatewayGuard guard = mock(GatewayGuard.class);
|
||||
OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class);
|
||||
ChatBillingService billing = mock(ChatBillingService.class);
|
||||
String userId = UUID.randomUUID().toString();
|
||||
Map<String, Object> key = Map.of("id", UUID.randomUUID().toString(), "userId", userId);
|
||||
Map<String, Object> model = Map.of("id", "empty-route", "providerId", "provider-relay", "upstreamModel", "empty-model", "mode", "sync");
|
||||
when(catalog.authenticateKey("test-key")).thenReturn(key);
|
||||
when(catalog.activeModel("empty-route")).thenReturn(model);
|
||||
when(catalog.keyCanInvoke(key, "empty-route")).thenReturn(true);
|
||||
when(provider.isConfigured("provider-relay")).thenReturn(true);
|
||||
Map<String, Object> upstreamResponse = new java.util.LinkedHashMap<>(Map.of(
|
||||
"id", "upstream-empty",
|
||||
"choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", ""), "finish_reason", "stop"))
|
||||
));
|
||||
when(provider.chat(eq("provider-relay"), eq("empty-model"), anyMap())).thenReturn(upstreamResponse);
|
||||
when(billing.reserve(anyMap(), anyMap(), any(), isNull())).thenReturn(new ChatBillingService.Ticket(UUID.randomUUID(), UUID.fromString(userId), ChatMeter.rates(null, true)));
|
||||
GatewayController controller = new GatewayController(catalog, provider, guard, billing, new ObjectMapper());
|
||||
|
||||
ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "empty-route", "messages", List.of(Map.of("role", "user", "content", "hello"))), null, new MockHttpServletResponse()));
|
||||
assertEquals("UPSTREAM_RESPONSE_INVALID", error.getCode());
|
||||
verify(billing, never()).settle(any(), anyMap(), anyInt(), any());
|
||||
verify(billing).fail(any(), eq("UPSTREAM_RESPONSE_INVALID"), anyString(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unimplementedMediaProviderDoesNotCreateFakeTask() {
|
||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||
|
||||
Reference in new issue
Block a user