feat:"拆分上游首字节/空闲/总期限并在执行期间续期预留与并发租约"
- OpenAiCompatibleProvider:总期限上限 3600 秒(默认仍 90),新增首字节与空闲超时,流式有数据即重置空闲窗口 - GatewayRequestExecutionService:调用期间定时续期钱包预留与 Redis 租约,结束即停 - ChatBillingService.renew 只延长 pending 预留;GatewayGuard 只续仍持有的租约,丢失不重建 - 新增本地 Docker 启动覆盖文件(预编译产物,仅监听 127.0.0.1) - 新增流式空闲/长流、续期、过期恢复不释放运行中调用、租约续期回归 验证:mvn -B -o clean verify 409 项通过(独立 PostgreSQL/Redis) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1a608f8ff8
commit
aef5356c54
12 files changed
+263
-10
No files matched your search
@@ -34,6 +34,16 @@ public class GatewayGuard {
|
||||
end
|
||||
return counts
|
||||
""", List.class);
|
||||
// Renew only a lease this request still owns; an expired or released token is never recreated.
|
||||
private static final DefaultRedisScript<Long> RENEW = new DefaultRedisScript<>("""
|
||||
local time = redis.call('TIME')
|
||||
local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
|
||||
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
|
||||
if not score or tonumber(score) <= now then return 0 end
|
||||
redis.call('ZADD', KEYS[1], 'XX', now + tonumber(ARGV[2]), ARGV[1])
|
||||
if redis.call('PTTL', KEYS[1]) < tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
||||
return 1
|
||||
""", Long.class);
|
||||
private static final DefaultRedisScript<Long> RELEASE = new DefaultRedisScript<>("return redis.call('ZREM', KEYS[1], ARGV[1])", Long.class);
|
||||
private final StringRedisTemplate redis;
|
||||
private final int requestsPerMinute;
|
||||
@@ -340,6 +350,15 @@ public class GatewayGuard {
|
||||
this.keySlot = keySlot;
|
||||
}
|
||||
|
||||
/** Extend every lease held by this admission; returns false if any lease was already lost. */
|
||||
@Override
|
||||
public boolean renew(long ttlMs) {
|
||||
if (closed.get()) return false;
|
||||
boolean held = true;
|
||||
for (Slot slot : new Slot[]{modelSlot, keySlot, userSlot}) if (slot != null && !slot.renew(ttlMs)) held = false;
|
||||
return held;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
@@ -356,6 +375,13 @@ public class GatewayGuard {
|
||||
|
||||
private Slot(String key, String token) { this.key = key; this.token = token; }
|
||||
|
||||
boolean renew(long ttlMs) {
|
||||
if (closed.get()) return false;
|
||||
if (key == null || token == null) return true;
|
||||
try { return execute(RENEW, key, token, Long.toString(Math.max(SLOT_TTL_MS, ttlMs))) == 1; }
|
||||
catch (GatewayLimitException exception) { return false; }
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed.compareAndSet(false, true)) return;
|
||||
|
||||
+3
@@ -2,6 +2,9 @@ package com.moyi.controlplane.gateway.request;
|
||||
|
||||
/** One request's releasable admission lease. */
|
||||
public interface ApiRequestAdmission extends AutoCloseable {
|
||||
/** Extend the lease while the request is still running. Returns false if it was already lost. */
|
||||
default boolean renew(long ttlMs) { return true; }
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
+35
-1
@@ -13,14 +13,33 @@ import org.springframework.stereotype.Service;
|
||||
/** Shared application boundary for synchronous and native-protocol model requests. */
|
||||
@Service
|
||||
public class GatewayRequestExecutionService {
|
||||
/** Wallet holds and Redis leases are short; a running call renews both on this cadence. */
|
||||
static final long RENEW_INTERVAL_MS = 60_000;
|
||||
static final long WALLET_HOLD_MS = 300_000;
|
||||
static final long LEASE_TTL_MS = 180_000;
|
||||
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GatewayRequestExecutionService.class);
|
||||
private final ChatBillingService billing;
|
||||
private final ApiRequestRiskPolicy risk;
|
||||
private final long renewIntervalMs;
|
||||
private final java.util.concurrent.ScheduledExecutorService renewals = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
var thread = new Thread(r, "request-lease-renewal"); thread.setDaemon(true); return thread;
|
||||
});
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired
|
||||
public GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk) {
|
||||
this(billing, risk, RENEW_INTERVAL_MS);
|
||||
}
|
||||
|
||||
GatewayRequestExecutionService(ChatBillingService billing, ApiRequestRiskPolicy risk, long renewIntervalMs) {
|
||||
this.billing = java.util.Objects.requireNonNull(billing, "billing");
|
||||
this.risk = java.util.Objects.requireNonNull(risk, "risk");
|
||||
if (renewIntervalMs < 1) throw new IllegalArgumentException("Renewal interval must be positive");
|
||||
this.renewIntervalMs = renewIntervalMs;
|
||||
}
|
||||
|
||||
@jakarta.annotation.PreDestroy
|
||||
void shutdown() { renewals.shutdownNow(); }
|
||||
|
||||
public <T> T execute(
|
||||
ApiRequestContext context,
|
||||
Map<String, Object> key,
|
||||
@@ -33,8 +52,11 @@ public class GatewayRequestExecutionService {
|
||||
Function<ApiException, T> deliverFailure
|
||||
) {
|
||||
risk.enforce(context);
|
||||
try (ApiRequestAdmission ignored = admission.get()) {
|
||||
try (ApiRequestAdmission lease = admission.get()) {
|
||||
ChatBillingService.Ticket ticket = billing.reserve(key, model, normalizedRequest, idempotencyKey, context.metadata());
|
||||
// Long calls outlive the initial hold and lease. Keep both alive until the work returns,
|
||||
// so expiry recovery cannot release a running call and other callers cannot take its slot.
|
||||
var renewal = renewals.scheduleWithFixedDelay(() -> renew(ticket, lease), renewIntervalMs, renewIntervalMs, java.util.concurrent.TimeUnit.MILLISECONDS);
|
||||
try {
|
||||
return work.execute(ticket);
|
||||
} catch (Exception failure) {
|
||||
@@ -46,10 +68,22 @@ public class GatewayRequestExecutionService {
|
||||
ApiException error = classifyFailure.apply(failure);
|
||||
billing.fail(ticket, error.getCode(), error.getMessage(), context.elapsedMillis());
|
||||
return deliverFailure.apply(error);
|
||||
} finally {
|
||||
renewal.cancel(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void renew(ChatBillingService.Ticket ticket, ApiRequestAdmission lease) {
|
||||
try {
|
||||
billing.renew(ticket.id(), WALLET_HOLD_MS);
|
||||
if (!lease.renew(LEASE_TTL_MS)) log.warn("Request concurrency lease could not be renewed; it may be reused before the call ends");
|
||||
} catch (RuntimeException exception) {
|
||||
// A missed renewal is retried on the next tick; the original hold still bounds recovery.
|
||||
log.warn("Request lease renewal failed ({})", exception.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
public BigDecimal settle(ChatBillingService.Ticket ticket, Map<String, Object> result, int latencyMs, Integer firstTokenMs) {
|
||||
// Validate and reduce to the only data required to price the call. The response body may contain
|
||||
// prompts, generated content, tool arguments or other sensitive data and is never recovery state.
|
||||
|
||||
+33
-8
@@ -30,6 +30,8 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class OpenAiCompatibleProvider {
|
||||
/** Upper bound for the configurable total deadline; leases are renewed at most this long. */
|
||||
public static final int MAX_TIMEOUT_SECONDS = 3600;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final HttpClient httpClient;
|
||||
private final JdbcTemplate jdbc;
|
||||
@@ -41,6 +43,10 @@ public class OpenAiCompatibleProvider {
|
||||
private final ConcurrentMap<String, AtomicInteger> keyCursors = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
// Long reasoning calls need a larger total deadline without letting a silent upstream hold
|
||||
// the request, its concurrency lease and wallet hold. Both default to the total deadline.
|
||||
private final long firstByteTimeoutMs;
|
||||
private final long idleTimeoutMs;
|
||||
@org.springframework.beans.factory.annotation.Autowired private UpstreamHealth health;
|
||||
|
||||
public OpenAiCompatibleProvider(
|
||||
@@ -54,7 +60,9 @@ public class OpenAiCompatibleProvider {
|
||||
this.jdbc = jdbc;
|
||||
this.secretCipher = secretCipher;
|
||||
this.environment = environment;
|
||||
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.timeoutSeconds = Math.max(1,Math.min(MAX_TIMEOUT_SECONDS,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.firstByteTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-first-byte-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||
this.idleTimeoutMs = 1000L*Math.max(1,Math.min(timeoutSeconds,environment.getProperty("moyi.gateway.upstream-idle-timeout-seconds",Integer.class,timeoutSeconds)));
|
||||
this.providers = Map.of(
|
||||
"provider-deepseek", configured("provider-deepseek", environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com"), environment),
|
||||
"provider-kimi", configured("provider-kimi", environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1"), environment),
|
||||
@@ -157,7 +165,7 @@ public class OpenAiCompatibleProvider {
|
||||
String json = writeJson(body);
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder()
|
||||
.uri(URI.create(base + path))
|
||||
.timeout(Duration.ofMillis(remaining))
|
||||
.timeout(Duration.ofMillis(streaming?Math.min(remaining,firstByteTimeoutMs):remaining))
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json));
|
||||
@@ -170,7 +178,7 @@ public class OpenAiCompatibleProvider {
|
||||
if(health!=null) health.record(healthRoute,apiKey,response.statusCode(),(int)TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started));
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
String details;
|
||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)))) {details=readErrorBody(bounded.body);}
|
||||
try(var bounded=new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),false)) {details=readErrorBody(bounded.body);}
|
||||
if (requiresLargerOutputBudget(details)) {
|
||||
throw new ApiException("INVALID_MAX_TOKENS", "上游模型要求 max_tokens 大于 1024,请提高输出 Token 上限后重试", 400);
|
||||
}
|
||||
@@ -190,7 +198,7 @@ public class OpenAiCompatibleProvider {
|
||||
response.body().close();
|
||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||
}
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)));
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-overallStarted)),streaming);
|
||||
} catch (ApiException exception) {
|
||||
throw exception;
|
||||
} catch (HttpTimeoutException | SocketTimeoutException exception) {
|
||||
@@ -233,11 +241,28 @@ public class OpenAiCompatibleProvider {
|
||||
private final InputStream body;
|
||||
private final AtomicBoolean expired=new AtomicBoolean();
|
||||
private final ScheduledFuture<?> deadline;
|
||||
private final boolean idleWatch;
|
||||
private volatile ScheduledFuture<?> idle;
|
||||
private volatile boolean closed;
|
||||
private Runnable afterClose=()->{};
|
||||
public StreamResponse onClose(Runnable callback) {afterClose=callback;return this;}
|
||||
private StreamResponse(InputStream body,long remainingMs) {
|
||||
this.body=body;
|
||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
||||
private StreamResponse(InputStream raw,long remainingMs,boolean streaming) {
|
||||
Runnable expire=() -> { expired.set(true); try { raw.close(); } catch(IOException ignored) {} };
|
||||
// The idle watchdog only matters when it is shorter than the remaining total deadline.
|
||||
this.idleWatch=streaming&&idleTimeoutMs<remainingMs;
|
||||
this.body=!idleWatch?raw:new FilterInputStream(raw) {
|
||||
@Override public int read() throws IOException { int value=super.read(); if(value!=-1) touch(expire); return value; }
|
||||
@Override public int read(byte[] buffer,int offset,int length) throws IOException { int count=super.read(buffer,offset,length); if(count>0) touch(expire); return count; }
|
||||
};
|
||||
this.deadline=deadlines.schedule(expire,remainingMs,TimeUnit.MILLISECONDS);
|
||||
if(idleWatch) touch(expire);
|
||||
}
|
||||
/** Received bytes prove the upstream is alive; restart the idle window. */
|
||||
private void touch(Runnable expire) {
|
||||
if(closed) return;
|
||||
ScheduledFuture<?> previous=idle;
|
||||
idle=deadlines.schedule(expire,idleTimeoutMs,TimeUnit.MILLISECONDS);
|
||||
if(previous!=null) previous.cancel(false);
|
||||
}
|
||||
public void events(EventConsumer consumer) throws IOException {
|
||||
events(consumer,true);
|
||||
@@ -248,7 +273,7 @@ public class OpenAiCompatibleProvider {
|
||||
consumer.accept(chunk);
|
||||
});
|
||||
}
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||
@Override public void close() { closed=true; deadline.cancel(false); if(idle!=null) idle.cancel(false); try { body.close(); } catch(IOException ignored) {} finally {afterClose.run();afterClose=()->{};} }
|
||||
}
|
||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||
|
||||
@@ -48,6 +48,15 @@ public class ChatBillingService {
|
||||
jdbc.update("UPDATE chat_billing_requests SET settlement_usage=?::jsonb, settlement_latency_ms=?, settlement_first_token_ms=?, expires_at=GREATEST(expires_at,now()+interval '5 minutes') WHERE id=? AND status='pending'",
|
||||
encode(normalizedUsage), Math.max(0,latencyMs), firstTokenMs, ticket.id());
|
||||
}
|
||||
/**
|
||||
* Extend a still-running call's hold so expiry recovery cannot release it mid-call.
|
||||
* Only pending tickets move, and the hold never shrinks below its current expiry.
|
||||
*/
|
||||
public boolean renew(UUID id, long holdMs) {
|
||||
long seconds=Math.max(60,(holdMs+999)/1000);
|
||||
return jdbc.update("UPDATE chat_billing_requests SET expires_at=GREATEST(expires_at,now()+make_interval(secs=>?)) WHERE id=? AND status='pending'",seconds,id)>0;
|
||||
}
|
||||
|
||||
@Transactional public void actualRoute(Ticket ticket,Map<String,Object> route) {
|
||||
var snapshots=jdbc.queryForList("SELECT procurement_pricing FROM model_price_history WHERE model_id=? ORDER BY id DESC LIMIT 1",route.get("id"));
|
||||
Object cost=snapshots.isEmpty()?null:snapshots.getFirst().get("procurement_pricing");
|
||||
|
||||
@@ -56,6 +56,12 @@ moyi:
|
||||
# provider's own response/limit.
|
||||
disable-model-concurrency: ${GATEWAY_DISABLE_MODEL_CONCURRENCY:false}
|
||||
queue-wait-ms: ${GATEWAY_QUEUE_WAIT_MS:30000}
|
||||
# Synchronous/stream upstream deadlines (seconds). Total is capped at 3600; first-byte and idle
|
||||
# default to the total and never exceed it. Running calls renew wallet holds and concurrency
|
||||
# leases, but a reverse proxy in front must also allow the longer read time.
|
||||
upstream-timeout-seconds: ${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}
|
||||
upstream-first-byte-timeout-seconds: ${GATEWAY_UPSTREAM_FIRST_BYTE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||
upstream-idle-timeout-seconds: ${GATEWAY_UPSTREAM_IDLE_TIMEOUT_SECONDS:${GATEWAY_UPSTREAM_TIMEOUT_SECONDS:90}}
|
||||
queue-poll-ms: ${GATEWAY_QUEUE_POLL_MS:100}
|
||||
tasks:
|
||||
max-queued-per-model: ${TASK_MAX_QUEUED_PER_MODEL:100}
|
||||
|
||||
@@ -450,6 +450,22 @@ class PlatformAdminPostgresTest {
|
||||
assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE operation='usage'",Integer.class));
|
||||
assertEquals(2,jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE status='failed' AND cost_amount=0",Integer.class));
|
||||
}
|
||||
@Test void renewedRunningCallIsNotReleasedByExpiryRecovery() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","renewal test",meta);
|
||||
var running=f.reserve(null);
|
||||
jdbc.update("UPDATE chat_billing_requests SET expires_at=now()+interval '5 seconds' WHERE id=?",running.id());
|
||||
assertTrue(f.billing().renew(running.id(),1_800_000));
|
||||
assertTrue(jdbc.queryForObject("SELECT expires_at>now()+interval '29 minutes' FROM chat_billing_requests WHERE id=?",Boolean.class,running.id()));
|
||||
// Renewal never shortens an existing hold.
|
||||
assertTrue(f.billing().renew(running.id(),60_000));
|
||||
assertTrue(jdbc.queryForObject("SELECT expires_at>now()+interval '29 minutes' FROM chat_billing_requests WHERE id=?",Boolean.class,running.id()));
|
||||
f.billing().recoverExpired();
|
||||
assertEquals("pending",jdbc.queryForObject("SELECT status FROM chat_billing_requests WHERE id=?",String.class,running.id()));
|
||||
assertTrue(jdbc.queryForObject("SELECT reserved_balance FROM user_wallets WHERE user_id=?",BigDecimal.class,memberId).signum()>0);
|
||||
f.billing().fail(running,"TEST_DONE","done",1);
|
||||
assertFalse(f.billing().renew(running.id(),1_800_000));
|
||||
}
|
||||
@Test void unexpectedProviderOverrunIsRecordedAsDebtAndBlocksNewCalls() {
|
||||
var f=chatFixture();
|
||||
wallet.grant(ownerToken,memberId,UUID.randomUUID(),"1","billing test",meta);
|
||||
|
||||
+15
@@ -73,6 +73,21 @@ class FairAdmissionRedisTest {
|
||||
keyHolder.close();
|
||||
}
|
||||
|
||||
@Test void renewalExtendsOnlyLeasesStillHeld() throws Exception {
|
||||
var admission = guard(0).acquire(model, UUID.randomUUID(), 0);
|
||||
String slots = prefix + "gateway:slots:model:fair-model";
|
||||
String token = redis.opsForZSet().range(slots, 0, 0).iterator().next();
|
||||
double before = redis.opsForZSet().score(slots, token);
|
||||
assertTrue(admission.renew(1_800_000));
|
||||
assertTrue(redis.opsForZSet().score(slots, token) > before + 1_000_000);
|
||||
redis.opsForZSet().remove(slots, token);
|
||||
// A lost lease is reported, never silently recreated on top of another caller's capacity.
|
||||
assertFalse(admission.renew(1_800_000));
|
||||
assertNull(redis.opsForZSet().score(slots, token));
|
||||
admission.close();
|
||||
assertFalse(admission.renew(1_800_000));
|
||||
}
|
||||
|
||||
private long zcard(String kind) {
|
||||
Long value = redis.opsForZSet().zCard(prefix + "gateway:" + kind + ":model:fair-model");
|
||||
return value == null ? 0 : value;
|
||||
|
||||
+20
@@ -96,6 +96,26 @@ class GatewayRequestExecutionServiceTest {
|
||||
verify(admission).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void runningCallRenewsItsHoldAndLeaseUntilTheWorkReturns() throws Exception {
|
||||
ChatBillingService billing = mock(ChatBillingService.class);
|
||||
var ticket = new ChatBillingService.Ticket(UUID.randomUUID(), userId, null);
|
||||
when(billing.reserve(key, model, normalized, null, context.metadata())).thenReturn(ticket);
|
||||
var service = new GatewayRequestExecutionService(billing, new ApiRequestRiskPolicy(ApiRequestRiskEngine.allowAll()), 20);
|
||||
ApiRequestAdmission admission = mock(ApiRequestAdmission.class);
|
||||
when(admission.renew(anyLong())).thenReturn(true);
|
||||
try {
|
||||
assertEquals("done", service.execute(context, key, model, normalized, null, () -> admission,
|
||||
ignored -> { Thread.sleep(150); return "done"; }, failure -> new ApiException("CALL_FAILED", "failed", 502), ApiException::getCode));
|
||||
verify(billing, atLeast(2)).renew(eq(ticket.id()), anyLong());
|
||||
verify(admission, atLeast(2)).renew(anyLong());
|
||||
// Renewal stops with the call; a finished request must not keep its hold alive.
|
||||
clearInvocations(billing);
|
||||
Thread.sleep(100);
|
||||
verify(billing, never()).renew(any(), anyLong());
|
||||
} finally { service.shutdown(); }
|
||||
}
|
||||
|
||||
@Test
|
||||
void workFailureIsRecordedAndAdmissionLeaseCloses() {
|
||||
ChatBillingService billing = mock(ChatBillingService.class);
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.moyi.controlplane.provider;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.security.SecretCipher;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
/** Long streams survive while bytes flow; a silent upstream ends at the idle window, not the total deadline. */
|
||||
class StreamDeadlineTest {
|
||||
private OpenAiCompatibleProvider provider(HttpServer server, int total, int idle) {
|
||||
var env = new MockEnvironment()
|
||||
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||
.withProperty("moyi.gateway.upstream-timeout-seconds", Integer.toString(total))
|
||||
.withProperty("moyi.gateway.upstream-idle-timeout-seconds", Integer.toString(idle));
|
||||
return new OpenAiCompatibleProvider(new ObjectMapper(), env, mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||
}
|
||||
|
||||
private HttpServer sse(long gapMs, int events, boolean finish) throws Exception {
|
||||
var server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
exchange.getResponseHeaders().set("Content-Type", "text/event-stream");
|
||||
exchange.sendResponseHeaders(200, 0);
|
||||
try (var out = exchange.getResponseBody()) {
|
||||
for (int i = 0; i < events; i++) {
|
||||
out.write(("data: {\"i\":" + i + "}\n\n").getBytes(StandardCharsets.UTF_8));
|
||||
out.flush();
|
||||
Thread.sleep(gapMs);
|
||||
}
|
||||
if (finish) out.write("data: [DONE]\n\n".getBytes(StandardCharsets.UTF_8));
|
||||
else Thread.sleep(5_000);
|
||||
} catch (Exception ignored) { }
|
||||
});
|
||||
server.start();
|
||||
return server;
|
||||
}
|
||||
|
||||
@Test void streamLongerThanTheIdleWindowSucceedsWhileEventsKeepArriving() throws Exception {
|
||||
var server = sse(400, 8, true);
|
||||
var provider = provider(server, 10, 1);
|
||||
try (var stream = provider.stream("provider-deepseek", "model", Map.of("messages", List.of()))) {
|
||||
var chunks = new ArrayList<Map<String,Object>>();
|
||||
long started = System.nanoTime();
|
||||
stream.events(chunks::add);
|
||||
assertEquals(8, chunks.size());
|
||||
assertTrue(System.nanoTime() - started > 2_000_000_000L, "the stream outlived the 1s idle window");
|
||||
} finally { provider.close(); server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void silentUpstreamTimesOutAtTheIdleWindow() throws Exception {
|
||||
var server = sse(0, 1, false);
|
||||
var provider = provider(server, 10, 1);
|
||||
try (var stream = provider.stream("provider-deepseek", "model", Map.of("messages", List.of()))) {
|
||||
long started = System.nanoTime();
|
||||
var error = assertThrows(ApiException.class, () -> stream.events(chunk -> { }));
|
||||
assertEquals("UPSTREAM_PROVIDER_TIMEOUT", error.getCode());
|
||||
long elapsedMs = (System.nanoTime() - started) / 1_000_000;
|
||||
assertTrue(elapsedMs < 4_000, "ended by the idle window, not the 10s total: " + elapsedMs);
|
||||
} finally { provider.close(); server.stop(0); }
|
||||
}
|
||||
|
||||
@Test void totalDeadlineIsCappedAndIdleNeverExceedsIt() throws Exception {
|
||||
var server = sse(0, 1, true);
|
||||
var env = new MockEnvironment()
|
||||
.withProperty("moyi.providers.deepseek.api-key", "local-fake-key")
|
||||
.withProperty("moyi.providers.deepseek.base-url", "http://127.0.0.1:" + server.getAddress().getPort() + "/v1")
|
||||
.withProperty("moyi.gateway.upstream-timeout-seconds", "999999")
|
||||
.withProperty("moyi.gateway.upstream-idle-timeout-seconds", "999999");
|
||||
var provider = new OpenAiCompatibleProvider(new ObjectMapper(), env, mock(JdbcTemplate.class), new SecretCipher("local-test-only"));
|
||||
try {
|
||||
assertEquals(OpenAiCompatibleProvider.MAX_TIMEOUT_SECONDS, org.springframework.test.util.ReflectionTestUtils.getField(provider, "timeoutSeconds"));
|
||||
assertEquals(1000L * OpenAiCompatibleProvider.MAX_TIMEOUT_SECONDS, org.springframework.test.util.ReflectionTestUtils.getField(provider, "idleTimeoutMs"));
|
||||
} finally { provider.close(); server.stop(0); }
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user