feat: stream chat and settle metered CNY usage atomically

This commit is contained in:
wlx committed 2026-09-20 01:36:22 +08:00
1 parent 7360cf3ebe
commit b46ee72f2d
28 files changed
+1098 -86

No files matched your search

@@ -3,13 +3,15 @@ package com.moyi.controlplane.gateway;
import com.moyi.controlplane.catalog.CatalogService;
import com.moyi.controlplane.common.ApiException;
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
import com.moyi.controlplane.usage.UsageAuditService;
import com.moyi.controlplane.wallet.ChatBillingService;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
@@ -24,13 +26,15 @@ public class GatewayController {
private final CatalogService catalogService;
private final OpenAiCompatibleProvider provider;
private final GatewayGuard guard;
private final UsageAuditService usage;
private final ChatBillingService billing;
private final ObjectMapper json;
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, UsageAuditService usage) {
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, ChatBillingService billing, ObjectMapper json) {
this.catalogService = catalogService;
this.provider = provider;
this.guard = guard;
this.usage = usage;
this.billing = billing;
this.json = json;
}
@GetMapping("/models")
@@ -55,7 +59,9 @@ public class GatewayController {
@PostMapping("/chat/completions")
public ResponseEntity<Map<String, Object>> chatCompletions(
@RequestHeader(value = "Authorization", required = false) String authorization,
@RequestBody Map<String, Object> input
@RequestBody Map<String, Object> input,
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
HttpServletResponse servletResponse
) {
Map<String, Object> key = requireKey(authorization);
long startedAt = System.currentTimeMillis();
@@ -64,34 +70,89 @@ public class GatewayController {
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404);
if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403);
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
if (Boolean.TRUE.equals(input.get("stream"))) throw new ApiException("STREAM_NOT_IMPLEMENTED", "流式转发将在下一节点开放", 501);
String providerId=String.valueOf(model.get("providerId"));
boolean mock="provider-mock".equals(providerId);
if(!mock && !provider.isConfigured(providerId)) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
ChatMeter.Request request=ChatMeter.request(input,json);
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
try (GatewayGuard.Slot slot = guard.acquire(model)) {
ResponseEntity<Map<String, Object>> result = invoke(key, modelId, model, input);
Map<String, Object> body = result.getBody();
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "succeeded", (int) (System.currentTimeMillis() - startedAt), number(body, "usage.prompt_tokens"), number(body, "usage.completion_tokens"), input, body, null, null);
return result;
} catch (ApiException exception) {
usage.recordCall(UUID.fromString(String.valueOf(key.get("userId"))), UUID.fromString(String.valueOf(key.get("id"))), modelId, String.valueOf(model.get("providerId")), "failed", (int) (System.currentTimeMillis() - startedAt), 0, 0, input, null, exception.getCode(), exception.getMessage());
throw exception;
var ticket=billing.reserve(key,model,request,idempotencyKey);
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
try {
catalogService.touchKey(key);
if(Boolean.TRUE.equals(request.body().get("stream"))) {
stream(model,request.body(),ticket,servletResponse,startedAt,mock);
return null;
}
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),request.body());
body.put("model",modelId);
requireCompletion(body);
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
} catch(Exception exception) {
ApiException error=exception instanceof ApiException e?e:new ApiException("CALL_INTERRUPTED","调用中断,请查看使用记录确认结算状态",502);
billing.fail(ticket,error.getCode(),error.getMessage(),elapsed(startedAt));
if(servletResponse.isCommitted()) {
try { event(servletResponse,Map.of("error",Map.of("code",error.getCode(),"message",error.getMessage()))); done(servletResponse); } catch(IOException ignored) { }
return null;
}
servletResponse.resetBuffer();
servletResponse.setContentType("application/json");
throw error;
}
}
}
private ResponseEntity<Map<String, Object>> invoke(Map<String, Object> key, String modelId, Map<String, Object> model, Map<String, Object> input) {
String providerId = String.valueOf(model.get("providerId"));
if (provider.isConfigured(providerId)) {
List<?> messages = input.get("messages") instanceof List<?> values ? values : List.of();
Map<String, Object> upstream = provider.chat(providerId, String.valueOf(model.get("upstreamModel")), messages);
upstream.put("model", modelId);
catalogService.touchKey(key);
return ResponseEntity.ok(upstream);
private void stream(Map<String,Object> model, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
String modelId=model.get("id").toString();
if(mock) {
var body=mockResponse(modelId,input);
prepareStream(response);
var choices=(List<?>)body.get("choices");
var message=((Map<?,?>)choices.getFirst()).get("message");
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",message))));
billing.settle(ticket,body,elapsed(started),elapsed(started));
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",Map.of(),"finish_reason","stop"))));
event(response,Map.of("choices",List.of(),"usage",body.get("usage"))); done(response); return;
}
if (!"provider-mock".equals(providerId)) {
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该供应商尚未配置密钥,不能执行真实调用", 503);
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
// Upstream HTTP errors are inspected before committing a 200/SSE response.
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),input)) {
var collector=new ChatStreamCollector(modelId,started);
prepareStream(response);
boolean[] disconnected={false};
upstream.events(chunk -> {
chunk.put("model",modelId);
collector.accept(chunk);
// A closed browser must not turn an otherwise completed provider call into free usage.
// Continue within the same hard deadline to obtain the final meter and settle once.
if(!disconnected[0]) try { event(response,chunk); } catch(IOException clientClosed) { disconnected[0]=true; }
});
var body=collector.result();
billing.settle(ticket,body,elapsed(started),collector.firstTokenMs());
if(!disconnected[0]) done(response);
}
}
private void prepareStream(HttpServletResponse response) {
response.setContentType("text/event-stream"); response.setCharacterEncoding("UTF-8");
response.setHeader("Cache-Control","no-cache"); response.setHeader("X-Accel-Buffering","no");
}
private void event(HttpServletResponse response,Object data) throws IOException {
response.getOutputStream().write(("data: "+json.writeValueAsString(data)+"\n\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
response.flushBuffer();
}
private void done(HttpServletResponse response) throws IOException {
response.getOutputStream().write("data: [DONE]\n\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)); response.flushBuffer();
}
private int elapsed(long started) { return (int)Math.min(Integer.MAX_VALUE,System.currentTimeMillis()-started); }
private void requireCompletion(Map<String,Object> body) {
if(body.containsKey("error") || !(body.get("choices") instanceof List<?> choices) || choices.isEmpty()
|| !(choices.getFirst() instanceof Map<?,?> choice) || choice.get("finish_reason")==null)
throw new ApiException("UPSTREAM_RESPONSE_INVALID","上游未返回完整的聊天结果,本次未扣费",502);
}
private Map<String,Object> mockResponse(String modelId,Map<String,Object> input) {
String prompt = extractPrompt(input.get("messages"));
String content = "MOYI Java Mock 已收到:" + (prompt.isBlank() ? "(空消息)" : prompt);
catalogService.touchKey(key);
Map<String, Object> response = new LinkedHashMap<>();
response.put("id", "chatcmpl-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12));
response.put("object", "chat.completion");
@@ -99,7 +160,7 @@ public class GatewayController {
response.put("model", modelId);
response.put("choices", List.of(Map.of("index", 0, "message", Map.of("role", "assistant", "content", content), "finish_reason", "stop")));
response.put("usage", Map.of("prompt_tokens", prompt.length(), "completion_tokens", content.length(), "total_tokens", prompt.length() + content.length()));
return ResponseEntity.status(HttpStatus.OK).body(response);
return response;
}
private Map<String, Object> requireKey(String authorization) {
@@ -117,12 +178,4 @@ public class GatewayController {
}).reduce((left, right) -> left + "\n" + right).orElse("");
}
private long number(Map<String, Object> body, String path) {
if (body == null) return 0;
Object usageValue = body.get("usage");
if (!(usageValue instanceof Map<?, ?> map)) return 0;
String key = path.substring(path.indexOf('.') + 1);
Object value = map.get(key);
return value instanceof Number number ? number.longValue() : 0;
}
}