feat: modularize gateway requests and recover settlements
This commit is contained in:
1 parent
7785cda6a6
commit
5533a483d9
40 files changed
+1598
-232
No files matched your search
+53
-30
@@ -4,7 +4,12 @@ import com.moyi.controlplane.catalog.CatalogService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.provider.OpenAiCompatibleProvider;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestContext;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestMetadata;
|
||||
import com.moyi.controlplane.gateway.request.ApiRequestProtocol;
|
||||
import com.moyi.controlplane.gateway.request.GatewayRequestExecutionService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
@@ -26,15 +31,16 @@ public class GatewayController {
|
||||
private final CatalogService catalogService;
|
||||
private final OpenAiCompatibleProvider provider;
|
||||
private final GatewayGuard guard;
|
||||
private final ChatBillingService billing;
|
||||
private final GatewayRequestExecutionService requests;
|
||||
private final ObjectMapper json;
|
||||
@org.springframework.beans.factory.annotation.Autowired private GatewayFailover failover;
|
||||
|
||||
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, ChatBillingService billing, ObjectMapper json) {
|
||||
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard,
|
||||
GatewayRequestExecutionService requests, ObjectMapper json) {
|
||||
this.catalogService = catalogService;
|
||||
this.provider = provider;
|
||||
this.guard = guard;
|
||||
this.billing = billing;
|
||||
this.requests = requests;
|
||||
this.json = json;
|
||||
}
|
||||
|
||||
@@ -73,15 +79,31 @@ public class GatewayController {
|
||||
@RequestHeader(value = "Authorization", required = false) String authorization,
|
||||
@RequestBody Map<String, Object> input,
|
||||
@RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey,
|
||||
HttpServletRequest request,
|
||||
HttpServletResponse servletResponse
|
||||
) {
|
||||
return executeChat(requireKey(authorization), input, idempotencyKey, servletResponse);
|
||||
return executeChat(requireKey(authorization), input, idempotencyKey, servletResponse,
|
||||
ApiRequestMetadata.fromHttp(request.getRemoteAddr()));
|
||||
}
|
||||
|
||||
// Both the public gateway and the authenticated playground share admission and billing.
|
||||
/** Direct-call compatibility for application tests and non-HTTP adapters. */
|
||||
public ResponseEntity<Map<String, Object>> chatCompletions(String authorization,
|
||||
Map<String, Object> input, String idempotencyKey, HttpServletResponse servletResponse) {
|
||||
return executeChat(requireKey(authorization), input, idempotencyKey, servletResponse, ApiRequestMetadata.internal());
|
||||
}
|
||||
|
||||
// Public API and Playground share the same request execution and risk boundary.
|
||||
public ResponseEntity<Map<String, Object>> executeChat(Map<String, Object> key,
|
||||
Map<String, Object> input, String idempotencyKey, HttpServletResponse servletResponse) {
|
||||
return executeChat(key, input, idempotencyKey, servletResponse, ApiRequestMetadata.internal());
|
||||
}
|
||||
|
||||
public ResponseEntity<Map<String, Object>> executeChat(Map<String, Object> key,
|
||||
Map<String, Object> input, String idempotencyKey, HttpServletResponse servletResponse, ApiRequestMetadata metadata) {
|
||||
long startedAt = System.currentTimeMillis();
|
||||
UUID keyId = UUID.fromString(String.valueOf(key.get("id")));
|
||||
// Reject request floods before catalog reads and payload normalization, including malformed requests.
|
||||
guard.checkRate(keyId, false);
|
||||
String modelId = String.valueOf(input.getOrDefault("model", ""));
|
||||
Map<String, Object> model = resolveModel(key, modelId);
|
||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线;请使用当前 API Key 的 /v1/models 返回的模型 ID", 404);
|
||||
@@ -92,27 +114,28 @@ public class GatewayController {
|
||||
boolean mock="provider-mock".equals(providerId);
|
||||
if(!mock && !provider.isConfigured(providerId, routeId, String.valueOf(model.get("upstreamModel")))) throw new ApiException("PROVIDER_NOT_CONFIGURED","该供应商尚未配置密钥,不能执行真实调用",503);
|
||||
ChatMeter.Request request=ChatMeter.request(input,json);
|
||||
guard.checkRate(UUID.fromString(String.valueOf(key.get("id"))), false);
|
||||
// Reserve both limits atomically. A queued request never consumes a
|
||||
// model slot while waiting for the API-Key slot (or vice versa).
|
||||
try (GatewayGuard.Admission admission = guard.acquire(model, UUID.fromString(String.valueOf(key.get("id"))), guard.queueWaitMs())) {
|
||||
var ticket=billing.reserve(key,model,request,idempotencyKey);
|
||||
ApiRequestContext context = new ApiRequestContext(
|
||||
UUID.fromString(String.valueOf(key.get("userId"))), keyId, ApiRequestProtocol.CHAT_COMPLETIONS,
|
||||
modelId, routeId, Boolean.TRUE.equals(request.body().get("stream")), request.reservedInputTokens(),
|
||||
request.maxOutputTokens(), metadata, System.nanoTime());
|
||||
return requests.execute(context, key, model, request, idempotencyKey, () -> {
|
||||
// A queued request never consumes one scarce slot while waiting for another.
|
||||
return guard.acquire(model, keyId, guard.queueWaitMs());
|
||||
}, ticket -> {
|
||||
servletResponse.setHeader("X-Request-Id",ticket.id().toString());
|
||||
try {
|
||||
catalogService.touchKey(key);
|
||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||
stream(key,model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
return null;
|
||||
}
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):failover!=null?failover.invoke(key,model,"chat",request.body(),ticket):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||
body.put("model",modelId);
|
||||
billing.observedUsage(ticket,"chat",body.get("usage"));
|
||||
requireCompletion(body);
|
||||
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
|
||||
} catch(Exception exception) {
|
||||
ApiException error=exception instanceof ApiException e?e:new ApiException("CALL_INTERRUPTED","调用中断,请查看使用记录确认结算状态",502);
|
||||
billing.fail(ticket,error.getCode(),error.getMessage(),elapsed(startedAt));
|
||||
catalogService.touchKey(key);
|
||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||
stream(key,model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
return null;
|
||||
}
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):failover!=null?failover.invoke(key,model,"chat",request.body(),ticket):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||
body.put("model",modelId);
|
||||
requests.observedUsage(ticket,"chat",body.get("usage"));
|
||||
requireCompletion(body);
|
||||
var cost=requests.settle(ticket,body,elapsed(startedAt),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
|
||||
}, exception -> exception instanceof ApiException api ? api
|
||||
: new ApiException("CALL_INTERRUPTED","调用中断,请查看使用记录确认结算状态",502), error -> {
|
||||
if(servletResponse.isCommitted()) {
|
||||
try { event(servletResponse,Map.of("error",Map.of("code",error.getCode(),"message",error.getMessage()))); done(servletResponse); } catch(IOException ignored) { }
|
||||
return null;
|
||||
@@ -120,8 +143,7 @@ public class GatewayController {
|
||||
servletResponse.resetBuffer();
|
||||
servletResponse.setContentType("application/json");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void stream(Map<String,Object> key,Map<String,Object> model, String modelId, String routeId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
@@ -131,7 +153,8 @@ public class GatewayController {
|
||||
var choices=(List<?>)body.get("choices");
|
||||
var message=((Map<?,?>)choices.getFirst()).get("message");
|
||||
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",message))));
|
||||
billing.settle(ticket,body,elapsed(started),elapsed(started));
|
||||
requests.observedUsage(ticket,"chat",body.get("usage"));
|
||||
requests.settle(ticket,body,elapsed(started),elapsed(started));
|
||||
event(response,Map.of("id",body.get("id"),"object","chat.completion.chunk","model",modelId,"choices",List.of(Map.of("index",0,"delta",Map.of(),"finish_reason","stop"))));
|
||||
event(response,Map.of("choices",List.of(),"usage",body.get("usage"))); done(response); return;
|
||||
}
|
||||
@@ -147,9 +170,9 @@ public class GatewayController {
|
||||
// A closed browser must not turn an otherwise completed provider call into free usage.
|
||||
// Continue within the same hard deadline to obtain the final meter and settle once.
|
||||
if(!disconnected[0]) try { event(response,chunk); } catch(IOException clientClosed) { disconnected[0]=true; }
|
||||
}); } finally {billing.observedUsage(ticket,"chat",collector.observedUsage());}
|
||||
}); } finally {requests.observedUsage(ticket,"chat",collector.observedUsage());}
|
||||
var body=collector.result();
|
||||
billing.settle(ticket,body,elapsed(started),collector.firstTokenMs());
|
||||
requests.settle(ticket,body,elapsed(started),collector.firstTokenMs());
|
||||
if(!disconnected[0]) done(response);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user