feat: extend enterprise gateway policies billing protocols and operations
This commit is contained in:
1 parent
1528b71882
commit
af43947b5d
67 files changed
+1822
-69
No files matched your search
+12
-6
@@ -28,6 +28,7 @@ public class GatewayController {
|
||||
private final GatewayGuard guard;
|
||||
private final ChatBillingService billing;
|
||||
private final ObjectMapper json;
|
||||
@org.springframework.beans.factory.annotation.Autowired private GatewayFailover failover;
|
||||
|
||||
public GatewayController(CatalogService catalogService, OpenAiCompatibleProvider provider, GatewayGuard guard, ChatBillingService billing, ObjectMapper json) {
|
||||
this.catalogService = catalogService;
|
||||
@@ -58,6 +59,9 @@ public class GatewayController {
|
||||
item.put("upstream_model", model.get("upstreamModel"));
|
||||
item.put("type", model.get("type"));
|
||||
item.put("mode", model.get("mode"));
|
||||
item.put("supported_protocols", model.getOrDefault("supportedProtocols",List.of()));
|
||||
item.put("execution_ready", model.getOrDefault("executionReady",false));
|
||||
item.put("billing_ready", model.getOrDefault("billingReady",false));
|
||||
return item;
|
||||
}).toList();
|
||||
catalogService.touchKey(key);
|
||||
@@ -82,6 +86,7 @@ public class GatewayController {
|
||||
Map<String, Object> model = resolveModel(key, modelId);
|
||||
if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线;请使用当前 API Key 的 /v1/models 返回的模型 ID", 404);
|
||||
if ("async".equals(model.get("mode"))) throw new ApiException("ASYNC_MODEL", "该模型需要使用异步任务接口", 400);
|
||||
if(!List.of("chat","text","code","multimodal","llm","reasoning","vision").contains(String.valueOf(model.get("type")))) throw new ApiException("WRONG_MODEL_INTERFACE","该模型需要使用模型目录标明的接口",400);
|
||||
String providerId=String.valueOf(model.get("providerId"));
|
||||
String routeId=String.valueOf(model.get("id"));
|
||||
boolean mock="provider-mock".equals(providerId);
|
||||
@@ -96,11 +101,12 @@ public class GatewayController {
|
||||
try {
|
||||
catalogService.touchKey(key);
|
||||
if(Boolean.TRUE.equals(request.body().get("stream"))) {
|
||||
stream(model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
stream(key,model,modelId,routeId,request.body(),ticket,servletResponse,startedAt,mock);
|
||||
return null;
|
||||
}
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||
Map<String,Object> body=mock?mockResponse(modelId,input):failover!=null?failover.invoke(key,model,"chat",request.body(),ticket):provider.chat(providerId,String.valueOf(model.get("upstreamModel")),routeId,request.body());
|
||||
body.put("model",modelId);
|
||||
billing.observedUsage(ticket,"chat",body.get("usage"));
|
||||
requireCompletion(body);
|
||||
var cost=billing.settle(ticket,body,elapsed(startedAt),null);
|
||||
return ResponseEntity.ok().header("X-Request-Id",ticket.id().toString()).header("X-Moyi-Cost-CNY",cost.toPlainString()).body(body);
|
||||
@@ -118,7 +124,7 @@ public class GatewayController {
|
||||
}
|
||||
}
|
||||
|
||||
private void stream(Map<String,Object> model, String modelId, String routeId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
private void stream(Map<String,Object> key,Map<String,Object> model, String modelId, String routeId, Map<String,Object> input, ChatBillingService.Ticket ticket, HttpServletResponse response, long started, boolean mock) throws IOException {
|
||||
if(mock) {
|
||||
var body=mockResponse(modelId,input);
|
||||
prepareStream(response);
|
||||
@@ -131,17 +137,17 @@ public class GatewayController {
|
||||
}
|
||||
// Keep admission, the concurrency slot and the upstream stream alive on this request thread.
|
||||
// Upstream HTTP errors are inspected before committing a 200/SSE response.
|
||||
try(var upstream=provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),routeId,input)) {
|
||||
try(var upstream=failover!=null?failover.stream(key,model,"chat",input,ticket):provider.stream(model.get("providerId").toString(),model.get("upstreamModel").toString(),routeId,input)) {
|
||||
var collector=new ChatStreamCollector(modelId,started);
|
||||
prepareStream(response);
|
||||
boolean[] disconnected={false};
|
||||
upstream.events(chunk -> {
|
||||
try { upstream.events(chunk -> {
|
||||
chunk.put("model",modelId);
|
||||
collector.accept(chunk);
|
||||
// A closed browser must not turn an otherwise completed provider call into free usage.
|
||||
// Continue within the same hard deadline to obtain the final meter and settle once.
|
||||
if(!disconnected[0]) try { event(response,chunk); } catch(IOException clientClosed) { disconnected[0]=true; }
|
||||
});
|
||||
}); } finally {billing.observedUsage(ticket,"chat",collector.observedUsage());}
|
||||
var body=collector.result();
|
||||
billing.settle(ticket,body,elapsed(started),collector.firstTokenMs());
|
||||
if(!disconnected[0]) done(response);
|
||||
|
||||
Reference in new issue
Block a user