feat: extend enterprise gateway policies billing protocols and operations
This commit is contained in:
1 parent
1528b71882
commit
af43947b5d
67 files changed
+1822
-69
No files matched your search
@@ -0,0 +1,45 @@
|
||||
package com.moyi.controlplane.gateway;
|
||||
import com.moyi.controlplane.catalog.CatalogService;
|
||||
import com.moyi.controlplane.provider.*;
|
||||
import com.moyi.controlplane.wallet.ChatBillingService;
|
||||
import java.util.*;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class GatewayFailover {
|
||||
private final CatalogService catalog;private final GatewayGuard guard;private final ChatBillingService billing;private final OpenAiCompatibleProvider provider;
|
||||
public GatewayFailover(CatalogService catalog,GatewayGuard guard,ChatBillingService billing,OpenAiCompatibleProvider provider) {this.catalog=catalog;this.guard=guard;this.billing=billing;this.provider=provider;}
|
||||
public List<Map<String,Object>> candidates(Map<String,Object> key,Map<String,Object> primary,String protocol) {
|
||||
var routes=new ArrayList<Map<String,Object>>();routes.add(primary);
|
||||
if(!(primary.get("capabilities") instanceof Map<?,?> cap)||!Boolean.TRUE.equals(cap.get("failover"))||primary.get("logicalModelId")==null) return routes;
|
||||
GatewayModelIds.authorizedModels(catalog,key).stream()
|
||||
.filter(m->!Objects.equals(m.get("id"),primary.get("id"))&&Objects.equals(m.get("logicalModelId"),primary.get("logicalModelId"))&&Objects.equals(m.get("upstreamModel"),primary.get("upstreamModel"))&&Objects.equals(m.get("type"),primary.get("type"))&&!"async".equals(m.get("mode")))
|
||||
.filter(m->"chat".equals(protocol)||NativeProtocolMeter.supports(m,protocol))
|
||||
.filter(m->provider.isConfigured(m.get("providerId").toString(),m.get("id").toString(),m.get("upstreamModel").toString()))
|
||||
.sorted(Comparator.comparingInt(m->m.get("priority") instanceof Number n?n.intValue():0)).limit(2).forEach(routes::add);
|
||||
return routes;
|
||||
}
|
||||
public Map<String,Object> invoke(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
|
||||
var routes=candidates(key,model,protocol);
|
||||
for(int i=0;i<routes.size();i++) {
|
||||
var route=routes.get(i);try(var slot=i==0?null:guard.acquire(route,1)) {
|
||||
if(i>0) billing.actualRoute(ticket,route);
|
||||
try {return "chat".equals(protocol)?provider.chat(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.invoke(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);}
|
||||
catch(UpstreamRejectedException e) {if(i==routes.size()-1) throw e;}
|
||||
}
|
||||
}
|
||||
throw new IllegalStateException("No route");
|
||||
}
|
||||
public OpenAiCompatibleProvider.StreamResponse stream(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
|
||||
var routes=candidates(key,model,protocol);
|
||||
for(int i=0;i<routes.size();i++) {
|
||||
var route=routes.get(i);GatewayGuard.Slot slot=i==0?null:guard.acquire(route,1);
|
||||
try {
|
||||
if(i>0) billing.actualRoute(ticket,route);
|
||||
var response="chat".equals(protocol)?provider.stream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.nativeStream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);
|
||||
if(slot!=null) response.onClose(slot::close);return response;
|
||||
} catch(RuntimeException e) {if(slot!=null) slot.close();if(!(e instanceof UpstreamRejectedException)||i==routes.size()-1) throw e;}
|
||||
}
|
||||
throw new IllegalStateException("No route");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user