45 lines
3.4 KiB
Java
45 lines
3.4 KiB
Java
package com.moyi.controlplane.gateway;
|
|||
|
|
import com.moyi.controlplane.catalog.CatalogService;
|
||
|
|
import com.moyi.controlplane.provider.*;
|
||
|
|
import com.moyi.controlplane.wallet.ChatBillingService;
|
||
|
|
import java.util.*;
|
||
|
|
import org.springframework.stereotype.Service;
|
||
|
|
|
||
|
|
@Service
|
||
|
|
public class GatewayFailover {
|
||
|
|
private final CatalogService catalog;private final GatewayGuard guard;private final ChatBillingService billing;private final OpenAiCompatibleProvider provider;
|
||
|
|
public GatewayFailover(CatalogService catalog,GatewayGuard guard,ChatBillingService billing,OpenAiCompatibleProvider provider) {this.catalog=catalog;this.guard=guard;this.billing=billing;this.provider=provider;}
|
||
|
|
public List<Map<String,Object>> candidates(Map<String,Object> key,Map<String,Object> primary,String protocol) {
|
||
|
|
var routes=new ArrayList<Map<String,Object>>();routes.add(primary);
|
||
|
|
if(!(primary.get("capabilities") instanceof Map<?,?> cap)||!Boolean.TRUE.equals(cap.get("failover"))||primary.get("logicalModelId")==null) return routes;
|
||
|
|
GatewayModelIds.authorizedModels(catalog,key).stream()
|
||
|
|
.filter(m->!Objects.equals(m.get("id"),primary.get("id"))&&Objects.equals(m.get("logicalModelId"),primary.get("logicalModelId"))&&Objects.equals(m.get("upstreamModel"),primary.get("upstreamModel"))&&Objects.equals(m.get("type"),primary.get("type"))&&!"async".equals(m.get("mode")))
|
||
|
|
.filter(m->"chat".equals(protocol)||NativeProtocolMeter.supports(m,protocol))
|
||
|
|
.filter(m->provider.isConfigured(m.get("providerId").toString(),m.get("id").toString(),m.get("upstreamModel").toString()))
|
||
|
|
.sorted(Comparator.comparingInt(m->m.get("priority") instanceof Number n?n.intValue():0)).limit(2).forEach(routes::add);
|
||
|
|
return routes;
|
||
|
|
}
|
||
|
|
public Map<String,Object> invoke(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
|
||
|
|
var routes=candidates(key,model,protocol);
|
||
|
|
for(int i=0;i<routes.size();i++) {
|
||
|
|
var route=routes.get(i);try(var slot=i==0?null:guard.acquire(route,1)) {
|
||
|
|
if(i>0) billing.actualRoute(ticket,route);
|
||
|
|
try {return "chat".equals(protocol)?provider.chat(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.invoke(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);}
|
||
|
|
catch(UpstreamRejectedException e) {if(i==routes.size()-1) throw e;}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
throw new IllegalStateException("No route");
|
||
|
|
}
|
||
|
|
public OpenAiCompatibleProvider.StreamResponse stream(Map<String,Object> key,Map<String,Object> model,String protocol,Map<String,Object> input,ChatBillingService.Ticket ticket) {
|
||
|
|
var routes=candidates(key,model,protocol);
|
||
|
|
for(int i=0;i<routes.size();i++) {
|
||
|
|
var route=routes.get(i);GatewayGuard.Slot slot=i==0?null:guard.acquire(route,1);
|
||
|
|
try {
|
||
|
|
if(i>0) billing.actualRoute(ticket,route);
|
||
|
|
var response="chat".equals(protocol)?provider.stream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input):provider.nativeStream(route.get("providerId").toString(),route.get("upstreamModel").toString(),route.get("id").toString(),input,protocol);
|
||
|
|
if(slot!=null) response.onClose(slot::close);return response;
|
||
|
|
} catch(RuntimeException e) {if(slot!=null) slot.close();if(!(e instanceof UpstreamRejectedException)||i==routes.size()-1) throw e;}
|
||
|
|
}
|
||
|
|
throw new IllegalStateException("No route");
|
||
|
|
}
|
||
|
|
}
|