feat: stream chat and settle metered CNY usage atomically
This commit is contained in:
1 parent
7360cf3ebe
commit
b46ee72f2d
28 files changed
+1098
-86
No files matched your search
+76
-9
@@ -8,10 +8,14 @@ import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
@@ -23,6 +27,8 @@ public class OpenAiCompatibleProvider {
|
||||
private final JdbcTemplate jdbc;
|
||||
private final SecretCipher secretCipher;
|
||||
private final Map<String, ProviderConfig> providers;
|
||||
private final ScheduledExecutorService deadlines = Executors.newSingleThreadScheduledExecutor(r -> { var t=new Thread(r,"upstream-deadlines"); t.setDaemon(true); return t; });
|
||||
private final int timeoutSeconds;
|
||||
|
||||
public OpenAiCompatibleProvider(
|
||||
ObjectMapper objectMapper,
|
||||
@@ -34,6 +40,7 @@ public class OpenAiCompatibleProvider {
|
||||
this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build();
|
||||
this.jdbc = jdbc;
|
||||
this.secretCipher = secretCipher;
|
||||
this.timeoutSeconds = Math.max(1,Math.min(90,environment.getProperty("moyi.gateway.upstream-timeout-seconds",Integer.class,90)));
|
||||
this.providers = Map.of(
|
||||
"provider-deepseek", new ProviderConfig(environment.getProperty("moyi.providers.deepseek.api-key", ""), environment.getProperty("moyi.providers.deepseek.base-url", "https://api.deepseek.com")),
|
||||
"provider-kimi", new ProviderConfig(environment.getProperty("moyi.providers.kimi.api-key", ""), environment.getProperty("moyi.providers.kimi.base-url", "https://api.moonshot.ai/v1")),
|
||||
@@ -46,29 +53,49 @@ public class OpenAiCompatibleProvider {
|
||||
return provider != null && !provider.apiKey().isBlank();
|
||||
}
|
||||
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, List<?> messages) {
|
||||
public Map<String, Object> chat(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
try(var response=open(providerId,upstreamModel,input,false)) {
|
||||
byte[] bytes=response.body.readNBytes(4*1024*1024+1);
|
||||
if(bytes.length>4*1024*1024) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
return objectMapper.readValue(bytes, new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||
} catch(ApiException e) { throw e; }
|
||||
catch(Exception e) { throw upstreamError(); }
|
||||
}
|
||||
|
||||
public StreamResponse stream(String providerId, String upstreamModel, Map<String,Object> input) {
|
||||
return open(providerId,upstreamModel,input,true);
|
||||
}
|
||||
|
||||
private StreamResponse open(String providerId, String upstreamModel, Map<String,Object> input, boolean streaming) {
|
||||
ProviderConfig provider = resolve(providerId);
|
||||
if (provider == null || provider.apiKey().isBlank()) {
|
||||
throw new ApiException("PROVIDER_NOT_CONFIGURED", "该模型的供应商尚未配置服务端 API Key", 503);
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
Map<String, Object> body = new LinkedHashMap<>(input);
|
||||
body.put("model", upstreamModel);
|
||||
body.put("messages", messages == null ? List.of() : messages);
|
||||
body.put("stream", false);
|
||||
body.put("stream", streaming);
|
||||
if(streaming) body.put("stream_options",Map.of("include_usage",true));
|
||||
else body.remove("stream_options");
|
||||
String json = writeJson(body);
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(normalizeUrl(provider.baseUrl()) + "/chat/completions"))
|
||||
.timeout(Duration.ofSeconds(90))
|
||||
.timeout(Duration.ofSeconds(timeoutSeconds))
|
||||
.header("Authorization", "Bearer " + provider.apiKey())
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json))
|
||||
.build();
|
||||
try {
|
||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
long started=System.nanoTime();
|
||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
if (response.statusCode() < 200 || response.statusCode() >= 300) {
|
||||
throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游模型供应商请求失败", 502);
|
||||
response.body().close();
|
||||
throw upstreamError();
|
||||
}
|
||||
return objectMapper.readValue(response.body(), new com.fasterxml.jackson.core.type.TypeReference<>() {});
|
||||
if(streaming && !response.headers().firstValue("Content-Type").orElse("").toLowerCase().startsWith("text/event-stream")) {
|
||||
response.body().close();
|
||||
throw new ApiException("UPSTREAM_STREAM_UNSUPPORTED","上游未返回 SSE 流式响应",502);
|
||||
}
|
||||
return new StreamResponse(response.body(),Math.max(1,timeoutSeconds*1000L-TimeUnit.NANOSECONDS.toMillis(System.nanoTime()-started)));
|
||||
} catch (ApiException exception) {
|
||||
throw exception;
|
||||
} catch (InterruptedException exception) {
|
||||
@@ -79,6 +106,46 @@ public class OpenAiCompatibleProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/** A hard body deadline is required: HttpRequest.timeout alone does not cover InputStream reads. */
|
||||
public final class StreamResponse implements AutoCloseable {
|
||||
private final InputStream body;
|
||||
private final AtomicBoolean expired=new AtomicBoolean();
|
||||
private final ScheduledFuture<?> deadline;
|
||||
private StreamResponse(InputStream body,long remainingMs) {
|
||||
this.body=body;
|
||||
this.deadline=deadlines.schedule(() -> { expired.set(true); try { body.close(); } catch(IOException ignored) {} },remainingMs,TimeUnit.MILLISECONDS);
|
||||
}
|
||||
public void events(EventConsumer consumer) throws IOException {
|
||||
var reader=new BufferedReader(new InputStreamReader(body,StandardCharsets.UTF_8));
|
||||
StringBuilder event=new StringBuilder(), line=new StringBuilder();
|
||||
long total=0;
|
||||
int c;
|
||||
// Bound both individual frames and the whole response before allocating unbounded strings.
|
||||
while((c=reader.read())!=-1) {
|
||||
if(expired.get()) throw new ApiException("UPSTREAM_PROVIDER_TIMEOUT","上游响应超时,本次未扣费",504);
|
||||
if(++total>4*1024*1024 || line.length()>262144 || event.length()>262144) throw new ApiException("UPSTREAM_RESPONSE_TOO_LARGE","上游响应超过限制",502);
|
||||
if(c=='\n') {
|
||||
String text=line.toString(); line.setLength(0);
|
||||
if(text.endsWith("\r")) text=text.substring(0,text.length()-1);
|
||||
if(text.isEmpty()) {
|
||||
if(!event.isEmpty()) {
|
||||
String data=event.substring(0,event.length()-1); event.setLength(0);
|
||||
if("[DONE]".equals(data)) return;
|
||||
Map<String,Object> chunk=objectMapper.readValue(data,new com.fasterxml.jackson.core.type.TypeReference<>(){});
|
||||
if(chunk.containsKey("error")) throw upstreamError();
|
||||
consumer.accept(chunk);
|
||||
}
|
||||
} else if(text.startsWith("data:")) event.append(text.substring(5).stripLeading()).append('\n');
|
||||
} else line.append((char)c);
|
||||
}
|
||||
throw new ApiException(expired.get()?"UPSTREAM_PROVIDER_TIMEOUT":"UPSTREAM_STREAM_INCOMPLETE","上游流式响应中断,本次未扣费",502);
|
||||
}
|
||||
@Override public void close() { deadline.cancel(false); try { body.close(); } catch(IOException ignored) {} }
|
||||
}
|
||||
@FunctionalInterface public interface EventConsumer { void accept(Map<String,Object> chunk) throws IOException; }
|
||||
@PreDestroy public void close() { deadlines.shutdownNow(); }
|
||||
private ApiException upstreamError() { return new ApiException("UPSTREAM_PROVIDER_ERROR","上游模型供应商请求失败",502); }
|
||||
|
||||
private String normalizeUrl(String value) {
|
||||
return value == null ? "" : value.replaceAll("/+$", "");
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user