From 5be06a97c1c80fecac00f4b5a4eb2a08ef2b5398 Mon Sep 17 00:00:00 2001 From: xpp Date: Wed, 21 Oct 2026 18:57:35 +0800 Subject: [PATCH] fix: retain async holds for unknown provider outcomes --- control-plane-java/README.md | 4 +- .../controlplane/catalog/CatalogService.java | 1 + .../transport/ProviderRequestException.java | 44 +++++ .../task/AsyncProviderClient.java | 78 +++++---- .../controlplane/task/AsyncTaskService.java | 109 +++++++----- .../controlplane/task/TaskOutcomeJournal.java | 91 ++++++++++ .../controlplane/usage/UsageAuditService.java | 14 +- .../V46__async_task_reconciliation.sql | 16 ++ .../task/AsyncProviderClientTest.java | 69 +++++++- .../task/AsyncTaskPostgresTest.java | 165 +++++++++++++++++- .../usage/UsageCapacityPostgresTest.java | 4 +- docs/21-api-request-modularization.md | 6 +- docs/22-async-task-safety.md | 48 ++++- prototype/src/components/UsageHistory.vue | 4 +- prototype/src/services/catalog.ts | 10 +- prototype/src/types.ts | 2 +- prototype/src/views/AdminConsole.vue | 4 +- prototype/src/views/UserConsole.vue | 13 +- 18 files changed, 574 insertions(+), 108 deletions(-) create mode 100644 control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/ProviderRequestException.java create mode 100644 control-plane-java/src/main/java/com/moyi/controlplane/task/TaskOutcomeJournal.java create mode 100644 control-plane-java/src/main/resources/db/migration/V46__async_task_reconciliation.sql diff --git a/control-plane-java/README.md b/control-plane-java/README.md index a30da0c..922c763 100644 --- a/control-plane-java/README.md +++ b/control-plane-java/README.md @@ -15,7 +15,9 @@ 协议字段映射留在各自 Meter/Adapter;`ApiRequestRiskRule` 只能读取最小化上下文并返回决策,不能调用 Provider 或改账。新增协议时应复用准入、风控和账务边界,而不是在 Controller 中另写一套钱包/重试流程。默认 `GATEWAY_RISK_MODE=shadow` 仅记录规则 ID,不改变调用;只有显式部署规则并设置 `enforce` 才会拒绝或限速。请求来源由服务端路由确定,不能信任 `X-Moyi-Source` 作为风控分类。 -账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;失败释放和成功结算沿用现有事务处理;不完整流或无效用量不得结算。上游提交结果未知时的异步核账状态机与异步创建幂等将在后续阶段单独处理。 +账务不变量:风险拒绝必须先于并发准入和钱包预留;预留使用价格快照;已确认失败的释放和成功结算沿用原子事务;不完整流或无效用量不得估算扣费。 + +异步创建支持用户级 `Idempotency-Key`(`V45`),同键同请求返回原任务,同键不同请求返回 409。异步上游受理未知、回执丢失、worker 中断或轮询期限到期时进入 `reconciling`(`V46`),保留冻结与预算占用,不自动重发/退款。已观察的生成结果先持久化再结算,账本失败只重试本地结算。发布时须先停旧 worker,不能混跑旧的未知结果退款逻辑。详见 `docs/22-async-task-safety.md`;受审计的人工核账入口仍属下一阶段。 同步和原生调用在结算前单独提交精简用量快照(Flyway `V44`)。快照已持久化而账本事务失败时,接口返回 `SETTLEMENT_PENDING`,冻结额度保留,由后台恢复任务重试结算。该快照不保存提示词、生成内容或密钥;不能将其当作响应重放存储。若数据库在快照写入前已不可用,则尚不能保证结果的持久恢复,必须结合上游记录核对。 diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java index 0e4f77d..a1a2327 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java @@ -383,6 +383,7 @@ public class CatalogService { result.put("tasks", Map.of( "queued", taskCounts.getOrDefault("queued", 0L), "running", taskCounts.getOrDefault("running", 0L), + "reconciling", taskCounts.getOrDefault("reconciling", 0L), "succeeded", taskCounts.getOrDefault("succeeded", 0L), "failed", taskCounts.getOrDefault("failed", 0L) )); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/ProviderRequestException.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/ProviderRequestException.java new file mode 100644 index 0000000..bbaeea8 --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/ProviderRequestException.java @@ -0,0 +1,44 @@ +package com.moyi.controlplane.provider.transport; + +import com.moyi.controlplane.common.ApiException; + +/** HTTP boundary facts, not a retry policy. Unknown acceptance must never authorize replay/refund. */ +public final class ProviderRequestException extends ApiException { + public enum Acceptance { NOT_SENT, REJECTED, UNKNOWN } + + private final Acceptance acceptance; + private final Integer upstreamStatus; + + private ProviderRequestException(String code, String message, int status, Acceptance acceptance, Integer upstreamStatus) { + super(code, message, status); + this.acceptance = acceptance; + this.upstreamStatus = upstreamStatus; + } + + public static ProviderRequestException notSent(String code, String message, int status) { + return new ProviderRequestException(code, message, status, Acceptance.NOT_SENT, null); + } + + public static ProviderRequestException unknown(String code, String message) { + return new ProviderRequestException(code, message, 502, Acceptance.UNKNOWN, null); + } + + public static ProviderRequestException http(int status, String message) { + String code = switch (status) { + case 401, 403 -> "UPSTREAM_AUTH_FAILED"; + case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; + case 429 -> "UPSTREAM_RATE_LIMITED"; + default -> "UPSTREAM_PROVIDER_ERROR"; + }; + // Timeout/conflict/5xx responses do not prove a generation was never accepted. + Acceptance acceptance = switch (status) { + case 400, 401, 402, 403, 404, 405, 413, 415, 422, 429 -> Acceptance.REJECTED; + default -> Acceptance.UNKNOWN; + }; + return new ProviderRequestException(code, message + "(HTTP " + status + ")", 502, acceptance, status); + } + + public Acceptance acceptance() { return acceptance; } + public Integer upstreamStatus() { return upstreamStatus; } + public boolean definitelyNotAccepted() { return acceptance != Acceptance.UNKNOWN; } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java index 905b286..4303579 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.security.SecretCipher; import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver; +import com.moyi.controlplane.provider.transport.ProviderRequestException; import java.net.URI; import java.net.InetAddress; import java.net.http.HttpClient; @@ -60,6 +61,22 @@ public class AsyncProviderClient { } public Map submit(String providerId, String upstreamModel, String routeId, String taskType, Map input) { + try { + return submitConfigured(providerId, upstreamModel, routeId, taskType, input); + } catch (ProviderRequestException exception) { + throw exception; + } catch (ApiException exception) { + // Only these local validation/configuration errors are known to precede generation submission. + if (List.of("PROVIDER_NOT_CONFIGURED", "UPSTREAM_KEY_NOT_AVAILABLE", "ASYNC_PROVIDER_NOT_IMPLEMENTED", + "MEDIA_INPUT_UNSUPPORTED", "INVALID_MEDIA_URL", "MEDIA_FETCH_FAILED", "MEDIA_TOO_LARGE", + "MEDIA_TYPE_UNSUPPORTED", "INVALID_TASK_INPUT").contains(exception.getCode())) { + throw ProviderRequestException.notSent(exception.getCode(), exception.getMessage(), exception.getStatus()); + } + throw ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", "上游任务提交结果尚未确认"); + } + } + + private Map submitConfigured(String providerId, String upstreamModel, String routeId, String taskType, Map input) { if (input == null) input = Map.of(); String prompt = String.valueOf(input.getOrDefault("prompt", "")); Provider provider = provider(providerId, routeId, upstreamModel); @@ -261,8 +278,8 @@ public class AsyncProviderClient { } private boolean isNotFound(ApiException exception) { - String message = exception.getMessage(); - return message != null && (message.contains("HTTP 404") || message.contains("HTTP 405")); + return exception instanceof ProviderRequestException failure + && (Integer.valueOf(404).equals(failure.upstreamStatus()) || Integer.valueOf(405).equals(failure.upstreamStatus())); } /** Normalize OpenAI image/audio response fields to the task result shape. */ @@ -287,19 +304,14 @@ public class AsyncProviderClient { } private Map requestAudio(String method, String url, String key, Map body, String message) { + boolean sending = false; try { HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT) .header("Authorization", "Bearer " + key).header("Content-Type", "application/json"); - HttpResponse response = client.send(builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(), HttpResponse.BodyHandlers.ofByteArray()); - if (response.statusCode() < 200 || response.statusCode() >= 300) { - String code = switch (response.statusCode()) { - case 401, 403 -> "UPSTREAM_AUTH_FAILED"; - case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; - case 429 -> "UPSTREAM_RATE_LIMITED"; - default -> "UPSTREAM_PROVIDER_ERROR"; - }; - throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502); - } + HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(); + sending = true; + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); byte[] bytes = response.body(); if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502); String contentType = response.headers().firstValue("Content-Type").orElse("audio/mpeg").split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT); @@ -309,8 +321,7 @@ public class AsyncProviderClient { result.put("mimeType", contentType); result.put("format", String.valueOf(body.getOrDefault("response_format", "mp3"))); return result; - } catch (ApiException exception) { throw exception; } - catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } + } catch (Exception exception) { throw requestFailure(exception, sending, message); } } /** @@ -322,6 +333,7 @@ public class AsyncProviderClient { */ private Map requestMultipart(String method, String url, String key, Map fields, String fileField, String media, String message) { + boolean sending = false; try { String boundary = "----moyi-" + java.util.UUID.randomUUID(); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -337,6 +349,7 @@ public class AsyncProviderClient { .header("Authorization", "Bearer " + key) .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build(); + sending = true; HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); @@ -345,18 +358,19 @@ public class AsyncProviderClient { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } return parsed; - } catch (ApiException exception) { throw exception; } - catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } + } catch (Exception exception) { throw requestFailure(exception, sending, message); } } private ApiException upstreamStatus(int status, String message) { - String code = switch (status) { - case 401, 403 -> "UPSTREAM_AUTH_FAILED"; - case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; - case 429 -> "UPSTREAM_RATE_LIMITED"; - default -> "UPSTREAM_PROVIDER_ERROR"; - }; - return new ApiException(code, message + "(HTTP " + status + ")", 502); + return ProviderRequestException.http(status, message); + } + + private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) { + if (exception instanceof InterruptedException) Thread.currentThread().interrupt(); + if (exception instanceof ProviderRequestException failure) return failure; + if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message); + if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus()); + return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502); } private void multipartField(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException { @@ -414,21 +428,16 @@ public class AsyncProviderClient { private record MediaBytes(byte[] bytes, String contentType, String filename) {} private Map request(String method, String url, String key, Map body, String message) { + boolean sending = false; try { HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(url)).timeout(UPSTREAM_REQUEST_TIMEOUT) .header("Authorization", "Bearer " + key).header("Content-Type", "application/json"); if ("POST".equals(method)) builder.POST(HttpRequest.BodyPublishers.ofString(write(body))); else builder.GET(); - HttpResponse response = client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() < 200 || response.statusCode() >= 300) { - String code = switch (response.statusCode()) { - case 401, 403 -> "UPSTREAM_AUTH_FAILED"; - case 402 -> "UPSTREAM_CREDIT_INSUFFICIENT"; - case 429 -> "UPSTREAM_RATE_LIMITED"; - default -> "UPSTREAM_PROVIDER_ERROR"; - }; - throw new ApiException(code, message + "(HTTP " + response.statusCode() + ")", 502); - } + HttpRequest request = builder.build(); + sending = true; + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); Object code = parsed.get("code"); if (parsed.containsKey("error") || code instanceof Number number && number.intValue() != 0) { @@ -437,8 +446,7 @@ public class AsyncProviderClient { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } return parsed; - } catch (ApiException exception) { throw exception; } - catch (Exception exception) { throw new ApiException("UPSTREAM_PROVIDER_ERROR", message, 502); } + } catch (Exception exception) { throw requestFailure(exception, sending, message); } } private Provider provider(String id) { diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java index ff536b9..4edbe15 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java @@ -13,6 +13,7 @@ import com.moyi.controlplane.gateway.request.ApiRequestProtocol; import com.moyi.controlplane.gateway.request.ApiRequestRiskEngine; import com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy; import com.moyi.controlplane.usage.UsageAuditService; +import com.moyi.controlplane.provider.transport.ProviderRequestException; import java.sql.ResultSet; import java.sql.SQLException; import java.time.OffsetDateTime; @@ -35,6 +36,7 @@ import org.springframework.transaction.annotation.Transactional; @Service public class AsyncTaskService { + private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(AsyncTaskService.class); private final JdbcTemplate jdbc; private final ObjectMapper objectMapper; private final CatalogService catalogService; @@ -49,6 +51,7 @@ public class AsyncTaskService { private final TransactionTemplate transactions; private final com.moyi.controlplane.gateway.EnterprisePolicies policies; private final ApiRequestRiskPolicy requestRisk; + private final TaskOutcomeJournal outcomes; /** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */ public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, @@ -104,6 +107,7 @@ public class AsyncTaskService { this.transactions = transactionManager == null ? null : new TransactionTemplate(transactionManager); this.policies = new com.moyi.controlplane.gateway.EnterprisePolicies(jdbc,objectMapper); this.requestRisk = java.util.Objects.requireNonNull(requestRisk, "requestRisk"); + this.outcomes = new TaskOutcomeJournal(jdbc, objectMapper, transactions, usage); } @Transactional @@ -177,7 +181,7 @@ public class AsyncTaskService { // model. Without the transaction advisory lock, concurrent creates // could all observe the same count and exceed maxQueuedPerModel. jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-queue:' || ?, 0))", modelId); - Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId); + Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running', 'reconciling')", Integer.class, modelId); if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429); @SuppressWarnings("unchecked") Map frozenPricing=(Map) model.get("pricing"); if(AsyncPriceSchedule.tokens(frozenPricing) && !"image".equals(taskType)) throw new ApiException("ASYNC_BILLING_UNSUPPORTED","该媒体类型的 Token 用量结算尚未接入",503); @@ -237,6 +241,11 @@ public class AsyncTaskService { @Scheduled(fixedDelayString = "${moyi.tasks.worker-interval-ms:1000}") public void completeMockTasks() { + // Retry local settlement from durable observations, never the upstream generation request. + for (var completion : outcomes.pendingCompletions()) { + try { completeObserved(completion); } + catch (RuntimeException exception) { log.warn("Async task {} settlement remains pending", completion.taskId()); } + } List taskIds = jdbc.query("SELECT id FROM model_tasks WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", (rs, row) -> rs.getString("id")); jdbc.update( "UPDATE model_tasks SET status = 'succeeded', progress = 100, result = CAST(CASE WHEN task_type = '3d' THEN ? ELSE ? END AS jsonb), updated_at = ?, completed_at = ? WHERE status IN ('queued', 'running') AND provider_task_id = 'mock' AND created_at < now() - INTERVAL '500 milliseconds'", @@ -244,7 +253,7 @@ public class AsyncTaskService { "{\"format\":\"mp4\",\"url\":\"https://mock.moyi.ai/tasks/result.mp4\"}", now(), now() ); taskIds.forEach(taskId -> usage.updateTask(taskId, "succeeded", Map.of("format", "result", "url", "https://mock.moyi.ai/tasks/" + taskId + "/result"), null, null)); - List> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50"); + List> pending = jdbc.queryForList("SELECT t.id, t.model_id, t.provider_task_id, t.provider_key_fingerprint, t.provider_deadline_at, t.created_at, m.provider_id, m.upstream_model FROM model_tasks t JOIN model_catalog m ON m.id=t.model_id WHERE t.status IN ('queued','running') AND t.completion_status IS NULL AND t.provider_task_id <> 'mock' AND (t.updated_at < now() - INTERVAL '5 seconds' OR t.provider_deadline_at <= now()) ORDER BY t.updated_at LIMIT 50"); for (Map row : pending) pollProviderTask(row); dispatchQueuedTasks(); } @@ -257,36 +266,43 @@ public class AsyncTaskService { */ private void dispatchQueuedTasks() { if (providerClient == null) return; - // A process can die after claiming a task but before receiving an - // upstream task id. Fail those claims after a grace period. Retrying - // them blindly could submit the same generation twice when the - // upstream accepted the request just before the process died. + // A lost submit response is ambiguous, not a failed generation. + // Quarantine stale claims without retrying or releasing financial holds. recoverInterruptedClaims(); for (int i = 0; i < 50; i++) { Map task = claimQueuedTask(); if (task == null) return; String taskId = String.valueOf(task.get("id")); - usage.updateTask(taskId, "running", null, null, null); + Map upstream; try { Map input = taskInput(task); - Map upstream = submitProviderTask(task, input); - // Some OpenAI-compatible image/audio endpoints return the - // generated media in the submit response rather than creating - // a pollable job. Persist that result and settle the task in - // the same worker transaction. + upstream = submitProviderTask(task, input); + } catch (RuntimeException exception) { + if (exception instanceof ProviderRequestException failure && failure.definitelyNotAccepted()) { + completeObserved(outcomes.observeTerminal(taskId, "failed", null, failure.getCode(), failure.getMessage())); + } else { + outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_ACCEPTANCE_UNKNOWN", "上游任务是否已受理尚未确认;冻结额度保留,请联系管理员核账,勿重复提交", false); + } + continue; + } + TaskOutcomeJournal.Completion completion = null; + try { + // Journal a received result before touching the ledger. A failed settlement + // must only retry locally, never generate again or release its reservation. if (upstream.get("_moyiInlineResult") instanceof Map inline) { Map result = new LinkedHashMap<>(); inline.forEach((key, value) -> result.put(String.valueOf(key), value)); - finishTask(taskId, "succeeded", 100, result, null, null); - continue; + completion = outcomes.observeTerminal(taskId, "succeeded", result, null, null); + } else { + OffsetDateTime submittedAt = now(); + outcomes.accepted(taskId, providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), + submittedAt, submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs))); } - OffsetDateTime submittedAt = now(); - jdbc.update("UPDATE model_tasks SET provider_task_id=?, provider_key_fingerprint=?, provider_deadline_at=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), upstream.get("_moyiUpstreamKeyFingerprint"), submittedAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)), submittedAt, taskId); } catch (RuntimeException exception) { - String code = exception instanceof ApiException error ? error.getCode() : "UPSTREAM_TASK_SUBMIT_FAILED"; - String message = exception instanceof ApiException error ? error.getMessage() : "上游异步任务创建失败"; - failTask(taskId, code, message); + outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_RECEIPT_UNKNOWN", "已调用上游,但任务回执无法确认或保存;冻结额度保留,等待核账,勿重复提交", false); } + // Deliberately outside the provider exception handler. + completeObserved(completion); } } @@ -306,18 +322,18 @@ public class AsyncTaskService { AND t.provider_task_id IS NULL AND m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL - AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status='running') < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16) + AND (SELECT count(*) FROM model_tasks ru WHERE ru.user_id=t.user_id AND ru.status IN ('running','reconciling')) < coalesce((SELECT max_concurrency FROM gateway_user_policies up WHERE up.user_id=t.user_id),16) AND ( ? <= 0 OR COALESCE(m.max_concurrency, ?) <= 0 OR (SELECT COUNT(*) FROM model_tasks running_model - WHERE running_model.model_id=t.model_id AND running_model.status='running') + WHERE running_model.model_id=t.model_id AND running_model.status IN ('running','reconciling')) < COALESCE(m.max_concurrency, ?) ) AND ( ? <= 0 OR (SELECT COUNT(*) FROM model_tasks running_key - WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running') + WHERE running_key.api_key_id=t.api_key_id AND running_key.status IN ('running','reconciling')) < ? ) ORDER BY t.created_at @@ -332,19 +348,26 @@ public class AsyncTaskService { if(!gatewayGuard.tryTaskUserSlot((UUID)row.get("api_key_id"),row.get("id").toString(),providerTimeoutMs+180000)) return null; try {jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));} catch(RuntimeException e) {gatewayGuard.releaseTaskUserSlot((UUID)row.get("user_id"),row.get("id").toString());throw e;} + usage.updateTask(row.get("id").toString(), "running", null, null, null); return row; }); } private void recoverInterruptedClaims() { List taskIds = inTransaction(() -> jdbc.queryForList( - "SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED", + "SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND completion_status IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED", String.class)); for (String taskId : taskIds) { - failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交"); + outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游提交回执丢失或进程中断;冻结额度保留,等待核账,勿重复提交", true); } } + private void completeObserved(TaskOutcomeJournal.Completion completion) { + if (completion == null) return; + finishTask(completion.taskId(), completion.status(), "succeeded".equals(completion.status()) ? 100 : 0, + completion.result(), completion.code(), completion.message()); + } + private void failTask(String taskId, String code, String message) { finishTask(taskId, "failed", 0, null, code, message); } @@ -354,8 +377,10 @@ public class AsyncTaskService { // crash after marking failed/succeeded leaves a hold that polling never // revisits. The conditional update also prevents duplicate settlement. inTransaction(() -> { - Map task=jdbc.queryForMap("SELECT status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId); - if(!List.of("queued","running").contains(task.get("status"))) return null; + Map task=jdbc.queryForMap("SELECT status,completion_status,pricing_snapshot::text AS pricing_snapshot FROM model_tasks WHERE id=? FOR UPDATE",taskId); + if(!List.of("queued","running","reconciling").contains(task.get("status"))) return null; + // A late failure must not override a durably observed success whose settlement is pending. + if (task.get("completion_status") != null && !status.equals(task.get("completion_status"))) return null; AsyncPriceSchedule.Bill bill=null; String finalCode=code,finalMessage=message; if("succeeded".equals(status) && task.get("pricing_snapshot")!=null) { @@ -363,7 +388,7 @@ public class AsyncTaskService { catch(ApiException e) { finalCode="TASK_USAGE_PENDING"; finalMessage="生成已完成,但上游计费用量未能核对;暂不扣费,等待核账"; } } int changed = jdbc.update( - "UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", + "UPDATE model_tasks SET status=?, progress=?, result=CAST(? AS jsonb), error=CAST(? AS jsonb), reconciliation_required_at=NULL, updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running','reconciling')", status, progress, result == null ? null : writeJson(result), finalCode == null ? null : writeJson(Map.of("code", finalCode, "message", finalMessage)), now(), now(), taskId); if (changed == 0) return null; @@ -397,6 +422,7 @@ public class AsyncTaskService { task.put("progress", resultSet.getInt("progress")); task.put("createdAt", resultSet.getObject("created_at", OffsetDateTime.class)); task.put("updatedAt", resultSet.getObject("updated_at", OffsetDateTime.class)); + task.put("requiresReconciliation", "reconciling".equals(resultSet.getString("status"))); putIfPresent(task, "costAmount", resultSet, "cost_amount"); String result = resultSet.getString("result"); if (result != null) task.put("result", readMap(result)); @@ -440,9 +466,12 @@ public class AsyncTaskService { private String providerTaskId(Map response) { Object direct = response.get("id"); - if (direct == null && response.get("data") instanceof Map data) direct = data.get("task_id"); - if (direct == null) throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回异步任务编号", 502); - return String.valueOf(direct); + if (direct == null) direct = response.get("task_id"); + if (direct == null && response.get("data") instanceof Map data) direct = data.get("task_id") == null ? data.get("id") : data.get("task_id"); + if (!(direct instanceof String value) || value.isBlank() || value.length() > 160 || value.chars().anyMatch(Character::isISOControl)) { + throw new ApiException("UPSTREAM_PROVIDER_ERROR", "上游没有返回有效的异步任务编号", 502); + } + return value; } private void pollProviderTask(Map row) { @@ -456,7 +485,7 @@ public class AsyncTaskService { deadline = createdAt.plus(java.time.Duration.ofMillis(providerTimeoutMs)); } if (!now().isBefore(deadline)) { - failTask(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;平台冻结额度已释放。上游可能仍在执行,请先核实后再提交"); + outcomes.requireReconciliation(taskId, "UPSTREAM_TASK_TIMEOUT", "异步任务查询已超时,执行结果未知;冻结额度保留,等待管理员核账,勿重复提交", false); return; } Map response; @@ -464,7 +493,7 @@ public class AsyncTaskService { response = providerClient.poll(String.valueOf(row.get("provider_id")), String.valueOf(row.get("model_id")), String.valueOf(row.get("upstream_model")), String.valueOf(row.get("provider_task_id")), row.get("provider_key_fingerprint") == null ? null : String.valueOf(row.get("provider_key_fingerprint"))); } catch (RuntimeException exception) { if (exception instanceof ApiException error && "UPSTREAM_KEY_NOT_AVAILABLE".equals(error.getCode())) { - failTask(taskId, error.getCode(), error.getMessage()); + outcomes.requireReconciliation(taskId, error.getCode(), "提交任务使用的上游凭据已不可用,执行结果未知;冻结额度保留,等待核账", false); return; } // Even a permanent authentication/configuration error only proves @@ -487,9 +516,9 @@ public class AsyncTaskService { if ("succeeded".equals(status) || "failed".equals(status) || "cancelled".equals(status)) { // Persistence failures must roll back and remain pollable, rather // than being mistaken for upstream failures and refunded. - finishTask(taskId, status, progress, result, + completeObserved(outcomes.observeTerminal(taskId, status, result, "failed".equals(status) ? "UPSTREAM_TASK_FAILED" : null, - "failed".equals(status) ? "上游异步任务失败" : null); + "failed".equals(status) ? "上游异步任务失败" : null)); } else { updatePollingState(taskId, progress, null, null); } @@ -497,7 +526,7 @@ public class AsyncTaskService { private void updatePollingState(String taskId, Integer progress, String code, String message) { inTransaction(() -> { - int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running')", + int changed = jdbc.update("UPDATE model_tasks SET status='running', progress=COALESCE(?,progress), error=CAST(? AS jsonb), updated_at=? WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL", progress, code == null ? null : writeJson(Map.of("code", code, "message", message, "retryable", true)), now(), taskId); if (changed > 0) usage.updateTask(taskId, "running", null, code, message); return null; @@ -515,10 +544,12 @@ public class AsyncTaskService { Object raw = response.get("status"); if (raw == null && response.get("data") instanceof Map data) raw = data.get("status"); String value = String.valueOf(raw == null ? "running" : raw).toLowerCase(java.util.Locale.ROOT); - if (value.contains("success") || value.equals("completed")) return "succeeded"; - if (value.contains("fail") || value.equals("error") || value.equals("banned")) return "failed"; - if (value.contains("cancel")) return "cancelled"; - return "running"; + return switch (value) { + case "success", "succeeded", "completed" -> "succeeded"; + case "failed", "failure", "error", "banned" -> "failed"; + case "cancelled", "canceled" -> "cancelled"; + default -> "running"; + }; } private int progress(Map response, String status) { diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/TaskOutcomeJournal.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/TaskOutcomeJournal.java new file mode 100644 index 0000000..7afac74 --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/TaskOutcomeJournal.java @@ -0,0 +1,91 @@ +package com.moyi.controlplane.task; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.moyi.controlplane.usage.UsageAuditService; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionTemplate; + +/** Durable provider observations only. No upstream calls, wallet mutation, retry or refund policy. */ +final class TaskOutcomeJournal { + record Completion(String taskId, String status, Map result, String code, String message) { } + + private final JdbcTemplate jdbc; + private final ObjectMapper mapper; + private final TransactionTemplate transactions; + private final UsageAuditService usage; + + TaskOutcomeJournal(JdbcTemplate jdbc, ObjectMapper mapper, TransactionTemplate transactions, UsageAuditService usage) { + this.jdbc = jdbc; + this.mapper = mapper; + this.transactions = transactions; + this.usage = usage; + } + + Completion observeTerminal(String taskId, String status, Map result, String code, String message) { + if (!List.of("succeeded", "failed", "cancelled").contains(status)) throw new IllegalArgumentException("Terminal outcome required"); + return transaction(() -> { + var row = jdbc.queryForMap("SELECT status,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE id=? FOR UPDATE", taskId); + if (!List.of("queued", "running", "reconciling").contains(row.get("status"))) return null; + if (row.get("completion_status") != null) return completion(taskId, row); + jdbc.update("UPDATE model_tasks SET completion_status=?,result=CAST(? AS jsonb),error=CAST(? AS jsonb),updated_at=now() WHERE id=?", + status, json(result), code == null ? null : json(Map.of("code", code, "message", message)), taskId); + return new Completion(taskId, status, result, code, message); + }); + } + + List pendingCompletions() { + return jdbc.queryForList("SELECT id,completion_status,result::text AS result,error::text AS error FROM model_tasks WHERE status IN ('running','reconciling') AND completion_status IS NOT NULL ORDER BY updated_at LIMIT 50") + .stream().map(row -> completion(String.valueOf(row.get("id")), row)).toList(); + } + + void accepted(String taskId, String providerTaskId, Object keyFingerprint, OffsetDateTime submittedAt, OffsetDateTime deadline) { + transaction(() -> { + int changed = jdbc.update("UPDATE model_tasks SET status='running',provider_task_id=?,provider_key_fingerprint=?,provider_deadline_at=?,error=NULL,reconciliation_required_at=NULL,updated_at=? WHERE id=? AND status IN ('running','reconciling') AND provider_task_id IS NULL AND completion_status IS NULL", + providerTaskId, keyFingerprint, deadline, submittedAt, taskId); + if (changed > 0) usage.updateTask(taskId, "running", null, null, null); + return null; + }); + } + + void requireReconciliation(String taskId, String code, String message, boolean interruptedClaim) { + transaction(() -> { + String claimGuard = interruptedClaim + ? " AND provider_task_id IS NULL AND updated_at < now()-interval '2 minutes'" : ""; + int changed = jdbc.update("UPDATE model_tasks SET status='reconciling',error=CAST(? AS jsonb),reconciliation_required_at=now(),updated_at=now() WHERE id=? AND status IN ('queued','running') AND completion_status IS NULL" + claimGuard, + json(Map.of("code", code, "message", message, "retryable", false, "requiresReconciliation", true)), taskId); + if (changed > 0) { + usage.updateTask(taskId, "reconciling", null, code, message); + jdbc.update("UPDATE usage_events SET cost_status='pending',cost_amount=NULL WHERE provider_task_id=? AND event_kind='task'", taskId); + } + return null; + }); + } + + private Completion completion(String taskId, Map row) { + Map error = map(row.get("error")); + return new Completion(taskId, String.valueOf(row.get("completion_status")), map(row.get("result")), + error == null ? null : (String) error.get("code"), error == null ? null : (String) error.get("message")); + } + + private Map map(Object value) { + if (value == null) return null; + try { return mapper.readValue(String.valueOf(value), new TypeReference<>() {}); } + catch (JsonProcessingException exception) { throw new IllegalStateException("Invalid persisted provider observation"); } + } + + private String json(Object value) { + if (value == null) return null; + try { return mapper.writeValueAsString(value); } + catch (JsonProcessingException exception) { throw new IllegalStateException("Provider observation cannot be encoded"); } + } + + private T transaction(Supplier work) { + return transactions == null ? work.get() : transactions.execute(status -> work.get()); + } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java index c88b036..884c3ce 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java @@ -157,15 +157,17 @@ public class UsageAuditService { WITH pending AS ( SELECT model_id, COUNT(*) FILTER (WHERE status='running') AS running, - COUNT(*) FILTER (WHERE status='queued') AS queued - FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id + COUNT(*) FILTER (WHERE status='queued') AS queued, + COUNT(*) FILTER (WHERE status='reconciling') AS reconciling + FROM model_tasks WHERE status IN ('queued','running','reconciling') GROUP BY model_id ) SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode, CASE WHEN m.max_concurrency = 0 THEN 0 ELSE COALESCE(m.max_concurrency, ?) END AS max_concurrency, (m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled, - COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued + COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued, + COALESCE(pending.reconciling, 0) AS reconciling FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id JOIN model_definitions d ON d.id=m.logical_model_id LEFT JOIN pending ON pending.model_id=m.id @@ -182,6 +184,7 @@ public class UsageAuditService { group.put("limit", modelGuardEnabled ? rs.getInt("max_concurrency") : 0); group.put("running", rs.getLong("running")); group.put("queued", rs.getLong("queued")); + group.put("reconciling", rs.getLong("reconciling")); return group; }, defaultModelLimit); List synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList(); @@ -193,12 +196,13 @@ public class UsageAuditService { group.put("running", current == null ? null : current.running()); group.put("queued", current == null ? null : current.queued()); } - // Keep disabled groups visible while their queued or running work still exists. + // Keep disabled groups visible while any work or unresolved financial hold still exists. groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled")) - || positive(group.get("running")) || positive(group.get("queued"))).toList(); + || positive(group.get("running")) || positive(group.get("queued")) || positive(group.get("reconciling"))).toList(); result.put("capacityGroups", groups); result.put("running", totalCapacity(groups, "running")); result.put("waiting", totalCapacity(groups, "queued")); + result.put("reconciling", totalCapacity(groups, "reconciling")); result.put("capacityConfig", Map.of( "queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(), "defaultModelConcurrency", defaultModelLimit, diff --git a/control-plane-java/src/main/resources/db/migration/V46__async_task_reconciliation.sql b/control-plane-java/src/main/resources/db/migration/V46__async_task_reconciliation.sql new file mode 100644 index 0000000..54f69a1 --- /dev/null +++ b/control-plane-java/src/main/resources/db/migration/V46__async_task_reconciliation.sql @@ -0,0 +1,16 @@ +-- Unknown provider acceptance is not a failed generation and must retain its financial holds. +ALTER TABLE model_tasks DROP CONSTRAINT model_tasks_status_check; +ALTER TABLE model_tasks ADD CONSTRAINT model_tasks_status_check + CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled')); +ALTER TABLE usage_events DROP CONSTRAINT usage_events_status_check; +ALTER TABLE usage_events ADD CONSTRAINT usage_events_status_check + CHECK (status IN ('queued','running','reconciling','succeeded','failed','cancelled')); +ALTER TABLE model_tasks ADD COLUMN reconciliation_required_at timestamptz; +-- A normalized observed terminal outcome survives a crash or transient ledger/audit write failure. +-- The result itself uses the existing result column; no second copy of generated media is stored. +ALTER TABLE model_tasks ADD COLUMN completion_status varchar(16) + CHECK (completion_status IN ('succeeded','failed','cancelled')); +CREATE INDEX model_tasks_pending_completion_idx ON model_tasks(updated_at) + WHERE completion_status IS NOT NULL AND status IN ('running','reconciling'); +CREATE INDEX model_tasks_reconciliation_idx ON model_tasks(reconciliation_required_at) + WHERE status='reconciling'; diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java index 69872a1..508eb10 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.security.SecretCipher; +import com.moyi.controlplane.provider.transport.ProviderRequestException; import com.sun.net.httpserver.HttpServer; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -207,7 +208,7 @@ class AsyncProviderClientTest { } @ParameterizedTest - @CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "503,UPSTREAM_PROVIDER_ERROR"}) + @CsvSource({"401,UPSTREAM_AUTH_FAILED", "403,UPSTREAM_AUTH_FAILED", "402,UPSTREAM_CREDIT_INSUFFICIENT", "429,UPSTREAM_RATE_LIMITED", "408,UPSTREAM_PROVIDER_ERROR", "409,UPSTREAM_PROVIDER_ERROR", "503,UPSTREAM_PROVIDER_ERROR"}) void httpErrorsExposeOnlySafeCodes(int status, String code) throws Exception { HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); server.createContext("/", exchange -> { @@ -221,6 +222,7 @@ class AsyncProviderClientTest { ApiException submit = assertThrows(ApiException.class, () -> client.submit("provider-vast", "v3.1-20260211", "3d", "chair")); assertEquals(code, submit.getCode()); assertEquals("上游 3D 任务创建失败(HTTP " + status + ")", submit.getMessage()); + assertEquals(status != 408 && status != 409 && status < 500, ((ProviderRequestException) submit).definitelyNotAccepted()); ApiException poll = assertThrows(ApiException.class, () -> client.poll("provider-vast", "task-test")); assertEquals(code, poll.getCode()); assertEquals("上游异步任务查询失败(HTTP " + status + ")", poll.getMessage()); @@ -229,6 +231,71 @@ class AsyncProviderClientTest { } } + @Test void successfulHttpResponseWithMalformedBodyHasUnknownAcceptance() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + byte[] body = "not-json; private upstream detail".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (var output = exchange.getResponseBody()) { output.write(body); } + }); + server.start(); + try { + var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort()); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair")); + assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance()); + org.junit.jupiter.api.Assertions.assertFalse(failure.getMessage().contains("private upstream detail")); + } finally { server.stop(0); } + } + + @ParameterizedTest + @CsvSource({"image,gpt-image-2", "video,sora-2", "audio,tts-1"}) + void compatibleSubmit5xxRemainsUnknownAndNeverUsesFallbackRoute(String type, String model) throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + var calls = new java.util.concurrent.atomic.AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); + exchange.getRequestBody().readAllBytes(); + byte[] body = "private account details".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(503, body.length); + try (var output = exchange.getResponseBody()) { output.write(body); } + }); + server.start(); + try { + var jdbc = mock(JdbcTemplate.class); + var cipher = new SecretCipher("test-encryption"); + when(jdbc.queryForMap(any(String.class), eq("relay"))).thenReturn(Map.of( + "code", "relay", "protocol", "openai-compatible", "base_url", "http://127.0.0.1:" + server.getAddress().getPort(), "credential_ciphertext", cipher.encrypt("test-key"))); + var client = new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class)); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("relay", model, type, "chair")); + assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance()); + assertEquals(1, calls.get()); + } finally { server.stop(0); } + } + + @Test void acceptedPostWithDroppedResponseIsUnknownAndIsNotTransparentlyRepeated() throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + var calls = new java.util.concurrent.atomic.AtomicInteger(); + server.createContext("/", exchange -> { + exchange.getRequestBody().readAllBytes(); + calls.incrementAndGet(); + exchange.close(); // Simulate acceptance followed by loss of the HTTP response. + }); + server.start(); + try { + var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort()); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "chair")); + assertEquals(ProviderRequestException.Acceptance.UNKNOWN, failure.acceptance()); + assertEquals(1, calls.get()); + } finally { server.stop(0); } + } + + @Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() { + var client = clientFor("http://127.0.0.1:1"); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private"))); + assertEquals(ProviderRequestException.Acceptance.NOT_SENT, failure.acceptance()); + assertEquals("INVALID_MEDIA_URL", failure.getCode()); + } + private AsyncProviderClient clientFor(String baseUrl) { JdbcTemplate jdbc = mock(JdbcTemplate.class); SecretCipher cipher = new SecretCipher("test-encryption"); diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java index 8d1111c..f6fc39f 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java @@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import com.moyi.controlplane.gateway.request.ApiRequestMetadata; +import com.moyi.controlplane.provider.transport.ProviderRequestException; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -42,6 +43,7 @@ class AsyncTaskPostgresTest { private AsyncProviderClient provider; private AsyncTaskService tasks; private CatalogService catalog; + private GatewayGuard guard; @BeforeEach void setup() { String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL"); @@ -52,7 +54,7 @@ class AsyncTaskPostgresTest { jdbc = new JdbcTemplate(source); usage = spy(new UsageAuditService(jdbc, json)); provider = mock(AsyncProviderClient.class); - GatewayGuard guard=mock(GatewayGuard.class); + guard=mock(GatewayGuard.class); when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true); catalog = mock(CatalogService.class); tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, @@ -152,7 +154,10 @@ class AsyncTaskPostgresTest { @ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"}) void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) { jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId); - when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503)); + when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow( + "PROVIDER_NOT_CONFIGURED".equals(code) + ? ProviderRequestException.notSent(code, "上游任务未接受", 503) + : ProviderRequestException.http(401, "上游任务未接受")); tasks.completeMockTasks(); tasks.completeMockTasks(); assertEquals("failed",status()); @@ -216,19 +221,167 @@ class AsyncTaskPostgresTest { assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId)); } - @Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() { + @Test void deadlineRetainsHoldWithoutClaimingTheUpstreamFailed() { jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId); poll(); poll(); - assertEquals("failed",status()); + assertEquals("reconciling",status()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); - assertEquals(0,wallet("reserved_balance").signum()); - assertEquals(0,cost().signum()); + assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance")); + assertNull(cost()); String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId); assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT")); assertTrue(error.contains("执行结果未知")); verifyNoInteractions(provider); } + @ParameterizedTest + @ValueSource(ints={408,409,500,502,503,504}) + void uncertainSubmitResponseNeverRefundsOrResubmits(int httpStatus) { + jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId); + when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString())) + .thenThrow(ProviderRequestException.http(httpStatus, "safe local message")); + tasks.completeMockTasks(); + tasks.completeMockTasks(); + assertEquals("reconciling", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertEquals(new BigDecimal("10.00000000"), wallet("balance")); + assertNull(cost()); + assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId)); + assertEquals("pending", jdbc.queryForObject("SELECT cost_status FROM usage_events WHERE provider_task_id=?", String.class, taskId)); + assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId)); + verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString()); + } + + @Test void lostSubmitResponseAndInterruptedClaimRequireReconciliationWithoutReplay() { + jdbc.update("UPDATE model_tasks SET provider_task_id=NULL,updated_at=now()-interval '3 minutes' WHERE id=?", taskId); + tasks.completeMockTasks(); + tasks.completeMockTasks(); + assertEquals("reconciling", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertNull(cost()); + verifyNoInteractions(provider); + } + + @Test void missingTaskReceiptRetainsFinancialAndBudgetHolds() { + var created = create("missing-receipt", Map.of("model", "model-mock-3d", "prompt", "chair")); + String id = String.valueOf(created.get("id")); + when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap())).thenReturn(Map.of("accepted", true)); + tasks.completeMockTasks(); + tasks.completeMockTasks(); + assertEquals("reconciling", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id)); + assertEquals(new BigDecimal("2.88000000"), wallet("reserved_balance")); + assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id)); + verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap()); + } + + @Test void genericSubmitFailureIsNotTreatedAsEvidenceOfRejection() { + jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId); + when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString())) + .thenThrow(new IllegalStateException("network connection lost; private details")); + tasks.completeMockTasks(); + assertEquals("reconciling", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertFalse(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?", String.class, taskId).contains("private details")); + } + + @Test void missingPollingCredentialDoesNotRefundAnAlreadyAcceptedJob() { + when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)) + .thenThrow(new ApiException("UPSTREAM_KEY_NOT_AVAILABLE", "sensitive detail", 503)); + poll(); + assertEquals("reconciling", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertNull(cost()); + } + + @Test void inlineSuccessSurvivesSettlementFailureAndRecoversWithoutRegenerating() { + jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?", taskId); + when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyString())) + .thenReturn(Map.of("_moyiInlineResult", Map.of("url", "https://example.test/generated.glb"))); + doThrow(new IllegalStateException("simulated audit failure")).when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull()); + assertThrows(IllegalStateException.class, () -> tasks.completeMockTasks()); + assertEquals("running", status()); + assertEquals("succeeded", jdbc.queryForObject("SELECT completion_status FROM model_tasks WHERE id=?", String.class, taskId)); + fail(); // A stale error cannot refund an already observed success before the retry settles it. + assertEquals("running", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertEquals(new BigDecimal("10.00000000"), wallet("balance")); + assertNull(cost()); + jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId); + doCallRealMethod().when(usage).updateTask(eq(taskId), eq("succeeded"), anyMap(), isNull(), isNull()); + tasks.completeMockTasks(); + tasks.completeMockTasks(); + assertEquals("succeeded", status()); + assertEquals(new BigDecimal("8.56000000"), wallet("balance")); + assertEquals(0, wallet("reserved_balance").signum()); + assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?", Integer.class, userId)); + verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyString()); + } + + @ParameterizedTest + @ValueSource(strings={"unsuccessful","not_failed","cancelling","unknown"}) + void ambiguousProviderStatusCannotTriggerSettlementOrRefund(String state) { + when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)) + .thenReturn(Map.of("status", state)); + poll(); + assertEquals("running", status()); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertNull(cost()); + } + + @Test void interruptedClaimCheckCannotQuarantineALaterAcceptedReceipt() { + var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage); + jdbc.update("UPDATE model_tasks SET updated_at=now()-interval '3 minutes' WHERE id=?", taskId); + journal.requireReconciliation(taskId, "STALE_WORKER", "stale claim check", true); + assertEquals("running", status(), "An upstream task ID proves this is no longer an interrupted claim"); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + } + + @Test void reconciliationAndUsageAuditRollBackTogetherWithoutTouchingTheHold() { + var journal = new TaskOutcomeJournal(jdbc, json, new org.springframework.transaction.support.TransactionTemplate(new DataSourceTransactionManager(jdbc.getDataSource())), usage); + doThrow(new IllegalStateException("audit offline")).when(usage).updateTask(eq(taskId), eq("reconciling"), isNull(), anyString(), anyString()); + assertThrows(IllegalStateException.class, () -> journal.requireReconciliation(taskId, "UNKNOWN", "result unknown", false)); + assertEquals("running", status()); + assertEquals("running", jdbc.queryForObject("SELECT status FROM usage_events WHERE provider_task_id=?", String.class, taskId)); + assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance")); + assertNull(cost()); + } + + @Test void failedTaskReplayDoesNotRequeueOrReserveAgain() { + var input = Map.of("model", "model-mock-3d", "prompt", "will be rejected"); + var first = create("rejected-task", input); + String id = String.valueOf(first.get("id")); + when(provider.submit(anyString(), anyString(), anyString(), anyString(), anyMap())) + .thenThrow(ProviderRequestException.http(429, "provider rejected")); + tasks.completeMockTasks(); + BigDecimal held = wallet("reserved_balance"); + var replay = create("rejected-task", input); + assertEquals(id, replay.get("id")); + assertEquals("failed", replay.get("status")); + assertEquals(held, wallet("reserved_balance")); + assertEquals(0, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds WHERE request_id=?", Integer.class, id)); + verify(provider, times(1)).submit(anyString(), anyString(), anyString(), anyString(), anyMap()); + } + + @Test void pendingReconciliationConsumesAsyncConcurrencyRatherThanOpeningAnotherSlot() { + tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, false, + guard, usage, provider, new DataSourceTransactionManager(jdbc.getDataSource()), + new com.moyi.controlplane.gateway.request.ApiRequestRiskPolicy(com.moyi.controlplane.gateway.request.ApiRequestRiskEngine.allowAll())); + jdbc.update("UPDATE model_tasks SET status='reconciling' WHERE id=?", taskId); + jdbc.update("UPDATE model_catalog SET max_concurrency=1 WHERE id='model-mock-3d'"); + String id = String.valueOf(create("next-while-unknown", Map.of("model", "model-mock-3d", "prompt", "another chair")).get("id")); + tasks.completeMockTasks(); + assertEquals("queued", jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?", String.class, id)); + verifyNoInteractions(provider); + } + + @Test void documentedSucceededStatusSettlesInsteadOfPollingForever() { + when(provider.poll("provider-vast", "model-mock-3d", "v3.1-20260211", "upstream-task", null)) + .thenReturn(Map.of("status", "succeeded", "url", "https://example.test/generated.glb")); + poll(); + assertEquals("succeeded", status()); + assertEquals(new BigDecimal("8.56000000"), wallet("balance")); + } + @Test void confirmedUpstreamFailureReleasesHoldImmediately() { when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed"))); poll(); diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/usage/UsageCapacityPostgresTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/usage/UsageCapacityPostgresTest.java index dc9246f..990ebf0 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/usage/UsageCapacityPostgresTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/usage/UsageCapacityPostgresTest.java @@ -39,7 +39,7 @@ class UsageCapacityPostgresTest { UUID user = UUID.randomUUID(), key = UUID.randomUUID(); jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'capacity-user','username','not-loginable','Capacity user')", user); jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Capacity test','local-test',?)", key, user, "c".repeat(64)); - for (String state : List.of("running", "queued", "succeeded")) { + for (String state : List.of("running", "queued", "reconciling", "succeeded")) { jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,created_at) VALUES (?,?,?,'capacity-route-105','3d',?,now()-interval '1 day')", "task-" + state, user, key, state); } } @@ -67,9 +67,11 @@ class UsageCapacityPostgresTest { Map async = groups.stream().filter(group -> group.get("id").equals("capacity-route-105")).findFirst().orElseThrow(); assertEquals(1L, async.get("running"), "A task older than ten minutes must remain in live capacity"); assertEquals(1L, async.get("queued"), "Running tasks must not also count as queued"); + assertEquals(1L, async.get("reconciling"), "Unknown upstream outcomes must remain visible, not be counted as failed or idle"); assertEquals(7, async.get("limit"), "Null route limits use the real configured model default"); assertEquals(3L, summary.get("running")); assertEquals(4L, summary.get("waiting")); + assertEquals(1L, summary.get("reconciling")); Map config = (Map) summary.get("capacityConfig"); assertEquals(4500L, config.get("queueWaitMs")); assertEquals(3, config.get("defaultKeyConcurrency")); diff --git a/docs/21-api-request-modularization.md b/docs/21-api-request-modularization.md index 348e5df..a40db33 100644 --- a/docs/21-api-request-modularization.md +++ b/docs/21-api-request-modularization.md @@ -63,8 +63,8 @@ PostgreSQL 集成测试由 `MOYI_AUTH_TEST_JDBC_URL` 启用,当前测试限制 ## 下一阶段优先级 -1. 异步创建 `Idempotency-Key`、请求哈希和数据库唯一性/锁语义,保证客户端重试不重复排队或预留。 -2. 异步上游提交结果未知的持久状态与核账入口:区分明确未受理、已受理可轮询、结果未知,禁止统一退款后自动重试。 -3. 执行中元数据和计量快照的崩溃窗口、重试退避、人工对账与待结算可视化。 +1. 已完成 `V45`:异步创建 `Idempotency-Key`、请求哈希和数据库唯一性/锁语义,客户端同键重试不重复排队或预留。 +2. 已完成 `V46` 的未知受理持久状态与冻结保护;审计化人工核账入口仍待实现。详情及本阶段 190 项后端验证结果见 `22-async-task-safety.md`。 +3. 已增加异步已观察终态的结算恢复与待核账提示;继续完善执行路由快照、重试退避、人工对账和不可原子提交窗口。 4. HTTP Transport 生命周期继续拆分;真实 Redis 并发、租约回收与队列公平性回归。 5. 再进入用户/角色/会话、管理操作与密钥生命周期的后续安全优化;本轮模块改造不替代完整审计。 diff --git a/docs/22-async-task-safety.md b/docs/22-async-task-safety.md index 3a22aea..4f270c6 100644 --- a/docs/22-async-task-safety.md +++ b/docs/22-async-task-safety.md @@ -12,6 +12,50 @@ 验证:Java 21 / Maven,隔离 PostgreSQL 15.18(不连接业务数据库),26 项针对性测试通过,0 失败、0 跳过。包含 8 路并发同键请求、重放、请求冲突、完成后路由下线重放、键摘要规范化与风控顺序。 -## 接续项 +## 阶段二:未知受理与可恢复结算(2026-10-21) -接续修复提交响应丢失、进程中断、轮询超时以及结算数据库失败的语义。未知结果不得自动重发生成请求或释放冻结余额;只有已证实的拒绝或终态失败才能进入正常释放流程。核账操作必须有证据和审计,不允许用直接修改钱包余额替代结算事务。 +### 模块边界 + +- `provider.transport.ProviderRequestException` 只描述 HTTP 边界事实:`NOT_SENT`、`REJECTED`、`UNKNOWN`,保留可安全展示的错误码及 HTTP 状态,不附带上游响应体、凭据或异常原文;不决定重试和退款。 +- `task.TaskOutcomeJournal` 持久化已观察的终态和待核账状态,不发送请求、不修改钱包、不制定退款策略。 +- `AsyncTaskService` 编排提交、只读轮询和本地结算。收到生成结果后先提交日志,再在原有钱包/任务/审计事务中结算;账本失败不能被提交请求异常处理器误认为上游生成失败。 + +### 状态与账务 + +| 观察到的事实 | 任务状态 / 后续动作 | 钱包与预算 | +| --- | --- | --- | +| 本地输入/配置校验失败、请求尚未发出 | `failed` | 原子释放预留,不扣费 | +| HTTP 400/401/402/403/404/405/413/415/422/429 明确拒绝 | `failed` | 原子释放预留,不自动重发 | +| HTTP 408/409、5xx、网络异常、成功响应不可解析 | `reconciling`,不自动重发 | 保留冻结及预算占用,等待证据核账 | +| 回执缺少合法任务 ID,或回执无法保存 | `reconciling` | 保留冻结,不假定生成失败 | +| worker 已认领、两分钟后仍无 ID 且无终态日志 | `reconciling` | 保留冻结,不重新派发 | +| 已受理任务的轮询期限到期,或原凭据已移除 | `reconciling`,停止自动轮询 | 保留冻结,不将查询失败视为生成失败 | +| 已收到上游终态,但本地结算失败 | 保存 `completion_status` 和归一化结果,下轮只重试本地结算 | 保留冻结,成功后只扣一次 | +| 已确认上游失败/取消 | 原子提交终态与审计 | 释放预留,只执行一次 | + +兼容视频路径的备用接口只在可结构化识别的 HTTP 404/405 拒绝后尝试,不再解析错误文案判断重试;5xx 或结果未知不换接口。轮询是只读操作,瞬时鉴权/限流/网络异常可继续尝试至持久化期限,但不能由此推导退款。 + +`V46` 扩展 `model_tasks` / `usage_events` 状态约束并增加终态观察日志。生成内容复用已有 `result` 列,不重复存两份。日志一旦观察到终态,晚到的冲突状态不能覆盖它;过期认领检查也不能覆盖已经保存的任务编号。 + +`reconciling` 仍计入异步模型/API Key/用户的数据库并发与队列容量,避免未知任务被无限替换为新任务。用户任务页展示“待核账”,提示额度仍冻结、勿重复提交;使用记录可筛选,管理员容量分组保留待核账计数,即使渠道已停用也不隐藏。 + +状态解析改为明确白名单:`success` / `succeeded` / `completed` 才能结算;不再把 `unsuccessful` 当成功,也不把 `cancelling` 当取消完成。其他未知状态继续只读查询,期限到后进入核账状态。 + +### 上线与核账边界 + +1. 本阶段只验证隔离测试数据库,没有迁移业务数据库或部署线上服务。发布时先停旧 worker,再迁移 `V45` / `V46` 并启新 worker,避免混跑旧的自动退款逻辑。 +2. 不回溯改写历史失败任务或重新冻结曾经释放的资金。历史的提交中断、超时、凭据丢失记录须独立审计,不能自动扣款。 +3. 待核账状态不会按 TTL 自动退款,不会自动重发。收集本地任务 ID、用户、时间、模型路由、预留金额、上游任务/账单记录;不要把完整凭据放进工单或日志。 +4. 暂不提供“无证据一键退款/重新生成”的管理接口。受审计的人工核账入口、关联上游任务编号、人工确认终态以及差额结算是下一阶段产品工作;在该入口完成前,只允许按受控运维流程核对证据,不直接修改钱包余额替代结算事务。 +5. 成功的 token 图片任务缺少可核验用量时,仍沿用既有 `TASK_USAGE_PENDING` 规则:保留结果、暂不扣费、释放该次预留并标记待核账。本阶段没有把“已知完成但计量缺失”与“是否受理未知”混为一类。 +6. HTTP 受理至观察日志提交之间仍有无法跨外部系统原子提交的窗口。数据库完全不可用时也不能承诺保存回执;恢复后转入待核账,而非声称精确恢复所有生成结果。 + +后续重点:审计化人工核账入口、异步路由/上游模型的执行快照、共享有界 HTTP Transport 与媒体下载安全、上游有证明的幂等能力适配。 + +### 最终验证 + +2026-10-21:Docker / Java 21 / Maven 3.9.9 执行 `mvn -B -o clean verify`,后端 190 项测试全部通过(0 失败、0 错误、0 跳过),Spring Boot JAR 构建成功。使用独立 PostgreSQL 15.18,Maven 容器共享测试数据库的 network namespace;未访问业务数据库或真实供应商 API。 + +前端 `npm run build`(含 `vue-tsc --noEmit`)和 `npm test` 通过,22 项测试全部通过。前端“待核账”提示已通过类型检查和构建,未将这些测试描述为完整浏览器交互验收。`git diff --check` 无错误。 + +重点回归覆盖:8 路同键并发、终态/失败后重放、不同请求冲突、HTTP 408/409/5xx、收到 POST 后断开响应且不重发、成功响应 JSON 无法解析、无任务 ID、旧认领检查竞态、凭据丢失、轮询期限、待核账并发占位、审计事务回滚、成功结果先保存后结算、晚到失败不得退款、结算恢复一次扣费,以及模糊状态白名单。 diff --git a/prototype/src/components/UsageHistory.vue b/prototype/src/components/UsageHistory.vue index a26b653..6e80a69 100644 --- a/prototype/src/components/UsageHistory.vue +++ b/prototype/src/components/UsageHistory.vue @@ -23,7 +23,7 @@ const number = (n: unknown) => n == null ? '—' : Number(n).toLocaleString('zh- const money = (n: unknown) => n == null ? '—' : `¥ ${Number(n).toLocaleString('zh-CN', { minimumFractionDigits: 2, maximumFractionDigits: 8 })}` const duration = (n: unknown) => n == null ? '—' : `${(Number(n) / 1000).toFixed(2)}s` const time = (value: string) => new Date(value).toLocaleString('zh-CN', { hour12: false }) -const statusName = (v: string) => ({ succeeded: '成功', failed: '失败', queued: '排队中', running: '执行中', cancelled: '已取消' })[v] || v +const statusName = (v: string) => ({ succeeded: '成功', failed: '失败', queued: '排队中', running: '执行中', reconciling: '待核账(额度仍冻结)', cancelled: '已取消' })[v] || v const modeName = (v?: string) => ({ sync: '同步', stream: '流式', async: '异步' })[v || ''] || '—' const sourceName = (v?: string) => ({ playground: '在线调试', api: 'API', unknown: '历史调用' })[v || 'unknown'] || '历史调用' const pretty = (v: unknown) => v == null ? '未保存内容' : JSON.stringify(v, null, 2) @@ -108,7 +108,7 @@ onUnmounted(() => { requestSequence++; selectedId.value = ''; if (timer) window. - + diff --git a/prototype/src/services/catalog.ts b/prototype/src/services/catalog.ts index 2b00842..9a1fb8c 100644 --- a/prototype/src/services/catalog.ts +++ b/prototype/src/services/catalog.ts @@ -122,7 +122,7 @@ export interface PlatformOverview { availabilityPercent?: number asyncTasks: number asyncSuccessRate?: number - tasks: { queued: number; running: number; succeeded: number; failed: number } + tasks: { queued: number; running: number; reconciling?: number; succeeded: number; failed: number } series: Array<{ date: string; count: number }> ledger: Array<{ id: string; title: string; amount: number; createdAt: string }> } @@ -172,8 +172,9 @@ export interface AdminUsageSummary { costToday: number costThisMonth: number series: Array<{ date: string; count: number }> - capacityGroups: Array<{ id: string; name: string; sourceLabel?: string; providerName?: string; mode?: string; routeEnabled?: boolean; running: number | null; limit: number; queued: number | null }> + capacityGroups: Array<{ id: string; name: string; sourceLabel?: string; providerName?: string; mode?: string; routeEnabled?: boolean; running: number | null; limit: number; queued: number | null; reconciling?: number }> running?: number | null + reconciling?: number | null capacityConfig?: { queueWaitMs: number defaultModelConcurrency: number @@ -231,7 +232,8 @@ export interface AsyncTaskResult { id: string model: string provider?: string - status: 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled' + status: 'queued' | 'running' | 'reconciling' | 'succeeded' | 'failed' | 'cancelled' + requiresReconciliation?: boolean progress?: number created_at?: string updated_at?: string @@ -240,7 +242,7 @@ export interface AsyncTaskResult { result?: { format?: string; url?: string; modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; mimeType?: string; creditsConsumed?: number; [key: string]: unknown } output?: { modelUrl?: string; previewUrl?: string; videoUrl?: string; imageUrl?: string; audioUrl?: string; url?: string; format?: string; mimeType?: string } creditsConsumed?: number - error?: { code?: number; message?: string } + error?: { code?: string | number; message?: string; retryable?: boolean; requiresReconciliation?: boolean } } /** OpenAI-compatible multimodal content. Keep text-only calls byte-for-byte compatible. */ diff --git a/prototype/src/types.ts b/prototype/src/types.ts index 4171cc3..2368648 100644 --- a/prototype/src/types.ts +++ b/prototype/src/types.ts @@ -4,7 +4,7 @@ import type { AuthRole } from './services/auth' export type Portal = 'user' | 'admin' export type ModelType = '文本' | '多模态' | '图像' | '3D' | '视频' | '音频' export type CallMode = '同步' | '流式' | '异步' -export type TaskStatus = '排队中' | '执行中' | '已完成' | '失败' +export type TaskStatus = '排队中' | '执行中' | '待核账' | '已完成' | '失败' export interface NavItem { id: string diff --git a/prototype/src/views/AdminConsole.vue b/prototype/src/views/AdminConsole.vue index 09d7351..db76342 100644 --- a/prototype/src/views/AdminConsole.vue +++ b/prototype/src/views/AdminConsole.vue @@ -202,8 +202,8 @@ function formatUsagePayload(value: unknown) { const queueGroups = computed(() => (adminSummary.value?.capacityGroups || []).map((group) => ({ ...group, type: `${group.providerName || '未知渠道'} · ${group.sourceLabel || '模型分组'}`, - status: group.running === null ? '未读取' : group.routeEnabled === false ? '已停用,仍有任务' : group.queued ? '排队中' : group.running ? '执行中' : '空闲', - warning: group.running === null || group.routeEnabled === false || !!group.queued, + status: group.reconciling ? `待核账 ${group.reconciling} 笔(额度仍冻结)` : group.running === null ? '未读取' : group.routeEnabled === false ? '已停用,仍有任务' : group.queued ? '排队中' : group.running ? '执行中' : '空闲', + warning: group.running === null || group.routeEnabled === false || !!group.queued || !!group.reconciling, }))) const { page: capacityPage, pageSize: capacityPageSize, pagedItems: capacityPageGroups } = usePagination(queueGroups) const configuredCapacityGroups = computed(() => adminSummary.value?.capacityGroups.filter(group => group.routeEnabled !== false) || []) diff --git a/prototype/src/views/UserConsole.vue b/prototype/src/views/UserConsole.vue index 55490ff..4d54c5d 100644 --- a/prototype/src/views/UserConsole.vue +++ b/prototype/src/views/UserConsole.vue @@ -235,6 +235,7 @@ function normalizeOverview(value: unknown): PlatformOverview { tasks: { queued: asNumber(rawTasks.queued), running: asNumber(rawTasks.running), + reconciling: asNumber(rawTasks.reconciling), succeeded: asNumber(rawTasks.succeeded), failed: asNumber(rawTasks.failed), }, @@ -504,6 +505,7 @@ function mapAsyncTask(task: AsyncTaskResult, model: ModelItem, taskPrompt?: stri const statusMap: Record = { queued: '排队中', running: '执行中', + reconciling: '待核账', succeeded: '已完成', failed: '失败', cancelled: '失败', @@ -516,7 +518,7 @@ function mapAsyncTask(task: AsyncTaskResult, model: ModelItem, taskPrompt?: stri status: statusMap[task.status], progress: task.status === 'succeeded' ? 100 : Math.round(task.progress || 0), createdAt: createdAt ? formatTaskTime(createdAt) : '刚刚', - duration: task.completedAt ? '已完成' : '计算中', + duration: task.completedAt ? '已完成' : task.status === 'reconciling' ? '等待核账' : '计算中', result: task.error?.message || (videoUrl ? '视频已生成' : imageUrl ? '图片已生成' : audioUrl ? '音频已生成' : modelUrl ? 'GLB 模型已生成' : resultUrl ? '文件已生成' : task.status === 'succeeded' ? '异步任务已完成' : task.status === 'running' ? `正在生成 · ${Math.round(task.progress || 0)}%` : '等待平台调度'), provider: task.provider, prompt: taskPrompt, @@ -557,13 +559,11 @@ async function pollTask(taskId: string, apiKey: string) { scheduleTaskPoll(taskId, apiKey, 2500) } else { taskPollers.delete(taskId) - // The async endpoint settles or releases the wallet only when the - // provider task reaches a terminal state. Refresh the overview here so - // the balance card and usage counters reflect that settlement immediately - // instead of waiting for a full page reload. + // Reconciliation stops automatic polling, but it is NOT a failure or a refund. + // Refresh financial state without encouraging another generation request. await refreshOverview() historyRevision.value++ - emit('notify', latest.status === 'succeeded' ? `${previous.name}完成,可以查看和保存结果` : `${previous.name}未能完成,请查看错误说明`) + emit('notify', latest.status === 'reconciling' ? `${previous.name}结果尚未确认,额度仍冻结,请联系管理员核账,勿重复提交` : latest.status === 'succeeded' ? `${previous.name}完成,可以查看和保存结果` : `${previous.name}未能完成,请查看错误说明`) } } catch (error) { taskPollers.delete(taskId) @@ -1010,6 +1010,7 @@ $env:OPENAI_API_KEY="你的 MOYI 用户 API Key" .token-limit { display: grid; gap: 8px; } .response-result > p, .response-content { white-space: pre-wrap; overflow-wrap: anywhere; max-width: 100%; } .response-result .status-chip.failed { color: #b42318; background: #fee4e2; } +.status-chip.待核账 { color: #b54708; background: #fffaeb; } .response-result p[role="alert"] { color: #b42318; } .playground-selectors { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; margin-bottom: 16px; } .playground-selectors label { display: grid; gap: 8px; font-size: 13px; min-width: 0; }