fix async dispatch stats and truthful status

This commit is contained in:
wlx committed 2026-09-27 19:55:17 +08:00
1 parent 9750aceb03
commit 8e9f0d52fd
4 files changed
+110 -48

No files matched your search

@@ -348,19 +348,46 @@ public class CatalogService {
auth.audit("catalog."+kind+"_deleted",admin.id(),new AuthService.RequestMeta(null,null),Map.of("id",id));
}
@Transactional(readOnly = true)
public Map<String, Object> overview(String token, AuthService auth) {
UserView user = auth.requireUser(token);
Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id());
Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id());
Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'task'", Integer.class, user.id());
Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id());
Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id());
Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'task'", Integer.class, user.id());
Map<String, Object> taskCounts = new LinkedHashMap<>();
jdbc.query("SELECT status, COUNT(*) AS count FROM model_tasks WHERE user_id = ? GROUP BY status", (rs, row) -> {
taskCounts.put(rs.getString("status"), rs.getLong("count"));
return null;
}, user.id());
Map<String, Object> result = new LinkedHashMap<>();
result.put("generatedAt", now());
result.put("callsToday", callsToday == null ? 0 : callsToday);
result.put("callsThisMonth", callsMonth == null ? 0 : callsMonth);
result.put("asyncTasks", tasks == null ? 0 : tasks);
result.put("tasks", Map.of("queued", 0, "running", 0, "succeeded", 0, "failed", 0));
result.put("series", List.of());
result.put("ledger", List.of());
result.put("tasks", Map.of(
"queued", taskCounts.getOrDefault("queued", 0L),
"running", taskCounts.getOrDefault("running", 0L),
"succeeded", taskCounts.getOrDefault("succeeded", 0L),
"failed", taskCounts.getOrDefault("failed", 0L)
));
result.put("series", jdbc.query(
"SELECT to_char(day, 'YYYY-MM-DD') AS day, COUNT(ue.id) AS count " +
"FROM generate_series(CURRENT_DATE - INTERVAL '6 days', CURRENT_DATE, INTERVAL '1 day') day " +
"LEFT JOIN usage_events ue ON ue.user_id = ? AND ue.deleted_at IS NULL AND ue.event_kind = 'call' " +
"AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
"GROUP BY day ORDER BY day",
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count")), user.id()));
Number averageLatency = jdbc.queryForObject(
"SELECT AVG(latency_ms) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded' AND latency_ms IS NOT NULL",
Number.class, user.id());
result.put("averageLatencyMs", averageLatency == null ? null : averageLatency.doubleValue());
Number finished = jdbc.queryForObject(
"SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status IN ('succeeded','failed','cancelled')",
Number.class, user.id());
Number successful = jdbc.queryForObject(
"SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded'",
Number.class, user.id());
result.put("availabilityPercent", finished == null || finished.longValue() == 0 ? null : successful.doubleValue() * 100.0 / finished.doubleValue());
var balances=jdbc.queryForList("SELECT balance, reserved_balance FROM user_wallets WHERE user_id=?",user.id());
var balance=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("balance");
var reserved=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("reserved_balance");
@@ -18,6 +18,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.beans.factory.annotation.Value;
@@ -33,6 +34,7 @@ public class AsyncTaskService {
private final CatalogService catalogService;
private final int maxQueuedPerModel;
private final int defaultModelConcurrency;
private final int defaultKeyConcurrency;
private final GatewayGuard gatewayGuard;
private final UsageAuditService usage;
private final AsyncProviderClient providerClient;
@@ -41,7 +43,7 @@ public class AsyncTaskService {
/** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */
public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService,
int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) {
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null);
this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, gatewayGuard, usage, null, null);
}
@org.springframework.beans.factory.annotation.Autowired
@@ -51,17 +53,19 @@ public class AsyncTaskService {
CatalogService catalogService,
@Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel,
@Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency,
@Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency,
GatewayGuard gatewayGuard,
UsageAuditService usage,
AsyncProviderClient providerClient,
PlatformTransactionManager transactionManager
) {
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive");
this.jdbc = jdbc;
this.objectMapper = objectMapper;
this.catalogService = catalogService;
this.maxQueuedPerModel = maxQueuedPerModel;
this.defaultModelConcurrency = defaultModelConcurrency;
this.defaultKeyConcurrency = defaultKeyConcurrency;
this.gatewayGuard = gatewayGuard;
this.usage = usage;
this.providerClient = providerClient;
@@ -165,10 +169,10 @@ public class AsyncTaskService {
private void dispatchQueuedTasks() {
if (providerClient == null) return;
// A process can die after claiming a task but before receiving an
// upstream task id. Return those claims to the queue after a short
// grace period so they neither consume a concurrency slot forever nor
// leave the user's reservation stuck.
jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'");
// upstream task id. Fail those claims after a grace period. Retrying
// them blindly could submit the same generation twice when the
// upstream accepted the request just before the process died.
recoverInterruptedClaims();
for (int i = 0; i < 50; i++) {
Map<String,Object> task = claimQueuedTask();
if (task == null) return;
@@ -183,38 +187,61 @@ public class AsyncTaskService {
);
jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId);
} catch (RuntimeException exception) {
jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'",
writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId);
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败");
failTask(taskId, "UPSTREAM_TASK_SUBMIT_FAILED", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage());
}
}
}
private Map<String,Object> claimQueuedTask() {
String sql = """
WITH candidate AS (
SELECT t.id
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0))
AND (SELECT COUNT(*) FROM model_tasks running
WHERE running.model_id=t.model_id AND running.status='running')
< COALESCE(m.max_concurrency, ?)
ORDER BY t.created_at
LIMIT 1
)
UPDATE model_tasks t
SET status='running', updated_at=now()
FROM candidate c
WHERE t.id=c.id AND t.status='queued'
RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt
""";
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency);
return rows.isEmpty() ? null : rows.getFirst();
return inTransaction(() -> {
// Keep the lock until the status update commits. The previous
// pg_try_advisory_xact_lock was executed in autocommit mode, so
// the lock ended before the UPDATE and did not coordinate two
// workers reliably.
jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))");
String sql = """
SELECT t.id, t.api_key_id, m.provider_id, m.upstream_model, t.task_type, t.prompt
FROM model_tasks t
JOIN model_catalog m ON m.id=t.model_id
JOIN model_providers p ON p.id=m.provider_id
WHERE t.status='queued'
AND t.provider_task_id IS NULL
AND m.status='active' AND m.deleted_at IS NULL
AND p.status='active' AND p.deleted_at IS NULL
AND (SELECT COUNT(*) FROM model_tasks running_model
WHERE running_model.model_id=t.model_id AND running_model.status='running')
< COALESCE(m.max_concurrency, ?)
AND (SELECT COUNT(*) FROM model_tasks running_key
WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running')
< ?
ORDER BY t.created_at
LIMIT 1
FOR UPDATE OF t SKIP LOCKED
""";
List<Map<String,Object>> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency);
if (rows.isEmpty()) return null;
Map<String,Object> row = rows.getFirst();
jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id"));
return row;
});
}
private void recoverInterruptedClaims() {
List<String> taskIds = inTransaction(() -> jdbc.queryForList(
"SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED",
String.class));
for (String taskId : taskIds) {
failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交");
}
}
private void failTask(String taskId, String code, String message) {
int changed = jdbc.update(
"UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')",
writeJson(Map.of("code", code, "message", message)), now(), now(), taskId);
if (changed == 0) return;
releaseTask(taskId);
usage.updateTask(taskId, "failed", null, code, message);
}
private Map<String, Object> requireKey(String authorization) {
@@ -393,6 +420,10 @@ public class AsyncTaskService {
}
}
private <T> T inTransaction(Supplier<T> work) {
return transactions == null ? work.get() : transactions.execute(status -> work.get());
}
private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); }
private void putIfPresent(Map<String,Object> target, String name, ResultSet resultSet, String column) throws SQLException {