From 8e9f0d52fdcf02acbdb01d379a898f7932892cf0 Mon Sep 17 00:00:00 2001 From: WangLX <519407855@qq.com> Date: Sun, 27 Sep 2026 19:55:17 +0800 Subject: [PATCH] fix async dispatch stats and truthful status --- .../controlplane/catalog/CatalogService.java | 39 ++++++-- .../controlplane/task/AsyncTaskService.java | 97 ++++++++++++------- prototype/src/types.ts | 2 +- prototype/src/views/AdminConsole.vue | 20 ++-- 4 files changed, 110 insertions(+), 48 deletions(-) diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java index 0bd6eed..1983ff2 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java @@ -348,19 +348,46 @@ public class CatalogService { auth.audit("catalog."+kind+"_deleted",admin.id(),new AuthService.RequestMeta(null,null),Map.of("id",id)); } + @Transactional(readOnly = true) public Map overview(String token, AuthService auth) { UserView user = auth.requireUser(token); - Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id()); - Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id()); - Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND event_kind = 'task'", Integer.class, user.id()); + Integer callsToday = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE", Integer.class, user.id()); + Integer callsMonth = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())", Integer.class, user.id()); + Integer tasks = jdbc.queryForObject("SELECT COUNT(*) FROM usage_events WHERE user_id = ? AND deleted_at IS NULL AND event_kind = 'task'", Integer.class, user.id()); + Map taskCounts = new LinkedHashMap<>(); + jdbc.query("SELECT status, COUNT(*) AS count FROM model_tasks WHERE user_id = ? GROUP BY status", (rs, row) -> { + taskCounts.put(rs.getString("status"), rs.getLong("count")); + return null; + }, user.id()); Map result = new LinkedHashMap<>(); result.put("generatedAt", now()); result.put("callsToday", callsToday == null ? 0 : callsToday); result.put("callsThisMonth", callsMonth == null ? 0 : callsMonth); result.put("asyncTasks", tasks == null ? 0 : tasks); - result.put("tasks", Map.of("queued", 0, "running", 0, "succeeded", 0, "failed", 0)); - result.put("series", List.of()); - result.put("ledger", List.of()); + result.put("tasks", Map.of( + "queued", taskCounts.getOrDefault("queued", 0L), + "running", taskCounts.getOrDefault("running", 0L), + "succeeded", taskCounts.getOrDefault("succeeded", 0L), + "failed", taskCounts.getOrDefault("failed", 0L) + )); + result.put("series", jdbc.query( + "SELECT to_char(day, 'YYYY-MM-DD') AS day, COUNT(ue.id) AS count " + + "FROM generate_series(CURRENT_DATE - INTERVAL '6 days', CURRENT_DATE, INTERVAL '1 day') day " + + "LEFT JOIN usage_events ue ON ue.user_id = ? AND ue.deleted_at IS NULL AND ue.event_kind = 'call' " + + "AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " + + "GROUP BY day ORDER BY day", + (rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count")), user.id())); + Number averageLatency = jdbc.queryForObject( + "SELECT AVG(latency_ms) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded' AND latency_ms IS NOT NULL", + Number.class, user.id()); + result.put("averageLatencyMs", averageLatency == null ? null : averageLatency.doubleValue()); + Number finished = jdbc.queryForObject( + "SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status IN ('succeeded','failed','cancelled')", + Number.class, user.id()); + Number successful = jdbc.queryForObject( + "SELECT COUNT(*) FROM usage_events WHERE user_id=? AND deleted_at IS NULL AND event_kind='call' AND status='succeeded'", + Number.class, user.id()); + result.put("availabilityPercent", finished == null || finished.longValue() == 0 ? null : successful.doubleValue() * 100.0 / finished.doubleValue()); var balances=jdbc.queryForList("SELECT balance, reserved_balance FROM user_wallets WHERE user_id=?",user.id()); var balance=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("balance"); var reserved=balances.isEmpty()?java.math.BigDecimal.ZERO:(java.math.BigDecimal)balances.getFirst().get("reserved_balance"); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java index cd0517b..e997bde 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java @@ -18,6 +18,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Supplier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.beans.factory.annotation.Value; @@ -33,6 +34,7 @@ public class AsyncTaskService { private final CatalogService catalogService; private final int maxQueuedPerModel; private final int defaultModelConcurrency; + private final int defaultKeyConcurrency; private final GatewayGuard gatewayGuard; private final UsageAuditService usage; private final AsyncProviderClient providerClient; @@ -41,7 +43,7 @@ public class AsyncTaskService { /** Compatibility constructor for isolated gateway tests that only exercise mock tasks. */ public AsyncTaskService(JdbcTemplate jdbc, ObjectMapper objectMapper, CatalogService catalogService, int maxQueuedPerModel, GatewayGuard gatewayGuard, UsageAuditService usage) { - this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, gatewayGuard, usage, null, null); + this(jdbc, objectMapper, catalogService, maxQueuedPerModel, 4, 4, gatewayGuard, usage, null, null); } @org.springframework.beans.factory.annotation.Autowired @@ -51,17 +53,19 @@ public class AsyncTaskService { CatalogService catalogService, @Value("${moyi.tasks.max-queued-per-model:100}") int maxQueuedPerModel, @Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency, + @Value("${moyi.gateway.default-key-concurrency:4}") int defaultKeyConcurrency, GatewayGuard gatewayGuard, UsageAuditService usage, AsyncProviderClient providerClient, PlatformTransactionManager transactionManager ) { - if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive"); + if (maxQueuedPerModel < 1 || defaultModelConcurrency < 1 || defaultKeyConcurrency < 1) throw new IllegalArgumentException("Task queue and concurrency limits must be positive"); this.jdbc = jdbc; this.objectMapper = objectMapper; this.catalogService = catalogService; this.maxQueuedPerModel = maxQueuedPerModel; this.defaultModelConcurrency = defaultModelConcurrency; + this.defaultKeyConcurrency = defaultKeyConcurrency; this.gatewayGuard = gatewayGuard; this.usage = usage; this.providerClient = providerClient; @@ -165,10 +169,10 @@ public class AsyncTaskService { private void dispatchQueuedTasks() { if (providerClient == null) return; // A process can die after claiming a task but before receiving an - // upstream task id. Return those claims to the queue after a short - // grace period so they neither consume a concurrency slot forever nor - // leave the user's reservation stuck. - jdbc.update("UPDATE model_tasks SET status='queued', updated_at=now() WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes'"); + // upstream task id. Fail those claims after a grace period. Retrying + // them blindly could submit the same generation twice when the + // upstream accepted the request just before the process died. + recoverInterruptedClaims(); for (int i = 0; i < 50; i++) { Map task = claimQueuedTask(); if (task == null) return; @@ -183,38 +187,61 @@ public class AsyncTaskService { ); jdbc.update("UPDATE model_tasks SET provider_task_id=?, updated_at=? WHERE id=? AND status='running'", providerTaskId(upstream), now(), taskId); } catch (RuntimeException exception) { - jdbc.update("UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status='running'", - writeJson(Map.of("message", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage())), now(), now(), taskId); - releaseTask(taskId); - usage.updateTask(taskId, "failed", null, "UPSTREAM_TASK_SUBMIT_FAILED", "上游异步任务创建失败"); + failTask(taskId, "UPSTREAM_TASK_SUBMIT_FAILED", exception.getMessage() == null ? "上游异步任务创建失败" : exception.getMessage()); } } } private Map claimQueuedTask() { - String sql = """ - WITH candidate AS ( - SELECT t.id - FROM model_tasks t - JOIN model_catalog m ON m.id=t.model_id - WHERE t.status='queued' - AND t.provider_task_id IS NULL - AND m.status='active' AND m.deleted_at IS NULL - AND pg_try_advisory_xact_lock(hashtextextended('moyi:async:' || t.model_id, 0)) - AND (SELECT COUNT(*) FROM model_tasks running - WHERE running.model_id=t.model_id AND running.status='running') - < COALESCE(m.max_concurrency, ?) - ORDER BY t.created_at - LIMIT 1 - ) - UPDATE model_tasks t - SET status='running', updated_at=now() - FROM candidate c - WHERE t.id=c.id AND t.status='queued' - RETURNING t.id, t.provider_id, t.upstream_model, t.task_type, t.prompt - """; - List> rows = jdbc.queryForList(sql, defaultModelConcurrency); - return rows.isEmpty() ? null : rows.getFirst(); + return inTransaction(() -> { + // Keep the lock until the status update commits. The previous + // pg_try_advisory_xact_lock was executed in autocommit mode, so + // the lock ended before the UPDATE and did not coordinate two + // workers reliably. + jdbc.queryForList("SELECT pg_advisory_xact_lock(hashtextextended('moyi:async-dispatch', 0))"); + String sql = """ + SELECT t.id, t.api_key_id, m.provider_id, m.upstream_model, t.task_type, t.prompt + FROM model_tasks t + JOIN model_catalog m ON m.id=t.model_id + JOIN model_providers p ON p.id=m.provider_id + WHERE t.status='queued' + AND t.provider_task_id IS NULL + AND m.status='active' AND m.deleted_at IS NULL + AND p.status='active' AND p.deleted_at IS NULL + AND (SELECT COUNT(*) FROM model_tasks running_model + WHERE running_model.model_id=t.model_id AND running_model.status='running') + < COALESCE(m.max_concurrency, ?) + AND (SELECT COUNT(*) FROM model_tasks running_key + WHERE running_key.api_key_id=t.api_key_id AND running_key.status='running') + < ? + ORDER BY t.created_at + LIMIT 1 + FOR UPDATE OF t SKIP LOCKED + """; + List> rows = jdbc.queryForList(sql, defaultModelConcurrency, defaultKeyConcurrency); + if (rows.isEmpty()) return null; + Map row = rows.getFirst(); + jdbc.update("UPDATE model_tasks SET status='running', updated_at=now() WHERE id=? AND status='queued'", row.get("id")); + return row; + }); + } + + private void recoverInterruptedClaims() { + List taskIds = inTransaction(() -> jdbc.queryForList( + "SELECT id FROM model_tasks WHERE status='running' AND provider_task_id IS NULL AND updated_at < now() - INTERVAL '2 minutes' FOR UPDATE SKIP LOCKED", + String.class)); + for (String taskId : taskIds) { + failTask(taskId, "UPSTREAM_TASK_SUBMIT_INTERRUPTED", "上游任务提交被中断,请确认上游记录后重新提交"); + } + } + + private void failTask(String taskId, String code, String message) { + int changed = jdbc.update( + "UPDATE model_tasks SET status='failed', error=CAST(? AS jsonb), updated_at=?, completed_at=? WHERE id=? AND status IN ('queued','running')", + writeJson(Map.of("code", code, "message", message)), now(), now(), taskId); + if (changed == 0) return; + releaseTask(taskId); + usage.updateTask(taskId, "failed", null, code, message); } private Map requireKey(String authorization) { @@ -393,6 +420,10 @@ public class AsyncTaskService { } } + private T inTransaction(Supplier work) { + return transactions == null ? work.get() : transactions.execute(status -> work.get()); + } + private UUID taskRequestId(String taskId) { return UUID.nameUUIDFromBytes(("moyi-async:" + taskId).getBytes(StandardCharsets.UTF_8)); } private void putIfPresent(Map target, String name, ResultSet resultSet, String column) throws SQLException { diff --git a/prototype/src/types.ts b/prototype/src/types.ts index b71e793..fdf8976 100644 --- a/prototype/src/types.ts +++ b/prototype/src/types.ts @@ -103,7 +103,7 @@ export interface ProviderItem { models: number latency: string successRate: string - status: '健康' | '波动' | '维护中' + status: '健康' | '已启用' | '波动' | '维护中' enabled: boolean code?: string protocol?: 'openai-compatible' | 'http' | 'sdk' diff --git a/prototype/src/views/AdminConsole.vue b/prototype/src/views/AdminConsole.vue index b21ab8f..2918e53 100644 --- a/prototype/src/views/AdminConsole.vue +++ b/prototype/src/views/AdminConsole.vue @@ -34,6 +34,7 @@ import { import type { AdminUser, ModelItem, ProviderItem } from '../types' import { catalogClient, type AdminAuditEvent, type AdminUsageSummary, type CatalogModel, type CatalogProvider, type UsageAuditRecord } from '../services/catalog' import type { AuthUser } from '../services/auth' +import { authClient } from '../services/auth' import SlidePanel from '../components/SlidePanel.vue' import UiModal from '../components/UiModal.vue' import CatalogAdmin from '../components/CatalogAdmin.vue' @@ -67,17 +68,19 @@ const usageDeleteTarget = ref(null) const usageBulkDeleteOpen = ref(false) const auditEvents = ref>([]) const overviewRefreshing = ref(false) +const verificationChannels = ref<{ email: boolean; phone: boolean } | null>(null) async function refreshOverviewData(notifyOnSuccess = false) { if (overviewRefreshing.value) return overviewRefreshing.value = true try { - const [providerResult, modelResult, userResult, auditResult, summaryResult] = await Promise.all([ + const [providerResult, modelResult, userResult, auditResult, summaryResult, verificationResult] = await Promise.all([ catalogClient.listAdminProviders(), catalogClient.listAdminModels(), catalogClient.listAdminUsers(), catalogClient.listAdminAudit(), catalogClient.getAdminUsageSummary().catch(() => null), + authClient.verificationChannels().catch(() => null), ]) providers.value = mapProviders(providerResult.providers, modelResult.models) models.value = mapModels(modelResult.models, providerResult.providers) @@ -88,6 +91,7 @@ async function refreshOverviewData(notifyOnSuccess = false) { publicModelCount.value = new Set(publishedRoutes.map((item) => item.logicalModelId)).size unpricedRouteCount.value = modelResult.models.filter((item) => item.status === 'active' && activeProviders.has(item.providerId) && activeDefinitions.has(item.logicalModelId) && !item.pricingConfigured).length adminSummary.value = summaryResult + verificationChannels.value = verificationResult users.value = userResult.users.map(mapUser) auditEvents.value = auditResult.events.map(mapAudit) catalogConnected.value = true @@ -225,9 +229,9 @@ function mapProviders(items: CatalogProvider[], modelItems: CatalogModel[]): Pro name: provider.name, capability: modelItems.filter((model) => model.providerId === provider.id).map((model) => model.type).join(' / ') || '待配置能力', models: modelItems.filter((model) => model.providerId === provider.id).length, - latency: '待采样', + latency: '未采样', successRate: '—', - status: provider.status === 'active' ? '健康' : '维护中', + status: provider.status === 'active' ? '已启用' : '维护中', enabled: provider.status === 'active', code: provider.code, protocol: provider.protocol, @@ -250,7 +254,7 @@ function mapModels(items: CatalogModel[], providersList: CatalogProvider[]): Mod type: typeLabels[model.type] || '文本', mode: modeLabels[model.mode] || '同步', description: model.description, - latency: model.mode === 'async' ? '异步任务' : '待采样', + latency: model.mode === 'async' ? '异步任务' : '未采样', specification: model.outputTypes.join(' / ') || '统一接口', availability: model.status === 'active' ? '在线' : '停用', accent: model.type === '3d' ? 'orange' : 'violet', @@ -272,9 +276,9 @@ function mapModels(items: CatalogModel[], providersList: CatalogProvider[]): Mod