package com.moyi.controlplane.task; import com.fasterxml.jackson.databind.ObjectMapper; import com.moyi.controlplane.catalog.CatalogService; import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.gateway.GatewayGuard; import com.moyi.controlplane.usage.UsageAuditService; import java.math.BigDecimal; import java.time.Duration; import java.time.OffsetDateTime; import java.util.Map; import java.util.UUID; import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import com.moyi.controlplane.gateway.request.ApiRequestMetadata; import org.flywaydb.core.Flyway; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.test.util.ReflectionTestUtils; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @EnabledIfEnvironmentVariable(named="MOYI_AUTH_TEST_JDBC_URL", matches="jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+") class AsyncTaskPostgresTest { private final ObjectMapper json = new ObjectMapper().findAndRegisterModules(); private final UUID userId = UUID.randomUUID(), keyId = UUID.randomUUID(); private final String taskId = "async-transaction-test"; private String schema; private DriverManagerDataSource base; private JdbcTemplate jdbc; private UsageAuditService usage; private AsyncProviderClient provider; private AsyncTaskService tasks; private CatalogService catalog; @BeforeEach void setup() { String url = System.getenv("MOYI_AUTH_TEST_JDBC_URL"); schema = "async_task_test_" + UUID.randomUUID().toString().replace("-", ""); base = new DriverManagerDataSource(url, "moyi_role_test", ""); var source = new DriverManagerDataSource(url + "?currentSchema=" + schema, "moyi_role_test", ""); Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).target("25").load().migrate(); jdbc = new JdbcTemplate(source); usage = spy(new UsageAuditService(jdbc, json)); provider = mock(AsyncProviderClient.class); GatewayGuard guard=mock(GatewayGuard.class); when(guard.tryTaskUserSlot(any(),anyString(),anyLong())).thenReturn(true); catalog = mock(CatalogService.class); tasks = new AsyncTaskService(jdbc, json, catalog, 100, 4, 4, 120_000, guard, usage, provider, new DataSourceTransactionManager(source)); jdbc.update("INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')", userId); jdbc.update("INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)", keyId, userId, "b".repeat(64)); jdbc.update("INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)", userId); jdbc.update("INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')", taskId,userId,keyId); usage.recordTask(userId,keyId,"model-mock-3d","provider-vast",taskId,"running",Map.of("prompt","chair"),null,null,null); Flyway.configure().dataSource(base).schemas(schema).defaultSchema(schema).load().migrate(); when(catalog.keyOwnerId(keyId)).thenReturn(userId); when(catalog.keyCanInvoke(anyMap(), eq("model-mock-3d"))).thenReturn(true); when(catalog.activeModel("model-mock-3d")).thenReturn(Map.of("id", "model-mock-3d", "providerId", "provider-vast", "type", "3d", "mode", "async", "pricing", Map.of("unit", "CNY / task", "price", 1.44))); } @AfterEach void cleanup() { if (base != null && schema != null) new JdbcTemplate(base).execute("DROP SCHEMA IF EXISTS " + schema + " CASCADE"); } private void fail() { ReflectionTestUtils.invokeMethod(tasks,"failTask",taskId,"PROVIDER_NOT_CONFIGURED","异步渠道尚未配置 API Key"); } private void poll() { Map row = jdbc.queryForMap("SELECT id,model_id,provider_task_id,provider_deadline_at,created_at,provider_key_fingerprint FROM model_tasks WHERE id=?", taskId); row.put("provider_id", "provider-vast"); row.put("upstream_model","v3.1-20260211"); ReflectionTestUtils.invokeMethod(tasks,"pollProviderTask",row); } private BigDecimal wallet(String column) { return jdbc.queryForObject("SELECT " + column + " FROM user_wallets WHERE user_id=?",BigDecimal.class,userId); } private String status() { return jdbc.queryForObject("SELECT status FROM model_tasks WHERE id=?",String.class,taskId); } private BigDecimal cost() { return jdbc.queryForObject("SELECT cost_amount FROM model_tasks WHERE id=?",BigDecimal.class,taskId); } private Map create(String idempotencyKey, Map input) { return tasks.createForKey(Map.of("id", keyId.toString()), "3d", input, idempotencyKey, ApiRequestMetadata.internal()); } @Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() { var input = Map.of("model", "model-mock-3d", "prompt", "one chair"); var first = create("client-operation-1", input); var replay = create("client-operation-1", new java.util.LinkedHashMap<>(input)); assertEquals(first.get("id"), replay.get("id")); String created = String.valueOf(first.get("id")); assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, created)); BigDecimal held = wallet("reserved_balance"); ReflectionTestUtils.invokeMethod(tasks, "finishTask", created, "succeeded", 100, Map.of("url", "https://example.test/result.glb"), null, null); when(catalog.activeModel("model-mock-3d")).thenReturn(null); var finishedReplay = create("client-operation-1", input); assertEquals(created, finishedReplay.get("id")); assertEquals("succeeded", finishedReplay.get("status")); assertTrue(wallet("reserved_balance").compareTo(held) < 0); assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE request_id IS NOT NULL AND user_id=?", Integer.class, userId)); String stored = jdbc.queryForObject("SELECT idempotency_hash FROM model_tasks WHERE id=?", String.class, created); assertEquals(64, stored.length()); assertNotEquals("client-operation-1", stored); } @Test void sameKeyDifferentRequestConflictsWithoutNewReservation() { create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "chair")); BigDecimal held = wallet("reserved_balance"); ApiException conflict = assertThrows(ApiException.class, () -> create("client-conflict", Map.of("model", "model-mock-3d", "prompt", "table"))); assertEquals("IDEMPOTENCY_CONFLICT", conflict.getCode()); assertEquals(409, conflict.getStatus()); assertEquals(held, wallet("reserved_balance")); assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class)); } @Test void concurrentSameKeyCreatesOneTaskOneAuditAndOneHold() throws Exception { var input = Map.of("model", "model-mock-3d", "prompt", "concurrent chair"); BigDecimal held = wallet("reserved_balance"); var start = new CountDownLatch(1); try (var executor = Executors.newFixedThreadPool(8)) { var results = new ArrayList>>(); for (int i = 0; i < 8; i++) results.add(executor.submit(() -> { assertTrue(start.await(10, TimeUnit.SECONDS)); return create("concurrent-operation", input); })); start.countDown(); var ids = new java.util.HashSet(); for (var future : results) ids.add(future.get(30, TimeUnit.SECONDS).get("id")); assertEquals(1, ids.size()); assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM usage_events WHERE provider_task_id=?", Integer.class, ids.iterator().next())); } assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL", Integer.class)); assertEquals(new BigDecimal("1.44000000"), wallet("reserved_balance").subtract(held)); assertEquals(1, jdbc.queryForObject("SELECT count(*) FROM gateway_budget_holds", Integer.class)); verifyNoInteractions(provider); } @ParameterizedTest @ValueSource(strings={"PROVIDER_NOT_CONFIGURED", "UPSTREAM_AUTH_FAILED"}) void submissionFailureReleasesExactlyOnceWithoutDebitingBalance(String code) { jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?",taskId); when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenThrow(new ApiException(code,"上游任务未接受",503)); tasks.completeMockTasks(); tasks.completeMockTasks(); assertEquals("failed",status()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(0,cost().signum()); assertEquals(0,jdbc.queryForObject("SELECT cost_amount FROM usage_events WHERE provider_task_id=?",BigDecimal.class,taskId).signum()); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId)); verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString(),anyString()); } @Test void refundAndTaskStateRollBackTogetherWhenAuditFails() { doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString()); assertThrows(IllegalStateException.class,this::fail); assertEquals("running",status()); assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance")); assertNull(cost()); doCallRealMethod().when(usage).updateTask(eq(taskId),eq("failed"),isNull(),anyString(),anyString()); fail(); fail(); assertEquals("failed",status()); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); } @Test void successfulSettlementRollsBackForRetryAndCannotBeRefundedByLateFailure() { when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","success","progress",100,"output",Map.of("model_url","https://example.test/model.glb")))); doThrow(new IllegalStateException("simulated database write failure")).when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull()); assertThrows(IllegalStateException.class,this::poll); assertEquals("running",status()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance")); assertNull(cost()); assertEquals(0,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId)); doCallRealMethod().when(usage).updateTask(eq(taskId),eq("succeeded"),anyMap(),isNull(),isNull()); poll(); poll(); fail(); assertEquals("succeeded",status()); assertEquals(new BigDecimal("8.56000000"),wallet("balance")); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(new BigDecimal("1.44000000"),cost()); assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId)); } @ParameterizedTest @ValueSource(strings={"UPSTREAM_PROVIDER_ERROR", "UPSTREAM_AUTH_FAILED", "UPSTREAM_RATE_LIMITED"}) void pollingErrorsKeepTheHoldAndClearOnRecovery(String code) { when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)) .thenThrow(new ApiException(code,"private upstream details",502)) .thenReturn(Map.of("data",Map.of("status","running","progress",75))); poll(); assertEquals("running",status()); assertEquals(new BigDecimal("1.44000000"),wallet("reserved_balance")); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertNull(cost()); String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId); assertTrue(error.contains("UPSTREAM_TASK_POLL_RETRY")); assertFalse(error.contains("private upstream details")); poll(); assertEquals("running",status()); assertEquals(75,jdbc.queryForObject("SELECT progress FROM model_tasks WHERE id=?",Integer.class,taskId)); assertNull(jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId)); assertNull(jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId)); } @Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed() { jdbc.update("UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?",taskId); poll(); poll(); assertEquals("failed",status()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(0,cost().signum()); String error = jdbc.queryForObject("SELECT error::text FROM model_tasks WHERE id=?",String.class,taskId); assertTrue(error.contains("UPSTREAM_TASK_TIMEOUT")); assertTrue(error.contains("执行结果未知")); verifyNoInteractions(provider); } @Test void confirmedUpstreamFailureReleasesHoldImmediately() { when(provider.poll("provider-vast","model-mock-3d","v3.1-20260211","upstream-task",null)).thenReturn(Map.of("data",Map.of("status","failed"))); poll(); assertEquals("failed",status()); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertEquals(0,cost().signum()); } @Test void migrationBackfillsOldAcceptedTasksAndNewSubmissionsUseConfiguredDeadline() { OffsetDateTime createdAt = jdbc.queryForObject("SELECT created_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId); OffsetDateTime migratedDeadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId); assertEquals(Duration.ofHours(1),Duration.between(createdAt,migratedDeadline)); jdbc.update("UPDATE model_tasks SET status='queued',provider_task_id=NULL,provider_deadline_at=NULL WHERE id=?",taskId); when(provider.submit(anyString(),anyString(),anyString(),anyString(),anyString())).thenReturn(Map.of("data",Map.of("task_id","upstream-task"))); tasks.completeMockTasks(); OffsetDateTime submittedAt = jdbc.queryForObject("SELECT updated_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId); OffsetDateTime deadline = jdbc.queryForObject("SELECT provider_deadline_at FROM model_tasks WHERE id=?",OffsetDateTime.class,taskId); assertEquals(Duration.ofMinutes(2),Duration.between(submittedAt,deadline)); tasks.completeMockTasks(); verify(provider,times(1)).submit(anyString(),anyString(),anyString(),anyString(),anyString()); verify(provider,never()).poll(anyString(),anyString()); assertEquals("running",status()); } @Test void imageTokensSettleSnapshotOnceAndReleaseUnusedReservation() throws Exception { var pricing=Map.of("unit","CNY / 1M tokens","input",20.16,"imageInput",32.256,"output",120.96,"taskBilling","openai-image-tokens"); jdbc.update("UPDATE model_tasks SET pricing_snapshot=CAST(? AS jsonb) WHERE id=?",json.writeValueAsString(pricing),taskId); var result=Map.of("url","https://example.test/image.png","usage",Map.of("input_tokens",50,"output_tokens",50,"input_tokens_details",Map.of("text_tokens",10,"image_tokens",40))); ReflectionTestUtils.invokeMethod(tasks,"finishTask",taskId,"succeeded",100,result,null,null); ReflectionTestUtils.invokeMethod(tasks,"finishTask",taskId,"succeeded",100,result,null,null); fail(); assertEquals(new BigDecimal("0.00753984"),cost()); assertEquals(new BigDecimal("9.99246016"),wallet("balance")); assertEquals(0,wallet("reserved_balance").signum()); assertEquals(1,jdbc.queryForObject("SELECT count(*) FROM wallet_entries WHERE user_id=?",Integer.class,userId)); assertEquals(50L,jdbc.queryForObject("SELECT input_units FROM usage_events WHERE provider_task_id=?",Long.class,taskId)); } @Test void completedImageWithoutUsageIsReturnedAndHeldForReconciliationWithoutCharging() throws Exception { var pricing=Map.of("unit","CNY / 1M tokens","input",1,"output",10,"taskBilling","gemini-image-tokens"); jdbc.update("UPDATE model_tasks SET pricing_snapshot=CAST(? AS jsonb) WHERE id=?",json.writeValueAsString(pricing),taskId); var result=Map.of("base64","png"); ReflectionTestUtils.invokeMethod(tasks,"finishTask",taskId,"succeeded",100,result,null,null); assertEquals("succeeded",status()); assertNull(cost()); assertEquals(new BigDecimal("10.00000000"),wallet("balance")); assertEquals(0,wallet("reserved_balance").signum()); assertEquals("pending",jdbc.queryForObject("SELECT cost_status FROM usage_events WHERE provider_task_id=?",String.class,taskId)); assertEquals("TASK_USAGE_PENDING",jdbc.queryForObject("SELECT error_code FROM usage_events WHERE provider_task_id=?",String.class,taskId)); assertTrue(jdbc.queryForObject("SELECT result::text FROM model_tasks WHERE id=?",String.class,taskId).contains("png")); } }