2026-09-28 04:55:00 +08:00
package com.moyi.controlplane.task ;
import com.fasterxml.jackson.databind.ObjectMapper ;
import com.moyi.controlplane.catalog.CatalogService ;
import com.moyi.controlplane.common.ApiException ;
import com.moyi.controlplane.gateway.GatewayGuard ;
import com.moyi.controlplane.usage.UsageAuditService ;
import java.math.BigDecimal ;
import java.time.Duration ;
import java.time.OffsetDateTime ;
import java.util.Map ;
import java.util.UUID ;
2026-10-21 18:41:21 +08:00
import java.util.ArrayList ;
import java.util.concurrent.CountDownLatch ;
import java.util.concurrent.Executors ;
import java.util.concurrent.TimeUnit ;
import com.moyi.controlplane.gateway.request.ApiRequestMetadata ;
2026-09-28 04:55:00 +08:00
import org.flywaydb.core.Flyway ;
import org.junit.jupiter.api.AfterEach ;
import org.junit.jupiter.api.BeforeEach ;
import org.junit.jupiter.api.Test ;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable ;
import org.junit.jupiter.params.ParameterizedTest ;
import org.junit.jupiter.params.provider.ValueSource ;
import org.springframework.jdbc.core.JdbcTemplate ;
import org.springframework.jdbc.datasource.DataSourceTransactionManager ;
import org.springframework.jdbc.datasource.DriverManagerDataSource ;
import org.springframework.test.util.ReflectionTestUtils ;
import static org.junit.jupiter.api.Assertions.* ;
import static org.mockito.ArgumentMatchers.* ;
import static org.mockito.Mockito.* ;
@EnabledIfEnvironmentVariable ( named = "MOYI_AUTH_TEST_JDBC_URL" , matches = "jdbc:postgresql://127\\.0\\.0\\.1:[0-9]+/.+" )
class AsyncTaskPostgresTest {
private final ObjectMapper json = new ObjectMapper (). findAndRegisterModules ();
private final UUID userId = UUID . randomUUID (), keyId = UUID . randomUUID ();
private final String taskId = "async-transaction-test" ;
private String schema ;
private DriverManagerDataSource base ;
private JdbcTemplate jdbc ;
private UsageAuditService usage ;
private AsyncProviderClient provider ;
private AsyncTaskService tasks ;
2026-10-21 18:41:21 +08:00
private CatalogService catalog ;
2026-09-28 04:55:00 +08:00
@BeforeEach void setup () {
String url = System . getenv ( "MOYI_AUTH_TEST_JDBC_URL" );
schema = "async_task_test_" + UUID . randomUUID (). toString (). replace ( "-" , "" );
base = new DriverManagerDataSource ( url , "moyi_role_test" , "" );
var source = new DriverManagerDataSource ( url + "?currentSchema=" + schema , "moyi_role_test" , "" );
Flyway . configure (). dataSource ( base ). schemas ( schema ). defaultSchema ( schema ). target ( "25" ). load (). migrate ();
jdbc = new JdbcTemplate ( source );
usage = spy ( new UsageAuditService ( jdbc , json ));
provider = mock ( AsyncProviderClient . class );
2026-10-13 19:08:50 +08:00
GatewayGuard guard = mock ( GatewayGuard . class );
when ( guard . tryTaskUserSlot ( any (), anyString (), anyLong ())). thenReturn ( true );
2026-10-21 18:41:21 +08:00
catalog = mock ( CatalogService . class );
tasks = new AsyncTaskService ( jdbc , json , catalog , 100 , 4 , 4 , 120_000 ,
2026-10-13 19:08:50 +08:00
guard , usage , provider , new DataSourceTransactionManager ( source ));
2026-09-28 04:55:00 +08:00
jdbc . update ( "INSERT INTO users(id,account,account_kind,password_hash,display_name) VALUES (?,'async-test','username','not-a-password','Test')" , userId );
jdbc . update ( "INSERT INTO platform_api_keys(id,user_id,name,key_prefix,key_hash) VALUES (?,?,'Test','test-prefix',?)" , keyId , userId , "b" . repeat ( 64 ));
jdbc . update ( "INSERT INTO user_wallets(user_id,balance,reserved_balance) VALUES (?,10,1.44)" , userId );
jdbc . update ( "INSERT INTO model_tasks(id,user_id,api_key_id,model_id,task_type,status,prompt,reserved_amount,provider_task_id) VALUES (?,?,?,'model-mock-3d','3d','running','chair',1.44,'upstream-task')" , taskId , userId , keyId );
usage . recordTask ( userId , keyId , "model-mock-3d" , "provider-vast" , taskId , "running" , Map . of ( "prompt" , "chair" ), null , null , null );
Flyway . configure (). dataSource ( base ). schemas ( schema ). defaultSchema ( schema ). load (). migrate ();
2026-10-21 18:41:21 +08:00
when ( catalog . keyOwnerId ( keyId )). thenReturn ( userId );
when ( catalog . keyCanInvoke ( anyMap (), eq ( "model-mock-3d" ))). thenReturn ( true );
when ( catalog . activeModel ( "model-mock-3d" )). thenReturn ( Map . of ( "id" , "model-mock-3d" , "providerId" , "provider-vast" , "type" , "3d" , "mode" , "async" , "pricing" , Map . of ( "unit" , "CNY / task" , "price" , 1 . 44 )));
2026-09-28 04:55:00 +08:00
}
@AfterEach void cleanup () {
if ( base != null && schema != null ) new JdbcTemplate ( base ). execute ( "DROP SCHEMA IF EXISTS " + schema + " CASCADE" );
}
private void fail () {
ReflectionTestUtils . invokeMethod ( tasks , "failTask" , taskId , "PROVIDER_NOT_CONFIGURED" , "异步渠道尚未配置 API Key" );
}
private void poll () {
2026-10-13 19:08:50 +08:00
Map < String , Object > row = jdbc . queryForMap ( "SELECT id,model_id,provider_task_id,provider_deadline_at,created_at,provider_key_fingerprint FROM model_tasks WHERE id=?" , taskId );
2026-09-28 04:55:00 +08:00
row . put ( "provider_id" , "provider-vast" );
2026-10-13 19:08:50 +08:00
row . put ( "upstream_model" , "v3.1-20260211" );
2026-09-28 04:55:00 +08:00
ReflectionTestUtils . invokeMethod ( tasks , "pollProviderTask" , row );
}
private BigDecimal wallet ( String column ) {
return jdbc . queryForObject ( "SELECT " + column + " FROM user_wallets WHERE user_id=?" , BigDecimal . class , userId );
}
private String status () {
return jdbc . queryForObject ( "SELECT status FROM model_tasks WHERE id=?" , String . class , taskId );
}
private BigDecimal cost () {
return jdbc . queryForObject ( "SELECT cost_amount FROM model_tasks WHERE id=?" , BigDecimal . class , taskId );
}
2026-10-21 18:41:21 +08:00
private Map < String , Object > create ( String idempotencyKey , Map < String , Object > input ) {
return tasks . createForKey ( Map . of ( "id" , keyId . toString ()), "3d" , input , idempotencyKey , ApiRequestMetadata . internal ());
}
@Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement () {
var input = Map . < String , Object > of ( "model" , "model-mock-3d" , "prompt" , "one chair" );
var first = create ( "client-operation-1" , input );
var replay = create ( "client-operation-1" , new java . util . LinkedHashMap <> ( input ));
assertEquals ( first . get ( "id" ), replay . get ( "id" ));
String created = String . valueOf ( first . get ( "id" ));
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM usage_events WHERE provider_task_id=?" , Integer . class , created ));
BigDecimal held = wallet ( "reserved_balance" );
ReflectionTestUtils . invokeMethod ( tasks , "finishTask" , created , "succeeded" , 100 , Map . of ( "url" , "https://example.test/result.glb" ), null , null );
when ( catalog . activeModel ( "model-mock-3d" )). thenReturn ( null );
var finishedReplay = create ( "client-operation-1" , input );
assertEquals ( created , finishedReplay . get ( "id" ));
assertEquals ( "succeeded" , finishedReplay . get ( "status" ));
assertTrue ( wallet ( "reserved_balance" ). compareTo ( held ) < 0 );
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM wallet_entries WHERE request_id IS NOT NULL AND user_id=?" , Integer . class , userId ));
String stored = jdbc . queryForObject ( "SELECT idempotency_hash FROM model_tasks WHERE id=?" , String . class , created );
assertEquals ( 64 , stored . length ());
assertNotEquals ( "client-operation-1" , stored );
}
@Test void sameKeyDifferentRequestConflictsWithoutNewReservation () {
create ( "client-conflict" , Map . of ( "model" , "model-mock-3d" , "prompt" , "chair" ));
BigDecimal held = wallet ( "reserved_balance" );
ApiException conflict = assertThrows ( ApiException . class ,
() -> create ( "client-conflict" , Map . of ( "model" , "model-mock-3d" , "prompt" , "table" )));
assertEquals ( "IDEMPOTENCY_CONFLICT" , conflict . getCode ());
assertEquals ( 409 , conflict . getStatus ());
assertEquals ( held , wallet ( "reserved_balance" ));
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL" , Integer . class ));
}
@Test void concurrentSameKeyCreatesOneTaskOneAuditAndOneHold () throws Exception {
var input = Map . < String , Object > of ( "model" , "model-mock-3d" , "prompt" , "concurrent chair" );
BigDecimal held = wallet ( "reserved_balance" );
var start = new CountDownLatch ( 1 );
try ( var executor = Executors . newFixedThreadPool ( 8 )) {
var results = new ArrayList < java . util . concurrent . Future < Map < String , Object >>> ();
for ( int i = 0 ; i < 8 ; i ++ ) results . add ( executor . submit (() -> {
assertTrue ( start . await ( 10 , TimeUnit . SECONDS ));
return create ( "concurrent-operation" , input );
}));
start . countDown ();
var ids = new java . util . HashSet < Object > ();
for ( var future : results ) ids . add ( future . get ( 30 , TimeUnit . SECONDS ). get ( "id" ));
assertEquals ( 1 , ids . size ());
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM usage_events WHERE provider_task_id=?" , Integer . class , ids . iterator (). next ()));
}
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM model_tasks WHERE idempotency_hash IS NOT NULL" , Integer . class ));
assertEquals ( new BigDecimal ( "1.44000000" ), wallet ( "reserved_balance" ). subtract ( held ));
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM gateway_budget_holds" , Integer . class ));
verifyNoInteractions ( provider );
}
2026-09-28 04:55:00 +08:00
@ParameterizedTest
@ValueSource ( strings = { "PROVIDER_NOT_CONFIGURED" , "UPSTREAM_AUTH_FAILED" })
void submissionFailureReleasesExactlyOnceWithoutDebitingBalance ( String code ) {
jdbc . update ( "UPDATE model_tasks SET status='queued',provider_task_id=NULL WHERE id=?" , taskId );
2026-10-13 19:08:50 +08:00
when ( provider . submit ( anyString (), anyString (), anyString (), anyString (), anyString ())). thenThrow ( new ApiException ( code , "上游任务未接受" , 503 ));
2026-09-28 04:55:00 +08:00
tasks . completeMockTasks ();
tasks . completeMockTasks ();
assertEquals ( "failed" , status ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( 0 , cost (). signum ());
assertEquals ( 0 , jdbc . queryForObject ( "SELECT cost_amount FROM usage_events WHERE provider_task_id=?" , BigDecimal . class , taskId ). signum ());
assertEquals ( 0 , jdbc . queryForObject ( "SELECT count(*) FROM wallet_entries WHERE user_id=?" , Integer . class , userId ));
2026-10-13 19:08:50 +08:00
verify ( provider , times ( 1 )). submit ( anyString (), anyString (), anyString (), anyString (), anyString ());
2026-09-28 04:55:00 +08:00
}
@Test void refundAndTaskStateRollBackTogetherWhenAuditFails () {
doThrow ( new IllegalStateException ( "simulated database write failure" )). when ( usage ). updateTask ( eq ( taskId ), eq ( "failed" ), isNull (), anyString (), anyString ());
assertThrows ( IllegalStateException . class , this :: fail );
assertEquals ( "running" , status ());
assertEquals ( new BigDecimal ( "1.44000000" ), wallet ( "reserved_balance" ));
assertNull ( cost ());
doCallRealMethod (). when ( usage ). updateTask ( eq ( taskId ), eq ( "failed" ), isNull (), anyString (), anyString ());
fail (); fail ();
assertEquals ( "failed" , status ());
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
}
@Test void successfulSettlementRollsBackForRetryAndCannotBeRefundedByLateFailure () {
2026-10-13 19:08:50 +08:00
when ( provider . poll ( "provider-vast" , "model-mock-3d" , "v3.1-20260211" , "upstream-task" , null )). thenReturn ( Map . of ( "data" , Map . of ( "status" , "success" , "progress" , 100 , "output" , Map . of ( "model_url" , "https://example.test/model.glb" ))));
2026-09-28 04:55:00 +08:00
doThrow ( new IllegalStateException ( "simulated database write failure" )). when ( usage ). updateTask ( eq ( taskId ), eq ( "succeeded" ), anyMap (), isNull (), isNull ());
assertThrows ( IllegalStateException . class , this :: poll );
assertEquals ( "running" , status ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertEquals ( new BigDecimal ( "1.44000000" ), wallet ( "reserved_balance" ));
assertNull ( cost ());
assertEquals ( 0 , jdbc . queryForObject ( "SELECT count(*) FROM wallet_entries WHERE user_id=?" , Integer . class , userId ));
doCallRealMethod (). when ( usage ). updateTask ( eq ( taskId ), eq ( "succeeded" ), anyMap (), isNull (), isNull ());
poll (); poll (); fail ();
assertEquals ( "succeeded" , status ());
assertEquals ( new BigDecimal ( "8.56000000" ), wallet ( "balance" ));
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( new BigDecimal ( "1.44000000" ), cost ());
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM wallet_entries WHERE user_id=?" , Integer . class , userId ));
}
@ParameterizedTest
@ValueSource ( strings = { "UPSTREAM_PROVIDER_ERROR" , "UPSTREAM_AUTH_FAILED" , "UPSTREAM_RATE_LIMITED" })
void pollingErrorsKeepTheHoldAndClearOnRecovery ( String code ) {
2026-10-13 19:08:50 +08:00
when ( provider . poll ( "provider-vast" , "model-mock-3d" , "v3.1-20260211" , "upstream-task" , null ))
2026-09-28 04:55:00 +08:00
. thenThrow ( new ApiException ( code , "private upstream details" , 502 ))
. thenReturn ( Map . of ( "data" , Map . of ( "status" , "running" , "progress" , 75 )));
poll ();
assertEquals ( "running" , status ());
assertEquals ( new BigDecimal ( "1.44000000" ), wallet ( "reserved_balance" ));
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertNull ( cost ());
String error = jdbc . queryForObject ( "SELECT error::text FROM model_tasks WHERE id=?" , String . class , taskId );
assertTrue ( error . contains ( "UPSTREAM_TASK_POLL_RETRY" ));
assertFalse ( error . contains ( "private upstream details" ));
poll ();
assertEquals ( "running" , status ());
assertEquals ( 75 , jdbc . queryForObject ( "SELECT progress FROM model_tasks WHERE id=?" , Integer . class , taskId ));
assertNull ( jdbc . queryForObject ( "SELECT error::text FROM model_tasks WHERE id=?" , String . class , taskId ));
assertNull ( jdbc . queryForObject ( "SELECT error_code FROM usage_events WHERE provider_task_id=?" , String . class , taskId ));
}
@Test void deadlineReleasesHoldWithoutClaimingTheUpstreamFailed () {
jdbc . update ( "UPDATE model_tasks SET provider_deadline_at=now()-interval '1 second' WHERE id=?" , taskId );
poll (); poll ();
assertEquals ( "failed" , status ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( 0 , cost (). signum ());
String error = jdbc . queryForObject ( "SELECT error::text FROM model_tasks WHERE id=?" , String . class , taskId );
assertTrue ( error . contains ( "UPSTREAM_TASK_TIMEOUT" ));
assertTrue ( error . contains ( "执行结果未知" ));
verifyNoInteractions ( provider );
}
@Test void confirmedUpstreamFailureReleasesHoldImmediately () {
2026-10-13 19:08:50 +08:00
when ( provider . poll ( "provider-vast" , "model-mock-3d" , "v3.1-20260211" , "upstream-task" , null )). thenReturn ( Map . of ( "data" , Map . of ( "status" , "failed" )));
2026-09-28 04:55:00 +08:00
poll ();
assertEquals ( "failed" , status ());
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertEquals ( 0 , cost (). signum ());
}
@Test void migrationBackfillsOldAcceptedTasksAndNewSubmissionsUseConfiguredDeadline () {
OffsetDateTime createdAt = jdbc . queryForObject ( "SELECT created_at FROM model_tasks WHERE id=?" , OffsetDateTime . class , taskId );
OffsetDateTime migratedDeadline = jdbc . queryForObject ( "SELECT provider_deadline_at FROM model_tasks WHERE id=?" , OffsetDateTime . class , taskId );
assertEquals ( Duration . ofHours ( 1 ), Duration . between ( createdAt , migratedDeadline ));
jdbc . update ( "UPDATE model_tasks SET status='queued',provider_task_id=NULL,provider_deadline_at=NULL WHERE id=?" , taskId );
2026-10-13 19:08:50 +08:00
when ( provider . submit ( anyString (), anyString (), anyString (), anyString (), anyString ())). thenReturn ( Map . of ( "data" , Map . of ( "task_id" , "upstream-task" )));
2026-09-28 04:55:00 +08:00
tasks . completeMockTasks ();
OffsetDateTime submittedAt = jdbc . queryForObject ( "SELECT updated_at FROM model_tasks WHERE id=?" , OffsetDateTime . class , taskId );
OffsetDateTime deadline = jdbc . queryForObject ( "SELECT provider_deadline_at FROM model_tasks WHERE id=?" , OffsetDateTime . class , taskId );
assertEquals ( Duration . ofMinutes ( 2 ), Duration . between ( submittedAt , deadline ));
tasks . completeMockTasks ();
2026-10-13 19:08:50 +08:00
verify ( provider , times ( 1 )). submit ( anyString (), anyString (), anyString (), anyString (), anyString ());
2026-09-28 04:55:00 +08:00
verify ( provider , never ()). poll ( anyString (), anyString ());
assertEquals ( "running" , status ());
}
2026-10-13 20:13:21 +08:00
@Test void imageTokensSettleSnapshotOnceAndReleaseUnusedReservation () throws Exception {
var pricing = Map . of ( "unit" , "CNY / 1M tokens" , "input" , 20 . 16 , "imageInput" , 32 . 256 , "output" , 120 . 96 , "taskBilling" , "openai-image-tokens" );
jdbc . update ( "UPDATE model_tasks SET pricing_snapshot=CAST(? AS jsonb) WHERE id=?" , json . writeValueAsString ( pricing ), taskId );
var result = Map . < String , Object > of ( "url" , "https://example.test/image.png" , "usage" , Map . of ( "input_tokens" , 50 , "output_tokens" , 50 , "input_tokens_details" , Map . of ( "text_tokens" , 10 , "image_tokens" , 40 )));
ReflectionTestUtils . invokeMethod ( tasks , "finishTask" , taskId , "succeeded" , 100 , result , null , null );
ReflectionTestUtils . invokeMethod ( tasks , "finishTask" , taskId , "succeeded" , 100 , result , null , null );
fail ();
assertEquals ( new BigDecimal ( "0.00753984" ), cost ());
assertEquals ( new BigDecimal ( "9.99246016" ), wallet ( "balance" ));
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( 1 , jdbc . queryForObject ( "SELECT count(*) FROM wallet_entries WHERE user_id=?" , Integer . class , userId ));
assertEquals ( 50L , jdbc . queryForObject ( "SELECT input_units FROM usage_events WHERE provider_task_id=?" , Long . class , taskId ));
}
@Test void completedImageWithoutUsageIsReturnedAndHeldForReconciliationWithoutCharging () throws Exception {
var pricing = Map . of ( "unit" , "CNY / 1M tokens" , "input" , 1 , "output" , 10 , "taskBilling" , "gemini-image-tokens" );
jdbc . update ( "UPDATE model_tasks SET pricing_snapshot=CAST(? AS jsonb) WHERE id=?" , json . writeValueAsString ( pricing ), taskId );
var result = Map . < String , Object > of ( "base64" , "png" );
ReflectionTestUtils . invokeMethod ( tasks , "finishTask" , taskId , "succeeded" , 100 , result , null , null );
assertEquals ( "succeeded" , status ());
assertNull ( cost ());
assertEquals ( new BigDecimal ( "10.00000000" ), wallet ( "balance" ));
assertEquals ( 0 , wallet ( "reserved_balance" ). signum ());
assertEquals ( "pending" , jdbc . queryForObject ( "SELECT cost_status FROM usage_events WHERE provider_task_id=?" , String . class , taskId ));
assertEquals ( "TASK_USAGE_PENDING" , jdbc . queryForObject ( "SELECT error_code FROM usage_events WHERE provider_task_id=?" , String . class , taskId ));
assertTrue ( jdbc . queryForObject ( "SELECT result::text FROM model_tasks WHERE id=?" , String . class , taskId ). contains ( "png" ));
}
2026-09-28 04:55:00 +08:00
}