diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d15e87c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: maven + directory: /control-plane-java + schedule: + interval: weekly + open-pull-requests-limit: 5 + - package-ecosystem: npm + directory: /prototype + schedule: + interval: weekly + open-pull-requests-limit: 5 + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml new file mode 100644 index 0000000..b9bd890 --- /dev/null +++ b/.github/workflows/verification.yml @@ -0,0 +1,66 @@ +name: Repository verification +on: + pull_request: + push: + branches: [master, main] +permissions: + contents: read +concurrency: + group: verify-${{ github.ref }} + cancel-in-progress: true +jobs: + backend: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:15.18 + env: + POSTGRES_USER: moyi_role_test + POSTGRES_DB: moyi_ci_test + POSTGRES_HOST_AUTH_METHOD: trust + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U moyi_role_test -d moyi_ci_test" + --health-interval 5s --health-timeout 5s --health-retries 10 + redis: + image: redis:7.4-alpine + ports: ['6380:6379'] + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MOYI_AUTH_TEST_JDBC_URL: jdbc:postgresql://127.0.0.1:5432/moyi_ci_test + MOYI_GATEWAY_TEST_REDIS_PORT: '6380' + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 + with: + distribution: temurin + java-version: '21' + cache: maven + - run: mvn -B -ntp -f control-plane-java/pom.xml clean verify + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: failure() + with: + name: backend-test-results + path: control-plane-java/target/surefire-reports/ + frontend: + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: prototype + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: '9' + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + cache: pnpm + cache-dependency-path: prototype/pnpm-lock.yaml + - run: pnpm install --frozen-lockfile + - run: pnpm run build + - run: pnpm test diff --git a/README.md b/README.md index 5344b0d..d36658d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ > > One Gateway for Every Model You Use. +## 当前工程状态 + +2026-10-21:当前产品为 Java/Vue 自研控制面与多模型网关,已补请求模块化、账务恢复、异步幂等和审计核账,并继续加固认证与 HTTP 边界。下文包含产品演进目标,不等于功能均已实现。**尚未完成完整 Codex WebSocket/续接、长请求续租、公平排队、账号全生命周期和媒体出站/DNS 绑定闭环。** + +逐项完成状态与剩余验收见 [安全与 CPR 任务追踪](docs/24-security-and-cpr-task-tracker.md),本轮设计与限制见 [认证及有界 HTTP 增量](docs/25-auth-and-bounded-transport.md)。本地验证通过不代表已部署或全仓库安全审计完成。 + ## 一、产品简介 模亦灵枢是由模灵科技与亦城财富等合作方共同推动的多模型智能服务平台。 diff --git a/control-plane-java/README.md b/control-plane-java/README.md index f08885c..564ed21 100644 --- a/control-plane-java/README.md +++ b/control-plane-java/README.md @@ -25,6 +25,10 @@ `provider.credentials.UpstreamCredentialResolver` 共享密钥池解析与轮转;`provider.transport.SseEventReader` 专门负责 SSE 解帧、UTF-8 校验和有界读取,不承担风控、路由或计费。完整模块边界、验证命令和后续风险清单见 `../docs/21-api-request-modularization.md`。 +`V49` 增加数据库共享登录窗口;所有新密码统一 8–256 位,改密撤销全部浏览器会话,登录/改密/重置通过用户行锁协调。初次创建管理员必须显式配置非默认密码;已存在管理员不会被启动配置重置。API Key 不随改密自动撤销。 + +`provider.transport.BoundedHttpTransport` 已供异步 JSON/multipart/音频/媒体读取复用:接收中限量、总期限覆盖响应体、取消订阅、不自动跟随重定向。它不等于 SSRF 已解决,DNS 预检与实际解析的绑定仍需补齐;同步/SSE 生命周期和 Codex 长请求亦未统一完成。详见 `../docs/25-auth-and-bounded-transport.md` 与 `../docs/24-security-and-cpr-task-tracker.md`。 + Flyway 脚本位于 `src/main/resources/db/migration/`,与现有 TypeScript 服务使用同一套基础表命名,后续迁移数据时可以按表逐步切换。 本目录的镜像由 `infra/moyi/docker-compose.yml` 构建。生产发布前应验证目标渠道能力、价格快照、Redis 准入和 PostgreSQL 集成测试;本地通过不代表线上部署验收。TypeScript Control Plane 保留为历史参考实现,不与 Java 后端同时承担同一钱包的写入。 diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java new file mode 100644 index 0000000..63eaef5 --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java @@ -0,0 +1,114 @@ +package com.moyi.controlplane.provider.transport; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** One bounded HTTP attempt. No credentials, route selection, protocol mapping, retry, risk or billing policy. */ +public final class BoundedHttpTransport { + private static final int ERROR_BODY_LIMIT = 64 * 1024; + private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NEVER).build(); + + public record Response(int statusCode, HttpHeaders headers, byte[] body) { + @Override public String toString() { return "TransportResponse[status=" + statusCode + ", bytes=" + body.length + "]"; } + } + public enum Reason { TIMEOUT, INTERRUPTED, BODY_TOO_LARGE, IO } + public static final class Failure extends IOException { + private final Reason reason; + private final Integer observedStatus; + private Failure(Reason reason, Integer status) { + super("Bounded HTTP attempt failed: " + reason); + this.reason = reason; this.observedStatus = status; + } + public Reason reason() { return reason; } + public Integer observedStatus() { return observedStatus; } + } + + public Response send(HttpRequest request, int maxResponseBytes) throws Failure { + if (maxResponseBytes < 1 || maxResponseBytes > 32 * 1024 * 1024) throw new IllegalArgumentException("Invalid response limit"); + Duration timeout = request.timeout().orElseThrow(() -> new IllegalArgumentException("An explicit total HTTP deadline is required")); + var status = new AtomicInteger(); + var subscriber = new AtomicReference(); + CompletableFuture> attempt = client.sendAsync(request, info -> { + status.set(info.statusCode()); + int limit = info.statusCode() >= 200 && info.statusCode() < 300 ? maxResponseBytes : Math.min(maxResponseBytes, ERROR_BODY_LIMIT); + var body = new LimitedBody(limit, info.statusCode(), info.headers().firstValueAsLong("Content-Length").orElse(-1)); + subscriber.set(body); + return body; + }); + try { + // Unlike a header-only timeout, this also bounds a server that drips or never finishes the body. + HttpResponse response = attempt.get(Math.max(1, timeout.toMillis()), TimeUnit.MILLISECONDS); + return new Response(response.statusCode(), response.headers(), response.body()); + } catch (TimeoutException exception) { + throw cancel(attempt, subscriber.get(), Reason.TIMEOUT, status.get()); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw cancel(attempt, subscriber.get(), Reason.INTERRUPTED, status.get()); + } catch (ExecutionException exception) { + Throwable cause = exception.getCause(); + while (cause instanceof java.util.concurrent.CompletionException && cause.getCause() != null) cause = cause.getCause(); + if (cause instanceof Failure failure) throw failure; + throw cancel(attempt, subscriber.get(), cause instanceof java.net.http.HttpTimeoutException ? Reason.TIMEOUT : Reason.IO, status.get()); + } + } + + private Failure cancel(CompletableFuture attempt, LimitedBody body, Reason reason, int status) { + Failure failure = new Failure(reason, status == 0 ? null : status); + if (body != null) body.cancel(failure); + attempt.cancel(true); + return failure; + } + + private static final class LimitedBody implements HttpResponse.BodySubscriber { + private final CompletableFuture completed = new CompletableFuture<>(); + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private final int limit, status; + private final long declaredLength; + private Flow.Subscription subscription; + private int received; + + LimitedBody(int limit, int status, long declaredLength) { this.limit = limit; this.status = status; this.declaredLength = declaredLength; } + @Override public CompletionStage getBody() { return completed; } + @Override public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + if (completed.isDone()) { subscription.cancel(); return; } + if (declaredLength > limit) { cancel(new Failure(Reason.BODY_TOO_LARGE, status)); return; } + subscription.request(1); + } + @Override public void onNext(List buffers) { + if (completed.isDone()) { if (subscription != null) subscription.cancel(); return; } + long chunkBytes = 0; + for (ByteBuffer buffer : buffers) chunkBytes += buffer.remaining(); + if (chunkBytes > limit - received) { cancel(new Failure(Reason.BODY_TOO_LARGE, status)); return; } + received += (int) chunkBytes; + byte[] part = new byte[8192]; + for (ByteBuffer buffer : buffers) while (buffer.hasRemaining()) { + int count = Math.min(part.length, buffer.remaining()); + buffer.get(part, 0, count); bytes.write(part, 0, count); + } + subscription.request(1); + } + @Override public void onError(Throwable error) { completed.completeExceptionally(new Failure(Reason.IO, status)); } + @Override public void onComplete() { completed.complete(bytes.toByteArray()); } + void cancel(Failure failure) { + completed.completeExceptionally(failure); + if (subscription != null) subscription.cancel(); + } + } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java index b9cb7de..2d700fa 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java @@ -7,11 +7,10 @@ import com.moyi.controlplane.common.ApiException; import com.moyi.controlplane.security.SecretCipher; import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver; import com.moyi.controlplane.provider.transport.ProviderRequestException; +import com.moyi.controlplane.provider.transport.BoundedHttpTransport; import java.net.URI; import java.net.InetAddress; -import java.net.http.HttpClient; import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.time.Duration; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -42,7 +41,7 @@ public class AsyncProviderClient { /** Keep rotation state across submit/poll calls; rebuilding a Provider on * every request otherwise selected the first upstream key forever. */ private final ConcurrentMap keyCursors = new ConcurrentHashMap<>(); - private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + private final BoundedHttpTransport transport = new BoundedHttpTransport(); public AsyncProviderClient(JdbcTemplate jdbc, SecretCipher cipher, ObjectMapper mapper, Environment environment) { this.jdbc = jdbc; this.cipher = cipher; this.mapper = mapper; this.environment = environment; @@ -321,7 +320,7 @@ public class AsyncProviderClient { .header("Authorization", "Bearer " + key).header("Content-Type", "application/json"); HttpRequest request = builder.POST(HttpRequest.BodyPublishers.ofString(write(body))).build(); sending = true; - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + var response = transport.send(request, 8 * 1024 * 1024); if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); byte[] bytes = response.body(); if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) throw new ApiException("UPSTREAM_MEDIA_TOO_LARGE", message + "(返回音频超过 8 MiB 或为空)", 502); @@ -361,7 +360,7 @@ public class AsyncProviderClient { .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(HttpRequest.BodyPublishers.ofByteArray(out.toByteArray())).build(); sending = true; - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + var response = transport.send(request, 8 * 1024 * 1024); if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); Object code = parsed.get("code"); @@ -379,6 +378,10 @@ public class AsyncProviderClient { private ProviderRequestException requestFailure(Exception exception, boolean sending, String message) { if (exception instanceof InterruptedException) Thread.currentThread().interrupt(); if (exception instanceof ProviderRequestException failure) return failure; + if (exception instanceof BoundedHttpTransport.Failure failure && failure.observedStatus() != null + && (failure.observedStatus() < 200 || failure.observedStatus() >= 300)) { + return ProviderRequestException.http(failure.observedStatus(), message); + } if (sending) return ProviderRequestException.unknown("UPSTREAM_PROVIDER_ERROR", message); if (exception instanceof ApiException error) return ProviderRequestException.notSent(error.getCode(), error.getMessage(), error.getStatus()); return ProviderRequestException.notSent("UPSTREAM_REQUEST_NOT_SENT", "请求准备失败,尚未提交上游生成", 502); @@ -409,7 +412,12 @@ public class AsyncProviderClient { } } HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(15)).GET().build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + BoundedHttpTransport.Response response; + try { response = transport.send(request, 8 * 1024 * 1024); } + catch (BoundedHttpTransport.Failure failure) { + if (failure.reason() == BoundedHttpTransport.Reason.BODY_TOO_LARGE) throw new ApiException("MEDIA_TOO_LARGE", "媒体输入超过 8 MiB", 400); + throw new ApiException("MEDIA_FETCH_FAILED", "无法在期限内读取媒体输入", 400); + } if (response.statusCode() < 200 || response.statusCode() >= 300) { throw new ApiException("MEDIA_FETCH_FAILED", "无法读取媒体输入(HTTP " + response.statusCode() + ")", 400); } @@ -447,7 +455,7 @@ public class AsyncProviderClient { else builder.GET(); HttpRequest request = builder.build(); sending = true; - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + var response = transport.send(request, 8 * 1024 * 1024); if (response.statusCode() < 200 || response.statusCode() >= 300) throw upstreamStatus(response.statusCode(), message); Map parsed = mapper.readValue(response.body(), new TypeReference<>() {}); Object code = parsed.get("code"); diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java new file mode 100644 index 0000000..25e19d9 --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java @@ -0,0 +1,32 @@ +package com.moyi.controlplane.gateway.request; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** Lightweight source dependency gates, not a substitute for design review or a sandbox. */ +class ModuleDependencyTest { + private final Path root = Path.of(System.getProperty("basedir", "."), "src/main/java/com/moyi/controlplane"); + + @Test void requestApplicationBoundaryDoesNotOwnHttpDatabaseOrProviderCredentials() throws Exception { + disallow(root.resolve("gateway/request"), List.of("java.net.http", "org.springframework.jdbc", ".security.SecretCipher", ".provider.credentials", ".task.AsyncProviderClient")); + } + @Test void transportDoesNotOwnRiskRoutingOrTheWallet() throws Exception { + disallow(root.resolve("provider/transport"), List.of(".gateway.", ".wallet.", ".auth.", ".task.", "org.springframework.jdbc", ".security.SecretCipher")); + } + @Test void taskSettlementDoesNotCallAnUpstream() throws Exception { + assertTrue(Files.exists(root.resolve("task/TaskSettlementService.java"))); + disallow(root.resolve("task/TaskSettlementService.java"), List.of("java.net.http", "AsyncProviderClient", "OpenAiCompatibleProvider")); + } + private void disallow(Path path, List forbidden) throws Exception { + assertTrue(Files.exists(path), "Expected source boundary: " + path); + try (var paths = Files.isDirectory(path) ? Files.walk(path) : java.util.stream.Stream.of(path)) { + for (Path file : paths.filter(p -> p.toString().endsWith(".java")).toList()) { + String source = Files.readString(file); + for (String dependency : forbidden) assertFalse(source.contains(dependency), file + " crosses boundary via " + dependency); + } + } + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/BoundedHttpTransportTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/BoundedHttpTransportTest.java new file mode 100644 index 0000000..28def8f --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/BoundedHttpTransportTest.java @@ -0,0 +1,113 @@ +package com.moyi.controlplane.provider.transport; + +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpRequest; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class BoundedHttpTransportTest { + private HttpServer server; + private final BoundedHttpTransport transport = new BoundedHttpTransport(); + private final CountDownLatch release = new CountDownLatch(1); + @BeforeEach void setup() throws Exception { server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); } + @AfterEach void cleanup() { release.countDown(); server.stop(0); } + private HttpRequest request(String path, long timeoutMs) { return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path)).timeout(Duration.ofMillis(timeoutMs)).GET().build(); } + + @Test void returnsBoundedBytesWithoutExposingBodyInDiagnostics() throws Exception { + server.createContext("/", exchange -> { + byte[] bytes = "private-provider-result".getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (var output = exchange.getResponseBody()) { output.write(bytes); } + }); server.start(); + var response = transport.send(request("/", 2000), 1024); + assertEquals("private-provider-result", new String(response.body(), StandardCharsets.UTF_8)); + assertFalse(response.toString().contains("private-provider-result")); + } + + @Test void oversizedChunkedResponseIsCancelledBeforeUnboundedBuffering() { + var calls = new AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { for (int i = 0; i < 100; i++) output.write(new byte[4096]); } + catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 2000), 8192)); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason()); + assertEquals(200, failure.observedStatus()); assertEquals(1, calls.get()); + } + + @Test void declaredOversizeIsRejectedWithoutWaitingForTheBody() { + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 1_000_000); + try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + exchange.close(); + }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 1000), 8192)); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason()); + } + + @Test void totalDeadlineIncludesBodyReadsAfterHeaders() { + var calls = new AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { + output.write('x'); output.flush(); + try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + } catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + long start = System.nanoTime(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/", 200), 8192)); + assertEquals(BoundedHttpTransport.Reason.TIMEOUT, failure.reason()); + assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < 2000); + assertEquals(1, calls.get()); + } + + @Test void errorBodiesUseASmallerLimitAndRetainOnlyObservedStatus() { + server.createContext("/", exchange -> { exchange.sendResponseHeaders(401, 1_000_000); exchange.close(); }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.send(request("/?private-query", 1000), 8 * 1024 * 1024)); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason()); + assertEquals(401, failure.observedStatus()); + assertFalse(failure.toString().contains("private-query")); + } + + @Test void redirectsAreReturnedAndNeverFollowed() throws Exception { + var redirected = new AtomicInteger(); + server.createContext("/start", exchange -> { exchange.getResponseHeaders().set("Location", "/internal"); exchange.sendResponseHeaders(302, -1); exchange.close(); }); + server.createContext("/internal", exchange -> { redirected.incrementAndGet(); exchange.sendResponseHeaders(200, -1); exchange.close(); }); server.start(); + assertEquals(302, transport.send(request("/start", 2000), 1024).statusCode()); + assertEquals(0, redirected.get()); + } + + @Test void interruptionCancelsTheAttemptAndRestoresInterruptFlag() throws Exception { + var arrived = new CountDownLatch(1); var result = new AtomicReference(); var interrupted = new AtomicReference(); + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 0); arrived.countDown(); + try { release.await(5, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + exchange.close(); + }); server.start(); + var worker = new Thread(() -> { + try { transport.send(request("/", 5000), 1024); } + catch (BoundedHttpTransport.Failure failure) { result.set(failure.reason()); interrupted.set(Thread.currentThread().isInterrupted()); } + }); worker.setDaemon(true); worker.start(); + try { + assertTrue(arrived.await(2, TimeUnit.SECONDS)); worker.interrupt(); worker.join(2000); + assertFalse(worker.isAlive()); assertEquals(BoundedHttpTransport.Reason.INTERRUPTED, result.get()); assertEquals(true, interrupted.get()); + } finally { release.countDown(); worker.interrupt(); } + } + + @Test void callersCannotOmitDeadlinesOrRequestUnboundedResponses() { + var request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:1/")).GET().build(); + assertThrows(IllegalArgumentException.class, () -> transport.send(request, 1024)); + assertThrows(IllegalArgumentException.class, () -> transport.send(request, Integer.MAX_VALUE)); + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java index d022393..9773418 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java @@ -298,6 +298,23 @@ class AsyncProviderClientTest { assertEquals("INVALID_CREDENTIAL_FINGERPRINT", assertThrows(ApiException.class, () -> client.requirePollableBinding("provider-vast", "route-a", "tripo", "3d", "test-key")).getCode()); } + @ParameterizedTest + @CsvSource({"200,UNKNOWN", "401,REJECTED"}) + void responseLimitKeepsTheObservedAcceptanceBoundary(int status, ProviderRequestException.Acceptance expected) throws Exception { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + var calls = new java.util.concurrent.atomic.AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); exchange.getRequestBody().readAllBytes(); + exchange.sendResponseHeaders(status, 9 * 1024 * 1024); exchange.close(); + }); server.start(); + try { + var client = clientFor("http://127.0.0.1:" + server.getAddress().getPort()); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", "test")); + assertEquals(expected, failure.acceptance()); + assertEquals(1, calls.get()); + } finally { server.stop(0); } + } + @Test void invalidMediaIsKnownToFailBeforeSendingGenerationRequest() { var client = clientFor("http://127.0.0.1:1"); var failure = assertThrows(ProviderRequestException.class, () -> client.submit("provider-vast", "tripo", "3d", Map.of("media_url", "http://127.0.0.1/private"))); diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java index e70df9c..0523bad 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncTaskPostgresTest.java @@ -127,6 +127,16 @@ class AsyncTaskPostgresTest { verifyNoInteractions(provider); } + @Test void taskReadsAreOwnerScopedAndCannotExposeAnotherUsersResult() { + UUID anotherUser = UUID.randomUUID(), anotherKey = UUID.randomUUID(); + when(catalog.keyOwnerId(anotherKey)).thenReturn(anotherUser); + var foreignKey = Map.of("id", anotherKey.toString()); + assertEquals("TASK_NOT_FOUND", assertThrows(ApiException.class, () -> tasks.getForKey(foreignKey, taskId)).getCode()); + when(catalog.authenticateKey("foreign-token")).thenReturn(foreignKey); + assertTrue(tasks.list("Bearer foreign-token", 100).isEmpty()); + assertEquals(taskId, tasks.getForKey(Map.of("id", keyId.toString()), taskId).get("id")); + } + @Test void idempotentReplayReturnsOriginalEvenAfterCompletionAndRouteRetirement() { var input = Map.of("model", "model-mock-3d", "prompt", "one chair"); var first = create("client-operation-1", input); diff --git a/docs/04-open-source-baseline-and-development-nodes.md b/docs/04-open-source-baseline-and-development-nodes.md index bc4ac03..c5d7a78 100644 --- a/docs/04-open-source-baseline-and-development-nodes.md +++ b/docs/04-open-source-baseline-and-development-nodes.md @@ -8,6 +8,8 @@ - 更新时间:2026-08-20 - 当前节点:D4,密钥与模型目录 Mock 闭环 +> 本文为 2026-08-20 的历史设计基线,保留当时的选型与节点记录,不代表最新实施状态。当前 Java/PostgreSQL/Redis 实现、已验证范围及 CPR 差距请以 `README.md` 和 `24-security-and-cpr-task-tracker.md` 为准。 + ## 一、开源项目使用决策 模亦灵枢不直接修改某个中转站项目并把它整体作为产品,而采用“开源网关能力 + MOYI 自研控制面与双端控制台”的组合方式。这样可以复用成熟的模型代理、虚拟密钥和用量能力,同时保留多类型模型、异步 3D 任务、额度台账和未来 MaaS 能力的产品自主性。 @@ -138,6 +140,6 @@ POST /v1/tasks | 邮件/短信 | 控制台验证码 | 服务商、模板、签名、发送限制和风控 | | 地区与合规 | 不对公网开放 | 服务地区、隐私政策、日志和内容安全要求 | -## 六、当前结论 +## 六、当时的阶段结论(历史记录) 当前 D3 已完成认证与权限基线。D4 已提供平台 Key 生命周期、统一模型目录、标准同步调用和异步 3D 任务接口,并完成首个 Tripo 真实 Provider Adapter 与文生 3D 验收。目录和真实任务归属当前仍使用内存仓库,PostgreSQL 表结构已准备;下一步进入 D5 调用计量,并在 D6 持久化异步任务、队列和结果文件。详见 [D4 密钥与模型目录实施计划](06-d4-integration-plan.md)。 diff --git a/docs/24-security-and-cpr-task-tracker.md b/docs/24-security-and-cpr-task-tracker.md new file mode 100644 index 0000000..ddf8404 --- /dev/null +++ b/docs/24-security-and-cpr-task-tracker.md @@ -0,0 +1,38 @@ +# 安全与 CPR 差距任务追踪 + +审查日期:2026-10-21。对照用户粘贴的 CPR 比对报告与本地代码,不将建议清单视为已完成验收。 + +## 已交付基线 + +- `5533a48`:gateway.request 请求边界、风险策略接口、可信来源元数据、共享凭据解析、有界 SSE 与同步结算恢复。默认 shadow,没有实际规则就不代表风控拦截已上线。 +- `7626584`:异步创建用户级幂等、哈希身份、事务锁、唯一性约束。 +- `5be06a9`:上游受理未知不重发不退款,持久终态日志和本地结算恢复。 +- `c716ec7`:人工核账、共享任务结算、provider/model 绑定、幂等审计回执、Redis 同任务续期。221 项后端与 25 项前端测试的结果仅对应该提交,不代表整个建议清单完成。 +- `d994ffb`:本轮认证修复,包含共享登录窗口、统一新密码策略、会话撤销与竞态保护、首次管理员配置检查及隔离界面回归。 + +## 待补任务与当前证据 + +| ID | 工作项 | 对照结果 / 当前状态 | 验收依据 | +| --- | --- | --- | --- | +| SEC-01 | 登录防爆破与账号/IP 限速 | 本轮已实现并验证:独立提交的账号/IP 固定窗口,未知账号同样限制 | 真实数据库并发、限额持久化、429/Retry-After,见 25 号文档 | +| SEC-02 | 所有角色统一新密码策略 | 本轮已实现并验证:所有角色新密码 8–256,保留既有有界哈希登录 | 各入口相同规则,KDF 参数上界,不静默截断 | +| SEC-03 | 改密与会话撤销、登录竞态 | 本轮已实现并验证:撤销全部会话、清 Cookie、行锁与二次校验 | 改密两种锁顺序、MFA 失败计数及审计回滚 | +| SEC-04 | 任务用户隔离与回执安全 | 已实现核心用户级查询/幂等/冻结/核账,并补跨用户读取回归;完整授权审计仍待继续 | 跨用户读取/列表不可见;核账拒绝普通用户 | +| SEC-05 | Provider 有界 HTTP Transport | 部分完成:异步 JSON/multipart/音频/媒体已接有界读取和总期限;同步/SSE 生命周期尚未统一抽取 | 超大 chunk、慢响应体、已观察拒绝状态、取消、重定向及原适配器回归 | +| SEC-06 | 媒体下载出站 / SSRF | 部分加固但未闭环:体积/期限/重定向有界,DNS 重绑定窗口和保留地址复核仍待补 | 私网/保留地址、DNS 绑定或可信出站代理尚待完整验证 | +| SEC-07 | 整体安全审查与生产验收 | 未完成:管理接口、注册/重置原子性、默认配置、依赖、备份恢复需继续 | 独立用例、部署与恢复演练,禁止宣称“全仓库安全” | +| COD-01 | Codex 协议能力及验收矩阵 | 未实现完整兼容;保留有限文本 Responses 的明确拒绝语义 | 指定客户端版本、模型、工具和计费联合验收 | +| COD-02 | Responses WebSocket 与响应归属续接 | 未实现,不可宣传完整 Codex 支持 | 原账号/凭据版本归属、断线、取消、租约、费用一致性 | +| COD-03 | 长请求连接/空闲/总期限与续租 | 未完成,不只修改 90 秒常量 | 并发租约、钱包预留恢复、长流/取消共同验证 | +| SCH-01 | 独立上游账号生命周期与调度 | 仅共享 Key 轮换/冷却/指纹,非完整账号实体 | 凭据版本、额度、模型范围、权重及解释能力 | +| SCH-02 | 公平排队 / 取消 / 共享等待预算 | 未完成,现有同步 Redis 竞争不保证 FIFO | 多客户端饥饿/FIFO/取消回收与横向并发验证 | +| OPS-01 | request/attempt 诊断、配置版本、质量门禁 | 部分:已有最终审计/健康/部分快照;本轮加依赖方向测试、CI/Dependabot 配置,远端未运行;逐 attempt/完整配置版本仍缺 | 本地门禁不等于 GitHub CI、CVE 扫描或真实协议验收 | +| OPS-02 | CPR 可选部署 / OAuth 账号池 | 未部署;属于可选方案,不默认新建生产基础设施 | 用户确认部署方式及授权来源后才接入;本地不发付费生成 | + +## 实施边界 + +已补 SEC-01/02/03,下一优先级是 SEC-06 出站/DNS 绑定、同步 Transport 与完整 provider 配置版本,再处理 COD-01/02/03 的协议/续接/长请求合同。Codex WebSocket 与 OAuth 不是添加 URL 即可安全完成的能力,须连同状态归属、计量、预留续期和客户端验收逐步实施。未部署、未真实供应商验证或待设计的项必须保持未完成标记。 + +本轮设计与验证细节见 `25-auth-and-bounded-transport.md`。这份表按用户粘贴报告的锁定基线追踪本地实施,不把旧的 CPR 版本号冒充实时最新版,也没有重新部署 CPR。只做本地 Git commit,不 push,不修改业务数据库。 + +2026-10-21 本轮最终验证:247 项后端测试全部通过(含真实 PostgreSQL/Redis,0 跳过),前端构建与 25 项测试通过,另完成隔离改密界面交互检查。新增 GitHub 配置只保存本地,远端未执行。本轮没有把剩余 COD/SCH/出站/运营验收项目标为完成。 diff --git a/docs/25-auth-and-bounded-transport.md b/docs/25-auth-and-bounded-transport.md new file mode 100644 index 0000000..b35f4c2 --- /dev/null +++ b/docs/25-auth-and-bounded-transport.md @@ -0,0 +1,44 @@ +# 认证安全与有界 HTTP 增量 + +日期:2026-10-21。范围是补齐用户清单中的可独立验证安全底座,不代表完整 Codex 兼容或全部 CPR 能力已完成;总任务状态见 `24-security-and-cpr-task-tracker.md`。 + +## 认证边界 + +- `LoginAttemptLimiter` 在昂贵密码校验前,使用 PostgreSQL 共享固定窗口:同一规范化账号每 60 秒最多 10 次,同一来源每 60 秒最多 50 次,包括未知账号、成功与失败尝试。超过返回 429 / `AUTH_RATE_LIMITED` 和 `Retry-After`。 +- `V49` 仅存 scope+身份摘要、计数和期限,不保存原始账号、IP 或密码。摘要是伪名化,不等于抵抗字典反推的匿名化。过期数据在后续尝试中分批清理;这是应用防爆破,不替代 WAF/网关整体流量防护。 +- 限速计数独立事务提交,失败登录/外层回滚不能抹去次数;稳定的桶锁顺序防死锁。先完成限速再进入登录事务,不让所有请求各占一条连接后再争抢 REQUIRES_NEW 连接。 +- 已知和未知账号均执行有界 PBKDF2 验证;未知账号使用非用户凭据的固定摘要形状,降低明显的是否存在计时差异,不承诺严格恒定响应时长。 +- 登录、改密、密码重置通过用户行锁协调。先登录的会话会被随后改密撤销;先改密后,旧密码不能继续创建有效会话。MFA 失败计数、动态码重放标记和恢复码消费与会话签发保持事务语义。 +- 改密撤销所有浏览器会话,包括当前会话;Controller 清理 Cookie,界面回到登录入口。仅改显示名称不撤销会话。API Key 不自动撤销,泄露时仍需单独停用或轮换。 +- `PasswordPolicy` 统一注册、管理端创建、所有角色改密/重置以及新建初始管理员的新密码规则:8–256 个 UTF-16 code unit、不能仅为空白,不静默裁切/修剪。超级管理员不再豁免长度策略。8 位是兼容当前产品规则,不是声称达到所有部署的最佳口令强度;更强策略、泄露口令检测、强制 MFA 仍需按部署要求补齐。 +- 兼容既有哈希和有界旧密码登录,不自动重置已有管理员。匹配端拒绝超大输入、异常盐/密钥尺寸和不合理迭代数,防止损坏哈希控制无界 KDF 工作。新建管理员必须显式配置密码,默认 `admin123` 及示例环境文件中的占位密码被拒绝;已有管理员启动不因缺少新的初始密码配置被重置。 + +注册/验证码整体流程、重置发送的账号枚举侧信道、管理敏感动作二次验证、会话管理面板等仍在剩余审查中;不能把本增量描述为“认证系统已无漏洞”。 + +## 有界 HTTP 模块 + +`provider.transport.BoundedHttpTransport` 只执行一次非流式 HTTP 传输,不选择账号/渠道、不读写钱包、不决定重试、也不携带业务风险策略: + +1. 强制调用方提供总期限,期限覆盖响应头和整个响应体;超时或线程中断时取消 future 和 body subscription,恢复中断标志。 +2. 使用限量 BodySubscriber,在继续累计之前检查 chunk 总量;声明 Content-Length 已超限时提前拒绝。错误体最多 64 KiB,不是“先读完整体再判断大小”。 +3. 禁止自动跟随重定向;返回已观察 HTTP 状态及字节,诊断对象不输出 URL、头、密钥或原始响应体。 +4. 异步 JSON、multipart、音频接入 8 MiB 上限;媒体输入也使用同一读取上限和 15 秒期限。媒体获取在生成发送前失败时不应被视为已发送生成请求。 +5. `AsyncProviderClient` 继续负责协议和受理语义:已观察到明确拒绝状态时保留该事实;成功响应超限、读取超时或接收结果未知时仍按 UNKNOWN 处理,不自动重发、不自动退款。原有 404/405 视频路径兼容边界不放宽。 + +这还不是全项目统一 Transport:同步/SSE 的现有有界读取和整体 deadline 仍在原 Provider 生命周期内;长请求续租、逐 attempt 交付状态、统一取消与跨路由共享预算未完成。 + +**媒体 SSRF 仍未闭环。** 本阶段约束了响应体、总期限和重定向,但旧 DNS 预检与 JDK HTTP 实际解析之间的重绑定窗口仍存在,保留地址范围也需完整复核。不得标记 SEC-06 完成;生产应通过可信出站代理/网络策略控制访问,后续实现可证明的 DNS 绑定或等价控制。没有擅自更改业务网络或把任意用户 URL 加入可信名单。 + +## 工程验证与限制 + +- 新增 3 项模块依赖方向测试:请求应用边界不直接持有 HTTP/JDBC/凭据;transport 不依赖风险/路由/钱包;结算模块不调用上游。它们是轻量源码门禁,不替代代码审查或沙箱。 +- 新增跨用户任务查询回归,其他用户不能读取任务结果或列出其任务。 +- `.github/workflows/verification.yml` 配置 Java 21、Node 22、锁文件安装、前后端构建与独立 PostgreSQL/Redis 测试;Actions 固定完整 commit SHA(2026-10-21 通过各上游 GitHub API 核对 v4 引用,pnpm 的 annotated tag 另行解引用),避免仅依赖可移动主版本标签。此项不等于审计了 Actions 全部源码。`.github/dependabot.yml` 配置 Maven/npm/Actions 更新检查。仅本地保存配置,未 push、未在 GitHub 运行,不能声称远端 CI 或依赖漏洞扫描已通过。 +- 本机 pnpm 包装器提示 esbuild 构建脚本待审批。没有全局关闭审批,也没有批准未知脚本;移除了本次自动生成的待审批占位配置。使用现有安装依赖执行 `npm run build` / `npm test`,前端构建与 25 项测试通过;CI 的锁文件安装仍需首次远端实际验证。 +- 使用 Browser 技能在全部 fetch 被拦截的隔离 fixture 中验证改密事件使设置页退回登录入口;密码是测试字符串,没有操作真实账号或浏览器会话。截图 `.tools/auth-security-ui-proof.png` 被忽略,不进入 Git。 + +最终全量验证于 2026-10-21 22:01(Asia/Shanghai)完成:Java 21 / Maven 3.9.9 的 `mvn -B -o clean verify`,**247 项测试,0 失败、0 错误、0 跳过**,Spring Boot JAR 成功构建。使用独立 PostgreSQL 15.18、Redis 7 容器和共享测试 network namespace,没有连接业务数据库。包含 9 项认证数据库安全回归、既有认证/角色/MFA/钱包/异步核账回归、8 项传输测试、2 项异步超限受理语义回归、3 项依赖方向门禁和跨用户任务读取回归。 + +前端 `npm run build`(vue-tsc + Vite)和 `npm test` 再次通过,25 项测试零失败;隔离界面验证通过。`git diff --check` 无错误。日志 `.tools/security-final-verify.log` 和截图被忽略,不进入提交。测试通过只证明本次本地已覆盖范围;CI 未远端执行,媒体 DNS 绑定、WebSocket/续接、长请求续租、公平调度、完整配置版本、依赖漏洞与备份恢复验收仍未完成。 + +不部署、不迁移业务数据库、不进行付费上游生成;只创建本地阶段性 Git commit,不 push。发布此版本前必须先迁移 V49,再启动新后端;首次启动须替换环境示例占位密码。不要将旧 TypeScript 参考服务当成与 Java 同时对外写账的第二套生产后端。