From 32f6f67b9f8b6b547624bf50ba8515a1f50fa9a1 Mon Sep 17 00:00:00 2001 From: xpp Date: Wed, 21 Oct 2026 22:49:57 +0800 Subject: [PATCH] =?UTF-8?q?fix:"=E4=BF=AE=E5=A4=8D=E5=AA=92=E4=BD=93?= =?UTF-8?q?=E4=B8=8B=E8=BD=BDDNS=E9=87=8D=E7=BB=91=E5=AE=9A=E5=B9=B6?= =?UTF-8?q?=E9=9A=94=E7=A6=BB=E5=87=BA=E7=AB=99=E4=BC=A0=E8=BE=93"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- control-plane-java/pom.xml | 7 + .../provider/media/MediaDownloader.java | 59 ++++ .../media/PublicMediaAddressPolicy.java | 53 ++++ .../transport/BoundedHttpTransport.java | 4 +- .../transport/PinnedHttpsTransport.java | 187 ++++++++++++ .../task/AsyncProviderClient.java | 54 +--- .../gateway/request/ModuleDependencyTest.java | 5 + .../provider/media/MediaDownloaderTest.java | 58 ++++ .../media/PublicMediaAddressPolicyTest.java | 65 ++++ .../transport/PinnedHttpsTransportTest.java | 279 ++++++++++++++++++ .../task/AsyncProviderClientTest.java | 18 ++ docs/24-security-and-cpr-task-tracker.md | 6 +- docs/25-auth-and-bounded-transport.md | 2 +- docs/26-media-dns-binding-and-acceptance.md | 46 +++ 15 files changed, 791 insertions(+), 56 deletions(-) create mode 100644 control-plane-java/src/main/java/com/moyi/controlplane/provider/media/MediaDownloader.java create mode 100644 control-plane-java/src/main/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicy.java create mode 100644 control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransport.java create mode 100644 control-plane-java/src/test/java/com/moyi/controlplane/provider/media/MediaDownloaderTest.java create mode 100644 control-plane-java/src/test/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicyTest.java create mode 100644 control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransportTest.java create mode 100644 docs/26-media-dns-binding-and-acceptance.md diff --git a/README.md b/README.md index d36658d..713572b 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ ## 当前工程状态 -2026-10-21:当前产品为 Java/Vue 自研控制面与多模型网关,已补请求模块化、账务恢复、异步幂等和审计核账,并继续加固认证与 HTTP 边界。下文包含产品演进目标,不等于功能均已实现。**尚未完成完整 Codex WebSocket/续接、长请求续租、公平排队、账号全生命周期和媒体出站/DNS 绑定闭环。** +2026-10-21:当前产品为 Java/Vue 自研控制面与多模型网关,已补请求模块化、账务恢复、异步幂等和审计核账,并继续加固认证与 HTTP 边界。本轮补齐了服务端参考图下载的 DNS 地址绑定与直连隔离;生产网络策略仍需验收。下文包含产品演进目标,不等于功能均已实现。**尚未完成完整 Codex WebSocket/续接、长请求续租、公平排队、账号全生命周期和生产安全验收。** -逐项完成状态与剩余验收见 [安全与 CPR 任务追踪](docs/24-security-and-cpr-task-tracker.md),本轮设计与限制见 [认证及有界 HTTP 增量](docs/25-auth-and-bounded-transport.md)。本地验证通过不代表已部署或全仓库安全审计完成。 +逐项完成状态与剩余验收见 [安全与 CPR 任务追踪](docs/24-security-and-cpr-task-tracker.md),认证边界见 [认证及有界 HTTP 增量](docs/25-auth-and-bounded-transport.md),最新出站加固与验收阻塞见 [媒体 DNS 绑定与验收状态](docs/26-media-dns-binding-and-acceptance.md)。本地验证通过不代表已部署或全仓库安全审计完成。 ## 一、产品简介 diff --git a/control-plane-java/pom.xml b/control-plane-java/pom.xml index 1d542c6..ab1dca5 100644 --- a/control-plane-java/pom.xml +++ b/control-plane-java/pom.xml @@ -17,6 +17,9 @@ 21 + + 5.6.5 + 5.4.4 @@ -53,6 +56,10 @@ org.springframework.security spring-security-crypto + + org.apache.httpcomponents.client5 + httpclient5 + org.postgresql postgresql diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/MediaDownloader.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/MediaDownloader.java new file mode 100644 index 0000000..cfea021 --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/MediaDownloader.java @@ -0,0 +1,59 @@ +package com.moyi.controlplane.provider.media; + +import com.moyi.controlplane.common.ApiException; +import com.moyi.controlplane.provider.transport.BoundedHttpTransport; +import com.moyi.controlplane.provider.transport.PinnedHttpsTransport; +import java.time.Duration; +import java.util.Locale; + +/** Public reference-image input only. No provider credentials, routing, retries or billing. */ +public final class MediaDownloader { + private static final int MAX_BYTES = 8 * 1024 * 1024; + private final PublicMediaAddressPolicy policy = new PublicMediaAddressPolicy(); + private final PinnedHttpsTransport transport; + private final PinnedHttpsTransport.AddressResolver resolver; + + public MediaDownloader() { this(new PinnedHttpsTransport(), new PublicMediaAddressPolicy()::resolve); } + MediaDownloader(PinnedHttpsTransport transport, PinnedHttpsTransport.AddressResolver resolver) { + this.transport = transport; + this.resolver = resolver; + } + + public record MediaBytes(byte[] bytes, String contentType, String filename) { + @Override public String toString() { return "MediaBytes[type=" + contentType + ", bytes=" + bytes.length + "]"; } + } + + public MediaBytes download(String value) { + java.net.URI uri; + try { uri = policy.requireUri(value); } + catch (IllegalArgumentException error) { throw invalidAddress(); } + BoundedHttpTransport.Response response; + try { response = transport.get(uri, resolver, Duration.ofSeconds(15), MAX_BYTES); } + catch (BoundedHttpTransport.Failure failure) { + if (failure.reason() == BoundedHttpTransport.Reason.INVALID_ADDRESS) throw invalidAddress(); + if (failure.reason() == BoundedHttpTransport.Reason.BODY_TOO_LARGE) throw new ApiException("MEDIA_TOO_LARGE", "媒体输入超过 8 MiB", 400); + throw new ApiException("MEDIA_FETCH_FAILED", "无法在期限内读取媒体输入", 400); + } + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new ApiException("MEDIA_FETCH_FAILED", "无法读取媒体输入(HTTP " + response.statusCode() + ")", 400); + } + if (response.body().length == 0) throw new ApiException("MEDIA_TOO_LARGE", "媒体输入为空或超过 8 MiB", 400); + String contentType = response.headers().firstValue("Content-Type").orElse("application/octet-stream") + .split(";", 2)[0].trim().toLowerCase(Locale.ROOT); + if (!contentType.matches("image/[a-z0-9.+-]{1,64}")) { + throw new ApiException("MEDIA_TYPE_UNSUPPORTED", "图像编辑和视频参考图必须是 image/* 文件", 400); + } + String extension = switch (contentType) { + case "image/jpeg" -> ".jpg"; + case "image/png" -> ".png"; + case "image/webp" -> ".webp"; + case "image/gif" -> ".gif"; + default -> ".bin"; + }; + return new MediaBytes(response.body(), contentType, "input" + extension); + } + + private ApiException invalidAddress() { + return new ApiException("INVALID_MEDIA_URL", "媒体地址必须是公开 HTTPS 地址,不能指向本机、内网或特殊用途地址", 400); + } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicy.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicy.java new file mode 100644 index 0000000..3cb6e4a --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicy.java @@ -0,0 +1,53 @@ +package com.moyi.controlplane.provider.media; + +import com.moyi.controlplane.provider.transport.PinnedHttpsTransport; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; + +/** Fail closed for non-global and special-purpose targets, including mixed DNS answer sets. */ +final class PublicMediaAddressPolicy { + URI requireUri(String value) { + try { + URI uri = URI.create(value); + if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null || uri.getHost().contains("%") + || uri.getUserInfo() != null || uri.getRawFragment() != null || uri.getPort() == 0 || uri.getPort() > 65535) { + throw new IllegalArgumentException(); + } + return uri; + } catch (RuntimeException error) { throw new IllegalArgumentException("Invalid public HTTPS media URL"); } + } + + InetAddress[] resolve(String host) throws IOException { + return requirePublic(InetAddress.getAllByName(host)); + } + + InetAddress[] requirePublic(InetAddress[] addresses) throws PinnedHttpsTransport.AddressRejectedException { + if (addresses == null || addresses.length == 0 || addresses.length > 16) throw new PinnedHttpsTransport.AddressRejectedException(); + for (var address : addresses) if (address == null || !isPublic(address)) throw new PinnedHttpsTransport.AddressRejectedException(); + return addresses.clone(); + } + + boolean isPublic(InetAddress address) { + byte[] raw = address.getAddress(); + int a = Byte.toUnsignedInt(raw[0]), b = Byte.toUnsignedInt(raw[1]); + if (raw.length == 4) { + int c = Byte.toUnsignedInt(raw[2]), d = Byte.toUnsignedInt(raw[3]); + return !(a == 0 || a == 10 || a == 127 || a >= 224 + || a == 100 && b >= 64 && b <= 127 + || a == 169 && b == 254 || a == 172 && b >= 16 && b <= 31 + || a == 192 && (b == 168 || b == 0 && (c == 0 || c == 2) || b == 88 && c == 99) + || a == 198 && (b == 18 || b == 19 || b == 51 && c == 100) + || a == 203 && b == 0 && c == 113 + || a == 168 && b == 63 && c == 129 && d == 16); // Azure platform virtual address + } + // Only native global unicast 2000::/3, excluding special/tunnel/documentation prefixes. + // This also rejects ULA, link/site local, multicast, compatible/mapped IPv4 and well-known NAT64. + if (raw.length != 16 || (a & 0xe0) != 0x20) return false; + int c = Byte.toUnsignedInt(raw[2]), d = Byte.toUnsignedInt(raw[3]); + return !(a == 0x20 && b == 0x01 && (c <= 1 || c == 0x0d && d == 0xb8) + || a == 0x20 && b == 0x02 // 6to4 can encode a private IPv4 target + || a == 0x3f && b == 0xfe // retired 6bone allocation + || a == 0x3f && b == 0xff && (c & 0xf0) == 0); // 3fff::/20 documentation + } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java index 63eaef5..fe52a7a 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/BoundedHttpTransport.java @@ -27,11 +27,11 @@ public final class BoundedHttpTransport { public record Response(int statusCode, HttpHeaders headers, byte[] body) { @Override public String toString() { return "TransportResponse[status=" + statusCode + ", bytes=" + body.length + "]"; } } - public enum Reason { TIMEOUT, INTERRUPTED, BODY_TOO_LARGE, IO } + public enum Reason { TIMEOUT, INTERRUPTED, BODY_TOO_LARGE, IO, INVALID_ADDRESS, BUSY } public static final class Failure extends IOException { private final Reason reason; private final Integer observedStatus; - private Failure(Reason reason, Integer status) { + Failure(Reason reason, Integer status) { super("Bounded HTTP attempt failed: " + reason); this.reason = reason; this.observedStatus = status; } diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransport.java b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransport.java new file mode 100644 index 0000000..0b24a3c --- /dev/null +++ b/control-plane-java/src/main/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransport.java @@ -0,0 +1,187 @@ +package com.moyi.controlplane.provider.transport; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Proxy; +import java.net.Socket; +import java.net.URI; +import java.net.UnknownHostException; +import java.net.http.HttpHeaders; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLContext; +import org.apache.hc.client5.http.DnsResolver; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.config.TlsConfig; +import org.apache.hc.client5.http.impl.DefaultSchemePortResolver; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; +import org.apache.hc.client5.http.impl.io.DefaultHttpClientConnectionOperator; +import org.apache.hc.client5.http.impl.io.ManagedHttpClientConnectionFactory; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; +import org.apache.hc.client5.http.ssl.DefaultClientTlsStrategy; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; +import org.apache.hc.client5.http.ssl.HostnameVerificationPolicy; +import org.apache.hc.core5.http.config.Http1Config; +import org.apache.hc.core5.io.CloseMode; +import org.apache.hc.core5.ssl.SSLContexts; +import org.apache.hc.core5.util.Timeout; +import static com.moyi.controlplane.provider.transport.BoundedHttpTransport.Reason; + +/** + * A credential-free HTTPS GET with per-attempt DNS binding. The caller supplies address policy; + * this module owns only bounded transport. Never rewrite the URI to an IP: TLS/SNI and Host use + * the original authority. No proxy, cookies, redirects, retry or connection reuse across attempts. + */ +public final class PinnedHttpsTransport { + private static final ExecutorService WORKERS = new ThreadPoolExecutor(0, 8, 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), task -> { + var thread = new Thread(task, "moyi-media-fetch"); + thread.setDaemon(true); + return thread; + }, new ThreadPoolExecutor.AbortPolicy()); + private final SSLContext sslContext; + private final ExecutorService workers; + + public PinnedHttpsTransport() { this(SSLContexts.createDefault(), WORKERS); } + PinnedHttpsTransport(SSLContext sslContext, ExecutorService workers) { + this.sslContext = sslContext; + this.workers = workers; + } + + @FunctionalInterface + public interface AddressResolver { InetAddress[] resolve(String host) throws IOException; } + public static final class AddressRejectedException extends IOException { + public AddressRejectedException() { super("Target address rejected"); } + } + + public BoundedHttpTransport.Response get(URI uri, AddressResolver resolver, Duration deadline, int maxBytes) + throws BoundedHttpTransport.Failure { + if (uri == null || !"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null + || uri.getUserInfo() != null || uri.getRawFragment() != null || uri.getHost().contains("%") + || uri.getPort() == 0 || uri.getPort() > 65535) throw new IllegalArgumentException("Invalid HTTPS target"); + if (maxBytes < 1 || maxBytes > 32 * 1024 * 1024 || deadline == null || deadline.isNegative() + || deadline.isZero() || deadline.compareTo(Duration.ofMinutes(1)) > 0) throw new IllegalArgumentException("Invalid transport limits"); + var request = new HttpGet(uri); + var observed = new AtomicInteger(); + Future attempt; + try { + attempt = workers.submit(() -> execute(uri, resolver, request, deadline, maxBytes, observed)); + } catch (RejectedExecutionException error) { + throw failure(Reason.BUSY, observed); + } + try { + return attempt.get(Math.max(1, deadline.toMillis()), TimeUnit.MILLISECONDS); + } catch (TimeoutException error) { + cancel(request, attempt); + throw failure(Reason.TIMEOUT, observed); + } catch (InterruptedException error) { + cancel(request, attempt); + Thread.currentThread().interrupt(); + throw failure(Reason.INTERRUPTED, observed); + } catch (ExecutionException error) { + cancel(request, attempt); + if (error.getCause() instanceof BoundedHttpTransport.Failure safe) throw safe; + if (error.getCause() instanceof AddressRejectedException) throw failure(Reason.INVALID_ADDRESS, observed); + if (error.getCause() instanceof java.net.SocketTimeoutException) throw failure(Reason.TIMEOUT, observed); + // Do not attach causes: TLS/DNS/parser exceptions can contain signed URLs or response data. + throw failure(Reason.IO, observed); + } + } + + private BoundedHttpTransport.Response execute(URI uri, AddressResolver resolver, HttpGet request, + Duration deadline, int maxBytes, AtomicInteger observed) throws IOException { + String host = unbracket(uri.getHost()); + InetAddress[] addresses = resolver.resolve(host); + if (addresses == null || addresses.length == 0 || addresses.length > 16 + || java.util.Arrays.stream(addresses).anyMatch(java.util.Objects::isNull)) throw new AddressRejectedException(); + // DNS lookup may ignore interruption. Once timed out, it must never resume with a late HTTP GET. + if (request.isCancelled() || Thread.currentThread().isInterrupted()) throw failure(Reason.INTERRUPTED, observed); + InetAddress[] pinned = addresses.clone(); + var timeout = Timeout.ofMilliseconds(Math.max(1, deadline.toMillis())); + var boundDns = new DnsResolver() { + @Override public InetAddress[] resolve(String target) throws UnknownHostException { + if (!host.equalsIgnoreCase(unbracket(target))) throw new UnknownHostException("Unexpected target"); + return pinned.clone(); + } + @Override public String resolveCanonicalHostname(String target) throws UnknownHostException { + if (!host.equalsIgnoreCase(unbracket(target))) throw new UnknownHostException("Unexpected target"); + return host; + } + }; + var tls = new DefaultClientTlsStrategy(sslContext, HostnameVerificationPolicy.BOTH, new DefaultHostnameVerifier()); + // A direct HTTP route alone is insufficient: new Socket() can still use the JVM's SOCKS selector. + var operator = new DefaultHttpClientConnectionOperator(ignored -> new Socket(Proxy.NO_PROXY), + DefaultSchemePortResolver.INSTANCE, boundDns, scheme -> "https".equalsIgnoreCase(scheme) ? tls : null); + var manager = new BasicHttpClientConnectionManager(operator, ManagedHttpClientConnectionFactory.builder() + .http1Config(Http1Config.custom().setMaxLineLength(8192).setMaxHeaderCount(64).build()).build()); + manager.setConnectionConfig(ConnectionConfig.custom().setConnectTimeout(timeout).setSocketTimeout(timeout).build()); + manager.setTlsConfig(TlsConfig.custom().setHandshakeTimeout(timeout).build()); + request.setConfig(RequestConfig.custom().setResponseTimeout(timeout).setConnectionRequestTimeout(timeout) + .setHardCancellationEnabled(true).setAuthenticationEnabled(false).setRedirectsEnabled(false).build()); + request.setHeader("Accept", "image/*"); + request.setHeader("Accept-Encoding", "identity"); + request.setHeader("Connection", "close"); + try (var client = HttpClients.custom().setConnectionManager(manager) + .setRoutePlanner(new DefaultRoutePlanner(DefaultSchemePortResolver.INSTANCE)) + .disableAutomaticRetries().disableRedirectHandling().disableCookieManagement() + .disableAuthCaching().disableContentCompression().disableDefaultUserAgent().build()) { + try { + var response = client.executeOpen(null, request, null); + try { + observed.set(response.getCode()); + var entity = response.getEntity(); + int limit = response.getCode() >= 200 && response.getCode() < 300 ? maxBytes : Math.min(maxBytes, 64 * 1024); + if (entity != null && entity.getContentLength() > limit) throw failure(Reason.BODY_TOO_LARGE, observed); + // Compressed images are already encoded; do not accept a decompression bomb or compressed MIME wrapper. + if (entity != null && entity.getContentEncoding() != null + && !"identity".equalsIgnoreCase(entity.getContentEncoding())) throw failure(Reason.IO, observed); + var body = new ByteArrayOutputStream(); + if (entity != null) { + var input = entity.getContent(); + byte[] chunk = new byte[8192]; + int count; + while ((count = input.read(chunk)) != -1) { + if (request.isCancelled() || Thread.currentThread().isInterrupted()) throw failure(Reason.INTERRUPTED, observed); + if (count > limit - body.size()) throw failure(Reason.BODY_TOO_LARGE, observed); + body.write(chunk, 0, count); + } + } + var headers = new LinkedHashMap>(); + for (var header : response.getHeaders()) headers.computeIfAbsent(header.getName(), ignored -> new ArrayList<>()).add(header.getValue()); + return new BoundedHttpTransport.Response(response.getCode(), HttpHeaders.of(headers, (key, value) -> true), body.toByteArray()); + } finally { + // Cancel BEFORE closing the entity: close/consume must not drain an oversized/never-ending body. + request.cancel(); + try { response.close(); } + catch (IOException expectedAfterCancellation) { /* Never replace the primary outcome with a closed-socket cleanup error. */ } + } + } finally { + request.cancel(); + client.close(CloseMode.IMMEDIATE); + } + } + } + + private static String unbracket(String host) { + return host.startsWith("[") && host.endsWith("]") ? host.substring(1, host.length() - 1) : host; + } + private static void cancel(HttpGet request, Future attempt) { request.cancel(); attempt.cancel(true); } + private static BoundedHttpTransport.Failure failure(Reason reason, AtomicInteger observed) { + return new BoundedHttpTransport.Failure(reason, observed.get() == 0 ? null : observed.get()); + } +} diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java index 2d700fa..8b53c31 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncProviderClient.java @@ -8,8 +8,9 @@ import com.moyi.controlplane.security.SecretCipher; import com.moyi.controlplane.provider.credentials.UpstreamCredentialResolver; import com.moyi.controlplane.provider.transport.ProviderRequestException; import com.moyi.controlplane.provider.transport.BoundedHttpTransport; +import com.moyi.controlplane.provider.media.MediaDownloader; +import com.moyi.controlplane.provider.media.MediaDownloader.MediaBytes; import java.net.URI; -import java.net.InetAddress; import java.net.http.HttpRequest; import java.time.Duration; import java.io.ByteArrayOutputStream; @@ -42,6 +43,7 @@ public class AsyncProviderClient { * every request otherwise selected the first upstream key forever. */ private final ConcurrentMap keyCursors = new ConcurrentHashMap<>(); private final BoundedHttpTransport transport = new BoundedHttpTransport(); + private final MediaDownloader mediaDownloader = new MediaDownloader(); public AsyncProviderClient(JdbcTemplate jdbc, SecretCipher cipher, ObjectMapper mapper, Environment environment) { this.jdbc = jdbc; this.cipher = cipher; this.mapper = mapper; this.environment = environment; @@ -221,7 +223,7 @@ public class AsyncProviderClient { String media=mediaUrl(input); if(media!=null) { try { - MediaBytes file=downloadMedia(media); + MediaBytes file=mediaDownloader.download(media); parts.add(Map.of("inlineData",Map.of("mimeType",file.contentType(),"data",Base64.getEncoder().encodeToString(file.bytes())))); } catch(ApiException e) { throw e; } catch(Exception e) { throw new ApiException("MEDIA_FETCH_FAILED","无法读取参考图",400); } @@ -351,7 +353,7 @@ public class AsyncProviderClient { multipartField(out, boundary, entry.getKey(), String.valueOf(entry.getValue())); } if (fileField != null && media != null) { - MediaBytes file = downloadMedia(media); + MediaBytes file = mediaDownloader.download(media); multipartFile(out, boundary, fileField, file.filename(), file.contentType(), file.bytes()); } out.write(("--" + boundary + "--\r\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); @@ -400,52 +402,6 @@ public class AsyncProviderClient { out.write("\r\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)); } - private MediaBytes downloadMedia(String value) throws IOException, InterruptedException { - URI uri = URI.create(value); - if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null || uri.getUserInfo() != null) { - throw new ApiException("INVALID_MEDIA_URL", "媒体地址必须是可公开访问的 HTTPS URL", 400); - } - for (InetAddress address : InetAddress.getAllByName(uri.getHost())) { - if (address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() - || address.isSiteLocalAddress() || address.isMulticastAddress()) { - throw new ApiException("INVALID_MEDIA_URL", "媒体地址不能指向内网或本机地址", 400); - } - } - HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(15)).GET().build(); - BoundedHttpTransport.Response response; - try { response = transport.send(request, 8 * 1024 * 1024); } - catch (BoundedHttpTransport.Failure failure) { - if (failure.reason() == BoundedHttpTransport.Reason.BODY_TOO_LARGE) throw new ApiException("MEDIA_TOO_LARGE", "媒体输入超过 8 MiB", 400); - throw new ApiException("MEDIA_FETCH_FAILED", "无法在期限内读取媒体输入", 400); - } - if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new ApiException("MEDIA_FETCH_FAILED", "无法读取媒体输入(HTTP " + response.statusCode() + ")", 400); - } - byte[] bytes = response.body(); - if (bytes.length == 0 || bytes.length > 8 * 1024 * 1024) { - throw new ApiException("MEDIA_TOO_LARGE", "媒体输入为空或超过 8 MiB", 400); - } - String contentType = response.headers().firstValue("Content-Type").orElse("application/octet-stream") - .split(";", 2)[0].trim().toLowerCase(java.util.Locale.ROOT); - if (!contentType.startsWith("image/")) { - throw new ApiException("MEDIA_TYPE_UNSUPPORTED", "图像编辑和视频参考图必须是 image/* 文件", 400); - } - String filename = "input" + extension(contentType); - return new MediaBytes(bytes, contentType, filename); - } - - private String extension(String contentType) { - return switch (contentType) { - case "image/jpeg" -> ".jpg"; - case "image/png" -> ".png"; - case "image/webp" -> ".webp"; - case "image/gif" -> ".gif"; - default -> ".bin"; - }; - } - - private record MediaBytes(byte[] bytes, String contentType, String filename) {} - private Map request(String method, String url, String key, Map body, String message) { boolean sending = false; try { diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java index 25e19d9..c359a1b 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/request/ModuleDependencyTest.java @@ -20,6 +20,11 @@ class ModuleDependencyTest { assertTrue(Files.exists(root.resolve("task/TaskSettlementService.java"))); disallow(root.resolve("task/TaskSettlementService.java"), List.of("java.net.http", "AsyncProviderClient", "OpenAiCompatibleProvider")); } + @Test void mediaInputDoesNotOwnCredentialsRoutingOrBilling() throws Exception { + disallow(root.resolve("provider/media"), List.of(".gateway.", ".wallet.", ".auth.", ".task.", + ".provider.credentials", "org.springframework.jdbc", ".security.SecretCipher", "java.net.http.HttpClient")); + assertFalse(Files.readString(root.resolve("task/AsyncProviderClient.java")).contains("InetAddress"), "DNS policy belongs to the media boundary"); + } private void disallow(Path path, List forbidden) throws Exception { assertTrue(Files.exists(path), "Expected source boundary: " + path); try (var paths = Files.isDirectory(path) ? Files.walk(path) : java.util.stream.Stream.of(path)) { diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/MediaDownloaderTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/MediaDownloaderTest.java new file mode 100644 index 0000000..da8f02f --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/MediaDownloaderTest.java @@ -0,0 +1,58 @@ +package com.moyi.controlplane.provider.media; + +import com.moyi.controlplane.common.ApiException; +import com.moyi.controlplane.provider.transport.BoundedHttpTransport; +import com.moyi.controlplane.provider.transport.PinnedHttpsTransport; +import java.net.URI; +import java.net.http.HttpHeaders; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class MediaDownloaderTest { + private final PinnedHttpsTransport transport = mock(PinnedHttpsTransport.class); + private final MediaDownloader downloader = new MediaDownloader(transport, host -> { throw new AssertionError("Unexpected network"); }); + + @Test void normalizesImagesAndPreservesBoundedTransportContract() throws Exception { + byte[] bytes = {1, 2, 3}; + when(transport.get(any(), any(), any(), anyInt())).thenReturn(response(200, "IMAGE/PNG; charset=utf-8", bytes)); + var image = downloader.download("https://media.test/a?signature=synthetic"); + assertArrayEquals(bytes, image.bytes()); assertEquals("image/png", image.contentType()); assertEquals("input.png", image.filename()); + verify(transport).get(eq(URI.create("https://media.test/a?signature=synthetic")), any(), eq(Duration.ofSeconds(15)), eq(8 * 1024 * 1024)); + assertFalse(image.toString().contains("synthetic")); + } + + @Test void invalidUrlsFailBeforeTransport() { + assertEquals("INVALID_MEDIA_URL", assertThrows(ApiException.class, () -> downloader.download("https://user:secret@media.test/a")).getCode()); + verifyNoInteractions(transport); + } + + @ParameterizedTest @CsvSource({"BODY_TOO_LARGE,MEDIA_TOO_LARGE", "INVALID_ADDRESS,INVALID_MEDIA_URL", "TIMEOUT,MEDIA_FETCH_FAILED", "INTERRUPTED,MEDIA_FETCH_FAILED", "BUSY,MEDIA_FETCH_FAILED", "IO,MEDIA_FETCH_FAILED"}) + void transportFailuresExposeOnlySafeDomainCodes(BoundedHttpTransport.Reason reason, String code) throws Exception { + var failure = mock(BoundedHttpTransport.Failure.class); when(failure.reason()).thenReturn(reason); + when(transport.get(any(), any(), any(), anyInt())).thenThrow(failure); + var error = assertThrows(ApiException.class, () -> downloader.download("https://media.test/a?signature=private")); + assertEquals(code, error.getCode()); assertFalse(error.getMessage().contains("private")); assertNull(error.getCause()); + } + + @ParameterizedTest @CsvSource({"302,image/png,MEDIA_FETCH_FAILED", "404,image/png,MEDIA_FETCH_FAILED", "200,text/html,MEDIA_TYPE_UNSUPPORTED", "200,image/invalid:type,MEDIA_TYPE_UNSUPPORTED"}) + void redirectsErrorsAndUnsupportedTypesAreRejected(int status, String type, String code) throws Exception { + when(transport.get(any(), any(), any(), anyInt())).thenReturn(response(status, type, new byte[] {1})); + assertEquals(code, assertThrows(ApiException.class, () -> downloader.download("https://media.test/a")).getCode()); + } + + @Test void emptyBodyIsRejected() throws Exception { + when(transport.get(any(), any(), any(), anyInt())).thenReturn(response(200, "image/png", new byte[0])); + assertEquals("MEDIA_TOO_LARGE", assertThrows(ApiException.class, () -> downloader.download("https://media.test/a")).getCode()); + } + + private BoundedHttpTransport.Response response(int status, String type, byte[] body) { + return new BoundedHttpTransport.Response(status, HttpHeaders.of(Map.of("Content-Type", List.of(type)), (key, value) -> true), body); + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicyTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicyTest.java new file mode 100644 index 0000000..49929d0 --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/provider/media/PublicMediaAddressPolicyTest.java @@ -0,0 +1,65 @@ +package com.moyi.controlplane.provider.media; + +import com.moyi.controlplane.provider.transport.PinnedHttpsTransport; +import java.net.InetAddress; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.junit.jupiter.api.Assertions.*; + +class PublicMediaAddressPolicyTest { + private final PublicMediaAddressPolicy policy = new PublicMediaAddressPolicy(); + + @ParameterizedTest + @ValueSource(strings = { + "0.0.0.0", "0.255.255.255", "10.0.0.1", "127.0.0.1", "127.255.255.254", + "100.64.0.1", "100.127.255.254", "169.254.169.254", "172.16.0.1", "172.31.255.254", + "192.0.0.1", "192.0.0.9", "192.0.2.1", "192.88.99.1", "192.168.1.1", + "198.18.0.1", "198.19.255.254", "198.51.100.1", "203.0.113.1", "224.0.0.1", + "239.255.255.255", "240.0.0.1", "255.255.255.255", "168.63.129.16", + "::", "::1", "::127.0.0.1", "::ffff:127.0.0.1", "::ffff:169.254.169.254", + "fc00::1", "fd00::1", "fe80::1", "fec0::1", "ff02::1", "64:ff9b::a00:1", "64:ff9b:1::1", + "100::1", "2001::1", "2001:1ff:ffff::1", "2001:2::1", "2001:20::1", "2001:db8::1", + "2002:7f00:1::1", "2002:0808:0808::1", "3ffe::1", "3fff::1", "3fff:fff:ffff::1", "4000::1" + }) + void rejectsNonGlobalSpecialAndTranslationTargets(String literal) throws Exception { + InetAddress address = InetAddress.getByName(literal); + assertFalse(policy.isPublic(address), literal); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[] {address})); + } + + @ParameterizedTest + @ValueSource(strings = {"1.1.1.1", "8.8.8.8", "100.63.255.254", "100.128.0.1", "172.15.255.254", "172.32.0.1", + "192.0.1.1", "198.17.255.254", "198.20.0.1", "223.255.255.254", "2606:4700:4700::1111", "2001:4860:4860::8888", "3fff:1000::1"}) + void permitsOrdinaryGlobalAddressesWithoutMakingInternetRequests(String literal) throws Exception { + var address = InetAddress.getByName(literal); + assertTrue(policy.isPublic(address), literal); + assertArrayEquals(new InetAddress[] {address}, policy.requirePublic(new InetAddress[] {address})); + } + + @Test void rejectsTheWholeMixedAnswerSetInEitherOrder() throws Exception { + var external = InetAddress.getByName("8.8.8.8"); + var internal = InetAddress.getByName("fd00::1"); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[] {external, internal})); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[] {internal, external})); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[0])); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[] {null})); + assertThrows(PinnedHttpsTransport.AddressRejectedException.class, () -> policy.requirePublic(new InetAddress[17])); + } + + @ParameterizedTest + @ValueSource(strings = {"http://example.test/a", "file:///etc/passwd", "data:image/png,a", "//example.test/a", + "https://user:secret@example.test/a", "https://example.test/a#fragment", "https://example.test:0/a", "https://example.test:65536/a", + "https://[fe80::1%25eth0]/a", "https://example.test\\@127.0.0.1/a", "https://example.test/%0d%0a/x\r\n", "https:///a", ""}) + void rejectsAmbiguousOrNonHttpsUris(String value) { + var failure = assertThrows(IllegalArgumentException.class, () -> policy.requireUri(value)); + assertFalse(failure.getMessage().contains("secret")); + assertNull(failure.getCause()); + } + + @Test void permitsSignedHttpsUrlsAndExplicitHttpsPortsWithoutRewritingAuthority() { + String value = "https://media.example.test:8443/a.png?signature=synthetic"; + assertEquals(value, policy.requireUri(value).toString()); + assertEquals("[2606:4700:4700::1111]", policy.requireUri("https://[2606:4700:4700::1111]/a").getHost()); + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransportTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransportTest.java new file mode 100644 index 0000000..d499ff0 --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/provider/transport/PinnedHttpsTransportTest.java @@ -0,0 +1,279 @@ +package com.moyi.controlplane.provider.transport; + +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.ExtendedSSLSession; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SNIHostName; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.*; + +class PinnedHttpsTransportTest { + @TempDir static Path temporary; + private static SSLContext serverTls, trustedTls; + private HttpsServer server; + private ExecutorService workers; + private PinnedHttpsTransport transport; + private final CountDownLatch release = new CountDownLatch(1); + + @BeforeAll static void createEphemeralTestCertificate() throws Exception { + // No committed key, no trust-all TLS: mint a .test certificate and trust only this test key. + String password = java.util.UUID.randomUUID().toString(); + Path store = temporary.resolve("media-test.p12"); + var process = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "keytool").toString(), + "-genkeypair", "-alias", "test", "-keyalg", "RSA", "-keysize", "2048", "-validity", "2", + "-dname", "CN=media.test", "-ext", "SAN=dns:media.test", "-storetype", "PKCS12", + "-keystore", store.toString(), "-storepass", password, "-keypass", password, "-noprompt") + .redirectErrorStream(true).start(); + assertTrue(process.waitFor(15, TimeUnit.SECONDS)); + assertEquals(0, process.exitValue(), () -> "Unable to create ephemeral TLS fixture"); + var keys = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(store)) { keys.load(input, password.toCharArray()); } + var kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keys, password.toCharArray()); + serverTls = SSLContext.getInstance("TLS"); + serverTls.init(kmf.getKeyManagers(), null, null); + var trust = KeyStore.getInstance(KeyStore.getDefaultType()); trust.load(null, null); + trust.setCertificateEntry("test", keys.getCertificate("test")); + var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); tmf.init(trust); + trustedTls = SSLContext.getInstance("TLS"); trustedTls.init(null, tmf.getTrustManagers(), null); + } + + @BeforeEach void setup() throws Exception { + server = HttpsServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.setHttpsConfigurator(new HttpsConfigurator(serverTls)); + workers = new ThreadPoolExecutor(0, 2, 5, TimeUnit.SECONDS, new SynchronousQueue<>(), + runnable -> { var thread = new Thread(runnable, "test-pinned-get"); thread.setDaemon(true); return thread; }); + transport = new PinnedHttpsTransport(trustedTls, workers); + } + @AfterEach void cleanup() throws Exception { + release.countDown(); workers.shutdownNow(); server.stop(0); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + private URI uri(String path) { return URI.create("https://media.test:" + server.getAddress().getPort() + path); } + private InetAddress[] loopback(String host) throws java.io.IOException { return new InetAddress[] {InetAddress.getByName("127.0.0.1")}; } + private BoundedHttpTransport.Response get(String path, long timeoutMs, int maxBytes) throws Exception { + return transport.get(uri(path), this::loopback, Duration.ofMillis(timeoutMs), maxBytes); + } + + @Test void bindsDnsOnceWhilePreservingHostSniAndCredentialIsolation() throws Exception { + var lookups = new AtomicInteger(); var host = new AtomicReference(); var sni = new AtomicReference(); + server.createContext("/", exchange -> { + host.set(exchange.getRequestHeaders().getFirst("Host")); + var session = (ExtendedSSLSession) ((com.sun.net.httpserver.HttpsExchange) exchange).getSSLSession(); + sni.set(((SNIHostName) session.getRequestedServerNames().getFirst()).getAsciiName()); + assertNull(exchange.getRequestHeaders().getFirst("Authorization")); + assertNull(exchange.getRequestHeaders().getFirst("Cookie")); + assertNull(exchange.getRequestHeaders().getFirst("Proxy-Authorization")); + assertEquals("identity", exchange.getRequestHeaders().getFirst("Accept-Encoding")); + exchange.getResponseHeaders().set("Content-Type", "image/png"); + byte[] bytes = "synthetic-image".getBytes(StandardCharsets.UTF_8); exchange.sendResponseHeaders(200, bytes.length); + try (var output = exchange.getResponseBody()) { output.write(bytes); } + }); server.start(); + var response = transport.get(uri("/?signature=synthetic-private-query"), target -> { + assertEquals("media.test", target); + if (lookups.incrementAndGet() != 1) throw new AssertionError("DNS was resolved twice"); + return loopback(target); + }, Duration.ofSeconds(3), 1024); + assertEquals(200, response.statusCode()); assertEquals(1, lookups.get()); + assertEquals("media.test:" + server.getAddress().getPort(), host.get()); assertEquals("media.test", sni.get()); + assertEquals("synthetic-image", new String(response.body(), StandardCharsets.UTF_8)); + assertFalse(response.toString().contains("synthetic")); + } + + @Test void aTrustedCertificateForAnotherHostnameStillFails() throws Exception { + var calls = new AtomicInteger(); + server.createContext("/", exchange -> { calls.incrementAndGet(); exchange.sendResponseHeaders(200, -1); exchange.close(); }); server.start(); + URI wrong = URI.create("https://wrong.test:" + server.getAddress().getPort() + "/?signature=private"); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.get(wrong, this::loopback, Duration.ofSeconds(3), 1024)); + assertEquals(BoundedHttpTransport.Reason.IO, failure.reason()); assertEquals(0, calls.get()); + assertNull(failure.getCause()); assertFalse(failure.toString().contains("private")); + } + + @Test void theDefaultTrustStoreDoesNotAcceptOurSelfSignedTestCertificate() throws Exception { + server.createContext("/", exchange -> { exchange.sendResponseHeaders(200, -1); exchange.close(); }); server.start(); + var strict = new PinnedHttpsTransport(org.apache.hc.core5.ssl.SSLContexts.createDefault(), workers); + assertEquals(BoundedHttpTransport.Reason.IO, assertThrows(BoundedHttpTransport.Failure.class, + () -> strict.get(uri("/"), this::loopback, Duration.ofSeconds(3), 1024)).reason()); + } + + @Test void redirectsAreNeverFollowed() throws Exception { + var calls = new AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); exchange.getResponseHeaders().set("Location", "https://127.0.0.1/private"); + exchange.sendResponseHeaders(302, -1); exchange.close(); + }); server.start(); + assertEquals(302, get("/", 3000, 1024).statusCode()); assertEquals(1, calls.get()); + } + + @Test void declaredOversizeDoesNotWaitForTheBodyOrDrainItOnClose() { + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 1_000_000); + try { release.await(3, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + exchange.close(); + }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 2000, 8192)); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason()); assertEquals(200, failure.observedStatus()); + } + + @Test void chunkedOversizeIsStoppedDuringRead() { + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { for (int i = 0; i < 100; i++) output.write(new byte[4096]); } + catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 3000, 8192)).reason()); + } + + @Test void deadlineIncludesSlowResponseBodyAndRestoresWorkerCapacity() throws Exception { + var arrived = new CountDownLatch(1); + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { + output.write('x'); output.flush(); arrived.countDown(); + try { release.await(5, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + } catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + long start = System.nanoTime(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 1000, 8192)); + assertEquals(BoundedHttpTransport.Reason.TIMEOUT, failure.reason()); assertEquals(200, failure.observedStatus()); + assertEquals(0, arrived.getCount()); assertTrue(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start) < 2500); + workers.shutdown(); assertTrue(workers.awaitTermination(2, TimeUnit.SECONDS)); + } + + @Test void lateDnsResolutionCannotStartHttpAfterTheDeadline() throws Exception { + var resolving = new CountDownLatch(1); var finished = new CountDownLatch(1); var calls = new AtomicInteger(); + server.createContext("/", exchange -> { calls.incrementAndGet(); exchange.sendResponseHeaders(200, -1); exchange.close(); }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> transport.get(uri("/"), host -> { + resolving.countDown(); + boolean interrupted = false; + while (release.getCount() != 0) try { release.await(1, TimeUnit.SECONDS); } + catch (InterruptedException ignoredForSimulatedNativeResolver) { interrupted = true; } + if (interrupted) Thread.currentThread().interrupt(); + finished.countDown(); return loopback(host); + }, Duration.ofMillis(200), 1024)); + assertEquals(BoundedHttpTransport.Reason.TIMEOUT, failure.reason()); assertEquals(0, resolving.getCount()); + release.countDown(); assertTrue(finished.await(2, TimeUnit.SECONDS)); + workers.shutdown(); assertTrue(workers.awaitTermination(2, TimeUnit.SECONDS)); assertEquals(0, calls.get()); + } + + @Test void interruptionCancelsSocketAndRestoresCallerInterruptFlag() throws Exception { + var arrived = new CountDownLatch(1); var observed = new AtomicReference(); var flag = new AtomicReference(); + server.createContext("/", exchange -> { + exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { + output.write('x'); output.flush(); arrived.countDown(); + try { release.await(5, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + } catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + var caller = new Thread(() -> { + try { get("/", 5000, 1024); } + catch (BoundedHttpTransport.Failure failure) { observed.set(failure.reason()); flag.set(Thread.currentThread().isInterrupted()); } + catch (Exception unexpected) { throw new AssertionError(unexpected); } + }); caller.setDaemon(true); caller.start(); + try { + assertTrue(arrived.await(3, TimeUnit.SECONDS)); caller.interrupt(); caller.join(2000); + assertFalse(caller.isAlive()); assertEquals(BoundedHttpTransport.Reason.INTERRUPTED, observed.get()); assertEquals(true, flag.get()); + } finally { release.countDown(); caller.interrupt(); } + } + + @Test void refusedOrDroppedRequestsAreNotAutomaticallyRetried() { + var calls = new AtomicInteger(); + server.createContext("/", exchange -> { calls.incrementAndGet(); exchange.close(); }); server.start(); + assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 3000, 1024)); assertEquals(1, calls.get()); + } + + @Test void compressedBodiesAreNotDecompressedOrTreatedAsImages() { + server.createContext("/", exchange -> { + exchange.getResponseHeaders().set("Content-Encoding", "gzip"); exchange.sendResponseHeaders(200, 0); + try (var output = exchange.getResponseBody()) { output.write(new byte[16]); } + catch (java.io.IOException expectedAfterCancellation) { } + }); server.start(); + assertEquals(BoundedHttpTransport.Reason.IO, assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 3000, 1024)).reason()); + } + + @Test void busyPoolRejectsImmediatelyRatherThanQueueingUnlimitedDnsRequests() throws Exception { + var occupied = new CountDownLatch(1); + var oneWorker = new ThreadPoolExecutor(0, 1, 5, TimeUnit.SECONDS, new SynchronousQueue<>()); + try { + oneWorker.submit(() -> { + occupied.countDown(); + try { release.await(5, TimeUnit.SECONDS); } catch (InterruptedException error) { Thread.currentThread().interrupt(); } + }); + assertTrue(occupied.await(2, TimeUnit.SECONDS)); + var rejected = new PinnedHttpsTransport(trustedTls, oneWorker); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> rejected.get(uri("/"), this::loopback, Duration.ofSeconds(3), 1024)); + assertEquals(BoundedHttpTransport.Reason.BUSY, failure.reason()); assertNull(failure.observedStatus()); + assertEquals(0, oneWorker.getQueue().size()); + } finally { release.countDown(); oneWorker.shutdownNow(); assertTrue(oneWorker.awaitTermination(2, TimeUnit.SECONDS)); } + } + + @Test void defaultSystemProxySelectorCannotBypassDnsBinding() throws Exception { + var original = java.net.ProxySelector.getDefault(); + var selections = new AtomicInteger(); + server.createContext("/", exchange -> { exchange.sendResponseHeaders(200, 1); exchange.getResponseBody().write('x'); exchange.close(); }); server.start(); + try { + java.net.ProxySelector.setDefault(new java.net.ProxySelector() { + @Override public java.util.List select(URI target) { + selections.incrementAndGet(); + return java.util.List.of(new java.net.Proxy(java.net.Proxy.Type.SOCKS, new InetSocketAddress("127.0.0.1", 1))); + } + @Override public void connectFailed(URI target, java.net.SocketAddress address, java.io.IOException failure) { } + }); + assertEquals(200, get("/", 3000, 1024).statusCode()); assertEquals(0, selections.get()); + } finally { java.net.ProxySelector.setDefault(original); } + } + + @Test void independentAttemptsRevalidateDnsAndNeverReuseCookiesOrThePreviousConnection() throws Exception { + var calls = new AtomicInteger(); var lookups = new AtomicInteger(); + server.createContext("/", exchange -> { + calls.incrementAndGet(); assertNull(exchange.getRequestHeaders().getFirst("Cookie")); + exchange.getResponseHeaders().set("Set-Cookie", "synthetic=value; Secure"); + exchange.sendResponseHeaders(200, 1); exchange.getResponseBody().write('x'); exchange.close(); + }); server.start(); + PinnedHttpsTransport.AddressResolver resolver = host -> { + if (lookups.incrementAndGet() == 3) throw new PinnedHttpsTransport.AddressRejectedException(); + return loopback(host); + }; + assertEquals(200, transport.get(uri("/"), resolver, Duration.ofSeconds(3), 1024).statusCode()); + assertEquals(200, transport.get(uri("/"), resolver, Duration.ofSeconds(3), 1024).statusCode()); + assertEquals(BoundedHttpTransport.Reason.INVALID_ADDRESS, assertThrows(BoundedHttpTransport.Failure.class, + () -> transport.get(uri("/"), resolver, Duration.ofSeconds(3), 1024)).reason()); + assertEquals(3, lookups.get()); assertEquals(2, calls.get()); + } + + @Test void errorBodyLimitRetainsStatusAndIsSmallerThanTheImageLimit() { + server.createContext("/", exchange -> { exchange.sendResponseHeaders(401, 1_000_000); exchange.close(); }); server.start(); + var failure = assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 3000, 8 * 1024 * 1024)); + assertEquals(BoundedHttpTransport.Reason.BODY_TOO_LARGE, failure.reason()); assertEquals(401, failure.observedStatus()); + } + + @Test void oversizedResponseHeadersAreNotBufferedWithoutLimits() { + server.createContext("/", exchange -> { + exchange.getResponseHeaders().set("X-Large", "x".repeat(16 * 1024)); + exchange.sendResponseHeaders(200, 1); exchange.getResponseBody().write('x'); exchange.close(); + }); server.start(); + assertEquals(BoundedHttpTransport.Reason.IO, assertThrows(BoundedHttpTransport.Failure.class, () -> get("/", 3000, 1024)).reason()); + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java index 9773418..37c3b2c 100644 --- a/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java +++ b/control-plane-java/src/test/java/com/moyi/controlplane/task/AsyncProviderClientTest.java @@ -322,6 +322,24 @@ class AsyncProviderClientTest { assertEquals("INVALID_MEDIA_URL", failure.getCode()); } + @ParameterizedTest + @CsvSource({"image,gpt-image-2", "video,sora-2", "image,gemini-3-pro-image-preview"}) + void localReferenceImageIsRejectedBeforeAnyGenerationPost(String type, String model) throws Exception { + var calls = new java.util.concurrent.atomic.AtomicInteger(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { calls.incrementAndGet(); exchange.sendResponseHeaders(500, -1); exchange.close(); }); server.start(); + try { + var jdbc = mock(JdbcTemplate.class); var cipher = new SecretCipher("test-encryption"); + when(jdbc.queryForMap(any(String.class), eq("relay"))).thenReturn(Map.of("code", "relay", "protocol", "openai-compatible", + "base_url", "http://127.0.0.1:" + server.getAddress().getPort(), "credential_ciphertext", cipher.encrypt("test-key"))); + var client = new AsyncProviderClient(jdbc, cipher, new ObjectMapper(), mock(Environment.class)); + var failure = assertThrows(ProviderRequestException.class, () -> client.submit("relay", model, type, + Map.of("prompt", "synthetic", "media_url", "https://127.0.0.1/private"))); + assertEquals(ProviderRequestException.Acceptance.NOT_SENT, failure.acceptance()); assertEquals("INVALID_MEDIA_URL", failure.getCode()); + assertEquals(0, calls.get()); + } finally { server.stop(0); } + } + private AsyncProviderClient clientFor(String baseUrl) { JdbcTemplate jdbc = mock(JdbcTemplate.class); SecretCipher cipher = new SecretCipher("test-encryption"); diff --git a/docs/24-security-and-cpr-task-tracker.md b/docs/24-security-and-cpr-task-tracker.md index ddf8404..fe95dc6 100644 --- a/docs/24-security-and-cpr-task-tracker.md +++ b/docs/24-security-and-cpr-task-tracker.md @@ -19,7 +19,7 @@ | SEC-03 | 改密与会话撤销、登录竞态 | 本轮已实现并验证:撤销全部会话、清 Cookie、行锁与二次校验 | 改密两种锁顺序、MFA 失败计数及审计回滚 | | SEC-04 | 任务用户隔离与回执安全 | 已实现核心用户级查询/幂等/冻结/核账,并补跨用户读取回归;完整授权审计仍待继续 | 跨用户读取/列表不可见;核账拒绝普通用户 | | SEC-05 | Provider 有界 HTTP Transport | 部分完成:异步 JSON/multipart/音频/媒体已接有界读取和总期限;同步/SSE 生命周期尚未统一抽取 | 超大 chunk、慢响应体、已观察拒绝状态、取消、重定向及原适配器回归 | -| SEC-06 | 媒体下载出站 / SSRF | 部分加固但未闭环:体积/期限/重定向有界,DNS 重绑定窗口和保留地址复核仍待补 | 私网/保留地址、DNS 绑定或可信出站代理尚待完整验证 | +| SEC-06 | 媒体下载出站 / SSRF | 服务端参考图下载的代码级 DNS 绑定已实现并通过本地回归;生产网络隔离与实际部署验收仍未完成 | 一次解析、整组地址校验、固定地址直连、原域名 TLS/SNI、禁代理与跳转;109 项新增回归,见 26 号文档 | | SEC-07 | 整体安全审查与生产验收 | 未完成:管理接口、注册/重置原子性、默认配置、依赖、备份恢复需继续 | 独立用例、部署与恢复演练,禁止宣称“全仓库安全” | | COD-01 | Codex 协议能力及验收矩阵 | 未实现完整兼容;保留有限文本 Responses 的明确拒绝语义 | 指定客户端版本、模型、工具和计费联合验收 | | COD-02 | Responses WebSocket 与响应归属续接 | 未实现,不可宣传完整 Codex 支持 | 原账号/凭据版本归属、断线、取消、租约、费用一致性 | @@ -31,8 +31,10 @@ ## 实施边界 -已补 SEC-01/02/03,下一优先级是 SEC-06 出站/DNS 绑定、同步 Transport 与完整 provider 配置版本,再处理 COD-01/02/03 的协议/续接/长请求合同。Codex WebSocket 与 OAuth 不是添加 URL 即可安全完成的能力,须连同状态归属、计量、预留续期和客户端验收逐步实施。未部署、未真实供应商验证或待设计的项必须保持未完成标记。 +已补 SEC-01/02/03 和 SEC-06 的本地下载代码边界,下一优先级是注册/密码重置事务审查、同步 Transport 与完整 provider 配置版本,再处理 COD-01/02/03 的协议/续接/长请求合同。SEC-06 的生产网络隔离仍属待验收。Codex WebSocket 与 OAuth 不是添加 URL 即可安全完成的能力,须连同状态归属、计量、预留续期和客户端验收逐步实施。未部署、未真实供应商验证或待设计的项必须保持未完成标记。 本轮设计与验证细节见 `25-auth-and-bounded-transport.md`。这份表按用户粘贴报告的锁定基线追踪本地实施,不把旧的 CPR 版本号冒充实时最新版,也没有重新部署 CPR。只做本地 Git commit,不 push,不修改业务数据库。 2026-10-21 本轮最终验证:247 项后端测试全部通过(含真实 PostgreSQL/Redis,0 跳过),前端构建与 25 项测试通过,另完成隔离改密界面交互检查。新增 GitHub 配置只保存本地,远端未执行。本轮没有把剩余 COD/SCH/出站/运营验收项目标为完成。 + +2026-10-21 媒体安全后续增量:使用 HttpClient 5.6.5 / HttpCore 5.4.4 重新执行 `mvn -B -o clean verify`,356 项后端测试通过(0 失败/错误/跳过),前端构建及 25 项测试通过。服务端本地下载不再使用 DNS 预检后交给 JDK 客户端二次解析的路径。用户指定的规格驱动验收尚未执行:`testingflow` 的 `.venv` 指向不存在的 Python,首个 `qa project list` 失败;没有绕过工具门禁,也没有自行批准规格。细节及下一步审阅范围见 `26-media-dns-binding-and-acceptance.md`。 diff --git a/docs/25-auth-and-bounded-transport.md b/docs/25-auth-and-bounded-transport.md index b35f4c2..21e9466 100644 --- a/docs/25-auth-and-bounded-transport.md +++ b/docs/25-auth-and-bounded-transport.md @@ -27,7 +27,7 @@ 这还不是全项目统一 Transport:同步/SSE 的现有有界读取和整体 deadline 仍在原 Provider 生命周期内;长请求续租、逐 attempt 交付状态、统一取消与跨路由共享预算未完成。 -**媒体 SSRF 仍未闭环。** 本阶段约束了响应体、总期限和重定向,但旧 DNS 预检与 JDK HTTP 实际解析之间的重绑定窗口仍存在,保留地址范围也需完整复核。不得标记 SEC-06 完成;生产应通过可信出站代理/网络策略控制访问,后续实现可证明的 DNS 绑定或等价控制。没有擅自更改业务网络或把任意用户 URL 加入可信名单。 +**该提交时的媒体 SSRF 尚未闭环。** 本阶段约束了响应体、总期限和重定向,但当时旧 DNS 预检与 JDK HTTP 实际解析之间的重绑定窗口仍存在。2026-10-21 的后续增量已用固定地址直连替换该路径,并补特殊地址范围回归,详见 `26-media-dns-binding-and-acceptance.md`;生产网络策略仍未验收。不能将本地下载修复扩大为全项目出站安全保证,也没有擅自更改业务网络或把任意用户 URL 加入可信名单。 ## 工程验证与限制 diff --git a/docs/26-media-dns-binding-and-acceptance.md b/docs/26-media-dns-binding-and-acceptance.md new file mode 100644 index 0000000..83fd4c1 --- /dev/null +++ b/docs/26-media-dns-binding-and-acceptance.md @@ -0,0 +1,46 @@ +# 媒体 DNS 绑定与功能验收状态 + +日期:2026-10-21。增量范围是服务端为 Gemini 图像输入、OpenAI 兼容图像编辑和视频参考图下载的 HTTPS 文件。不是完整生产安全审计,不涉及 CPR 部署、业务数据迁移或付费上游生成。 + +## 已实现边界 + +- `provider.media.PublicMediaAddressPolicy`:校验 HTTPS URI,拒绝 userinfo、fragment、zone ID 与非法端口。DNS 答案整组校验;只要存在非公开/特殊用途地址就全部拒绝,不选择其中的公网地址继续。答案为空、含 null 或超过 16 个地址同样拒绝。IPv4 覆盖本机、私网、共享地址、链路本地、基准测试、文档、组播、保留地址和平台虚拟地址;IPv6 仅接受原生 global-unicast 候选,排除特殊、文档、6to4、已退役 6bone 等前缀。策略采用保守拒绝,某些特殊用途的实际可达地址也可能被拒绝。 +- `provider.transport.PinnedHttpsTransport`:每次调用只向系统解析一次,经策略验证后使用这份不可变地址集合连接。HTTP 库自己的 DNS 回调只返回该集合,不调用系统 DNS,也不反向解析。URI 始终保留原域名,Host、TLS 证书链、主机名校验和 SNI 不替换为 IP;证书校验未放宽。 +- 路由层固定直连,底层 Socket 显式 `Proxy.NO_PROXY`,防止 JVM 的系统 SOCKS 代理绕过本地 DNS 绑定。没有自动跳转、HTTP 重试、Cookie 管理、认证缓存或跨下载连接复用;不接受任意请求头、上游 Key 或浏览器 Cookie。 +- 15 秒总期限覆盖 DNS、连接、TLS 和整个响应体;超时/中断取消 socket,调用方中断标志保留。DNS 原生实现可能不能及时中断,迟到的解析结果会检查取消状态,不能在超时后再发起 HTTP。最多 8 个后台工作线程、无等待队列;饱和直接安全失败,避免无限积压。 +- 成功体最多 8 MiB,错误体最多 64 KiB;声明超限提前拒绝,chunked 在读取过程中计数。HTTP/1 响应行最多 8192 字符、头数量最多 64。禁自动解压,拒绝非 identity 的 Content-Encoding。先取消再关闭,关闭时不排空超大/无限响应,清理异常不覆盖主要超限原因。 +- `provider.media.MediaDownloader` 只负责文件输入、MIME 类型及域错误映射;`AsyncProviderClient` 不再自行执行 DNS 策略。模块不依赖凭据、路由、风控、认证或账务,新增依赖方向检查。媒体下载失败仍是生成之前的 `NOT_SENT`,不制造“上游已受理未知”的结果。 + +## 依赖与本地验证 + +新模块使用 Apache HttpClient **5.6.5** 和配套 HttpCore **5.4.4**,在 POM 中固定属性,不沿用 Spring Boot 3.4.5 管理的旧组合,也不采用 5.7 alpha。版本在 2026-10-21 通过 Maven Central metadata/POM 与 Apache 官方发布记录核对;这不是完整依赖漏洞扫描或零漏洞承诺。参考资料: + +- Apache 发布记录:`https://hc.apache.org/news.html` +- HttpClient 父 POM:`https://repo.maven.apache.org/maven2/org/apache/httpcomponents/client5/httpclient5-parent/5.6.5/httpclient5-parent-5.6.5.pom` + +最终后端命令为 `mvn -B -o clean verify`,Java 21 / Maven 3.9.9 容器,共享本机依赖缓存。**356 项测试,0 失败、0 错误、0 跳过,JAR 构建成功**;结束时间 2026-10-21 22:44:03(Asia/Shanghai)。相对前一基线增加 109 项:76 项 URI/地址策略、13 项下载域边界、16 项真实本地 HTTPS 传输、3 项适配器发送前语义,以及 1 项模块依赖方向检查。 + +TLS 测试用 keytool 临时生成 `.test` 证书,仅测试客户端信任该证书;默认客户端必须拒绝它,可信但域名不匹配同样拒绝。测试还验证原 Host/SNI、DNS 只查一次、后续下载重新校验、Cookie 不复用、系统 SOCKS 代理不被读取、重定向不跟随、超限/响应头/压缩/慢体/中断/DNS 迟到和无队列饱和。没有通过关闭证书校验来获得绿灯。 + +数据库测试使用本轮独立 PostgreSQL 15.18 和 Redis 7 容器,共享隔离 network namespace,未映射业务数据库端口、未连接业务库。前端 `npm run build` 和 `npm test` 也通过,25 项测试零失败。日志 `.tools/media-dns-final-verify.log` 与构建目录忽略,不进入 Git。提交前再执行 `git diff --check`、逐路径检查差异与暂存区,只本地提交,不 push。 + +## 尚未完成的安全边界 + +本增量只证明上述本地下载路径。传给供应商由其下载的 URL、同步聊天输入中的第三方 URL、浏览器展示结果文件、管理员配置的 Provider Base URL 不属于这条下载路径;不能将它们标成已通过该策略。未知 NAT64/自定义翻译前缀、透明代理、组织路由将公网 IP 引向内部服务等部署行为,需要网络 egress ACL/防火墙控制;地址表也要随地址分配变化维护。生产出站允许范围、部署连通性和真实供应商兼容性均未验收。 + +图片目前校验媒体类型和字节大小,不解码文件、扫描内容或限制解码后像素;MIME 声明不是内容真实性保证。并发数 8 是保守传输资源上限,不是租户公平队列;长请求续租、完整 Provider 配置版本、WebSocket 续接、公平调度与备份恢复仍在 24 号任务表中保持待完成。 + +## 规格驱动验收的真实状态 + +已按用户指定的 `spec-driven-acceptance-testing` 读取完整技能、方法决策表、字段格式、框架 README 和模板。技能记录的 `E:/PROJECT/xpp_dev_agent` 在当前主机不存在,实际框架位于 `D:/PROJECT/xpp_dev_agent/dev-test`,档案目录位于同级 `qa-projects`。 + +首先执行了框架规定的项目发现命令: + +```powershell +# 工作目录 D:\PROJECT\xpp_dev_agent\dev-test +.\.venv\Scripts\python.exe -m qa project list +``` + +退出码 **1**,提示 Python 路径 `C:/Users/xpp/.conda/envs/ctf/python.exe` 不存在;`.venv/pyvenv.cfg` 仍记录旧的 E 盘创建路径。因此尚未完成项目登记/体检,**未创建或批准业务规格、未执行 `qa run --project`,没有验收报告可交付**。没有手动读取档案代替项目命令、改写共享 Python 环境、绕过批准门禁,或把本轮 Maven/前端测试冒称规格驱动验收。 + +后续先确认并修复共享框架环境,成功运行项目发现/体检;项目不存在时按 `qa project init` 建档并只填写源码核实的事实。首批建议审阅范围是隔离环境下匿名访问拒绝、普通用户/管理员权限、任务跨用户隔离、改密撤销会话、错误响应不暴露凭据,以及媒体失败不发起生成。规格始终先 draft,必须向用户展示具体输入、预期与清理步骤,获明确批准后才记录批准并执行。变更真实账号、付费模型调用和生产环境测试不在当前授权范围。