From c7278c00a650c4071eef883dfaf1fc895376a585 Mon Sep 17 00:00:00 2001 From: WangLX <519407855@qq.com> Date: Thu, 10 Sep 2026 18:10:31 +0800 Subject: [PATCH] feat: package Beijing test deployment with HTTPS and preflight checks --- .gitattributes | 5 + .gitignore | 5 + README.md | 1 + control-plane-java/.dockerignore | 2 + control-plane-java/Dockerfile.runtime | 9 + .../Dockerfile.runtime.dockerignore | 3 + .../controlplane/catalog/CatalogService.java | 2 +- .../gateway/GatewayController.java | 3 + .../controlplane/gateway/GatewayGuard.java | 14 +- .../controlplane/task/AsyncTaskService.java | 3 + .../controlplane/usage/UsageAuditService.java | 8 +- .../src/main/resources/application.yml | 11 +- .../gateway/GatewayDeploymentTest.java | 76 +++++ .../gateway/PostgresBindingTest.java | 49 ++++ docs/09-java-control-plane-node.md | 6 +- docs/10-beijing-test-environment-checklist.md | 16 +- docs/11-server-test-deployment.md | 191 ++++++++++++ infra/moyi/.env.cloud.example | 74 +++-- infra/moyi/README.md | 11 +- infra/moyi/deploy.py | 274 ++++++++++++++++++ infra/moyi/docker-compose.cloud.yml | 86 +++++- infra/moyi/docker-compose.https.yml | 17 ++ infra/moyi/install-docker-ubuntu.sh | 40 +++ infra/moyi/nginx/http.conf | 7 + infra/moyi/nginx/https-fallback.conf | 7 + infra/moyi/nginx/https.conf.template | 19 ++ infra/moyi/nginx/routes.conf | 40 +++ infra/moyi/tests/test_deploy.py | 139 +++++++++ prototype/.dockerignore | 2 + prototype/Dockerfile.runtime | 5 + prototype/Dockerfile.runtime.dockerignore | 4 + prototype/build-release.mjs | 6 + scripts/package-deployment.ps1 | 72 +++++ 33 files changed, 1153 insertions(+), 54 deletions(-) create mode 100644 .gitattributes create mode 100644 control-plane-java/Dockerfile.runtime create mode 100644 control-plane-java/Dockerfile.runtime.dockerignore create mode 100644 control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java create mode 100644 control-plane-java/src/test/java/com/moyi/controlplane/gateway/PostgresBindingTest.java create mode 100644 docs/11-server-test-deployment.md create mode 100644 infra/moyi/deploy.py create mode 100644 infra/moyi/docker-compose.https.yml create mode 100644 infra/moyi/install-docker-ubuntu.sh create mode 100644 infra/moyi/nginx/http.conf create mode 100644 infra/moyi/nginx/https-fallback.conf create mode 100644 infra/moyi/nginx/https.conf.template create mode 100644 infra/moyi/nginx/routes.conf create mode 100644 infra/moyi/tests/test_deploy.py create mode 100644 prototype/Dockerfile.runtime create mode 100644 prototype/Dockerfile.runtime.dockerignore create mode 100644 prototype/build-release.mjs create mode 100644 scripts/package-deployment.ps1 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fa22992 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.conf text eol=lf +*.template text eol=lf +Dockerfile* text eol=lf diff --git a/.gitignore b/.gitignore index c9267f0..8e7fedd 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ prototype/.env .tools/ control-plane-java/.tools/ control-plane-java/target/ +artifacts/ +infra/moyi/certs/ +infra/moyi/__pycache__/ +infra/moyi/tests/__pycache__/ +infra/moyi/.deploy-mode diff --git a/README.md b/README.md index b52c290..2eb96be 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,7 @@ MaaS 模型服务与应用开发 - [首批模型供应商接入节点](docs/07-provider-integration.md) - [Java 后端与容器化部署基线](docs/08-java-container-deployment.md) - [Java Control Plane 节点一:基础控制面](docs/09-java-control-plane-node.md) +- [北京服务器测试部署与发布包](docs/11-server-test-deployment.md) ## 前端原型 diff --git a/control-plane-java/.dockerignore b/control-plane-java/.dockerignore index b41e45f..32bcca4 100644 --- a/control-plane-java/.dockerignore +++ b/control-plane-java/.dockerignore @@ -2,3 +2,5 @@ .idea target *.log +.env* +.tools diff --git a/control-plane-java/Dockerfile.runtime b/control-plane-java/Dockerfile.runtime new file mode 100644 index 0000000..8994ac6 --- /dev/null +++ b/control-plane-java/Dockerfile.runtime @@ -0,0 +1,9 @@ +ARG JAVA_RUNTIME_IMAGE=eclipse-temurin:21-jre-alpine +FROM ${JAVA_RUNTIME_IMAGE} +WORKDIR /app +RUN addgroup -S moyi && adduser -S -G moyi moyi +COPY --chown=moyi:moyi target/moyi-control-plane.jar /app/moyi-control-plane.jar +USER moyi +ENV JAVA_OPTS="-Xms256m -Xmx1536m -XX:+ExitOnOutOfMemoryError" +EXPOSE 8787 +ENTRYPOINT ["sh", "-c", "exec java $JAVA_OPTS -jar /app/moyi-control-plane.jar"] diff --git a/control-plane-java/Dockerfile.runtime.dockerignore b/control-plane-java/Dockerfile.runtime.dockerignore new file mode 100644 index 0000000..c67f62e --- /dev/null +++ b/control-plane-java/Dockerfile.runtime.dockerignore @@ -0,0 +1,3 @@ +** +!target/ +!target/moyi-control-plane.jar diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java index 10f6346..c112ead 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/catalog/CatalogService.java @@ -147,7 +147,7 @@ public class CatalogService { OffsetDateTime now = now(); jdbc.update( "INSERT INTO platform_api_keys (id, user_id, project_id, name, key_prefix, key_hash, scopes, model_ids, status, created_at) VALUES (?, ?, ?, ?, ?, ?, CAST(? AS jsonb), CAST(? AS jsonb), 'active', ?)", - id, user.id(), projectId, name, plaintext.substring(0, 16), sha256(plaintext), writeJson(scopes), writeJson(modelIds), now + UUID.fromString(id), user.id(), projectId, name, plaintext.substring(0, 16), sha256(plaintext), writeJson(scopes), writeJson(modelIds), now ); Map result = new LinkedHashMap<>(); result.put("key", plaintext); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayController.java b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayController.java index 1003d41..2f0bd3f 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayController.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayController.java @@ -86,6 +86,9 @@ public class GatewayController { catalogService.touchKey(key); return ResponseEntity.ok(upstream); } + if (!"provider-mock".equals(providerId)) { + throw new ApiException("PROVIDER_NOT_CONFIGURED", "该供应商尚未配置密钥,不能执行真实调用", 503); + } String prompt = extractPrompt(input.get("messages")); String content = "MOYI Java Mock 已收到:" + (prompt.isBlank() ? "(空消息)" : prompt); catalogService.touchKey(key); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java index 18941e6..ca816c2 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/gateway/GatewayGuard.java @@ -25,6 +25,7 @@ public class GatewayGuard { private final int requestsPerMinute; private final int readsPerMinute; private final int defaultModelConcurrency; + private final String redisPrefix; private static DefaultRedisScript script(String location) { DefaultRedisScript script = new DefaultRedisScript<>(); @@ -37,7 +38,8 @@ public class GatewayGuard { StringRedisTemplate redis, @Value("${moyi.gateway.requests-per-minute:60}") int requestsPerMinute, @Value("${moyi.gateway.reads-per-minute:300}") int readsPerMinute, - @Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency + @Value("${moyi.gateway.default-model-concurrency:4}") int defaultModelConcurrency, + @Value("${moyi.gateway.redis-prefix:moyi:}") String redisPrefix ) { if (requestsPerMinute < 1 || readsPerMinute < 1 || defaultModelConcurrency < 1) { throw new IllegalArgumentException("Gateway limits must be positive"); @@ -46,10 +48,14 @@ public class GatewayGuard { this.requestsPerMinute = requestsPerMinute; this.readsPerMinute = readsPerMinute; this.defaultModelConcurrency = defaultModelConcurrency; + if (redisPrefix == null || !redisPrefix.matches("[A-Za-z0-9:_-]{2,80}:")) { + throw new IllegalArgumentException("Redis prefix must end in ':' and contain only letters, numbers, ':', '_' or '-'"); + } + this.redisPrefix = redisPrefix; } public void checkRate(UUID keyId, boolean readOnly) { - String bucket = "moyi:gateway:rate:" + (readOnly ? "read:" : "invoke:") + keyId; + String bucket = redisPrefix + "gateway:rate:" + (readOnly ? "read:" : "invoke:") + keyId; long retryMs = execute(RATE, bucket, Integer.toString(readOnly ? readsPerMinute : requestsPerMinute), "60000"); if (retryMs > 0) { throw new GatewayLimitException("RATE_LIMITED", "请求过于频繁,请稍后重试", 429, (int) ((retryMs + 999) / 1000)); @@ -62,7 +68,7 @@ public class GatewayGuard { } public Slot acquire(Map model) { - String bucket = "moyi:gateway:slots:" + model.get("id"); + String bucket = redisPrefix + "gateway:slots:" + model.get("id"); String token = UUID.randomUUID().toString(); long acquired = execute(ACQUIRE, bucket, Integer.toString(modelConcurrency(model)), Long.toString(SLOT_TTL_MS), token); if (acquired != 1) throw new GatewayLimitException("MODEL_BUSY", "当前模型并发已满,请稍后重试", 429, 1); @@ -71,7 +77,7 @@ public class GatewayGuard { private long execute(DefaultRedisScript script, String key, String... args) { try { - Long value = redis.execute(script, List.of(key), args); + Long value = redis.execute(script, List.of(key), (Object[]) args); if (value != null) return value; } catch (DataAccessException exception) { log.warn("Gateway guard Redis operation failed ({})", exception.getClass().getSimpleName()); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java index 24aee9f..851fd65 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/task/AsyncTaskService.java @@ -56,6 +56,9 @@ public class AsyncTaskService { if (model == null) throw new ApiException("MODEL_NOT_FOUND", "模型不存在或已下线", 404); if (!catalogService.keyCanInvoke(key, modelId)) throw new ApiException("MODEL_FORBIDDEN", "当前 Key 无权调用该模型", 403); if (!taskType.equals(model.get("type")) || !"async".equals(model.get("mode"))) throw new ApiException("NOT_ASYNC_MEDIA_MODEL", "该模型不是对应的异步媒体模型", 400); + if (!"provider-mock".equals(model.get("providerId"))) { + throw new ApiException("ASYNC_PROVIDER_NOT_IMPLEMENTED", "Java 版尚未接入该供应商的异步执行器", 501); + } Integer queued = jdbc.queryForObject("SELECT COUNT(*) FROM model_tasks WHERE model_id = ? AND status IN ('queued', 'running')", Integer.class, modelId); if (queued != null && queued >= maxQueuedPerModel) throw new ApiException("TASK_QUEUE_FULL", "该模型当前排队已满,请稍后重试", 429); String prompt = String.valueOf(input.getOrDefault("prompt", "")).trim(); diff --git a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java index 47e6e0c..5c092a6 100644 --- a/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java +++ b/control-plane-java/src/main/java/com/moyi/controlplane/usage/UsageAuditService.java @@ -223,7 +223,13 @@ public class UsageAuditService { try { JsonNode sanitized = sanitize(objectMapper.valueToTree(value)); String text = objectMapper.writeValueAsString(sanitized); - return text.length() > MAX_PAYLOAD_CHARS ? text.substring(0, MAX_PAYLOAD_CHARS) + "…" : text; + if (text.length() <= MAX_PAYLOAD_CHARS) return text; + // Preserve valid JSON for PostgreSQL jsonb even when payloads are large. + ObjectNode summary = objectMapper.createObjectNode(); + summary.put("_truncated", true); + summary.put("_originalChars", text.length()); + summary.put("preview", text.substring(0, 8000)); + return objectMapper.writeValueAsString(summary); } catch (JsonProcessingException exception) { return "{\"_unserializable\":true}"; } } diff --git a/control-plane-java/src/main/resources/application.yml b/control-plane-java/src/main/resources/application.yml index d90c928..b6eee8b 100644 --- a/control-plane-java/src/main/resources/application.yml +++ b/control-plane-java/src/main/resources/application.yml @@ -10,12 +10,18 @@ spring: minimum-idle: ${DB_POOL_MIN_IDLE:2} data: redis: - url: ${SPRING_DATA_REDIS_URL:redis://localhost:6379} + host: ${MOYI_REDIS_HOST:localhost} + port: ${MOYI_REDIS_PORT:6379} + username: ${MOYI_REDIS_USERNAME:} + password: ${MOYI_REDIS_PASSWORD:} + database: ${MOYI_REDIS_DATABASE:0} + ssl: + enabled: ${MOYI_REDIS_SSL:false} connect-timeout: 2s timeout: 2s flyway: enabled: ${FLYWAY_ENABLED:true} - baseline-on-migrate: true + baseline-on-migrate: ${FLYWAY_BASELINE_ON_MIGRATE:false} server: address: ${CONTROL_PLANE_HOST:0.0.0.0} @@ -34,6 +40,7 @@ management: moyi: gateway: + redis-prefix: ${MOYI_REDIS_KEY_PREFIX:moyi:} requests-per-minute: ${GATEWAY_REQUESTS_PER_MINUTE:60} reads-per-minute: ${GATEWAY_READS_PER_MINUTE:300} default-model-concurrency: ${GATEWAY_DEFAULT_MODEL_CONCURRENCY:4} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java new file mode 100644 index 0000000..6de2bba --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/GatewayDeploymentTest.java @@ -0,0 +1,76 @@ +package com.moyi.controlplane.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.moyi.controlplane.catalog.CatalogService; +import com.moyi.controlplane.common.ApiException; +import com.moyi.controlplane.provider.OpenAiCompatibleProvider; +import com.moyi.controlplane.task.AsyncTaskService; +import com.moyi.controlplane.usage.UsageAuditService; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.jdbc.core.JdbcTemplate; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class GatewayDeploymentTest { + @Test + void redisRateKeysAreEnvironmentScoped() { + StringRedisTemplate redis = mock(StringRedisTemplate.class); + doReturn(0L).when(redis).execute(any(RedisScript.class), anyList(), any(Object[].class)); + UUID keyId = UUID.randomUUID(); + new GatewayGuard(redis, 60, 300, 4, "moyi:test:").checkRate(keyId, false); + verify(redis).execute(any(RedisScript.class), eq(List.of("moyi:test:gateway:rate:invoke:" + keyId)), any(Object[].class)); + } + + @Test + void invalidRedisPrefixIsRejected() { + assertThrows(IllegalArgumentException.class, () -> new GatewayGuard(mock(StringRedisTemplate.class), 60, 300, 4, "")); + } + + @Test + void unconfiguredRealProviderNeverReturnsMockSuccess() { + CatalogService catalog = mock(CatalogService.class); + GatewayGuard guard = mock(GatewayGuard.class); + OpenAiCompatibleProvider provider = mock(OpenAiCompatibleProvider.class); + UsageAuditService usage = mock(UsageAuditService.class); + Map key = Map.of("id", UUID.randomUUID().toString(), "userId", UUID.randomUUID().toString()); + when(catalog.authenticateKey("test-key")).thenReturn(key); + when(catalog.activeModel("model-real")).thenReturn(Map.of("id", "model-real", "providerId", "provider-deepseek", "mode", "sync")); + when(catalog.keyCanInvoke(key, "model-real")).thenReturn(true); + GatewayController controller = new GatewayController(catalog, provider, guard, usage); + ApiException error = assertThrows(ApiException.class, () -> controller.chatCompletions("Bearer test-key", Map.of("model", "model-real"))); + assertEquals("PROVIDER_NOT_CONFIGURED", error.getCode()); + assertEquals(503, error.getStatus()); + verify(provider, never()).chat(anyString(), anyString(), anyList()); + } + + @Test + void explicitMockProviderStillWorks() { + CatalogService catalog = mock(CatalogService.class); + Map key = Map.of("id", UUID.randomUUID().toString(), "userId", UUID.randomUUID().toString()); + when(catalog.authenticateKey("test-key")).thenReturn(key); + when(catalog.activeModel("mock")).thenReturn(Map.of("id", "mock", "providerId", "provider-mock", "mode", "sync")); + when(catalog.keyCanInvoke(key, "mock")).thenReturn(true); + GatewayController controller = new GatewayController(catalog, mock(OpenAiCompatibleProvider.class), mock(GatewayGuard.class), mock(UsageAuditService.class)); + assertEquals(200, controller.chatCompletions("Bearer test-key", Map.of("model", "mock")).getStatusCode().value()); + } + + @Test + void unimplementedMediaProviderDoesNotCreateFakeTask() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + CatalogService catalog = mock(CatalogService.class); + Map key = Map.of("id", UUID.randomUUID().toString()); + when(catalog.authenticateKey("test-key")).thenReturn(key); + when(catalog.activeModel("tripo")).thenReturn(Map.of("id", "tripo", "providerId", "provider-vast", "type", "3d", "mode", "async")); + when(catalog.keyCanInvoke(key, "tripo")).thenReturn(true); + AsyncTaskService tasks = new AsyncTaskService(jdbc, new ObjectMapper(), catalog, 100, mock(GatewayGuard.class), mock(UsageAuditService.class)); + ApiException error = assertThrows(ApiException.class, () -> tasks.create("Bearer test-key", "3d", Map.of("model", "tripo", "prompt", "test"))); + assertEquals(501, error.getStatus()); + verifyNoInteractions(jdbc); + } +} diff --git a/control-plane-java/src/test/java/com/moyi/controlplane/gateway/PostgresBindingTest.java b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/PostgresBindingTest.java new file mode 100644 index 0000000..10168f6 --- /dev/null +++ b/control-plane-java/src/test/java/com/moyi/controlplane/gateway/PostgresBindingTest.java @@ -0,0 +1,49 @@ +package com.moyi.controlplane.gateway; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.moyi.controlplane.auth.AuthService; +import com.moyi.controlplane.catalog.CatalogService; +import com.moyi.controlplane.security.SecretCipher; +import com.moyi.controlplane.usage.UsageAuditService; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class PostgresBindingTest { + @Test + void keyIdUsesNativeUuidBinding() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + AuthService auth = mock(AuthService.class); + when(auth.requireUser("session")).thenReturn(new AuthService.UserView(UUID.randomUUID(), "admin", "username", "Admin", "admin", "active", null, null)); + doAnswer(call -> { + Object[] bindings = (Object[]) call.getRawArguments()[1]; + assertInstanceOf(UUID.class, bindings[0]); + return 1; + }).when(jdbc).update(startsWith("INSERT INTO platform_api_keys"), any(Object[].class)); + doReturn(Map.of("name", "test")).when(jdbc).queryForObject(anyString(), any(RowMapper.class), any(Object[].class)); + new CatalogService(jdbc, new ObjectMapper(), mock(SecretCipher.class)).createKey("session", Map.of("name", "test"), auth); + verify(jdbc).update(startsWith("INSERT INTO platform_api_keys"), any(Object[].class)); + } + + @Test + void oversizedAuditPayloadIsStillValidJsonb() { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + ObjectMapper mapper = new ObjectMapper(); + doAnswer(call -> { + Object[] bindings = (Object[]) call.getRawArguments()[1]; + String payload = (String) bindings[11]; + assertTrue(payload.length() <= 32000); + assertTrue(mapper.readTree(payload).get("_truncated").asBoolean()); + return 1; + }).when(jdbc).update(startsWith("INSERT INTO usage_events"), any(Object[].class)); + new UsageAuditService(jdbc, mapper).recordCall(UUID.randomUUID(), UUID.randomUUID(), "model", "provider", "succeeded", 5, 0, 0, + List.of("x".repeat(8000), "y".repeat(8000), "z".repeat(8000), "w".repeat(8000), "v".repeat(8000)), null, null, null); + verify(jdbc).update(startsWith("INSERT INTO usage_events"), any(Object[].class)); + } +} diff --git a/docs/09-java-control-plane-node.md b/docs/09-java-control-plane-node.md index 7676bb8..6fcb94d 100644 --- a/docs/09-java-control-plane-node.md +++ b/docs/09-java-control-plane-node.md @@ -74,12 +74,12 @@ Invoke-WebRequest http://localhost:8080/health 本节点完成的是“控制面基础闭环 + 同步供应商转发 + Redis 网关保护 + 持久化任务队列”,不是生产网关完成版: - 数据已写入 PostgreSQL;Redis 已用于按 API Key 限流、按模型并发租约和 Redis Lua 原子操作。默认同步调用每 Key 每分钟 60 次、模型并发 4,均可通过环境变量调整;Redis 不可用时网关返回 503,不会绕过保护直接请求上游; -- `/v1/chat/completions` 已支持 DeepSeek、Kimi、GLM 的 OpenAI 兼容转发;管理员在目录中新增带凭证的兼容供应商后,也可按模型绑定动态渠道;未配置 Key 时返回 Mock,流式响应暂未开放; -- `/v1/3d/tasks` 和 `/v1/video/tasks` 已提供数据库持久化 Mock worker,真实供应商执行器尚未替换; +- `/v1/chat/completions` 已支持 DeepSeek、Kimi、GLM 的 OpenAI 兼容非流式转发;真实供应商未配置 Key 时返回 503,只有 `provider-mock` 可以返回 Mock;流式响应暂未开放; +- `/v1/3d/tasks` 和 `/v1/video/tasks` 已提供数据库持久化 Mock worker;真实供应商执行器尚未替换,返回 501 而非伪造任务成功; - Seedance、Tripo/VAST 的异步任务和 OSS 结果存储尚未迁移到 Java; - 异步任务已持久化并提供按模型的排队上限(默认 100,可由 `TASK_MAX_QUEUED_PER_MODEL` 调整),但真实供应商执行器、优先级调度、价格/额度扣减和计费账单仍沿用下一节点计划; - 模型调用记录会自动保存同步请求/响应和异步任务入队结果,输入输出中的 `authorization`、`api_key`、`token`、`secret`、`password` 等字段会脱敏,单个文本字段最多 8000 字符、单个 JSON 载荷最多 32000 字符;管理员可筛选、查看、备注、软删除和导出 CSV; -- Docker 镜像需要在具备 Maven 和 Docker daemon 的环境中构建验证。 +- 服务器测试部署参照 `11-server-test-deployment.md`;预编译包附带 JAR 和静态文件,服务器只需运行时基础镜像,不需要 Maven/Node。 ## 下一节点 diff --git a/docs/10-beijing-test-environment-checklist.md b/docs/10-beijing-test-environment-checklist.md index 296d81c..8b60328 100644 --- a/docs/10-beijing-test-environment-checklist.md +++ b/docs/10-beijing-test-environment-checklist.md @@ -2,6 +2,8 @@ 本文用于第一轮真实模型联调,不代表最终生产架构。当前约定为:应用服务器放在阿里云北京地域,RDS、Tair 和 OSS 优先复用北京资源。 +已确认:Ubuntu 24.04 / 4C8G,RDS PostgreSQL 15,Tair 标准架构,域名 `moyi.soulofmodel.com`。当前执行手册以 [服务器测试部署手册](11-server-test-deployment.md) 为准。 + ## 推荐规格 本次测试环境采用 **4 核 8 GiB 轻量应用服务器**,运行 Java Control Plane、Vue/Nginx、Docker Compose、日志和基础监控。RDS、Tair、OSS 在云上托管后,应用服务器不承担数据库和缓存的内存压力;这个规格可以为公开测试、构建镜像和异步任务突发保留更充足的余量。 @@ -15,18 +17,18 @@ ├─ frontend (Nginx,公网 80/443) └─ moyi-api (Java,仅内网 8787) ├─ 北京 RDS PostgreSQL:业务、Key、模型目录、用量审计 - ├─ 北京 Tair/Redis:会话、限流、并发、队列状态、幂等 - └─ 北京 OSS:图片、视频、3D 结果文件和上传文件 + ├─ 北京 Tair/Redis:按 Key 限流、按模型并发租约 + └─ 北京 OSS:预留,Java 结果文件落盘尚未接入 ``` -应用服务器、RDS、Tair 尽量放在同一地域和同一 VPC。RDS 和 Tair 的白名单只允许应用服务器的内网地址;安全组只开放 80/443,8787 不对公网开放。生产阶段再补域名、HTTPS、备案、备份和高可用。 +轻量服务器需要配置到 RDS/Tair 所属 VPC 的内网互通,并添加私网 IP 白名单。SSH 隧道测试仅需受限 22 端口;域名测试先配置 HTTPS 和受限 80/443。8787 不对公网开放。迁移前先备份,高可用留待生产设计。 ## 现在必须确认 - 轻量应用服务器的操作系统、CPU 架构、磁盘容量、Docker/Compose 版本和私网 IP。 - RDS **数据库引擎和版本**、北京地域/可用区、内网地址和端口、数据库名、应用账号,以及白名单配置。当前 Java 工程使用 PostgreSQL 驱动和 PostgreSQL Flyway 迁移;如果现有 RDS 是 MySQL,需要先做数据库适配,不能直接填连接串。 - Tair 实例的内网地址和端口、Redis 兼容模式、是否启用 TLS、账号/密码或 ACL、白名单。Tair 不需要创建“库表”;Redis 的逻辑 DB/Key 空间由应用使用,当前连接默认使用 DB 0。 -- OSS 的北京 Region、Bucket、内网 Endpoint、读写权限和生命周期。AccessKey 建议使用 RAM 子账号或云凭据注入,不要提交到 Git。 +- OSS 当前可后置;后续接入时再提供北京 Region、Bucket、内网 Endpoint、读写权限和生命周期,不要提前交付 AccessKey。 ## 真实联调前再提供 @@ -41,6 +43,6 @@ ## 验收顺序 1. 配置 RDS/Tair/OSS 白名单并从服务器验证内网连通。 -2. 复制 `infra/moyi/.env.cloud.example` 为服务器上的 `.env.cloud`,填写连接配置和随机密钥。 -3. 启动 `docker compose -f docker-compose.cloud.yml up -d --build`,确认 `/health`、登录、Key 创建和模型目录。 -4. 先用一个同步文本/多模态模型验收输入、输出、额度和调用记录,再验收 Tripo/VAST 异步任务、队列、OSS 文件和失败重试。 +2. 在 `infra/moyi` 执行 `python3 deploy.py init`,填写连接配置;脚本生成随机安全密钥。 +3. 使用 `deploy.py check/up/smoke` 完成配置检查、启动与验收,参见最新手册。 +4. 本次先验收同步调用、Key 和审计;真实异步任务、OSS、可靠重试及完整计费是后续节点。 diff --git a/docs/11-server-test-deployment.md b/docs/11-server-test-deployment.md new file mode 100644 index 0000000..d1eb836 --- /dev/null +++ b/docs/11-server-test-deployment.md @@ -0,0 +1,191 @@ +# 北京测试服务器部署手册 + +## 1. 已确认的环境与本次边界 + +| 项目 | 本次配置 | +| --- | --- | +| 应用服务器 | 阿里云轻量,北京,4 核 8 GiB,Ubuntu 24.04,70 GiB 系统盘 | +| 公网地址 | `39.96.90.106` | +| 私网地址 | 截图为 `172.17.24.125`;配置内网互通后再次核对 | +| 域名 | `moyi.soulofmodel.com`,不修改 `www` 或主域名原系统 | +| 数据库 | 现有北京 RDS PostgreSQL 15,新增独立数据库 `moyi_control` | +| 缓存 | 现有北京 Tair 标准架构,专用账号及 `moyi:test:` Key 前缀 | +| 对象存储 | 北京 OSS 可预留新 Bucket,本次 Java 部署暂不需要 OSS 密钥 | + +这是内部测试发布,不是完整生产版。可验收登录、邀请码、目录、项目、Key 权限、非流式同步转发、限流、模型并发和调用审计。以下尚未完成:Java 真实 Tripo/Seedance 异步执行、OSS 文件保存、流式转发、完整计费扣减、分布式任务抢占和可靠重试。 + +真实供应商没有配置密钥时返回 503,不再返回 Mock 成功。只有明确绑定 `provider-mock` 的模型可以走 Mock;真实异步供应商返回 501。已有种子目录和价格只是联调数据,实际模型 ID/能力须由供应商控制台核验后修改。 + +注意更正早期说明:当前登录会话和任务状态保存于 PostgreSQL;Tair 已实现的是按 Key 限流、按模型并发租约,不是已完成的 MQ、幂等或异步调度平台。不需要现在购买 MQ。 + +## 2. 你需要在阿里云完成的配置 + +### 网络互通,先确认对旧系统的影响 + +1. 确认新服务器、RDS、Tair 在同一阿里云账号和北京地域。 +2. 轻量控制台的“内网互通”中关联 RDS/Tair 所属 VPC;如属于不同 VPC,分别设置。 +3. **首次在该地域开通内网互通可能导致该地域所有轻量服务器停机约 1 分钟**,包括现有小程序服务器。先检查是否已开通,在维护时间执行,不要为本项目随意取消旧连接。 +4. 开通后重新核对新服务器私网 IP,把它以 `/32` 加入 RDS、Tair 的新增白名单分组 `moyi-test`。保留旧系统的白名单,不要覆盖或放行 `0.0.0.0/0`。 +5. 不要为了连接方便给 RDS/Tair 开公网地址;同地域不等于已内网互通。 + +依据:[阿里云轻量内网互通](https://help.aliyun.com/zh/simple-application-server/user-guide/manage-service-interconnection/)。 + +### 防火墙和网段 + +- SSH 22 仅允许办公公网 IP;先保持当前会话并验证新登录通路,再收紧来源,避免锁出。 +- SSH 隧道模式不需要开放 80/443/8080。HTTPS 模式开放 80、443;内部测试阶段建议也仅允许测试人员公网 IP。 +- Java 8787、数据库端口、6379 不发布到公网;8080 始终只绑定 `127.0.0.1`。 +- 不只依赖 UFW:Docker 发布端口的流量可能绕过 UFW,阿里云防火墙来源限制必须配置。[Docker 官方说明](https://docs.docker.com/engine/install/ubuntu/) +- 当前服务器在 `172.17.*` 私网,而 Docker 默认桥接网络也可能使用 `172.17.0.0/16`。安装前后查看 `ip route`;若重叠,先调整 Docker 地址池/默认桥接网段,再启动。不要直接覆盖已有 `/etc/docker/daemon.json`。 +- 本 Compose 显式使用 `10.253.240.0/24`,但这只是待确认候选值。须确保不与服务器、RDS、Tair、现有 Docker/VPC 路由重叠;必要时修改 `MOYI_DOCKER_SUBNET`。脚本会检查该子网与已解析的 RDS/Tair 地址是否重叠,不代替完整路由审查。 + +### RDS PostgreSQL 15 + +1. 在现有 RDS 实例创建专用数据库 `moyi_control` 和专用账号 `moyi_app`,**不在小程序库里直接添加本项目的表**。 +2. 确保 `moyi_app` 在这个库中有连接、建表、建索引和读写权限。PG15 中需确认 `public` schema 的 CREATE 权限或让该账号拥有新库;不能只授予 SELECT/INSERT。 +3. 新数据库第一次启动由 Flyway 自动建表。配置关闭自动 baseline,非空未受 Flyway 管理的库会拒绝迁移,不要为绕过错误而启用 baseline。 +4. 数据库修改前创建 RDS 备份,并核对可恢复性;不调整原系统账号、权限或数据库。 +5. 提供内网地址、实际端口、库名和账号。密码只填写服务器 `.env.cloud`。若启用数据库 TLS,按 RDS 配置 JDBC 参数及 CA 校验,不默认假设端口/TLS。 + +### Tair 标准架构 + +1. 新建本项目专用账号,保留旧系统账号与密码。不要重置共享实例的默认密码。 +2. 使用内网地址、标准架构连接,按实例实际情况填写 TLS、端口、账号和密码。 +3. 无需创建表。使用 `moyi:test:` 前缀;支持 ACL Key 规则时,限制为此命名空间。可使用受支持的独立逻辑 DB,但它不等于独立实例或资源隔离。 +4. 网关需要 PING、Lua 脚本执行以及 INCR、PTTL、PEXPIRE、TIME、ZADD、ZCARD、ZREM、ZREMRANGEBYSCORE 等操作。只读账号不够。预检查仅验证 TCP,应用就绪检查再验证连接与鉴权,Mock 模型调用才验证真实 Lua 权限。 +5. **不要执行 FLUSHDB、FLUSHALL,不要修改共享实例的全局配置。** 共享实例的内存、CPU、驱逐策略仍共享,测试流量要有限额。 + +### 域名和 DV 证书 + +1. 在 `soulofmodel.com` 的当前 DNS 托管处添加 A 记录:主机记录 `moyi`,值 `39.96.90.106`,TTL 默认 10 分钟。不修改 `www`、`@`、MX 或 DNS 服务器。 +2. 申请覆盖 **`moyi.soulofmodel.com`** 的单域名 DV 证书即可。DV 满足当前企业测试的 HTTPS 需求,OV/EV 主要增加组织身份核验,不是当前必需项。[阿里云选型说明](https://help.aliyun.com/zh/ssl-certificate/functions-and-features) +3. 使用证书控制台给出的 DNS 验证记录,不删除原业务验证记录;下载 **Nginx 格式**证书。 +4. 将包含完整链的 PEM 和对应私钥放到服务器 `infra/moyi/certs/fullchain.pem`、`privkey.pem`,私钥权限 600。不要发到聊天或 Git。 +5. 现有域名已备案并接入阿里云时,子域名通常无需单独备案;仍需核对主体、服务内容和页面备案信息。备案不是模型供应商授权或 AI 服务合规的替代。[子域名备案说明](https://help.aliyun.com/zh/icp-filing/basic-icp-service/support/for-the-record-domain-faq/) + +### OSS 本次可以后置 + +可先创建北京私有 Bucket,例如自定义的 `moyi-test-assets-*`;名称须全局唯一。Java 的结果文件落盘尚未完成,本次不需要创建或提供 AccessKey,更不要为赶部署设置公共读写。后续接入时再配置仅目标 Bucket 的 RAM 权限、签名 URL、生命周期和测试/生产隔离。 + +## 3. 部署包与服务器初始化 + +发布压缩包位于开发机 `artifacts/moyi-test-<版本>.tar.gz`,旁边有 `.sha256`。包内包含已编译的 Java 21 JAR、Vue 静态文件、运行时 Dockerfile、Compose、环境变量模板、Nginx、部署脚本和逐文件校验清单。 + +它不是离线 Docker 镜像包。服务器不用安装 Java/Maven/Node,但首次需访问镜像仓库下载 `eclipse-temurin:21-jre-alpine` 和 `nginx:1.27-alpine`。若网络不可用,应由开发机/CI 构建并推送阿里云 ACR 或交付 `docker save` 镜像包;不要使用来源不明的镜像代理。运行时镜像地址可以通过环境模板改为已同步的可信镜像地址。 + +将压缩包及 SHA256 文件通过 SCP/SFTP 上传到服务器同一目录。下面操作在服务器执行;`<版本>` 替换成实际文件名中的版本: + +```bash +sha256sum -c moyi-test-<版本>.tar.gz.sha256 +sudo mkdir -p /opt/moyi/releases/<版本> +sudo tar -xzf moyi-test-<版本>.tar.gz -C /opt/moyi/releases/<版本> +cd /opt/moyi/releases/<版本>/infra/moyi +sudo bash install-docker-ubuntu.sh +sudo python3 deploy.py verify +sudo python3 deploy.py init +sudo nano .env.cloud +``` + +安装脚本仅针对 Ubuntu 24.04,使用 Docker 官方 APT 源。发现已有 Docker 时不覆盖安装;发现冲突包时停止,不自动卸载。仓库不可达时报错停止,再单独配置可信来源,不会关闭 TLS 校验。不要为执行安装脚本更改现有服务器上的 Nginx、UFW 或其他应用。 + +`init` 自动生成随机管理员初始密码、密码 Pepper 和供应商凭证加密密钥,配置文件权限 600;已存在配置时拒绝覆盖。账号仍是已约定的 `adimin`,**云端不使用 `admin123`**。初始密码在服务器 `.env.cloud` 中查看,不会打印到日志。 + +## 4. 环境变量怎么填 + +| 分类 | 变量 | 本次要求 | +| --- | --- | --- | +| RDS | `MOYI_DB_URL`、`MOYI_DB_USERNAME`、`MOYI_DB_PASSWORD` | 必填;新建 PostgreSQL 数据库 | +| Tair | `MOYI_REDIS_HOST/PORT/USERNAME/PASSWORD/DATABASE/SSL` | 地址、密码必填,其他按实例信息 | +| 隔离 | `MOYI_REDIS_KEY_PREFIX`、`MOYI_DOCKER_SUBNET` | 专用前缀,网段不冲突 | +| 安全 | `AUTH_PASSWORD_PEPPER`、`CREDENTIAL_ENCRYPTION_KEY`、`ADMIN_PASSWORD` | init 已生成;升级必须保留 | +| HTTPS | `MOYI_DOMAIN` | 已预填 `moyi.soulofmodel.com`;证书另放 certs | +| 容量 | `DB_POOL_*`、`GATEWAY_*`、`TASK_*`、`JAVA_OPTS` | 先用测试默认值,不代表已压测容量 | +| 供应商 | `DEEPSEEK_*`、`KIMI_*`、`ZHIPU_*` | 首次可空,在管理端配置更直观;环境密钥优先级更高 | +| 后置 | `OSS_*`、`TRIPO_API_KEY`、`VOLCENGINE_API_KEY` | 预留,不注入本次 Java 服务,留空 | + +密码包含 `$`、`#` 或空格时用单引号包住,例如 `MOYI_REDIS_PASSWORD='实际密码'`。若密码本身包含单引号,按 Compose dotenv 转义规则处理。**不要 `source .env.cloud`,不要把完整 `docker compose config` 输出发到聊天**,其中可能含真实密钥。使用 `deploy.py check` 只输出检查结论。 + +旧的 `MOYI_REDIS_URL` 已改为分离字段,避免用户名/密码的 URL 编码问题。旧云配置升级时逐项补齐,不直接复制覆盖。Cookie 安全属性由模式决定,不再通过随意设置公网 HTTP 来降低安全性。 + +## 5. 启动方式二选一 + +### A. 暂无证书:SSH 隧道内部测试 + +```bash +sudo python3 deploy.py check --mode tunnel --network +sudo python3 deploy.py up --mode tunnel --accept-migrations +sudo python3 deploy.py smoke --mode tunnel --login +``` + +在你自己的电脑打开一个终端并保持连接(替换 `SSH用户名`,不要默认假设是 root): + +```powershell +ssh -N -L 8080:127.0.0.1:8080 SSH用户名@39.96.90.106 +``` + +浏览器访问 `http://localhost:8080`。无需开放公网 8080,登录流量通过 SSH 加密。若电脑 8080 已占用,应同时调整配置中的本地端口和 CORS,或选择另一个转发端口后同步 CORS。 + +### B. 域名和证书就绪:HTTPS 测试 + +```bash +sudo chmod 600 certs/privkey.pem +sudo python3 deploy.py check --mode https --network +sudo python3 deploy.py up --mode https --accept-migrations +sudo python3 deploy.py smoke --mode https --login +``` + +通过阿里云防火墙允许的测试人员访问 `https://moyi.soulofmodel.com`,模型 API 使用 `https://moyi.soulofmodel.com/v1`。HTTPS 模式自动开启 Secure Cookie;证书错误时拒绝启动。不要关闭客户端证书校验。 + +`up` 先编译运行时镜像,再启动后端、检查 RDS/Tair 就绪、校验 Nginx,最后启动前端。数据库迁移必须明确带 `--accept-migrations`;配置错指其他库仍可能造成改动,执行前须核对并备份。后端不会对公网发布端口。 + +## 6. 查看状态、测试与排障 + +```bash +sudo python3 deploy.py status +sudo python3 deploy.py logs +sudo python3 deploy.py smoke +sudo python3 deploy.py stop +``` + +成功启动后会保存模式,后续命令默认沿用;第一次启动失败时请显式带 `--mode`。`stop` 只停止容器,不删数据库、缓存或镜像。 + +验收顺序: + +1. 登录与退出、刷新恢复会话;管理员创建邀请码,普通用户注册。 +2. 创建个人/项目 Key,编辑模型权限,禁用 Key 后访问被拒绝。 +3. 用明确标记 Mock 的文本模型检查 Tair Lua 限流和并发,再配置一个真实同步供应商、小额验证请求与响应。 +4. 查看调用记录输入输出、错误、按 Key 筛选及导出;测试数据也可能含个人信息,仅管理员可访问,注意日志留存与脱敏。 +5. Java 流式调用仍返回 501;真实 3D/视频接口仍返回 501,不属于本次可用能力。 + +`smoke` 默认不调用任何付费模型;`--login` 会使用初始管理员凭据验证 Cookie 并产生登录审计。如已修改管理员密码,不再使用初始凭据进行该测试。 + +常见失败: + +- RDS/Tair TCP 不通:检查 VPC 内网互通、私网 IP、白名单和网段冲突;不要改成全网白名单。 +- 后端 unhealthy:通过 logs 检查数据库账号权限、Flyway、Tair TLS/密码。`/health` 是基本状态,容器就绪依据 `/actuator/health/readiness`,外部不能访问 actuator。 +- 镜像下载失败:检查服务器到官方镜像仓库的连接,下一步交付 ACR/离线镜像;不要重复更换应用端口。 +- 页面能开但登录失败:检查所用模式和浏览器域名,HTTPS 模式不能用 IP HTTP 登录。 +- 真实模型 503:先在管理端配置正确供应商 Key、Base URL、上游模型 ID,不是服务器配置不足。 + +## 7. 证书更新、升级与回退 + +本包提供有效期检查及安全重载,**不自动购买、签发或续期证书**。开启阿里云证书到期通知,续签后替换服务器 `certs/fullchain.pem`、`privkey.pem`,执行: + +```bash +sudo python3 deploy.py check --mode https +sudo python3 deploy.py reload-tls --mode https +``` + +检查会提示 30 天内到期,证书不足 1 天有效期时拒绝;`nginx -t` 成功后才 reload。保留旧证书备份以便更新失败时恢复。 + +升级时每个版本解压到独立 `releases/<版本>` 目录,**保留相同 Compose 项目名**,安全复制旧 `.env.cloud` 和证书到新版本,并补齐新增配置项。不要重新生成 Pepper/加密密钥,否则会影响密码校验和供应商凭据解密。预编译发布包需将旧环境中的 `MOYI_IMAGE_TAG` 改为新 `release-manifest.json` 的 `release` 值,并使用 `Dockerfile.runtime`。 + +执行新版本 `check`、备份 RDS 后再 `up`。构建失败时旧容器仍运行;数据库迁移及后端更新后发生的失败不能承诺自动回退。只有确认旧程序与新数据库结构兼容,才能回到旧版本目录用相同项目名启动;不兼容时需从 RDS 备份恢复到另一个测试库并重新验证。禁止用 `docker system prune`、`down -v`、删除旧库等方式“修复”发布失败。 + +## 8. 还需提供的信息 + +- RDS 内网地址、端口、新库和账号名、所属 VPC;Tair 内网地址、端口、账号、TLS 状态及 VPC。 +- DNS A 记录是否完成,DV 证书是否已签发并上传到服务器。 +- SSH 登录用户名、端口以及你已配置的密钥连接方式;不要发送密码或私钥。 + +密码/模型 Key 只由你填写 `.env.cloud` 或管理端。未获得这些信息和服务器访问授权前,本包只能在本机完成构建与配置测试,不能宣称已部署到云服务器。 diff --git a/infra/moyi/.env.cloud.example b/infra/moyi/.env.cloud.example index c382ee0..bb3244a 100644 --- a/infra/moyi/.env.cloud.example +++ b/infra/moyi/.env.cloud.example @@ -1,38 +1,66 @@ -# Copy to .env.cloud on the target server. Never commit the real file. +# Run: python3 deploy.py init. Never commit .env.cloud. +# Single-quote passwords with '$', '#', spaces, etc. Do NOT source this file. +COMPOSE_PROJECT_NAME=moyi-test MOYI_HTTP_PORT=8080 -CONTROL_PLANE_CORS_ORIGINS=https://moyi.example.com +MOYI_DOMAIN=moyi.soulofmodel.com +CONTROL_PLANE_CORS_ORIGINS=http://localhost:8080 +# Select an unused subnet outside all server/RDS/Tair VPC ranges. +MOYI_DOCKER_SUBNET=10.253.240.0/24 -# Alibaba Cloud RDS PostgreSQL private endpoint. -MOYI_DB_URL=jdbc:postgresql://rds-private-endpoint:1921/moyi_control +# Source defaults. Release initializer selects runtime Dockerfiles and tag. +MOYI_IMAGE_TAG=dev +MOYI_API_DOCKERFILE=Dockerfile +MOYI_FRONTEND_DOCKERFILE=Dockerfile +JAVA_RUNTIME_IMAGE=eclipse-temurin:21-jre-alpine +NGINX_RUNTIME_IMAGE=nginx:1.27-alpine + +# REQUIRED: dedicated EMPTY RDS PostgreSQL database, never the mini-app DB. +# Confirm port in RDS console (often 5432 or 1921). MySQL is NOT supported. +MOYI_DB_URL=jdbc:postgresql://CHANGE_ME_RDS_HOST:5432/moyi_control MOYI_DB_USERNAME=moyi_app -MOYI_DB_PASSWORD=replace-with-a-long-random-password +MOYI_DB_PASSWORD='CHANGE_ME_RDS_PASSWORD' +DB_POOL_MAX_SIZE=10 +DB_POOL_MIN_IDLE=2 -# Alibaba Cloud Redis/Tair private endpoint. Use TLS parameters if enabled by the instance. -MOYI_REDIS_URL=rediss://:replace-with-redis-password@redis-private-endpoint:6379/0 +# REQUIRED: Tair standard endpoint or cluster PROXY, NOT cluster direct. +# Separate fields avoid URI-encoding passwords. Match TLS to the instance. +MOYI_REDIS_HOST=CHANGE_ME_TAIR_HOST +MOYI_REDIS_PORT=6379 +MOYI_REDIS_USERNAME= +MOYI_REDIS_PASSWORD='CHANGE_ME_TAIR_PASSWORD' +MOYI_REDIS_DATABASE=0 +MOYI_REDIS_SSL=false +MOYI_REDIS_KEY_PREFIX=moyi:test: +# Generated by init. Keep stable and back up securely across upgrades. +AUTH_PASSWORD_PEPPER=GENERATE_AUTH_PEPPER +CREDENTIAL_ENCRYPTION_KEY=GENERATE_CREDENTIAL_KEY +ADMIN_ACCOUNT=adimin +ADMIN_PASSWORD=GENERATE_ADMIN_PASSWORD +ADMIN_DISPLAY_NAME=MOYI Admin +SESSION_TTL_HOURS=168 +JAVA_OPTS='-Xms256m -Xmx1536m -XX:+ExitOnOutOfMemoryError' GATEWAY_REQUESTS_PER_MINUTE=60 GATEWAY_READS_PER_MINUTE=300 GATEWAY_DEFAULT_MODEL_CONCURRENCY=4 TASK_MAX_QUEUED_PER_MODEL=100 MOYI_TASK_WORKER_INTERVAL_MS=1000 -SESSION_TTL_HOURS=168 -AUTH_PASSWORD_PEPPER=replace-with-a-long-random-secret-at-least-32-chars -CREDENTIAL_ENCRYPTION_KEY=replace-with-a-long-random-secret-for-provider-credentials -ADMIN_ACCOUNT=adimin -ADMIN_PASSWORD=replace-with-a-strong-initial-password -ADMIN_DISPLAY_NAME=平台管理员 -AUTH_SECURE_COOKIE=true -# 仅使用公网 IP + HTTP 临时验收时改为 false;正式域名 HTTPS 必须保持 true。 +# OPTIONAL: use admin UI OR environment for one synchronous provider. +# Environment credentials take precedence over admin UI credentials. +DEEPSEEK_API_KEY= +DEEPSEEK_BASE_URL=https://api.deepseek.com +KIMI_API_KEY= +KIMI_BASE_URL=https://api.moonshot.ai/v1 +ZHIPU_API_KEY= +ZHIPU_BASE_URL=https://open.bigmodel.cn/api/paas/v4 -OSS_ENDPOINT= -OSS_REGION= +# RESERVED ONLY: Java Tripo/Seedance + OSS are NOT yet implemented. +# Leave these secrets empty; Compose intentionally does not inject them yet. +TRIPO_API_KEY= +VOLCENGINE_API_KEY= +OSS_REGION=cn-beijing +OSS_ENDPOINT=oss-cn-beijing-internal.aliyuncs.com OSS_BUCKET= OSS_ACCESS_KEY_ID= OSS_ACCESS_KEY_SECRET= - -TRIPO_API_KEY= -DEEPSEEK_API_KEY= -KIMI_API_KEY= -ZHIPU_API_KEY= -VOLCENGINE_API_KEY= diff --git a/infra/moyi/README.md b/infra/moyi/README.md index 8ce8839..052965c 100644 --- a/infra/moyi/README.md +++ b/infra/moyi/README.md @@ -1,10 +1,17 @@ # MOYI 容器化部署基线 +## 北京服务器测试部署(当前入口) + +当前已确认 Ubuntu 24.04 / 4C8G、RDS PostgreSQL 15、Tair 标准架构、`moyi.soulofmodel.com`。 +完整阿里云配置、发布包、环境变量、证书、启动和回退说明见 [服务器测试部署手册](../../docs/11-server-test-deployment.md)。 +推荐 `python3 deploy.py init` 生成配置,使用 `deploy.py check/up/smoke`,无证书时默认 SSH 隧道,有证书时添加 `--mode https`。 +发布包由 `scripts/package-deployment.ps1` 构建,只包含编译产物及非敏感配置,不包含真实 `.env.cloud` 或证书。 + 这套 Compose 用于 MOYI 的 Java 后端迁移基线:Vue 前端以 Nginx 生产构建运行,Java Control Plane 以 Spring Boot 运行,PostgreSQL 和 Redis 使用独立持久化卷。OSS 不在 Compose 中自建,生产环境连接云厂商 OSS;本地如需模拟对象存储,可另行增加 MinIO profile。 ## 当前迁移状态 -`control-plane-java/` 当前提供 Java 21 / Spring Boot 的认证、模型目录、项目、Key、权限和最小 OpenAI 兼容接口。真实供应商适配、Redis 队列/限流、异步任务和计费完成并通过 UAT 前,不要直接把生产流量切到 Java 服务。 +`control-plane-java/` 提供认证、模型目录、项目、Key、非流式同步供应商转发、Tair 网关保护及审计。真实异步执行、流式响应、OSS 和计费仍未完成,不要直接切入生产流量。 ## 启动 @@ -26,7 +33,7 @@ Invoke-WebRequest http://localhost:8080/health ## 云上托管依赖 -RDS、Redis 和 OSS 不需要部署在轻量应用服务器上。云上使用托管服务时,复制 `.env.cloud.example` 为 `.env.cloud`,填写 RDS/Redis 私网连接信息,然后执行: +RDS、Tair 和 OSS 不部署在轻量应用服务器上。填写 `.env.cloud` 后优先使用 `deploy.py` 预检查。以下原始命令仅用于理解底层 Compose 行为;默认只在服务器回环地址监听: ```powershell docker compose --env-file .env.cloud -f docker-compose.cloud.yml build diff --git a/infra/moyi/deploy.py b/infra/moyi/deploy.py new file mode 100644 index 0000000..717d7b4 --- /dev/null +++ b/infra/moyi/deploy.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""MOYI test deployment helper. Python 3 standard library only.""" +import argparse +import hashlib +import http.cookiejar +import ipaddress +import json +import os +from pathlib import Path +import re +import secrets +import socket +import stat +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent + + +def fail(message): + raise ValueError(message) + + +def execute(command, capture=False, data=None): + result = subprocess.run(command, cwd=HERE, input=data, capture_output=capture, + text=True, encoding="utf-8", errors="replace") + if result.returncode: + # Compose error output can contain expanded credentials. Do not echo it. + fail("Command failed: " + " ".join(command[:3]) + ". Check local configuration/permissions; do not share secrets.") + return result.stdout if capture else "" + + +def initialize(): + destination = HERE / ".env.cloud" + if destination.exists(): + fail(".env.cloud already exists; refusing to overwrite credentials.") + content = (HERE / ".env.cloud.example").read_text(encoding="utf-8-sig") + for marker in ("GENERATE_AUTH_PEPPER", "GENERATE_CREDENTIAL_KEY", "GENERATE_ADMIN_PASSWORD"): + content = content.replace(marker, secrets.token_hex(32)) + manifest = ROOT / "release-manifest.json" + if manifest.exists(): + release = json.loads(manifest.read_text(encoding="utf-8")) + content = content.replace("MOYI_IMAGE_TAG=dev", "MOYI_IMAGE_TAG=" + release["release"]) + content = content.replace("MOYI_API_DOCKERFILE=Dockerfile\n", "MOYI_API_DOCKERFILE=Dockerfile.runtime\n") + content = content.replace("MOYI_FRONTEND_DOCKERFILE=Dockerfile\n", "MOYI_FRONTEND_DOCKERFILE=Dockerfile.runtime\n") + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as output: + output.write(content) + (HERE / "certs").mkdir(mode=0o700, exist_ok=True) + print("Created .env.cloud (0600). Secrets generated but NOT printed. Edit database/Tair settings locally.") + + +def compose_command(env_file, mode): + command = ["docker", "compose", "--env-file", str(env_file), "-f", "docker-compose.cloud.yml"] + if mode == "https": + command += ["-f", "docker-compose.https.yml"] + return command + + +def load_config(env_file, mode): + if not env_file.is_file(): + fail("Environment file missing. Run: python3 deploy.py init") + if os.name != "nt" and stat.S_IMODE(env_file.stat().st_mode) & 0o077: + fail("Environment file must be private. Run chmod 600 on it.") + config = execute(compose_command(env_file, mode) + ["config", "--format", "json"], capture=True) + resolved = json.loads(config) + # Compose serializes literal '$' as '$$' for re-parseable config output. + for service in resolved.get("services", {}).values(): + for key, value in service.get("environment", {}).items(): + if isinstance(value, str): + service["environment"][key] = value.replace("$$", "$") + return resolved + + +def validate_config(config, mode): + services = config["services"] + api = services["moyi-api"]["environment"] + required = ("SPRING_DATASOURCE_URL", "SPRING_DATASOURCE_USERNAME", "SPRING_DATASOURCE_PASSWORD", + "MOYI_REDIS_HOST", "MOYI_REDIS_PASSWORD", "AUTH_PASSWORD_PEPPER", + "CREDENTIAL_ENCRYPTION_KEY", "ADMIN_PASSWORD") + for key in required: + value = str(api.get(key, "")) + if not value or any(marker in value.upper() for marker in ("CHANGE_ME", "GENERATE_", "REPLACE-WITH")): + fail("Missing/placeholder setting: " + key) + for key, minimum in (("AUTH_PASSWORD_PEPPER", 32), ("CREDENTIAL_ENCRYPTION_KEY", 32), ("ADMIN_PASSWORD", 16)): + if len(str(api[key])) < minimum: + fail(key + " is too short (minimum " + str(minimum) + " characters).") + if api["AUTH_PASSWORD_PEPPER"] == api["CREDENTIAL_ENCRYPTION_KEY"]: + fail("Auth pepper and credential encryption key must differ.") + url = str(api["SPRING_DATASOURCE_URL"]) + if not url.startswith("jdbc:postgresql://"): + fail("This release requires RDS PostgreSQL (confirmed target: 15), not MySQL.") + database = urllib.parse.urlsplit(url[5:]) + if not database.hostname or database.username or database.password or database.path in ("", "/", "/postgres", "/template0", "/template1"): + fail("Use a dedicated PostgreSQL database and separate username/password fields.") + if not re.fullmatch(r"[A-Za-z0-9:_-]{2,80}:", str(api.get("MOYI_REDIS_KEY_PREFIX", ""))): + fail("Set a dedicated Redis key prefix ending in ':'.") + if str(api.get("MOYI_REDIS_SSL", "")).lower() not in ("true", "false"): + fail("MOYI_REDIS_SSL must be true or false.") + if services["moyi-api"].get("ports"): + fail("Backend ports must not be published.") + ports = services["frontend"].get("ports", []) + if mode == "tunnel" and any(p.get("host_ip") != "127.0.0.1" for p in ports): + fail("Tunnel mode must bind only to 127.0.0.1.") + if mode == "https": + domain = services["frontend"]["environment"].get("MOYI_DOMAIN", "") + if not re.fullmatch(r"(?=.{1,253}$)[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?", domain) or "." not in domain: + fail("MOYI_DOMAIN must be a hostname, without protocol/port/path.") + if str(api.get("AUTH_SECURE_COOKIE")).lower() != "true": + fail("HTTPS requires secure cookies.") + elif str(api.get("AUTH_SECURE_COOKIE")).lower() != "false": + fail("Local SSH-tunnel HTTP requires secure cookies disabled.") + subnet = ipaddress.ip_network(config["networks"]["default"]["ipam"]["config"][0]["subnet"]) + return api, database, subnet + + +def verify_release(): + manifest_path = ROOT / "release-manifest.json" + if not manifest_path.exists(): + print("Source checkout: no precompiled release manifest.") + return + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for name, expected in manifest["sha256"].items(): + path = (ROOT / name).resolve() + if not path.is_relative_to(ROOT.resolve()) or not path.is_file(): + fail("Release file missing or outside package: " + name) + if hashlib.sha256(path.read_bytes()).hexdigest() != expected: + fail("Release checksum mismatch: " + name) + print("Release checksums OK: " + manifest["release"]) + + +def check_certificate(domain): + certificate = HERE / "certs/fullchain.pem" + key = HERE / "certs/privkey.pem" + if not certificate.is_file() or not key.is_file(): + fail("HTTPS requires certs/fullchain.pem and certs/privkey.pem.") + if os.name != "nt" and stat.S_IMODE(key.stat().st_mode) & 0o077: + fail("TLS private key must have mode 0600.") + # -checkhost output alone is not portable as an exit-status assertion. + details = execute(["openssl", "x509", "-in", str(certificate), "-noout", "-checkhost", domain], capture=True) + if "does match certificate" not in details: + fail("TLS certificate does not cover MOYI_DOMAIN.") + execute(["openssl", "x509", "-in", str(certificate), "-noout", "-checkend", "86400"], capture=True) + public_cert = execute(["openssl", "x509", "-in", str(certificate), "-noout", "-pubkey"], capture=True) + public_key = execute(["openssl", "pkey", "-in", str(key), "-pubout", "-passin", "pass:"], capture=True) + if public_cert.strip() != public_key.strip(): + fail("TLS certificate and private key do not match.") + renewal = subprocess.run(["openssl", "x509", "-in", str(certificate), "-noout", "-checkend", "2592000"], capture_output=True) + print("TLS certificate/key OK." if renewal.returncode == 0 else "WARNING: TLS certificate expires within 30 days; renew it.") + + +def check_network(api, database, subnet): + targets = (("RDS", database.hostname, database.port or 5432), + ("Tair", api["MOYI_REDIS_HOST"], int(api.get("MOYI_REDIS_PORT", 6379)))) + for label, host, port in targets: + try: + addresses = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + if any(ipaddress.ip_address(item[4][0]) in subnet for item in addresses): + fail(label + " address overlaps MOYI_DOCKER_SUBNET. Select a non-overlapping Docker subnet.") + with socket.create_connection((host, port), timeout=5): + pass + except OSError: + fail(label + " TCP check failed. Check private-network interconnection and whitelist.") + print(label + " TCP reachable (authentication/TLS checked after application startup).") + + +def preflight(env_file, mode, network=False): + verify_release() + config = load_config(env_file, mode) + api, database, subnet = validate_config(config, mode) + if mode == "https": + check_certificate(config["services"]["frontend"]["environment"]["MOYI_DOMAIN"]) + if network: + check_network(api, database, subnet) + print("Configuration OK. No credentials printed. Mode: " + mode) + return config + + +def smoke(config, mode, login=False): + ports = config["services"]["frontend"]["ports"] + port = next(str(p["published"]) for p in ports if p.get("host_ip") == "127.0.0.1") + base = "http://127.0.0.1:" + port if mode == "tunnel" else "https://" + config["services"]["frontend"]["environment"]["MOYI_DOMAIN"] + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())) + + def request(path, expected=200, payload=None): + body = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request(base + path, data=body, headers={"Content-Type": "application/json"}) + try: + response = opener.open(req, timeout=15) + except urllib.error.HTTPError as error: + response = error + if response.code != expected: + fail("Smoke check failed: " + path + " HTTP " + str(response.code)) + return response.read() + + if b'&2 + exit 1 +fi +. /etc/os-release +if [ "${ID:-}" != ubuntu ] || [ "${VERSION_ID:-}" != 24.04 ]; then + printf '%s\n' 'This installer targets the confirmed Ubuntu 24.04 server only.' >&2 + exit 1 +fi +if command -v docker >/dev/null 2>&1; then + docker version + docker compose version + printf '%s\n' 'Docker already installed; existing packages and settings were left untouched.' + exit 0 +fi +for package in docker.io docker-compose docker-compose-v2 podman-docker containerd runc; do + if dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q 'install ok installed'; then + printf 'Existing package %s found. Stop and review; nothing will be removed.\n' "$package" >&2 + exit 1 + fi +done +if [ -e /etc/apt/sources.list.d/docker.sources ] || [ -e /etc/apt/sources.list.d/docker.list ]; then + printf '%s\n' 'Docker APT configuration exists. Review it before continuing.' >&2 + exit 1 +fi +apt-get update +apt-get install -y ca-certificates curl python3 openssl +install -m 0755 -d /etc/apt/keyrings +curl -fsSL --connect-timeout 10 --max-time 60 https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc +printf 'Types: deb\nURIs: https://download.docker.com/linux/ubuntu\nSuites: noble\nComponents: stable\nArchitectures: %s\nSigned-By: /etc/apt/keyrings/docker.asc\n' "$(dpkg --print-architecture)" > /etc/apt/sources.list.d/docker.sources +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +systemctl enable --now docker +docker version +docker compose version +printf '%s\n' 'Installed. No firewall rules, Docker address pools, users or existing applications were changed.' diff --git a/infra/moyi/nginx/http.conf b/infra/moyi/nginx/http.conf new file mode 100644 index 0000000..34c07c7 --- /dev/null +++ b/infra/moyi/nginx/http.conf @@ -0,0 +1,7 @@ +server { + listen 80 default_server; + server_name _; + root /usr/share/nginx/html; + index index.html; + include /etc/nginx/moyi-routes.conf; +} diff --git a/infra/moyi/nginx/https-fallback.conf b/infra/moyi/nginx/https-fallback.conf new file mode 100644 index 0000000..04f29b2 --- /dev/null +++ b/infra/moyi/nginx/https-fallback.conf @@ -0,0 +1,7 @@ +# Unknown HTTP hosts cannot reach the application. +server { + listen 80 default_server; + server_name _; + location = /nginx-health { access_log off; return 200 "ok\n"; } + location / { return 444; } +} diff --git a/infra/moyi/nginx/https.conf.template b/infra/moyi/nginx/https.conf.template new file mode 100644 index 0000000..b94fcfb --- /dev/null +++ b/infra/moyi/nginx/https.conf.template @@ -0,0 +1,19 @@ +server { + listen 80; + server_name ${MOYI_DOMAIN}; + location = /nginx-health { access_log off; return 200 "ok\n"; } + location / { return 308 https://${MOYI_DOMAIN}$request_uri; } +} + +server { + listen 443 ssl; + server_name ${MOYI_DOMAIN}; + ssl_certificate /etc/nginx/certs/fullchain.pem; + ssl_certificate_key /etc/nginx/certs/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:MOYI:10m; + ssl_session_timeout 1d; + root /usr/share/nginx/html; + index index.html; + include /etc/nginx/moyi-routes.conf; +} diff --git a/infra/moyi/nginx/routes.conf b/infra/moyi/nginx/routes.conf new file mode 100644 index 0000000..14d2d1a --- /dev/null +++ b/infra/moyi/nginx/routes.conf @@ -0,0 +1,40 @@ +server_tokens off; +client_max_body_size 20m; +add_header X-Content-Type-Options nosniff always; +add_header Referrer-Policy strict-origin-when-cross-origin always; +add_header X-Frame-Options SAMEORIGIN always; +# Do not log prompts, credentials or query strings. +access_log off; +location = /nginx-health { access_log off; return 200 "ok\n"; } +location /actuator { return 404; } +location ~ /\. { return 404; } +location = /health { + proxy_pass http://moyi-api:8787/health; +} +location /api/ { + proxy_pass http://moyi-api:8787; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 5s; + proxy_read_timeout 120s; + proxy_next_upstream off; +} +location /v1/ { + proxy_pass http://moyi-api:8787; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_cache off; + proxy_connect_timeout 5s; + proxy_read_timeout 120s; + proxy_send_timeout 120s; + proxy_next_upstream off; +} +location / { try_files $uri $uri/ /index.html; } diff --git a/infra/moyi/tests/test_deploy.py b/infra/moyi/tests/test_deploy.py new file mode 100644 index 0000000..9456eb3 --- /dev/null +++ b/infra/moyi/tests/test_deploy.py @@ -0,0 +1,139 @@ +import importlib.util +import json +import shutil +import subprocess +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +DIRECTORY = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("deploy", DIRECTORY / "deploy.py") +deploy = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(deploy) + + +def configuration(): + return { + "services": { + "moyi-api": {"environment": { + "SPRING_DATASOURCE_URL": "jdbc:postgresql://rds.internal:5432/moyi_control", + "SPRING_DATASOURCE_USERNAME": "moyi_app", + "SPRING_DATASOURCE_PASSWORD": "test-only-rds-password", + "MOYI_REDIS_HOST": "redis.internal", + "MOYI_REDIS_PASSWORD": "test-only-tair-password", + "MOYI_REDIS_KEY_PREFIX": "moyi:test:", + "MOYI_REDIS_SSL": "false", + "AUTH_PASSWORD_PEPPER": "p" * 64, + "CREDENTIAL_ENCRYPTION_KEY": "k" * 64, + "ADMIN_PASSWORD": "test-only-admin-password", + "AUTH_SECURE_COOKIE": "false" + }}, + "frontend": {"ports": [{"host_ip": "127.0.0.1", "published": "8080", "target": 80}], "environment": {"MOYI_DOMAIN": "moyi.soulofmodel.com"}} + }, + "networks": {"default": {"ipam": {"config": [{"subnet": "10.253.240.0/24"}]}}} + } + + +class DeploymentTests(unittest.TestCase): + def test_tunnel_config(self): + deploy.validate_config(configuration(), "tunnel") + + def test_reject_placeholder(self): + cfg = configuration() + cfg["services"]["moyi-api"]["environment"]["MOYI_REDIS_HOST"] = "CHANGE_ME_TAIR_HOST" + with self.assertRaisesRegex(ValueError, "placeholder"): + deploy.validate_config(cfg, "tunnel") + + def test_reject_mysql(self): + cfg = configuration() + cfg["services"]["moyi-api"]["environment"]["SPRING_DATASOURCE_URL"] = "jdbc:mysql://db/app" + with self.assertRaisesRegex(ValueError, "PostgreSQL"): + deploy.validate_config(cfg, "tunnel") + + def test_reject_public_http(self): + cfg = configuration() + cfg["services"]["frontend"]["ports"][0]["host_ip"] = "0.0.0.0" + with self.assertRaisesRegex(ValueError, "127.0.0.1"): + deploy.validate_config(cfg, "tunnel") + + def test_reject_public_backend(self): + cfg = configuration() + cfg["services"]["moyi-api"]["ports"] = [{"published": "8787"}] + with self.assertRaisesRegex(ValueError, "Backend"): + deploy.validate_config(cfg, "tunnel") + + def test_https_requires_secure_cookie(self): + with self.assertRaisesRegex(ValueError, "secure cookies"): + deploy.validate_config(configuration(), "https") + + def test_https_domain_and_cookie(self): + cfg = configuration() + cfg["services"]["moyi-api"]["environment"]["AUTH_SECURE_COOKIE"] = "true" + deploy.validate_config(cfg, "https") + cfg["services"]["frontend"]["environment"]["MOYI_DOMAIN"] = "example.com; return 200;" + with self.assertRaisesRegex(ValueError, "hostname"): + deploy.validate_config(cfg, "https") + + def test_initialization_never_overwrites_secrets(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / ".env.cloud.example").write_text((DIRECTORY / ".env.cloud.example").read_text()) + with patch.object(deploy, "HERE", directory), patch.object(deploy, "ROOT", directory): + deploy.initialize() + before = (directory / ".env.cloud").read_bytes() + self.assertNotIn(b"GENERATE_AUTH_PEPPER", before) + with self.assertRaisesRegex(ValueError, "overwrite"): + deploy.initialize() + self.assertEqual(before, (directory / ".env.cloud").read_bytes()) + + def test_manifest_detects_tampering(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + (directory / "sample").write_bytes(b"original") + digest = deploy.hashlib.sha256(b"original").hexdigest() + (directory / "release-manifest.json").write_text(json.dumps({"release": "test", "sha256": {"sample": digest}})) + with patch.object(deploy, "ROOT", directory): + deploy.verify_release() + (directory / "sample").write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + deploy.verify_release() + + def test_compose_paths_and_password_not_in_arguments(self): + command = deploy.compose_command(Path("settings"), "https") + self.assertIn("docker-compose.https.yml", command) + self.assertNotIn("password", " ".join(command)) + + @unittest.skipUnless(shutil.which("docker"), "Docker CLI required, daemon not needed") + def test_real_compose_interpolation_and_https_overlay(self): + content = (DIRECTORY / ".env.cloud.example").read_text(encoding="utf-8") + replacements = { + "CHANGE_ME_RDS_HOST": "rds.internal", "CHANGE_ME_RDS_PASSWORD": "test-db-$value#literal", + "CHANGE_ME_TAIR_HOST": "redis.internal", "CHANGE_ME_TAIR_PASSWORD": "test-cache-$value#literal", + "GENERATE_AUTH_PEPPER": "p" * 64, "GENERATE_CREDENTIAL_KEY": "k" * 64, + "GENERATE_ADMIN_PASSWORD": "test-only-admin-password", + } + for before, after in replacements.items(): + content = content.replace(before, after) + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / ".env.cloud" + path.write_text(content, encoding="utf-8") + path.chmod(0o600) + for mode in ("tunnel", "https"): + cfg = deploy.load_config(path, mode) + deploy.validate_config(cfg, mode) + api = cfg["services"]["moyi-api"]["environment"] + self.assertEqual(api["MOYI_REDIS_PASSWORD"], "test-cache-$value#literal") + self.assertEqual(api["SPRING_DATASOURCE_PASSWORD"], "test-db-$value#literal") + self.assertNotIn("MOYI_REDIS_PASSWORD", cfg["services"]["frontend"].get("environment", {})) + self.assertNotIn("OSS_ACCESS_KEY_SECRET", api) + if mode == "https": + ports = cfg["services"]["frontend"]["ports"] + self.assertEqual({str(p["published"]) for p in ports}, {"80", "443", "8080"}) + defaults = [v for v in cfg["services"]["frontend"]["volumes"] if v["target"] == "/etc/nginx/conf.d/default.conf"] + self.assertEqual(len(defaults), 1) + self.assertTrue(defaults[0]["source"].endswith("https-fallback.conf")) + + +if __name__ == "__main__": + unittest.main() diff --git a/prototype/.dockerignore b/prototype/.dockerignore index 6a46de5..a2adaea 100644 --- a/prototype/.dockerignore +++ b/prototype/.dockerignore @@ -3,3 +3,5 @@ node_modules dist *.log +.env* +!.env.example diff --git a/prototype/Dockerfile.runtime b/prototype/Dockerfile.runtime new file mode 100644 index 0000000..bdbd128 --- /dev/null +++ b/prototype/Dockerfile.runtime @@ -0,0 +1,5 @@ +ARG NGINX_RUNTIME_IMAGE=nginx:1.27-alpine +FROM ${NGINX_RUNTIME_IMAGE} +COPY nginx.conf /etc/nginx/conf.d/default.conf +COPY dist /usr/share/nginx/html +EXPOSE 80 443 diff --git a/prototype/Dockerfile.runtime.dockerignore b/prototype/Dockerfile.runtime.dockerignore new file mode 100644 index 0000000..0db2aee --- /dev/null +++ b/prototype/Dockerfile.runtime.dockerignore @@ -0,0 +1,4 @@ +** +!dist/ +!dist/** +!nginx.conf diff --git a/prototype/build-release.mjs b/prototype/build-release.mjs new file mode 100644 index 0000000..5d20bc0 --- /dev/null +++ b/prototype/build-release.mjs @@ -0,0 +1,6 @@ +import { build } from 'vite' + +// Build only against the same-origin Java API. Ignore developer dotenv files. +process.env.VITE_CONTROL_PLANE_URL = '/' +process.env.VITE_AUTH_MODE = 'remote' +await build({ envDir: false }) diff --git a/scripts/package-deployment.ps1 b/scripts/package-deployment.ps1 new file mode 100644 index 0000000..8c85706 --- /dev/null +++ b/scripts/package-deployment.ps1 @@ -0,0 +1,72 @@ +param( + [string]$MavenCommand = 'mvn', + [string]$JavaHome = $env:JAVA_HOME +) +$ErrorActionPreference = 'Stop' +$repo = Split-Path $PSScriptRoot -Parent +Set-Location $repo +if ((git status --porcelain)) { throw 'Commit the source changes before packaging. The release must identify a clean commit.' } +$commit = (git rev-parse HEAD).Trim() +$releaseId = (Get-Date -Format 'yyyyMMdd-HHmmss') + '-' + $commit.Substring(0, 8) +if ($JavaHome) { $env:JAVA_HOME = $JavaHome } +& $MavenCommand -B -ntp '-Dmaven.repo.local=.tools/m2' -f control-plane-java/pom.xml clean verify +if ($LASTEXITCODE -ne 0) { throw 'Java verification failed; no release created.' } + +# Never load developer .env files while building distributable browser assets. +Push-Location prototype +try { + $env:VITE_CONTROL_PLANE_URL = '/' + $env:VITE_AUTH_MODE = 'remote' + & node node_modules/vue-tsc/bin/vue-tsc.js --noEmit + if ($LASTEXITCODE -ne 0) { throw 'Vue type check failed.' } + & node build-release.mjs + if ($LASTEXITCODE -ne 0) { throw 'Vue build failed.' } +} finally { Pop-Location } + +$artifactRoot = Join-Path $repo 'artifacts' +New-Item -ItemType Directory -Force -Path $artifactRoot | Out-Null +$staging = Join-Path $artifactRoot ('moyi-test-' + $releaseId) +New-Item -ItemType Directory -Path $staging | Out-Null +# Copy tracked deployment files only, never .env, certificates or .git. +$tracked = @(git ls-files infra/moyi docs/11-server-test-deployment.md) +foreach ($relative in $tracked) { + if ($relative -match '(^|/)(\.env\.cloud|\.deploy-mode|certs)(/|$)') { throw 'Sensitive path in release allowlist.' } + $destination = Join-Path $staging $relative + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + $text = [IO.File]::ReadAllText((Join-Path $repo $relative)).Replace("`r`n", "`n") + [IO.File]::WriteAllText($destination, $text, [Text.UTF8Encoding]::new($false)) +} +foreach ($relative in @('control-plane-java/Dockerfile.runtime', 'control-plane-java/Dockerfile.runtime.dockerignore', 'prototype/Dockerfile.runtime', 'prototype/Dockerfile.runtime.dockerignore', 'prototype/nginx.conf')) { + $destination = Join-Path $staging $relative + New-Item -ItemType Directory -Force -Path (Split-Path $destination -Parent) | Out-Null + [IO.File]::WriteAllText($destination, [IO.File]::ReadAllText((Join-Path $repo $relative)).Replace("`r`n", "`n"), [Text.UTF8Encoding]::new($false)) +} +New-Item -ItemType Directory -Path (Join-Path $staging 'control-plane-java/target') | Out-Null +Copy-Item -LiteralPath 'control-plane-java/target/moyi-control-plane.jar' -Destination (Join-Path $staging 'control-plane-java/target/moyi-control-plane.jar') +Copy-Item -LiteralPath 'prototype/dist' -Destination (Join-Path $staging 'prototype/dist') -Recurse +Copy-Item -LiteralPath 'docs/11-server-test-deployment.md' -Destination (Join-Path $staging 'DEPLOYMENT.md') +$hashes = [ordered]@{} +foreach ($file in Get-ChildItem -LiteralPath $staging -File -Recurse | Sort-Object FullName) { + $relative = $file.FullName.Substring($staging.Length + 1).Replace('\', '/') + $hashes[$relative] = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash.ToLowerInvariant() +} +$manifest = [ordered]@{ + release = $releaseId + commit = $commit + builtAt = (Get-Date).ToUniversalTime().ToString('o') + kind = 'precompiled-jar-and-static-assets-not-offline-docker-images' + javaRelease = 21 + database = 'PostgreSQL 15' + redis = 'Tair standard' + checks = @('maven-clean-verify', 'vue-tsc', 'vite-build') + sha256 = $hashes +} +[IO.File]::WriteAllText((Join-Path $staging 'release-manifest.json'), ($manifest | ConvertTo-Json -Depth 5), [Text.UTF8Encoding]::new($false)) +$archive = $staging + '.tar.gz' +& tar -czf $archive -C $staging . +if ($LASTEXITCODE -ne 0) { throw 'Archive creation failed.' } +$checksum = (Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() +[IO.File]::WriteAllText(($archive + '.sha256'), "$checksum $([IO.Path]::GetFileName($archive))`n", [Text.UTF8Encoding]::new($false)) +Write-Output "Release: $archive" +Write-Output "SHA256: $checksum" +Write-Output 'Contains compiled artifacts, not Docker images. First deployment needs registry connectivity.'