fix: classify upstream failures and prevent empty billed responses
This commit is contained in:
1 parent
b455d8d79f
commit
9acc69d4c7
28 files changed
+1449
-131
No files matched your search
+69
-10
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.moyi.controlplane.auth.AuthService;
|
||||
import com.moyi.controlplane.common.ApiException;
|
||||
import com.moyi.controlplane.gateway.GatewayGuard;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
@@ -21,16 +22,31 @@ import org.springframework.dao.EmptyResultDataAccessException;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
|
||||
@Service
|
||||
public class UsageAuditService {
|
||||
private static final int MAX_PAYLOAD_CHARS = 32_000;
|
||||
private final JdbcTemplate jdbc;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final GatewayGuard gatewayGuard;
|
||||
private final int asyncQueueLimit;
|
||||
private final long asyncWorkerIntervalMs;
|
||||
|
||||
public UsageAuditService(JdbcTemplate jdbc, ObjectMapper objectMapper) {
|
||||
this(jdbc, objectMapper, null, 100, 1000);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public UsageAuditService(JdbcTemplate jdbc, ObjectMapper objectMapper, GatewayGuard gatewayGuard,
|
||||
@Value("${moyi.tasks.max-queued-per-model:100}") int asyncQueueLimit,
|
||||
@Value("${moyi.tasks.worker-interval-ms:1000}") long asyncWorkerIntervalMs) {
|
||||
this.jdbc = jdbc;
|
||||
this.objectMapper = objectMapper;
|
||||
this.gatewayGuard = gatewayGuard;
|
||||
this.asyncQueueLimit = asyncQueueLimit;
|
||||
this.asyncWorkerIntervalMs = asyncWorkerIntervalMs;
|
||||
}
|
||||
|
||||
public void startChat(UUID id, UUID userId, UUID keyId, Map<String,Object> model, Map<String,Object> input) {
|
||||
@@ -105,7 +121,6 @@ public class UsageAuditService {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("callsToday", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= CURRENT_DATE"));
|
||||
result.put("callsThisMonth", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND event_kind = 'call' AND created_at >= date_trunc('month', now())"));
|
||||
result.put("waiting", count("SELECT COUNT(*) FROM usage_events WHERE deleted_at IS NULL AND status IN ('queued','running')"));
|
||||
result.put("costToday", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= CURRENT_DATE"));
|
||||
result.put("costThisMonth", decimal("SELECT COALESCE(SUM(cost_amount),0) FROM usage_events WHERE deleted_at IS NULL AND status = 'succeeded' AND created_at >= date_trunc('month', now())"));
|
||||
result.put("series", jdbc.query(
|
||||
@@ -114,26 +129,70 @@ public class UsageAuditService {
|
||||
"LEFT JOIN usage_events ue ON ue.deleted_at IS NULL AND ue.created_at >= day AND ue.created_at < day + INTERVAL '1 day' " +
|
||||
"GROUP BY day ORDER BY day",
|
||||
(rs, row) -> Map.of("date", rs.getString("day"), "count", rs.getLong("count"))));
|
||||
result.put("capacityGroups", jdbc.query(
|
||||
"SELECT m.id, m.public_name, COALESCE(m.max_concurrency, 0) AS max_concurrency, " +
|
||||
"COUNT(*) FILTER (WHERE ue.status = 'running') AS running, " +
|
||||
"COUNT(*) FILTER (WHERE ue.status IN ('queued','running')) AS queued " +
|
||||
"FROM model_catalog m LEFT JOIN usage_events ue ON ue.model_id = m.id " +
|
||||
"AND ue.deleted_at IS NULL AND ue.created_at >= now() - INTERVAL '10 minutes' " +
|
||||
"WHERE m.deleted_at IS NULL AND m.status = 'active' " +
|
||||
"GROUP BY m.id, m.public_name, m.max_concurrency ORDER BY m.public_name LIMIT 100",
|
||||
int defaultModelLimit = gatewayGuard == null ? 4 : gatewayGuard.defaultModelConcurrency();
|
||||
List<Map<String, Object>> groups = jdbc.query("""
|
||||
WITH pending AS (
|
||||
SELECT model_id,
|
||||
COUNT(*) FILTER (WHERE status='running') AS running,
|
||||
COUNT(*) FILTER (WHERE status='queued') AS queued
|
||||
FROM model_tasks WHERE status IN ('queued','running') GROUP BY model_id
|
||||
)
|
||||
SELECT m.id, m.public_name, m.source_label, p.name AS provider_name, m.call_mode,
|
||||
COALESCE(NULLIF(m.max_concurrency, 0), ?) AS max_concurrency,
|
||||
(m.status='active' AND m.deleted_at IS NULL AND p.status='active' AND p.deleted_at IS NULL
|
||||
AND d.status='active' AND d.deleted_at IS NULL) AS route_enabled,
|
||||
COALESCE(pending.running, 0) AS running, COALESCE(pending.queued, 0) AS queued
|
||||
FROM model_catalog m JOIN model_providers p ON p.id=m.provider_id
|
||||
JOIN model_definitions d ON d.id=m.logical_model_id
|
||||
LEFT JOIN pending ON pending.model_id=m.id
|
||||
ORDER BY m.public_name, m.source_label, m.id
|
||||
""",
|
||||
(rs, row) -> {
|
||||
Map<String, Object> group = new LinkedHashMap<>();
|
||||
group.put("id", rs.getString("id"));
|
||||
group.put("name", rs.getString("public_name"));
|
||||
group.put("sourceLabel", rs.getString("source_label"));
|
||||
group.put("providerName", rs.getString("provider_name"));
|
||||
group.put("mode", rs.getString("call_mode"));
|
||||
group.put("routeEnabled", rs.getBoolean("route_enabled"));
|
||||
group.put("limit", rs.getInt("max_concurrency"));
|
||||
group.put("running", rs.getLong("running"));
|
||||
group.put("queued", rs.getLong("queued"));
|
||||
return group;
|
||||
}));
|
||||
}, defaultModelLimit);
|
||||
List<String> synchronousIds = groups.stream().filter(group -> !"async".equals(group.get("mode"))).map(group -> group.get("id").toString()).toList();
|
||||
GatewayGuard.CapacitySnapshot snapshot = gatewayGuard == null
|
||||
? new GatewayGuard.CapacitySnapshot(false, Map.of()) : gatewayGuard.capacitySnapshot(synchronousIds);
|
||||
for (Map<String, Object> group : groups) {
|
||||
if ("async".equals(group.get("mode"))) continue;
|
||||
GatewayGuard.Capacity current = snapshot.groups().get(group.get("id").toString());
|
||||
group.put("running", current == null ? null : current.running());
|
||||
group.put("queued", current == null ? null : current.queued());
|
||||
}
|
||||
// Keep disabled groups visible while their queued or running work still exists.
|
||||
groups = groups.stream().filter(group -> Boolean.TRUE.equals(group.get("routeEnabled"))
|
||||
|| positive(group.get("running")) || positive(group.get("queued"))).toList();
|
||||
result.put("capacityGroups", groups);
|
||||
result.put("running", totalCapacity(groups, "running"));
|
||||
result.put("waiting", totalCapacity(groups, "queued"));
|
||||
result.put("capacityConfig", Map.of(
|
||||
"queueWaitMs", gatewayGuard == null ? 30000 : gatewayGuard.queueWaitMs(),
|
||||
"defaultModelConcurrency", defaultModelLimit,
|
||||
"defaultKeyConcurrency", gatewayGuard == null ? 4 : gatewayGuard.defaultKeyConcurrency(),
|
||||
"asyncQueueLimit", asyncQueueLimit,
|
||||
"asyncWorkerIntervalMs", asyncWorkerIntervalMs,
|
||||
"synchronousSnapshotAvailable", snapshot.available()
|
||||
));
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean positive(Object value) { return value instanceof Number number && number.longValue() > 0; }
|
||||
|
||||
private Long totalCapacity(List<Map<String, Object>> groups, String field) {
|
||||
if (groups.stream().anyMatch(group -> group.get(field) == null)) return null;
|
||||
return groups.stream().mapToLong(group -> ((Number) group.get(field)).longValue()).sum();
|
||||
}
|
||||
|
||||
private long count(String sql) {
|
||||
Number value = jdbc.queryForObject(sql, Number.class);
|
||||
return value == null ? 0L : value.longValue();
|
||||
|
||||
Reference in new issue
Block a user