throughput: paced downstream rate, with the qualifier that makes it honest

A throughput number reports the smallest limit on the path, and the sender's own
ceiling is one of the candidates. If the server was asked for 50 Mbps and 50
Mbps arrived, the network was never the constraint and "50 Mbps" says nothing
about it. So the result always carries limited_by and measures_network, and a
finding is raised only when the path is actually implicated.

Loss is computed against the *sender's* count, not the requested rate: the
server reports what it put on the wire, and the gap is the loss. A receiver
alone cannot tell "the network dropped it" from "the sender never sent it", and
guessing turns a healthy server-side limit into a phantom network fault. The
count is stored per action, not per packet — half a million packets of structs
would turn a measurement into memory exhaustion.

Sending is paced rather than flat out. An unpaced burst measures the server's
NIC and the first queue it meets, then collapses into loss that reads as a
network fault. The schedule is absolute rather than sleep-per-packet, which
would accumulate scheduler error and drift the rate down over a ten-second run.

Throughput gets its own grant budget sized from the request, so every other
action stays bounded at 8 MiB. When the byte cap binds before the clock does,
the *duration* is shortened and reported, rather than the run being truncated
halfway: promising thirty seconds and delivering twenty-one is the same
information with a surprise attached, and it keeps "the clock ended the run" as
the normal case — the only case where the rate is a clean property of the path.

That last behaviour came out of a test that failed honestly: 30 s at 100 Mbps
needs 375 MB against a 256 MB cap.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

echolot-app/core-engine/src/main/kotlin/app/echo_lot/engine/ThroughputMeasurement.kt

@@ -0,0 +1,201 @@
+// SPDX-FileCopyrightText: 2026 Echolot contributors
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package app.echo_lot.engine
+
+import app.echo_lot.measurement.*
+import app.echo_lot.protocol.ControlClient
+import app.echo_lot.protocol.ProbeSession
+import app.echo_lot.protocol.Wire
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.encodeToJsonElement
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+
+/**
+ * Downstream throughput: the server sends at a paced rate for a bounded time and the client
+ * measures what arrives (`perf.throughput_udp`).
+ *
+ * The number this produces is only meaningful with a qualifier attached, and getting that
+ * qualifier right is most of the work here. A throughput test reports the *smallest* limit on the
+ * path, and the sender's own ceiling is one of the candidates: if the server was asked for 50 Mbps
+ * and 50 Mbps arrived, the network was never the constraint and "50 Mbps" says nothing about it.
+ * Reporting that as a capacity measurement would be a confident lie, so the result always carries
+ * [ThroughputMetrics.limitedBy] and a finding is only raised when the network is actually
+ * implicated.
+ *
+ * Comparing against the *sender's* count rather than the requested rate is the other half: the
+ * server reports how much it actually put on the wire, and the gap between that and what arrived
+ * is the loss. A receiver alone cannot tell "the network dropped it" from "the sender never sent
+ * it", and guessing turns a healthy server-side limit into a phantom network fault.
+ */
+class ThroughputMeasurement(private val ids: IdSource) {
+
+    private val json = Json { encodeDefaults = true; explicitNulls = true }
+
+    fun run(
+        credential: String,
+        sessionId: String,
+        control: ControlClient,
+        probe: ProbeSession,
+        sessionRef: String,
+        durationS: Int = 5,
+        kbps: Int = 50_000,
+        sizeBytes: Int = 1200,
+    ): Pair<Test, List<Finding>> {
+        val testId = ids.uuid()
+        val started = ids.monoNs()
+
+        val reply = runCatching {
+            control.action(
+                credential, sessionId,
+                """{"action":"throughput","direction":"down","duration_s":$durationS,""" +
+                    """"kbps":$kbps,"size_bytes":$sizeBytes}""",
+            )
+        }
+        if (reply.isFailure) {
+            return Test(
+                id = testId, type = TestType.PERF_THROUGHPUT_UDP, sessionRef = sessionRef, tier = Tier.APP,
+                startedMonoNs = started, endedMonoNs = ids.monoNs(),
+                status = TestStatus.UNSUPPORTED,
+                error = TestError("action_refused", reply.exceptionOrNull()?.message ?: "throughput refused"),
+            ) to emptyList()
+        }
+
+        // The server may have shortened the run to fit its own byte budget; listen for what it
+        // actually promised, not for what we asked.
+        val plannedMs = parseInt(reply.getOrNull(), "duration_ms") ?: (durationS * 1000)
+
+        // A margin past the planned end so the tail of the run is not counted as loss: packets
+        // still in flight when we stop listening were not dropped, they were merely late.
+        val received = probe.collectGranted(plannedMs + 1_500L)
+            .filter { it.type == Wire.TYPE_THROUGHPUT_DATA }
+
+        val bytes = received.sumOf { it.sizeBytes.toLong() }
+        val spanNs = if (received.size >= 2) {
+            received.maxOf { it.tRxNs } - received.minOf { it.tRxNs }
+        } else {
+            0L
+        }
+        // Measured over the arrival span rather than our listening window, which includes the
+        // request round trip and the trailing margin and would understate the rate.
+        val receivedKbps = if (spanNs > 0) (bytes * 8 * 1_000_000 / spanNs).toInt() else 0
+
+        val sender = senderReport(control, credential, sessionId)
+        val sentPackets = sender?.packets ?: 0
+        val lossPct = if (sentPackets > 0) {
+            round2((sentPackets - received.size).coerceAtLeast(0) * 100.0 / sentPackets)
+        } else {
+            null
+        }
+
+        // Only a run the *clock* ended measured the network. One stopped by our own byte budget
+        // or rate ceiling measured this server.
+        val limitedBy = sender?.limitedBy ?: "unknown"
+        val networkLimited = limitedBy == "duration" &&
+            sender != null && receivedKbps > 0 && receivedKbps < sender.kbps * 9 / 10
+
+        val metrics = json.encodeToJsonElement(
+            ThroughputMetrics(
+                requestedKbps = kbps,
+                plannedDurationMs = plannedMs,
+                packetsReceived = received.size,
+                bytesReceived = bytes,
+                receivedKbps = receivedKbps,
+                senderPackets = sender?.packets,
+                senderBytes = sender?.bytes,
+                senderKbps = sender?.kbps,
+                lossPct = lossPct,
+                limitedBy = limitedBy,
+                measuresNetwork = networkLimited,
+            ),
+        ) as JsonObject
+
+        val findings = ArrayList<Finding>()
+        when {
+            sender == null -> Unit // no sender report: nothing can be concluded, so nothing is
+            received.isEmpty() -> findings.add(
+                finding(
+                    "perf.throughput_no_delivery", Category.PERFORMANCE, Severity.HIGH, testId,
+                    "No throughput traffic arrived",
+                    "The server sent ${sender.packets} packets and none arrived. This is a " +
+                        "connectivity fault rather than a slow link.",
+                ),
+            )
+            networkLimited -> findings.add(
+                finding(
+                    "perf.throughput_below_offered", Category.PERFORMANCE, Severity.LOW, testId,
+                    "Downstream throughput ${receivedKbps / 1000} Mbit/s, below the " +
+                        "${sender.kbps / 1000} Mbit/s offered",
+                    "The server sent at ${sender.kbps / 1000} Mbit/s for the full run and " +
+                        "${receivedKbps / 1000} Mbit/s arrived" +
+                        (lossPct?.let { ", losing $it % of packets" } ?: "") +
+                        ". The path could not carry what was offered.",
+                ),
+            )
+        }
+
+        return Test(
+            id = testId, type = TestType.PERF_THROUGHPUT_UDP, sessionRef = sessionRef, tier = Tier.APP,
+            startedMonoNs = started, endedMonoNs = ids.monoNs(),
+            status = if (received.isEmpty()) TestStatus.FAILED else TestStatus.OK,
+            metrics = metrics,
+        ) to findings
+    }
+
+    private data class SenderReport(
+        val packets: Int, val bytes: Long, val kbps: Int, val limitedBy: String,
+    )
+
+    /** The server's own account of the run, from the observations API. */
+    private fun senderReport(
+        control: ControlClient, credential: String, sessionId: String,
+    ): SenderReport? = runCatching {
+        val arr = Json.parseToJsonElement(control.observations(credential, sessionId))
+            .jsonObject["throughput"]?.jsonArray ?: return null
+        val last = arr.lastOrNull()?.jsonObject ?: return null
+        SenderReport(
+            packets = last["packets"]?.jsonPrimitive?.content?.toIntOrNull() ?: 0,
+            bytes = last["bytes"]?.jsonPrimitive?.content?.toLongOrNull() ?: 0,
+            kbps = last["kbps"]?.jsonPrimitive?.content?.toIntOrNull() ?: 0,
+            limitedBy = last["limited_by"]?.jsonPrimitive?.content ?: "unknown",
+        )
+    }.getOrNull()
+
+    private fun parseInt(body: String?, key: String): Int? =
+        body?.let { Regex("\"$key\"\\s*:\\s*(-?\\d+)").find(it)?.groupValues?.get(1)?.toIntOrNull() }
+
+    private fun finding(code: String, cat: Category, sev: Severity, testId: String, title: String, desc: String) =
+        Finding(
+            id = ids.uuid(), code = code, category = cat, severity = sev, confidence = Confidence.HIGH,
+            title = title, description = desc, evidenceRefs = listOf(EvidenceRef(testId)),
+        )
+
+    private fun round2(v: Double) = Math.round(v * 100.0) / 100.0
+}
+
+/** Metrics for perf.throughput_udp. */
+@Serializable
+data class ThroughputMetrics(
+    @SerialName("requested_kbps") val requestedKbps: Int,
+    @SerialName("planned_duration_ms") val plannedDurationMs: Int,
+    @SerialName("packets_received") val packetsReceived: Int,
+    @SerialName("bytes_received") val bytesReceived: Long,
+    @SerialName("received_kbps") val receivedKbps: Int,
+    @SerialName("sender_packets") val senderPackets: Int? = null,
+    @SerialName("sender_bytes") val senderBytes: Long? = null,
+    @SerialName("sender_kbps") val senderKbps: Int? = null,
+    /** Against the sender's count, so a server-side limit is never counted as network loss. */
+    @SerialName("loss_pct") val lossPct: Double? = null,
+    /** What ended the run: duration | budget | rate | send_error | unknown. */
+    @SerialName("limited_by") val limitedBy: String,
+    /**
+     * Whether this number says anything about the network. False when the sender's own ceiling
+     * was the binding constraint — in which case the rate is a property of the test, not the path.
+     */
+    @SerialName("measures_network") val measuresNetwork: Boolean,
+)

echolot-app/core-protocol/src/main/kotlin/app/echo_lot/protocol/Wire.kt

@@ -38,6 +38,9 @@ object Wire {
      */
     const val TYPE_FRAG_DATA: Int = 0x0D
 
+    /** One packet of a sustained-rate downstream run. */
+    const val TYPE_THROUGHPUT_DATA: Int = 0x0E
+
     /** The 8-byte on-the-wire prefix = first 16 hex chars of the session id, decoded. */
     fun wirePrefix(sessionId: String): ByteArray {
         require(sessionId.length >= 16) { "session id too short" }