diff --git a/echolot-app/core-engine/src/main/kotlin/app/echo_lot/engine/DownstreamMeasurement.kt b/echolot-app/core-engine/src/main/kotlin/app/echo_lot/engine/DownstreamMeasurement.kt index 942c7dc..226cdfd 100644 --- a/echolot-app/core-engine/src/main/kotlin/app/echo_lot/engine/DownstreamMeasurement.kt +++ b/echolot-app/core-engine/src/main/kotlin/app/echo_lot/engine/DownstreamMeasurement.kt @@ -37,6 +37,120 @@ class DownstreamMeasurement(private val ids: IdSource) { /** How long to wait for a granted burst after the server accepts the action. */ private val collectWindowMs = 4_000L + /** + * Shorter, but long enough to cover the first_last mode's deliberate 250 ms hold plus a + * reassembly. A fragment burst is one datagram: it is here quickly or not at all. + */ + private val fragWindowMs = 1_500L + + /** + * Asks the server to send one deliberately-fragmented datagram per ordering, and reports + * which orderings survive the path. + * + * Kernel fragmentation always emits fragments in order, first one first, so an oversized + * datagram can only answer "do fragments get through at all". The interesting fault is about + * ordering: only the *first* fragment carries the UDP ports, so a stateful firewall that has + * not seen it has nothing to match the rest against, and many drop them. That failure is + * invisible to every in-order test and shows up in the field as "large DNS answers fail here" + * or "the tunnel breaks when the MTU drops". + */ + fun fragmentOrdering( + credential: String, + sessionId: String, + control: ControlClient, + probe: ProbeSession, + sessionRef: String, + sizeBytes: Int = 2000, + fragBytes: Int = 576, + ): Pair> { + val testId = ids.uuid() + val started = ids.monoNs() + val delivered = LinkedHashMap() + val fragmentCounts = LinkedHashMap() + var unsupported = false + + for (mode in FRAG_MODES) { + val reply = runCatching { + control.action( + credential, sessionId, + """{"action":"frag_send","size_bytes":$sizeBytes,"mode":"$mode","frag_bytes":$fragBytes}""", + ) + } + if (reply.isFailure) { + // A server without a raw socket says so; that is a missing capability, not a + // property of the network, and must not be recorded as a failed delivery. + unsupported = true + break + } + parseInt(reply.getOrNull(), "fragments")?.let { fragmentCounts[mode] = it } + // The burst is already on the wire when the action returns (it is sent + // synchronously), so anything that survived is either here or lost. + val got = probe.collectGranted(fragWindowMs).any { it.type == Wire.TYPE_FRAG_DATA } + delivered[mode] = got + } + + if (unsupported) { + return Test( + id = testId, type = TestType.MTU_FRAG_ORDERING, sessionRef = sessionRef, tier = Tier.APP, + startedMonoNs = started, endedMonoNs = ids.monoNs(), + status = TestStatus.UNSUPPORTED, + error = TestError("no_raw_socket", "this server cannot craft fragments"), + ) to emptyList() + } + + val metrics = json.encodeToJsonElement( + FragOrderingMetrics( + sizeBytes = sizeBytes, + fragBytes = fragBytes, + fragmentsPerBurst = fragmentCounts, + deliveredByMode = delivered, + inOrderDelivered = delivered[FRAG_IN_ORDER] == true, + reorderedDelivered = delivered[FRAG_REVERSED] == true, + delayedFirstDelivered = delivered[FRAG_FIRST_LAST] == true, + ), + ) as JsonObject + + val findings = ArrayList() + val inOrder = delivered[FRAG_IN_ORDER] == true + val reversed = delivered[FRAG_REVERSED] == true + val firstLast = delivered[FRAG_FIRST_LAST] == true + + if (!inOrder) { + findings.add( + finding( + "mtu.fragments_blocked", Category.MTU, Severity.MEDIUM, testId, + "IP fragments do not reach this device", + "A fragmented datagram sent in the normal order never arrived. Anything that " + + "relies on fragmentation — large DNS answers over UDP, some VPN traffic — " + + "will fail here rather than slow down.", + ), + ) + } else if (!reversed || !firstLast) { + // The precise and useful finding: fragments work, but only if they arrive tidily. + val which = buildList { + if (!reversed) add("out of order") + if (!firstLast) add("with the first fragment delayed") + }.joinToString(" or ") + findings.add( + finding( + "mtu.fragment_reorder_sensitive", Category.MTU, Severity.LOW, testId, + "Fragments are dropped when they arrive $which", + "In-order fragments are delivered, but the same datagram sent $which is not. " + + "Something on the path only reassembles when the first fragment (the one " + + "carrying the UDP ports) arrives first — typical of a stateful firewall " + + "or NAT. It works until the network reorders, then fails intermittently, " + + "which is the hardest kind of fault to chase.", + ), + ) + } + return Test( + id = testId, type = TestType.MTU_FRAG_ORDERING, sessionRef = sessionRef, tier = Tier.APP, + startedMonoNs = started, endedMonoNs = ids.monoNs(), + status = if (inOrder) TestStatus.OK else TestStatus.PARTIAL, + metrics = metrics, + ) to findings + } + /** * Runs all three against an already-primed session. * @@ -66,6 +180,16 @@ class DownstreamMeasurement(private val ids: IdSource) { tests.add(df.test); tests.add(frag.test); tests.add(train.test) + // Fragment ordering only makes sense once we know fragments arrive at all; when they do + // not, the ordering variants would all report "not delivered" and read as three faults + // instead of one. + if (frag.largestDelivered != null) { + val (fragTest, fragFindings) = + fragmentOrdering(credential, sessionId, control, probe, sessionRef) + tests.add(fragTest) + findings.addAll(fragFindings) + } + // A downstream MTU below the classic 1500-byte Ethernet payload is worth saying out loud: // it is the usual cause of "small requests work, large responses hang". val pathMtu = df.largestDelivered @@ -310,6 +434,11 @@ class DownstreamMeasurement(private val ids: IdSource) { /** IPv4 (20) + UDP (8). The v6 case is 48; reported per-family once v6 sessions land. */ const val IP_UDP_OVERHEAD4 = 28 + const val FRAG_IN_ORDER = "in_order" + const val FRAG_REVERSED = "reversed" + const val FRAG_FIRST_LAST = "first_last" + val FRAG_MODES = listOf(FRAG_IN_ORDER, FRAG_REVERSED, FRAG_FIRST_LAST) + /** Straddles the usual suspects: 1500 Ethernet, 1492 PPPoE, 1400-ish tunnels. */ val DEFAULT_SIZES = listOf(600, 1200, 1372, 1400, 1450, 1472, 1500, 2000, 4000) @@ -330,6 +459,18 @@ data class BigSendMetrics( @SerialName("path_mtu_bytes") val pathMtuBytes: Int? = null, ) +/** Metrics for mtu.frag_ordering. */ +@Serializable +data class FragOrderingMetrics( + @SerialName("size_bytes") val sizeBytes: Int, + @SerialName("frag_bytes") val fragBytes: Int, + @SerialName("fragments_per_burst") val fragmentsPerBurst: Map, + @SerialName("delivered_by_mode") val deliveredByMode: Map, + @SerialName("in_order_delivered") val inOrderDelivered: Boolean, + @SerialName("reordered_delivered") val reorderedDelivered: Boolean, + @SerialName("delayed_first_delivered") val delayedFirstDelivered: Boolean, +) + /** Metrics for train.udp_downstream. */ @Serializable data class DownTrainMetrics( diff --git a/echolot-app/core-measurement/src/main/kotlin/app/echo_lot/measurement/Test.kt b/echolot-app/core-measurement/src/main/kotlin/app/echo_lot/measurement/Test.kt index 3393dcd..c6a6d0e 100644 --- a/echolot-app/core-measurement/src/main/kotlin/app/echo_lot/measurement/Test.kt +++ b/echolot-app/core-measurement/src/main/kotlin/app/echo_lot/measurement/Test.kt @@ -77,6 +77,8 @@ object TestType { const val MTU_BLACKHOLE = "mtu.blackhole" const val MTU_MSS_OBSERVED = "mtu.mss_observed" const val MTU_FRAG_DELIVERY = "mtu.frag_delivery" + /** Whether fragments survive arriving out of order, not merely whether they survive. */ + const val MTU_FRAG_ORDERING = "mtu.frag_ordering" // nat const val NAT_STUN_5780 = "nat.stun_5780" const val NAT_MAPPING_LIFETIME_UDP = "nat.mapping_lifetime_udp" diff --git a/echolot-app/core-protocol/src/main/kotlin/app/echo_lot/protocol/Wire.kt b/echolot-app/core-protocol/src/main/kotlin/app/echo_lot/protocol/Wire.kt index 2462f33..7969abe 100644 --- a/echolot-app/core-protocol/src/main/kotlin/app/echo_lot/protocol/Wire.kt +++ b/echolot-app/core-protocol/src/main/kotlin/app/echo_lot/protocol/Wire.kt @@ -32,6 +32,12 @@ object Wire { const val TYPE_DOWNTRAIN_DATA: Int = 0x06 const val TYPE_BIG_SEND: Int = 0x0C + /** + * A datagram the server deliberately fragmented. Its arrival IS the measurement: it can only + * be delivered if every fragment survived the path and the local stack reassembled them. + */ + const val TYPE_FRAG_DATA: Int = 0x0D + /** The 8-byte on-the-wire prefix = first 16 hex chars of the session id, decoded. */ fun wirePrefix(sessionId: String): ByteArray { require(sessionId.length >= 16) { "session id too short" } diff --git a/server/cmd/echolot-server/main.go b/server/cmd/echolot-server/main.go index ababd1a..b150506 100644 --- a/server/cmd/echolot-server/main.go +++ b/server/cmd/echolot-server/main.go @@ -112,6 +112,14 @@ func serve(cfg *config.Config) error { } caps := []string{"udp-probe", "delayed-echo", "connect-back", "http-echo", "downtrain", "big-send"} + // Crafted fragments need a raw socket. Advertised only when one can actually be opened — + // a capability we cannot deliver turns a missing feature into a failed measurement. + rawFrag := dataplane.RawFragSupported() + if rawFrag { + caps = append(caps, "frag-send") + } else { + slog.Info("frag-send unavailable: no raw socket (needs CAP_NET_RAW)") + } if len(config.Addrs(cfg.TCPListen)) > 0 { caps = append(caps, "tcp-echo", "tls-echo") } @@ -154,6 +162,11 @@ func serve(cfg *config.Config) error { AppRange: appRange, PublicControlURL: publicControlURL(cfg), } + // Left nil when there is no raw socket, so the handler answers "not implemented" with a + // reason rather than failing somewhere deeper. + if rawFrag { + ctl.FragSend = dp.FragSend + } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() diff --git a/server/internal/control/control.go b/server/internal/control/control.go index 6d4e7b5..6db09d0 100644 --- a/server/internal/control/control.go +++ b/server/internal/control/control.go @@ -59,6 +59,9 @@ type Server struct { BigSend func(sess *session.Session, g *session.Grant, sizes []int, df bool) ([]dataplane.BigSendResult, error) // Runs stores uploaded measurement documents (may be nil: uploads unsupported). Runs *runs.Store + // FragSend emits one datagram as hand-built IP fragments in a chosen order (may be nil: + // needs a raw socket, so it is unavailable to an unprivileged server). + FragSend func(sess *session.Session, g *session.Grant, sizeBytes int, mode dataplane.FragMode, fragSize int) (dataplane.FragResult, error) // EgressMTU reports the server's own measured egress path MTU (0 = unknown). With DF set // we cannot emit a datagram larger than this, so requested sizes above it are refused up // front and reported as such — the client must not read that as a downstream path limit. @@ -241,6 +244,8 @@ func (s *Server) actions(w http.ResponseWriter, r *http.Request) { IntervalUs int `json:"interval_us"` SizesBytes []int `json:"sizes_bytes"` DF *bool `json:"df"` + Mode string `json:"mode"` + FragBytes int `json:"frag_bytes"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "bad body"}) @@ -373,6 +378,45 @@ func (s *Server) actions(w http.ResponseWriter, r *http.Request) { "grant": map[string]any{"max_bytes": g.MaxBytes, "max_kbps": g.MaxKbps}, }) + case "frag_send": + if s.FragSend == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{ + "error": "frag_send needs a raw socket, which this server does not have", + }) + return + } + size := clamp(req.SizeBytes, 1600, 8000) // must exceed the path MTU or nothing fragments + mode := dataplane.FragMode(req.Mode) + switch mode { + case dataplane.FragInOrder, dataplane.FragReversed, dataplane.FragFirstLast: + default: + mode = dataplane.FragInOrder + } + fragBytes := clamp(req.FragBytes, 8, 1400) + g := sess.NewGrant(actionID, int64(size), 0, session.DefaultGrantLimits) + if g == nil { + writeJSON(w, http.StatusConflict, noDataPlaneYet) + return + } + // Synchronous: the whole burst is a few kB and at most a few hundred milliseconds, and + // the caller wants to know it was actually emitted before it starts listening. An + // asynchronous send would make "nothing arrived" ambiguous between a path drop and a + // send that never happened — the one distinction this test exists to make. + result, err := s.FragSend(sess, g, size, mode, fragBytes) + slog.Info("frag_send finished", "action", actionID, "mode", mode, + "size", size, "fragments", result.Fragments, "err", err) + if err != nil { + writeJSON(w, http.StatusConflict, map[string]any{ + "error": err.Error(), "action_id": actionID, "result": result, + }) + return + } + writeJSON(w, http.StatusAccepted, map[string]any{ + "action_id": actionID, "mode": string(mode), "size_bytes": size, + "frag_bytes": fragBytes, "fragments": result.Fragments, + "grant": map[string]any{"max_bytes": g.MaxBytes, "max_kbps": g.MaxKbps}, + }) + default: writeJSON(w, http.StatusBadRequest, map[string]string{"error": "unknown or unimplemented action"}) } diff --git a/server/internal/dataplane/frag_linux.go b/server/internal/dataplane/frag_linux.go new file mode 100644 index 0000000..02c45f2 --- /dev/null +++ b/server/internal/dataplane/frag_linux.go @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +//go:build linux + +package dataplane + +import ( + "encoding/binary" + "fmt" + "net/netip" + "sync/atomic" + "syscall" + "time" + + "echo-lot.app/server/internal/session" +) + +// Crafted IPv4 fragmentation (spec §5 frag_send). +// +// Letting the kernel fragment an oversized datagram — which is what big_send with df=false does — +// answers one question: do fragments get through at all. It cannot answer the more interesting +// one, because the kernel always emits fragments in order, first one first. +// +// The classic middlebox fault is precisely about that ordering. Only the *first* fragment carries +// the UDP header, and therefore the ports; a stateful firewall or NAT that has not seen it has no +// flow to match later fragments against. Plenty of implementations drop them. Others hold them +// briefly and reassemble; others leak. The difference is invisible to any test that sends +// fragments in order, and it shows up in the real world as "large DNS answers fail on this +// network" or "the VPN works until the MTU drops". +// +// So this builds the fragments by hand and controls their order and timing. That needs a raw +// socket (CAP_NET_RAW); when we do not have one the capability is not advertised, rather than +// advertised and failing later. + +// FragMode is how a fragmented datagram is put on the wire. +type FragMode string + +const ( + // FragInOrder is the baseline: first fragment first, as the kernel would. A path that fails + // this fails everything, and it tells the others apart from a path that drops all fragments. + FragInOrder FragMode = "in_order" + // FragReversed sends the last fragment first. This is the one that finds stateful devices + // which need the first fragment to build state. + FragReversed FragMode = "reversed" + // FragFirstLast holds the first fragment back until the others have arrived, which tests + // whether the path buffers non-first fragments at all and for how long. + FragFirstLast FragMode = "first_last" +) + +var fragIPID atomic.Uint32 + +// RawFragSupported reports whether crafted fragments can actually be sent here. +// +// Checked by opening the socket rather than by inspecting capabilities: the question is "will +// this work", and a permission model has more ways to say no than a capability bit has to say yes +// (user namespaces, seccomp, LSM). Advertising a capability we cannot deliver would turn a +// missing feature into a failed measurement. +func RawFragSupported() bool { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW) + if err != nil { + return false + } + _ = syscall.Close(fd) + return true +} + +// FragResult is what happened to one crafted fragment burst. +type FragResult struct { + Mode FragMode `json:"mode"` + SizeBytes int `json:"size_bytes"` + Fragments int `json:"fragments"` + Sent bool `json:"sent"` + Err string `json:"err,omitempty"` +} + +// FragSend emits one ELT1 packet of sizeBytes as hand-built IPv4 fragments, in the given order. +// +// The datagram is assembled whole and then cut up, so what the client reassembles — if it +// reassembles — is a normal, HMAC-valid packet indistinguishable from any other. That matters: +// the client must not be able to tell a crafted fragment burst from a kernel one, or it would be +// measuring our sender rather than the path. +func (s *Server) FragSend( + sess *session.Session, g *session.Grant, sizeBytes int, mode FragMode, fragSize int, +) (FragResult, error) { + res := FragResult{Mode: mode, SizeBytes: sizeBytes} + + target := sess.DataSource() + if !target.IsValid() { + return res, fmt.Errorf("no observed data-plane source") + } + if !target.Addr().Unmap().Is4() { + // IPv6 has no in-network fragmentation: only the source may fragment, via an extension + // header. Worth building, but it is a different mechanism and belongs in its own code + // path rather than pretending this one covers it. + return res, fmt.Errorf("crafted fragmentation is IPv4-only for now") + } + conn := s.connFor(target, sess.DataLocal()) + if conn == nil { + return res, fmt.Errorf("no data-plane socket matches target family") + } + local := sess.DataLocal() + if !local.IsValid() { + return res, fmt.Errorf("session has no recorded local address") + } + + if sizeBytes < HeaderSize+8 { + sizeBytes = HeaderSize + 8 + } + if sizeBytes > 8000 { + sizeBytes = 8000 + } + if !g.Allow(sizeBytes) { + return res, fmt.Errorf("grant exhausted") + } + + // The ELT1 packet, signed exactly as any other, then wrapped in UDP. + payload := make([]byte, sizeBytes-HeaderSize) + binary.BigEndian.PutUint32(payload[0:4], uint32(sizeBytes)) + copy(payload[4:], mode) + elt := s.buildPacket(sess, TypeFragData, 0, payload) + + udp := buildUDP(local, target, elt) + + // Fragment offsets are in 8-byte units, so every fragment except the last must be a multiple + // of 8. A payload that is not is not an error — it is a fragment that no host will reassemble. + if fragSize <= 0 { + fragSize = 576 + } + fragSize = (fragSize / 8) * 8 + if fragSize < 8 { + fragSize = 8 + } + + fragments := splitIPv4(local.Addr(), target.Addr(), udp, fragSize, uint16(fragIPID.Add(1))) + res.Fragments = len(fragments) + + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW) + if err != nil { + res.Err = err.Error() + return res, err + } + defer syscall.Close(fd) + if err := syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1); err != nil { + res.Err = err.Error() + return res, err + } + + dst := syscall.SockaddrInet4{} + copy(dst.Addr[:], target.Addr().Unmap().AsSlice()) + + send := func(pkt []byte) error { return syscall.Sendto(fd, pkt, 0, &dst) } + + switch mode { + case FragReversed: + for i := len(fragments) - 1; i >= 0; i-- { + if err := send(fragments[i]); err != nil { + res.Err = err.Error() + return res, err + } + time.Sleep(time.Millisecond) + } + case FragFirstLast: + for i := 1; i < len(fragments); i++ { + if err := send(fragments[i]); err != nil { + res.Err = err.Error() + return res, err + } + time.Sleep(time.Millisecond) + } + // Long enough to be a real test of whether anything holds fragments, short enough to stay + // inside the usual 30-second reassembly timeout by a wide margin. + time.Sleep(250 * time.Millisecond) + if err := send(fragments[0]); err != nil { + res.Err = err.Error() + return res, err + } + default: + for _, f := range fragments { + if err := send(f); err != nil { + res.Err = err.Error() + return res, err + } + time.Sleep(time.Millisecond) + } + } + res.Sent = true + return res, nil +} + +// buildUDP wraps a payload in a UDP header with a computed checksum. +// +// The checksum is optional in IPv4 and it would be less code to send zero, but a zero-checksum +// datagram is dropped by some middleboxes — and that drop would be recorded as a fragmentation +// failure, which is exactly the wrong conclusion. +func buildUDP(src, dst netip.AddrPort, payload []byte) []byte { + out := make([]byte, 8+len(payload)) + binary.BigEndian.PutUint16(out[0:2], src.Port()) + binary.BigEndian.PutUint16(out[2:4], dst.Port()) + binary.BigEndian.PutUint16(out[4:6], uint16(8+len(payload))) + copy(out[8:], payload) + + // Pseudo-header + UDP header + data, per RFC 768. + var sum uint32 + s4, d4 := src.Addr().Unmap().As4(), dst.Addr().Unmap().As4() + for _, b := range [][]byte{s4[:], d4[:]} { + sum += uint32(binary.BigEndian.Uint16(b[0:2])) + sum += uint32(binary.BigEndian.Uint16(b[2:4])) + } + sum += uint32(syscall.IPPROTO_UDP) + sum += uint32(len(out)) + for i := 0; i+1 < len(out); i += 2 { + sum += uint32(binary.BigEndian.Uint16(out[i : i+2])) + } + if len(out)%2 == 1 { + sum += uint32(out[len(out)-1]) << 8 + } + for sum>>16 != 0 { + sum = (sum & 0xFFFF) + (sum >> 16) + } + ck := ^uint16(sum) + if ck == 0 { + ck = 0xFFFF // 0 means "no checksum" in IPv4; the all-ones form is the same value + } + binary.BigEndian.PutUint16(out[6:8], ck) + return out +} + +// splitIPv4 cuts a UDP datagram into IPv4 fragments of at most fragSize payload bytes each. +// +// Every fragment carries the same IP ID — that is what marks them as one datagram — and every one +// but the last sets MF. The kernel fills in the header checksum and total length for us under +// IP_HDRINCL (raw(7)); the ID it only fills when zero, which is why it is set explicitly here. +func splitIPv4(src, dst netip.Addr, udp []byte, fragSize int, id uint16) [][]byte { + s4, d4 := src.Unmap().As4(), dst.Unmap().As4() + var out [][]byte + for off := 0; off < len(udp); off += fragSize { + end := off + fragSize + if end > len(udp) { + end = len(udp) + } + chunk := udp[off:end] + more := end < len(udp) + + hdr := make([]byte, 20, 20+len(chunk)) + hdr[0] = 0x45 // IPv4, 5 words of header + hdr[1] = 0 // DSCP/ECN + binary.BigEndian.PutUint16(hdr[2:4], uint16(20+len(chunk))) + binary.BigEndian.PutUint16(hdr[4:6], id) + flagsOff := uint16(off / 8) + if more { + flagsOff |= 0x2000 // MF + } + binary.BigEndian.PutUint16(hdr[6:8], flagsOff) + hdr[8] = 64 // TTL + hdr[9] = syscall.IPPROTO_UDP + // hdr[10:12] checksum left zero: the kernel computes it under IP_HDRINCL. + copy(hdr[12:16], s4[:]) + copy(hdr[16:20], d4[:]) + out = append(out, append(hdr, chunk...)) + } + return out +} diff --git a/server/internal/dataplane/frag_linux_test.go b/server/internal/dataplane/frag_linux_test.go new file mode 100644 index 0000000..4e76fc6 --- /dev/null +++ b/server/internal/dataplane/frag_linux_test.go @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +//go:build linux + +package dataplane + +import ( + "encoding/binary" + "net/netip" + "testing" +) + +// Fragment headers are the kind of thing that is either exactly right or silently useless: a +// wrong offset unit, a missing MF bit or a bad checksum produces packets that leave the machine +// and are dropped by the receiver's IP stack without a word. Nothing downstream would notice — +// the client would simply record "fragments do not get through", which is a wrong answer rather +// than a missing one. Hence these check the bytes. + +func testAddrs() (netip.AddrPort, netip.AddrPort) { + return netip.MustParseAddrPort("192.0.2.1:8442"), netip.MustParseAddrPort("198.51.100.9:41000") +} + +func TestSplitCoversThePayloadExactlyOnce(t *testing.T) { + src, dst := testAddrs() + udp := buildUDP(src, dst, make([]byte, 2000)) + + frags := splitIPv4(src.Addr(), dst.Addr(), udp, 576, 0x1234) + if len(frags) < 3 { + t.Fatalf("expected several fragments for %d bytes, got %d", len(udp), len(frags)) + } + + // Reassemble the way a receiver would: place each fragment's payload at its offset. + rebuilt := make([]byte, len(udp)) + covered := make([]bool, len(udp)) + for _, f := range frags { + flagsOff := binary.BigEndian.Uint16(f[6:8]) + off := int(flagsOff&0x1FFF) * 8 + body := f[20:] + if off+len(body) > len(udp) { + t.Fatalf("fragment at offset %d overruns the datagram", off) + } + for i, b := range body { + if covered[off+i] { + t.Fatalf("byte %d delivered twice", off+i) + } + covered[off+i] = true + rebuilt[off+i] = b + } + } + for i, c := range covered { + if !c { + t.Fatalf("byte %d was never sent", i) + } + } + for i := range udp { + if rebuilt[i] != udp[i] { + t.Fatalf("reassembled byte %d differs", i) + } + } +} + +func TestFragmentHeadersAreWellFormed(t *testing.T) { + src, dst := testAddrs() + udp := buildUDP(src, dst, make([]byte, 3000)) + frags := splitIPv4(src.Addr(), dst.Addr(), udp, 800, 0xBEEF) + + for i, f := range frags { + if got := f[0]; got != 0x45 { + t.Errorf("fragment %d: version/IHL = %#x, want 0x45", i, got) + } + if got := f[9]; got != 17 { + t.Errorf("fragment %d: protocol = %d, want 17 (UDP)", i, got) + } + if got := binary.BigEndian.Uint16(f[4:6]); got != 0xBEEF { + t.Errorf("fragment %d: IP ID = %#x — all fragments of one datagram must share it", i, got) + } + if got := binary.BigEndian.Uint16(f[2:4]); int(got) != len(f) { + t.Errorf("fragment %d: total length = %d, actual %d", i, got, len(f)) + } + flagsOff := binary.BigEndian.Uint16(f[6:8]) + mf := flagsOff&0x2000 != 0 + wantMF := i < len(frags)-1 + if mf != wantMF { + t.Errorf("fragment %d: MF = %v, want %v", i, mf, wantMF) + } + } +} + +// Offsets are counted in 8-byte units, so every fragment but the last must be a multiple of 8. +// A 100-byte "fragment size" that silently becomes 100 bytes on the wire produces a datagram no +// host will ever reassemble. +func TestNonFinalFragmentsAreEightByteMultiples(t *testing.T) { + src, dst := testAddrs() + udp := buildUDP(src, dst, make([]byte, 2500)) + for _, size := range []int{8, 100, 576, 999, 1400} { + frags := splitIPv4(src.Addr(), dst.Addr(), udp, (size/8)*8, 1) + for i, f := range frags[:len(frags)-1] { + if body := len(f) - 20; body%8 != 0 { + t.Errorf("size %d: non-final fragment %d carries %d bytes, not a multiple of 8", + size, i, body) + } + } + } +} + +// The UDP checksum is optional in IPv4, and sending zero would be less code — but a +// zero-checksum datagram is dropped by some middleboxes, and that drop would be recorded as a +// fragmentation failure. So it must be present and correct. +func TestUDPChecksumVerifies(t *testing.T) { + src, dst := testAddrs() + for _, n := range []int{0, 1, 7, 8, 100, 1001} { // odd lengths exercise the tail-byte path + udp := buildUDP(src, dst, make([]byte, n)) + if got := binary.BigEndian.Uint16(udp[6:8]); got == 0 { + t.Fatalf("payload %d: checksum is zero, which means 'not computed'", n) + } + if sum := verifyUDPChecksum(src.Addr(), dst.Addr(), udp); sum != 0xFFFF { + t.Errorf("payload %d: checksum does not verify (one's complement sum %#x)", n, sum) + } + if got := binary.BigEndian.Uint16(udp[4:6]); int(got) != len(udp) { + t.Errorf("payload %d: UDP length field %d, actual %d", n, got, len(udp)) + } + } +} + +func TestUDPPortsComeFromTheSessionAddresses(t *testing.T) { + src, dst := testAddrs() + udp := buildUDP(src, dst, []byte("x")) + if got := binary.BigEndian.Uint16(udp[0:2]); got != src.Port() { + t.Errorf("source port = %d, want %d", got, src.Port()) + } + // The destination port must be the client's observed source port, or the datagram arrives + // at the machine and is discarded before any socket sees it. + if got := binary.BigEndian.Uint16(udp[2:4]); got != dst.Port() { + t.Errorf("destination port = %d, want %d", got, dst.Port()) + } +} + +// Recomputes the one's complement sum over the pseudo-header and datagram; a correct checksum +// makes the total 0xFFFF. +func verifyUDPChecksum(src, dst netip.Addr, udp []byte) uint16 { + var sum uint32 + s4, d4 := src.Unmap().As4(), dst.Unmap().As4() + for _, b := range [][]byte{s4[:], d4[:]} { + sum += uint32(binary.BigEndian.Uint16(b[0:2])) + sum += uint32(binary.BigEndian.Uint16(b[2:4])) + } + sum += 17 + sum += uint32(len(udp)) + for i := 0; i+1 < len(udp); i += 2 { + sum += uint32(binary.BigEndian.Uint16(udp[i : i+2])) + } + if len(udp)%2 == 1 { + sum += uint32(udp[len(udp)-1]) << 8 + } + for sum>>16 != 0 { + sum = (sum & 0xFFFF) + (sum >> 16) + } + return uint16(sum) +} diff --git a/server/internal/dataplane/frag_other.go b/server/internal/dataplane/frag_other.go new file mode 100644 index 0000000..7978fb8 --- /dev/null +++ b/server/internal/dataplane/frag_other.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +//go:build !linux + +package dataplane + +import ( + "fmt" + + "echo-lot.app/server/internal/session" +) + +// Crafting IP fragments needs a raw socket and Linux's IP_HDRINCL semantics. Off Linux the +// capability is simply not advertised, so a client never asks for it — better than answering +// with a measurement we cannot actually make. + +type FragMode string + +const ( + FragInOrder FragMode = "in_order" + FragReversed FragMode = "reversed" + FragFirstLast FragMode = "first_last" +) + +type FragResult struct { + Mode FragMode `json:"mode"` + SizeBytes int `json:"size_bytes"` + Fragments int `json:"fragments"` + Sent bool `json:"sent"` + Err string `json:"err,omitempty"` +} + +func RawFragSupported() bool { return false } + +func (s *Server) FragSend( + sess *session.Session, g *session.Grant, sizeBytes int, mode FragMode, fragSize int, +) (FragResult, error) { + return FragResult{Mode: mode, SizeBytes: sizeBytes}, + fmt.Errorf("crafted fragmentation is only implemented on Linux") +} diff --git a/server/internal/dataplane/udp.go b/server/internal/dataplane/udp.go index 4bfed45..223ccb0 100644 --- a/server/internal/dataplane/udp.go +++ b/server/internal/dataplane/udp.go @@ -35,6 +35,8 @@ const ( // Server->client under an asymmetric grant (spec §3.4/§5). TypeDownTrainData = 0x06 TypeBigSend = 0x0C + // TypeFragData is delivered only after IP reassembly, so its arrival IS the measurement. + TypeFragData = 0x0D ) type Server struct { @@ -232,6 +234,17 @@ func (s *Server) send(conn *net.UDPConn, raddr netip.AddrPort, sess *session.Ses // EMSGSIZE means our own egress MTU refused the datagram, which is a different fact from the // client not receiving it. func (s *Server) sendErr(conn *net.UDPConn, raddr netip.AddrPort, sess *session.Session, typ byte, seq uint32, payload []byte) error { + pkt := s.buildPacket(sess, typ, seq, payload) + _, err := conn.WriteToUDPAddrPort(pkt, raddr) + return err +} + +// buildPacket assembles and signs an ELT1 packet without sending it. +// +// Split out for the crafted-fragment path, which needs the bytes so it can cut them up itself. +// What arrives after reassembly must be indistinguishable from an ordinary packet, or the client +// would be measuring our sender rather than the path — so it goes through exactly this function. +func (s *Server) buildPacket(sess *session.Session, typ byte, seq uint32, payload []byte) []byte { pkt := make([]byte, HeaderSize+len(payload)) copy(pkt[0:4], Magic) pkt[4] = typ @@ -247,8 +260,7 @@ func (s *Server) sendErr(conn *net.UDPConn, raddr netip.AddrPort, sess *session. mac.Write(pkt[0:28]) mac.Write(payload) copy(pkt[28:32], mac.Sum(nil)[:4]) - _, err := conn.WriteToUDPAddrPort(pkt, raddr) - return err + return pkt } func hexByte(hi, lo byte) byte {