server: DF-mode big_send + uploaded-run storage with an operator policy

big_send now forces the Don't-Fragment bit for the whole burst by default, so
the largest size that arrives IS the downstream path MTU rather than "fragments
got through" — two different measurements the schema already separates. Sizes
above our own egress MTU (from the startup self-test) are refused up front and
reported as max_df_bytes, because absence caused by our kernel must not be read
as a limit of the client's path.

Uploads: one JSON file per run under the state dir, with the policy the operator
actually cares about — who may upload (off / anonymous / account), how large,
how long to keep, and the least anonymization accepted. The profile advertises
all of it so the app can present the switch honestly instead of discovering the
rules by failing. `account` refuses today rather than falling back to anonymous:
picking the strict setting before OIDC lands must not silently mean the loose one.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
mrambossek
2026-08-01 10:26:19 +02:00
co-authored by Claude Fable 5
parent 7e1015c211
commit 2521d39989
19 changed files with 1172 additions and 31 deletions
+61
View File
@@ -0,0 +1,61 @@
// SPDX-FileCopyrightText: 2026 Echolot contributors
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build linux
package dataplane
import (
"net"
"syscall"
)
// PMTUD socket-option values. Go's syscall package exports IP_MTU_DISCOVER and
// IPV6_MTU_DISCOVER but not the IP_PMTUDISC_* values, so they are spelled out
// here (include/linux/in.h, in6.h — stable ABI, same reasoning as the client's
// OsAbi.kt).
const (
pmtudiscWant = 0 // per-route default: fragment locally when needed
pmtudiscDo = 2 // always set DF: oversized sends fail with EMSGSIZE, never fragment
)
// withDF runs fn with the Don't-Fragment bit forced on for conn, then puts the
// socket back the way it was found.
//
// The socket is shared by every session on that address family, so the caller
// must hold Server.dfMu: a concurrent big_send must not silently ride along
// with someone else's DF window (or, worse, clear it mid-flight).
func withDF(conn *net.UDPConn, fn func() error) error {
raw, err := conn.SyscallConn()
if err != nil {
return err
}
v4 := conn.LocalAddr().(*net.UDPAddr).IP.To4() != nil
level, opt := syscall.IPPROTO_IPV6, syscall.IPV6_MTU_DISCOVER
if v4 {
level, opt = syscall.IPPROTO_IP, syscall.IP_MTU_DISCOVER
}
var setErr error
prev := pmtudiscWant
if err := raw.Control(func(fd uintptr) {
if p, e := syscall.GetsockoptInt(int(fd), level, opt); e == nil {
prev = p
}
setErr = syscall.SetsockoptInt(int(fd), level, opt, pmtudiscDo)
}); err != nil {
return err
}
if setErr != nil {
return setErr
}
defer func() {
_ = raw.Control(func(fd uintptr) {
_ = syscall.SetsockoptInt(int(fd), level, opt, prev)
})
}()
return fn()
}
// dfSupported reports whether withDF can actually set the DF bit here.
const dfSupported = true
+16
View File
@@ -0,0 +1,16 @@
// SPDX-FileCopyrightText: 2026 Echolot contributors
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build !linux
package dataplane
import "net"
// Forcing DF per-socket is Linux-specific (IP_MTU_DISCOVER). Off Linux the
// send still happens — just without the guarantee that nothing fragmented it,
// so the caller must report the result as fragment-delivery evidence rather
// than a path-MTU measurement. See dfSupported.
func withDF(conn *net.UDPConn, fn func() error) error { return fn() }
const dfSupported = false
+59 -22
View File
@@ -52,10 +52,25 @@ func (s *Server) DownTrain(sess *session.Session, g *session.Grant, count, sizeB
return sent, nil
}
// BigSend transmits one datagram per requested size, largest-first metadata intact, so the client
// can see which sizes survive the *downstream* path — the mtu.pmtud_down / mtu.blackhole evidence.
// The client cannot produce this itself: only the far end can emit a large packet toward it.
func (s *Server) BigSend(sess *session.Session, g *session.Grant, sizes []int) ([]int, error) {
// BigSendResult records what happened to one requested size. `Sent` false with an EMSGSIZE-ish
// Err means *we* could not put it on the wire (the datagram exceeds our own egress MTU with DF
// set) — the client must not read its absence as a path limit, so this is reported, not hidden.
type BigSendResult struct {
SizeBytes int `json:"size_bytes"`
Seq int `json:"seq"`
Sent bool `json:"sent"`
Err string `json:"err,omitempty"`
}
// BigSend transmits one datagram per requested size so the client can see which sizes survive the
// *downstream* path — the mtu.pmtud_down / mtu.frag_delivery evidence. The client cannot produce
// this itself: only the far end can emit a large packet toward it.
//
// With df set, the DF bit is forced for the whole burst, so nothing fragments and the largest
// size that arrives IS the downstream path MTU. Without it, the kernel fragments freely and the
// result only says whether fragments get through — a different (also useful) measurement, and
// the reason the two are separate test types.
func (s *Server) BigSend(sess *session.Session, g *session.Grant, sizes []int, df bool) ([]BigSendResult, error) {
target := sess.DataSource()
if !target.IsValid() {
return nil, fmt.Errorf("no observed data-plane source")
@@ -64,24 +79,46 @@ func (s *Server) BigSend(sess *session.Session, g *session.Grant, sizes []int) (
if conn == nil {
return nil, fmt.Errorf("no data-plane socket matches target family")
}
attempted := make([]int, 0, len(sizes))
for i, size := range sizes {
if size < HeaderSize+8 {
size = HeaderSize + 8
results := make([]BigSendResult, 0, len(sizes))
burst := func() error {
for i, size := range sizes {
if size < HeaderSize+8 {
size = HeaderSize + 8
}
if size > 9000 { // jumbo ceiling; beyond this the kernel will refuse anyway
size = 9000
}
if !g.Allow(size) {
break
}
payload := make([]byte, size-HeaderSize)
// Echo the intended size into the payload so a truncated/fragmented arrival is
// still attributable to the size we meant to send.
binary.BigEndian.PutUint32(payload[0:4], uint32(size))
err := s.sendErr(conn, target, sess, TypeBigSend, uint32(i), payload)
results = append(results, BigSendResult{
SizeBytes: size, Seq: i, Sent: err == nil, Err: errString(err),
})
time.Sleep(20 * time.Millisecond) // keep bursts from being read as congestion loss
}
if size > 9000 { // jumbo ceiling; beyond this the kernel will refuse anyway
size = 9000
}
if !g.Allow(size) {
break
}
payload := make([]byte, size-HeaderSize)
// Echo the intended size into the payload so a truncated/fragmented arrival is
// still attributable to the size we meant to send.
binary.BigEndian.PutUint32(payload[0:4], uint32(size))
s.send(conn, target, sess, TypeBigSend, uint32(i), payload)
attempted = append(attempted, size)
time.Sleep(20 * time.Millisecond) // keep bursts from being read as congestion loss
return nil
}
return attempted, nil
if df && dfSupported {
s.dfMu.Lock()
defer s.dfMu.Unlock()
if err := withDF(conn, burst); err != nil {
return results, err
}
return results, nil
}
return results, burst()
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}
+13 -1
View File
@@ -45,6 +45,10 @@ type Server struct {
mu sync.Mutex
conns []*net.UDPConn
// dfMu serialises DF windows: the listening socket is shared by every session on that
// family, so two concurrent big_sends must not overlap their DF on/off transitions.
dfMu sync.Mutex
}
// Serve runs the read loop for one socket; call once per bound address.
@@ -203,6 +207,13 @@ func (s *Server) timesyncResp(conn *net.UDPConn, raddr netip.AddrPort, sess *ses
}
func (s *Server) send(conn *net.UDPConn, raddr netip.AddrPort, sess *session.Session, typ byte, seq uint32, payload []byte) {
_ = s.sendErr(conn, raddr, sess, typ, seq, payload)
}
// sendErr is send with the write error surfaced. Only the DF-mode big_send cares: there an
// EMSGSIZE means our own egress MTU refused the datagram, which is a different fact from the
// client not receiving it.
func (s *Server) sendErr(conn *net.UDPConn, raddr netip.AddrPort, sess *session.Session, typ byte, seq uint32, payload []byte) error {
pkt := make([]byte, HeaderSize+len(payload))
copy(pkt[0:4], Magic)
pkt[4] = typ
@@ -218,7 +229,8 @@ func (s *Server) send(conn *net.UDPConn, raddr netip.AddrPort, sess *session.Ses
mac.Write(pkt[0:28])
mac.Write(payload)
copy(pkt[28:32], mac.Sum(nil)[:4])
_, _ = conn.WriteToUDPAddrPort(pkt, raddr)
_, err := conn.WriteToUDPAddrPort(pkt, raddr)
return err
}
func hexByte(hi, lo byte) byte {