From 4ae744aae59c2f2d13a325867a44e6bb96cc9c6e Mon Sep 17 00:00:00 2001 From: mrambossek Date: Fri, 31 Jul 2026 20:44:38 +0200 Subject: [PATCH] =?UTF-8?q?server:=20self-test=20=E2=80=94=20sysctl=20audi?= =?UTF-8?q?t=20+=20egress-MTU=20self-proof=20("server=20proven=20good")?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A measurement server must prove its own host isn't distorting results: - sysctl audit (/proc/sys): flags accept_ra on a static host, ICMP redirects, ICMP rate-limiting of the server's own errors, and disabled TCP options — each a measurement-fidelity hazard, with the "why". - egress-MTU self-proof: DF PMTUD probe (IP_MTU_DISCOVER + getsockopt IP_MTU, no root — Linux-only, stub elsewhere) to external anchors. If the server's own uplink is below 1500, client MTU tests measure THIS server, so we say so. Exposed at GET /admin/selftest (full report) and as server_selftest {mtu_ok, sysctl_ok} in the profile so clients can trust or skip MTU tests. Recommended deploy/99-echolot-sysctl.conf + README section. Co-Authored-By: Claude Opus 5 --- server/README.md | 20 ++++ server/cmd/echolot-server/main.go | 34 +++++++ server/deploy/99-echolot-sysctl.conf | 40 ++++++++ server/internal/config/config.go | 3 + server/internal/control/control.go | 24 ++++- server/internal/selftest/mtu_linux.go | 103 +++++++++++++++++++++ server/internal/selftest/mtu_other.go | 13 +++ server/internal/selftest/selftest.go | 126 ++++++++++++++++++++++++++ 8 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 server/deploy/99-echolot-sysctl.conf create mode 100644 server/internal/selftest/mtu_linux.go create mode 100644 server/internal/selftest/mtu_other.go create mode 100644 server/internal/selftest/selftest.go diff --git a/server/README.md b/server/README.md index b95513d..eb10a61 100644 --- a/server/README.md +++ b/server/README.md @@ -53,6 +53,26 @@ All configurable via `ECHOLOT_*_LISTEN`. Plus: Deliberately out of scope here: an echo listener on 443 (to detect port-based egress filtering) — that genuinely needs 443 and belongs on a dedicated IP, not on a host running a reverse proxy. +## Host tuning (measurement fidelity) + +A measurement server must not let the kernel distort what clients observe. Apply the +recommended sysctls and the daemon will confirm the host is clean: + +```sh +sudo cp deploy/99-echolot-sysctl.conf /etc/sysctl.d/ && sudo sysctl --system +``` + +The daemon **self-tests at startup and via `GET /admin/selftest`** (localhost): +- **sysctl audit** — flags settings that would distort results (RA acceptance on a static host, + ICMP redirects, ICMP rate-limiting of the server's own errors, disabled TCP options). +- **egress-MTU self-proof** — DF-probes external anchors (`ECHOLOT_MTU_PROBE_TARGETS`, + default 1.1.1.1 + a v6 anchor) and reads the discovered path MTU. If the server's *own* uplink + can't carry 1500, client MTU results would measure this server, not the client — so the profile + exposes `server_selftest.mtu_ok` and the log warns loudly. + +Both signals ride in `GET /v1/profile` as `server_selftest` so a client can trust — or skip — +MTU testing accordingly. + ## Run in Docker (config via env) ```sh diff --git a/server/cmd/echolot-server/main.go b/server/cmd/echolot-server/main.go index a489895..268c443 100644 --- a/server/cmd/echolot-server/main.go +++ b/server/cmd/echolot-server/main.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "os/signal" "path/filepath" "strconv" + "sync/atomic" "syscall" "time" @@ -37,6 +39,7 @@ import ( "echo-lot.app/server/internal/config" "echo-lot.app/server/internal/control" "echo-lot.app/server/internal/dataplane" + "echo-lot.app/server/internal/selftest" "echo-lot.app/server/internal/selfupdate" "echo-lot.app/server/internal/session" "echo-lot.app/server/internal/store" @@ -138,11 +141,42 @@ func serve(cfg *config.Config) error { }(addr, ln) } + // Self-test: prove the host is a clean measurement target. Sysctl audit is + // instant; the egress-MTU proof does network round trips, so publish the + // sysctl-only report immediately and swap in the full one when it lands. + var selftestPtr atomic.Pointer[selftest.Report] + initial := selftest.Report{Sysctls: selftest.Sysctls()} + selftestPtr.Store(&initial) + for _, c := range initial.Sysctls { + if c.Severity == selftest.Warn { + slog.Warn("sysctl not measurement-clean", "sysctl", c.Name, "got", c.Got, "want", c.Want, "why", c.Why) + } + } + go func() { + r := selftest.Run(config.Addrs(cfg.MTUProbeTargets)) + selftestPtr.Store(&r) + for _, m := range r.EgressMTU { + if !m.FullMTU { + slog.Warn("egress MTU below 1500 — client MTU results measure THIS server, not the client", + "target", m.Target, "discovered_mtu", m.DiscoveredMTU, "err", m.Err) + } + } + slog.Info("self-test complete", "sysctl_ok", r.SysctlOK, "mtu_ok", r.MTUOK) + }() + ctl.ProvenGood = func() (mtuOK, sysctlOK bool) { + r := selftestPtr.Load() + return r.MTUOK, r.SysctlOK + } + // Admin/health (plain HTTP, localhost by default; spec §7) admin := http.NewServeMux() admin.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, `{"ok":true,"version":%q}`, Version) }) + admin.HandleFunc("GET /admin/selftest", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(selftestPtr.Load()) + }) // TODO(spec §7): enrollment token management + device list. Until the // admin UI exists, mint tokens with: echolot-admin (or curl on this // listener once the endpoint lands). diff --git a/server/deploy/99-echolot-sysctl.conf b/server/deploy/99-echolot-sysctl.conf new file mode 100644 index 0000000..a63c798 --- /dev/null +++ b/server/deploy/99-echolot-sysctl.conf @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2026 Echolot contributors +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Recommended sysctls for an Echolot probe-server host: keep the kernel from +# silently altering what clients measure. Install with: +# sudo cp 99-echolot-sysctl.conf /etc/sysctl.d/ +# sudo sysctl --system +# The daemon audits these at startup and via GET /admin/selftest; anything not +# set here shows up as a "not measurement-clean" warning. + +# Static-addressed host: never let a Router Advertisement mutate our routing. +# (Echolot's whole job is detecting broken RAs — the server must be immune.) +net.ipv6.conf.all.accept_ra = 0 +net.ipv6.conf.default.accept_ra = 0 + +# Don't let ICMP redirects rewrite our routing mid-measurement, and don't +# emit redirects (we're an endpoint, not a router). +net.ipv4.conf.all.accept_redirects = 0 +net.ipv4.conf.default.accept_redirects = 0 +net.ipv6.conf.all.accept_redirects = 0 +net.ipv4.conf.all.send_redirects = 0 +net.ipv4.conf.default.send_redirects = 0 + +# Don't throttle the server's own ICMP errors (dest-unreachable/frag-needed/ +# time-exceeded) — throttling produces false loss/black-hole readings when +# clients probe toward this server. +net.ipv4.icmp_ratelimit = 0 + +# These are usually already correct; pinned so the server can honestly +# negotiate/reflect them (a missing option in a client's evidence is then the +# path's fault, not ours). +net.ipv4.tcp_sack = 1 +net.ipv4.tcp_timestamps = 1 +net.ipv4.tcp_window_scaling = 1 +net.ipv4.ip_no_pmtu_disc = 0 +net.ipv4.icmp_echo_ignore_all = 0 + +# Loose reverse-path filtering suits a multi-IP measurement host (strict mode +# can drop alt-address / asymmetric replies used by STUN 5780). +net.ipv4.conf.all.rp_filter = 2 diff --git a/server/internal/config/config.go b/server/internal/config/config.go index d268058..3b0a3ca 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -32,6 +32,8 @@ type Config struct { // Optional cleartext HTTP-echo listener (spec §4 plaintext-path test). // Default empty = off; it exposes only POST /v1/echo, no auth, no secrets. HTTPEchoListen string // ECHOLOT_HTTP_ECHO_LISTEN / --http-echo-listen + // Comma-separated anchors for the egress-MTU self-proof (host or ip). + MTUProbeTargets string // ECHOLOT_MTU_PROBE_TARGETS / --mtu-probe-targets // Admin UI / health listener (spec §7: localhost-only by default) AdminListen string // ECHOLOT_ADMIN_LISTEN / --admin-listen @@ -75,6 +77,7 @@ func Load(args []string) (*Config, *Actions, error) { fs.StringVar(&c.DNSListen, "dns-listen", envOr("DNS_LISTEN", ""), "canary-DNS listen address(es) udp+tcp/53, comma-separated; empty disables (spec §6.1)") fs.StringVar(&c.CanaryZone, "canary-zone", envOr("CANARY_ZONE", ""), "authoritative canary zone, e.g. c.echo-lot.app") fs.StringVar(&c.HTTPEchoListen, "http-echo-listen", envOr("HTTP_ECHO_LISTEN", ""), "optional CLEARTEXT http-echo listen address(es); empty disables (spec §4)") + fs.StringVar(&c.MTUProbeTargets, "mtu-probe-targets", envOr("MTU_PROBE_TARGETS", "1.1.1.1,2606:4700:4700::1111"), "egress-MTU self-proof anchors, comma-separated") fs.StringVar(&c.AdminListen, "admin-listen", envOr("ADMIN_LISTEN", "127.0.0.1:8444"), "admin/health listen address (keep localhost)") fs.StringVar(&c.StateDir, "state-dir", envOr("STATE_DIR", defaultStateDir()), "state directory (device store, generated TLS)") fs.StringVar(&c.Name, "name", envOr("NAME", "echolot"), "server profile name") diff --git a/server/internal/control/control.go b/server/internal/control/control.go index e45cdfb..cfcef9f 100644 --- a/server/internal/control/control.go +++ b/server/internal/control/control.go @@ -53,6 +53,11 @@ type Server struct { CanaryQueries func(sessionPrefix string) any // CanaryZone is surfaced in the profile so the app knows what to query. CanaryZone string + // ProvenGood reports the server's self-test signal (may be nil). Surfaced + // in the profile so a client can trust — or skip — MTU tests: if the + // server's own egress isn't full-MTU, client MTU results measure the + // server, not the client. + ProvenGood func() (mtuOK, sysctlOK bool) } func (s *Server) Handler() http.Handler { @@ -78,6 +83,16 @@ func (s *Server) EchoHandler() http.Handler { return mux } +// selftestSignal is the compact "server proven good" object for the profile. +// mtu_ok=false tells a client its MTU results would measure this server. +func selftestSignal(f func() (bool, bool)) map[string]any { + if f == nil { + return map[string]any{"mtu_ok": nil, "sysctl_ok": nil} + } + mtuOK, sysctlOK := f() + return map[string]any{"mtu_ok": mtuOK, "sysctl_ok": sysctlOK} +} + // sessionAuth resolves {id} and requires the bearer to be the owning device. func (s *Server) sessionAuth(w http.ResponseWriter, r *http.Request) *session.Session { dev := s.Store.DeviceByCredential(bearer(r)) @@ -264,10 +279,11 @@ func (s *Server) profile(w http.ResponseWriter, r *http.Request) { "tcp_port": s.TCPPort, "stun_port": s.StunPort, }}, - "pins": []string{"pin-sha256:" + s.PinB64}, - "next_pins": []string{}, - "canary_zone": s.CanaryZone, - "limits": map[string]any{"max_kbps": 50000, "max_session_s": 900}, + "pins": []string{"pin-sha256:" + s.PinB64}, + "next_pins": []string{}, + "canary_zone": s.CanaryZone, + "server_selftest": selftestSignal(s.ProvenGood), + "limits": map[string]any{"max_kbps": 50000, "max_session_s": 900}, }) } diff --git a/server/internal/selftest/mtu_linux.go b/server/internal/selftest/mtu_linux.go new file mode 100644 index 0000000..d6e0dff --- /dev/null +++ b/server/internal/selftest/mtu_linux.go @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +//go:build linux + +package selftest + +import ( + "net" + "net/netip" + "syscall" + "time" +) + +// Linux IP-level constants for PMTU discovery. Not all are exported by the +// stdlib syscall package across versions, so they are pinned here (stable +// kernel ABI) — same rationale as the prober's OsAbi. +const ( + ipMTUDiscover = 10 // IP_MTU_DISCOVER + ipMTU = 14 // IP_MTU + ipPMTUDiscDo = 2 // IP_PMTUDISC_DO (set DF, honor PMTU) + ipv6MTUDiscover = 23 // IPV6_MTU_DISCOVER + ipv6MTU = 24 // IPV6_MTU + ipv6PMTUDiscDo = 2 // IPV6_PMTUDISC_DO +) + +// probeEgressMTU sends a DF-flagged full-size UDP datagram toward target and +// reads back the kernel's discovered path MTU. A reduction below 1500 means +// the SERVER's own uplink can't carry full-size packets — so client MTU +// results would measure the server, not the client. No root, no raw socket: +// IP_MTU_DISCOVER + a getsockopt on IP_MTU, mirroring the prober's approach. +func probeEgressMTU(target string) MTUResult { + res := MTUResult{Target: target} + addr, err := netip.ParseAddr(target) + if err != nil { + // allow "host" that resolves + ips, e := net.LookupIP(target) + if e != nil || len(ips) == 0 { + res.Err = "resolve: " + errStr(err) + return res + } + addr, _ = netip.AddrFromSlice(ips[0]) + } + addr = addr.Unmap() + + is4 := addr.Is4() + fam := syscall.AF_INET6 + if is4 { + fam = syscall.AF_INET + } + fd, err := syscall.Socket(fam, syscall.SOCK_DGRAM, 0) + if err != nil { + res.Err = "socket: " + errStr(err) + return res + } + defer syscall.Close(fd) + + if is4 { + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, ipMTUDiscover, ipPMTUDiscDo) + } else { + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_IPV6, ipv6MTUDiscover, ipv6PMTUDiscDo) + } + + // Full-size probe: 1500 total − IP/UDP headers (28 v4, 48 v6). + payload := 1472 + sa := sockaddr(addr, 33434) + if !is4 { + payload = 1452 + } + // A DF send larger than the local MTU fails immediately with EMSGSIZE; a + // path reduction updates IP_MTU after the ICMP frag-needed returns, so we + // send, briefly wait, and read the discovered MTU. + _ = syscall.Sendto(fd, make([]byte, payload), 0, sa) + time.Sleep(700 * time.Millisecond) + _ = syscall.Sendto(fd, make([]byte, payload), 0, sa) // second send observes any reduction + + level, opt := syscall.IPPROTO_IP, ipMTU + if !is4 { + level, opt = syscall.IPPROTO_IPV6, ipv6MTU + } + mtu, err := syscall.GetsockoptInt(fd, level, opt) + if err != nil || mtu <= 0 { + res.Err = "getsockopt IP_MTU: " + errStr(err) + return res + } + res.DiscoveredMTU = mtu + res.FullMTU = mtu >= 1500 + return res +} + +func sockaddr(a netip.Addr, port int) syscall.Sockaddr { + if a.Is4() { + return &syscall.SockaddrInet4{Port: port, Addr: a.As4()} + } + return &syscall.SockaddrInet6{Port: port, Addr: a.As16()} +} + +func errStr(err error) string { + if err == nil { + return "nil" + } + return err.Error() +} diff --git a/server/internal/selftest/mtu_other.go b/server/internal/selftest/mtu_other.go new file mode 100644 index 0000000..20221ad --- /dev/null +++ b/server/internal/selftest/mtu_other.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +//go:build !linux + +package selftest + +// probeEgressMTU: PMTUD via IP_MTU_DISCOVER is Linux-specific. Off-Linux the +// self-test reports MTU as unproven rather than guessing (the daemon runs on +// Linux in production; this keeps dev builds compiling). +func probeEgressMTU(target string) MTUResult { + return MTUResult{Target: target, Err: "egress MTU probe is Linux-only"} +} diff --git a/server/internal/selftest/selftest.go b/server/internal/selftest/selftest.go new file mode 100644 index 0000000..7e2736e --- /dev/null +++ b/server/internal/selftest/selftest.go @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package selftest lets the daemon prove its own host is a clean measurement +// target: the kernel isn't silently altering what clients measure, and the +// server's own egress reaches full MTU. If the server side is already broken, +// client-side results (especially MTU/PMTUD) measure the server, not the +// client — so the daemon says so. +package selftest + +import ( + "os" + "strconv" + "strings" +) + +// Severity of a check result. +type Severity string + +const ( + OK Severity = "ok" + Warn Severity = "warn" +) + +// Check is one sysctl (or derived) assertion. +type Check struct { + Name string `json:"name"` + Got string `json:"got"` + Want string `json:"want"` + Severity Severity `json:"severity"` + Why string `json:"why"` +} + +// MTUResult is one egress path-MTU probe outcome. +type MTUResult struct { + Target string `json:"target"` + DiscoveredMTU int `json:"discovered_mtu"` + FullMTU bool `json:"full_mtu"` // >= 1500 + Err string `json:"err,omitempty"` +} + +// Report is the whole self-test. +type Report struct { + Sysctls []Check `json:"sysctls"` + EgressMTU []MTUResult `json:"egress_mtu"` + // SysctlOK / MTUOK are the compact "server proven good" signals; the + // profile surfaces these so a client can skip MTU tests the server can't + // support honestly. + SysctlOK bool `json:"sysctl_ok"` + MTUOK bool `json:"mtu_ok"` +} + +// readSysctl reads /proc/sys/. Empty string if unavailable. +func readSysctl(name string) string { + p := "/proc/sys/" + strings.ReplaceAll(name, ".", "/") + b, err := os.ReadFile(p) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// sysctlChecks are the measurement-fidelity assertions. Each closure returns +// OK/Warn given the read value; a missing value (non-Linux / restricted) is +// reported as Warn "unreadable" but never fatal. +var sysctlChecks = []struct { + name string + want string + why string + ok func(v string) bool +}{ + {"net.ipv6.conf.all.accept_ra", "0", "static v6 host must not let RAs mutate routing (the very thing Echolot detects)", eq("0")}, + {"net.ipv4.conf.all.accept_redirects", "0", "ICMP redirects could alter routing mid-measurement", eq("0")}, + {"net.ipv4.conf.all.send_redirects", "0", "an endpoint should not emit ICMP redirects", eq("0")}, + {"net.ipv4.icmp_echo_ignore_all", "0", "server must answer ping so clients can measure to it", eq("0")}, + {"net.ipv4.ip_no_pmtu_disc", "0", "server must honor path MTU on its own sends", eq("0")}, + {"net.ipv4.tcp_sack", "1", "so a missing SACK in mss_observed is the path's fault, not the server's", eq("1")}, + {"net.ipv4.tcp_timestamps", "1", "so TCP-timestamp absence reflects the path, not the server", eq("1")}, + {"net.ipv4.tcp_window_scaling", "1", "so wscale absence reflects the path, not the server", eq("1")}, + {"net.ipv4.icmp_ratelimit", "0", "nonzero throttles the server's ICMP errors → false loss/black-hole readings", eq("0")}, +} + +func eq(want string) func(string) bool { return func(v string) bool { return v == want } } + +// Sysctls runs the sysctl audit. +func Sysctls() []Check { + out := make([]Check, 0, len(sysctlChecks)) + for _, c := range sysctlChecks { + got := readSysctl(c.name) + sev := Warn + switch { + case got == "": + got = "(unreadable)" + case c.ok(got): + sev = OK + } + out = append(out, Check{Name: c.name, Got: got, Want: c.want, Severity: sev, Why: c.why}) + } + return out +} + +// Run performs the full self-test: sysctl audit + egress MTU probes to the +// given targets (each "host" — port is irrelevant for PMTUD). +func Run(mtuTargets []string) Report { + r := Report{Sysctls: Sysctls()} + r.SysctlOK = true + for _, c := range r.Sysctls { + if c.Severity == Warn { + r.SysctlOK = false + } + } + r.MTUOK = true + for _, t := range mtuTargets { + res := probeEgressMTU(t) + r.EgressMTU = append(r.EgressMTU, res) + if !res.FullMTU { + r.MTUOK = false + } + } + if len(r.EgressMTU) == 0 { + r.MTUOK = false // couldn't prove it + } + return r +} + +var _ = strconv.Atoi