echolot/server/cmd/echolot-server/main.go

// SPDX-FileCopyrightText: 2026 Echolot contributors
// SPDX-License-Identifier: GPL-3.0-or-later

// echolot-server: the probe server (spec: docs/probe-protocol.md).
//
// Modes:
//   - container ("docker mode"): autodetected (or --docker / ECHOLOT_DOCKER=1);
//     config comes from ECHOLOT_* env vars; systemd/self-update are refused.
//   - native: same flags/env, plus --install-systemd / --uninstall-systemd
//     and opt-in --self-update against a Gitea releases API.
package main

import (
	"bufio"
	"context"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"log/slog"
	"math/big"
	"net"
	"net/http"
	"net/netip"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"sync/atomic"
	"syscall"
	"time"

	"echo-lot.app/server/internal/acmehttp"
	"echo-lot.app/server/internal/adminauth"
	"echo-lot.app/server/internal/adminui"
	"echo-lot.app/server/internal/canarydns"
	"echo-lot.app/server/internal/certreload"
	"echo-lot.app/server/internal/compat"
	"echo-lot.app/server/internal/config"
	"echo-lot.app/server/internal/control"
	"echo-lot.app/server/internal/dataplane"
	"echo-lot.app/server/internal/oidc"
	"echo-lot.app/server/internal/runs"
	"echo-lot.app/server/internal/selftest"
	"echo-lot.app/server/internal/selfupdate"
	"echo-lot.app/server/internal/session"
	"echo-lot.app/server/internal/store"
	"echo-lot.app/server/internal/stun"
	"echo-lot.app/server/internal/system"
	"echo-lot.app/server/internal/tcpecho"
)

// Version is stamped via -ldflags "-X main.Version=v1.2.3" in CI.
var Version = "dev"

func main() {
	if err := run(); err != nil {
		slog.Error("fatal", "err", err)
		os.Exit(1)
	}
}

func run() error {
	cfg, actions, err := config.Load(os.Args[1:])
	if err != nil {
		return err
	}
	control.Version = Version

	switch {
	case actions.Help:
		// Compatibility shim for one release.
		//
		// Serving became an explicit verb, but self-update is run by the *old* binary — so the
		// repair added to the updater cannot fix the very update that installs the new one. A
		// unit written before this change starts us with no arguments, and without this branch
		// the service would simply stop working, unattended, on a host nobody is watching.
		//
		// Only when systemd started us: INVOCATION_ID is set by systemd for every service
		// invocation and by nothing else, so a person at a terminal still gets usage. Remove
		// this once no deployment predates --serve.
		if os.Getenv("INVOCATION_ID") != "" {
			slog.Warn("started by systemd with no verb — this unit predates --serve; " +
				"repairing it and serving anyway")
			if repaired, err := system.RepairExecStart(); err != nil {
				slog.Error("could not repair the unit; fix ExecStart by hand", "err", err)
			} else if repaired {
				slog.Info("systemd unit updated to pass --serve")
			}
			return serve(cfg)
		}
		config.Usage(os.Stderr)
		os.Exit(2)
		return nil
	case actions.Version:
		fmt.Println(Version)
		return nil
	case actions.InstallSystemd:
		// The unit runs this same binary in serve mode; host config comes
		// from /etc/echolot-server.env. A self-update timer is installed
		// only when an update API is configured.
		return system.InstallSystemd(cfg.SelfUpdateAPI)
	case actions.UninstallSystemd:
		return system.UninstallSystemd()
	case actions.SetAdminPassword:
		return setAdminPassword(cfg)
	case actions.MintEnrollToken != "":
		return mintEnrollToken(cfg, actions.MintEnrollToken)
	case actions.SelfUpdate:
		return selfupdate.Run(cfg.SelfUpdateAPI, Version)
	}
	return serve(cfg)
}

// mintEnrollToken prints a §2.1 bootstrap link for a new device.
//
// The link is assembled here rather than by hand because it has to carry the public URL and the
// base64 SPKI pin percent-encoded correctly, and a pin wrong by one character fails later as an
// inscrutable TLS error rather than as a bad pin.
func mintEnrollToken(cfg *config.Config, note string) error {
	st, err := store.Open(cfg.StateDir)
	if err != nil {
		return fmt.Errorf("state store: %w", err)
	}
	cert, err := loadOrCreateCert(cfg)
	if err != nil {
		return fmt.Errorf("tls: %w", err)
	}
	pin, err := control.SpkiPinB64(cert)
	if err != nil {
		return fmt.Errorf("pin: %w", err)
	}
	tok, err := st.NewEnrollToken(24*time.Hour, note)
	if err != nil {
		return err
	}
	base := cfg.PublicControlURL
	if base == "" {
		return fmt.Errorf("set ECHOLOT_PUBLIC_URL so the link can say where to connect")
	}
	fmt.Println(control.EnrollmentURI(base, pin, tok))
	// stderr, so piping the command somewhere yields the link alone.
	fmt.Fprintln(os.Stderr, "\nSingle use, valid 24 hours. Treat it like a password until spent.")
	return nil
}

// controlURL is the address devices connect to: the hostname that selects the pinned certificate.
//
// Falls back to the public URL when no separate control hostname is configured, so a server that
// does not share the admin port keeps answering discovery with something usable.
func controlURL(cfg *config.Config) string {
	if cfg.ControlHostname == "" {
		return cfg.PublicControlURL
	}
	return "https://" + cfg.ControlHostname
}

func serve(cfg *config.Config) error {
	slog.Info("echolot-server starting", "version", Version, "mode",
		map[bool]string{true: "container", false: "native"}[cfg.Docker],
		"state_dir", cfg.StateDir)

	st, err := store.Open(cfg.StateDir)
	if err != nil {
		return fmt.Errorf("state store: %w", err)
	}
	cert, err := loadOrCreateCert(cfg)
	if err != nil {
		return fmt.Errorf("tls: %w", err)
	}
	pin, err := control.SpkiPinB64(cert)
	if err != nil {
		return err
	}
	slog.Info("control-plane certificate", "pin-sha256", pin)

	sessions := session.NewManager(15 * time.Minute)
	dp := &dataplane.Server{Sessions: sessions}
	// TCP echo shares the control cert for its elt-echo TLS variant.
	tcpSrv := &tcpecho.Server{
		TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12},
	}

	caps := []string{"udp-probe", "delayed-echo", "connect-back", "http-echo", "downtrain", "big-send", "throughput"}
	// Crafted fragments need a raw socket. Advertised only when one can actually be opened —
	// a capability we cannot deliver turns a missing feature into a failed measurement.
	rawFrag := dataplane.RawFragSupported()
	if rawFrag {
		caps = append(caps, "frag-send")
	} else {
		slog.Info("frag-send unavailable: no raw socket (needs CAP_NET_RAW)")
	}
	if len(config.Addrs(cfg.TCPListen)) > 0 {
		caps = append(caps, "tcp-echo", "tls-echo")
	}

	// Uploaded-run storage. A failure here is not fatal: measurement still works, uploads just
	// stay unavailable and say so in the profile.
	runStore, err := runs.Open(cfg.StateDir, runs.Policy{
		Mode:             runs.Mode(cfg.UploadsMode),
		MaxBytes:         cfg.UploadMaxBytes,
		RetentionDays:    cfg.UploadRetentionDays,
		MaxRunsPerDevice: cfg.UploadMaxRuns,
		MinAnonymization: cfg.UploadMinAnon,
	})
	if err != nil {
		slog.Warn("uploaded-run storage unavailable — uploads disabled", "err", err)
		runStore = nil
	} else {
		slog.Info("uploads", "mode", cfg.UploadsMode, "min_anonymization", cfg.UploadMinAnon,
			"retention_days", cfg.UploadRetentionDays, "max_runs_per_device", cfg.UploadMaxRuns)
	}

	// A malformed window is fatal rather than ignored: an operator who set a restriction must
	// not end up running without one because of a typo.
	appRange, err := compat.ParseRange(cfg.MinAppVersion, cfg.MaxAppVersion)
	if err != nil {
		return fmt.Errorf("app version window: %w", err)
	}
	slog.Info("client compatibility", "accepts_app", appRange.String(),
		"protocol", control.ProtocolVersion, "schema", control.SchemaVersion)

	// Identity is optional. Without an issuer the server simply has no sign-in, and
	// uploads=account can never be satisfied — which is the honest outcome, not a silent
	// downgrade to anonymous.
	// One verifier per issuer. An IdP may mint a distinct issuer per application — Authentik
	// derives it from the application slug — and a token's `iss` must match whoever signed it.
	// Each verifier accepts only the client belonging to its own issuer, so a token minted for
	// the phone cannot be replayed at the admin login and vice versa.
	var idp, adminIdP *oidc.Verifier
	appIssuer := cfg.OIDCAppIssuer
	if appIssuer == "" {
		appIssuer = cfg.OIDCIssuer // IdPs with one global issuer
	}
	if appIssuer != "" && cfg.OIDCAppClientID != "" {
		idp = oidc.New(oidc.Config{
			Issuer: appIssuer, AppClientID: cfg.OIDCAppClientID, AdminGroup: cfg.OIDCAdminGroup,
		}, nil)
		slog.Info("identity: app client", "issuer", appIssuer, "client_id", cfg.OIDCAppClientID)
	}
	if cfg.OIDCIssuer != "" && cfg.OIDCClientID != "" {
		adminIdP = oidc.New(oidc.Config{
			Issuer: cfg.OIDCIssuer, ClientID: cfg.OIDCClientID, AdminGroup: cfg.OIDCAdminGroup,
		}, nil)
		slog.Info("identity: admin client", "issuer", cfg.OIDCIssuer,
			"client_id", cfg.OIDCClientID, "admin_group", cfg.OIDCAdminGroup)
		if cfg.OIDCAdminGroup == "" {
			slog.Warn("no admin group set: nobody will be an admin via OIDC " +
				"(set ECHOLOT_OIDC_ADMIN_GROUP)")
		}
	}
	if idp == nil && adminIdP == nil && cfg.UploadsMode == string(runs.ModeAccount) {
		slog.Warn("uploads=account but no identity provider is configured — " +
			"every upload will be refused")
	}

	ip4, ip6, ip4Alt, ip6Alt := cfg.MeasurementAddrs()
	ctl := &control.Server{
		IP4: ip4, IP6: ip6, IP4Alt: ip4Alt, IP6Alt: ip6Alt,
		Store: st, Sessions: sessions, Name: cfg.Name,
		UDPPort: mustPort(firstAddr(cfg.UDPListen)), TCPPort: mustPort(firstAddr(cfg.TCPListen)),
		StunPort: mustPort(firstAddr(cfg.StunListen)), PinB64: pin, CertChain: cert.Certificate,
		DelayedEcho:      dp.SendDelayedEcho,
		DownTrain:        dp.DownTrain,
		BigSend:          dp.BigSend,
		TCPRecent:        func(ip string) any { return tcpSrv.RecentFor(ip) },
		Runs:             runStore,
		AppRange:         appRange,
		PublicControlURL: publicControlURL(cfg),
		OIDC:             idp,
		AdminOIDC:        adminIdP,
	}
	// Left nil when there is no raw socket, so the handler answers "not implemented" with a
	// reason rather than failing somewhere deeper.
	if rawFrag {
		ctl.FragSend = dp.FragSend
	}
	ctl.DownThroughput = dp.DownThroughput

	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()
	ctlAddrs := config.Addrs(cfg.ControlListen)
	udpAddrs := config.Addrs(cfg.UDPListen)
	errCh := make(chan error, len(ctlAddrs)+len(udpAddrs)+2)

	// Control plane (HTTPS, pin-based trust) — one shared server, one
	// listener per configured address; Shutdown closes them all.
	ctlSrv := &http.Server{
		Handler:           ctl.Handler(),
		TLSConfig:         &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12},
		ReadHeaderTimeout: 10 * time.Second,
	}
	for _, addr := range ctlAddrs {
		ln, err := net.Listen("tcp", addr)
		if err != nil {
			return fmt.Errorf("control listen %s: %w", addr, err)
		}
		go func(a string, l net.Listener) {
			errCh <- fmt.Errorf("control %s: %w", a, ctlSrv.ServeTLS(l, "", ""))
		}(addr, ln)
	}

	// Self-test: prove the host is a clean measurement target. Sysctl audit is
	// instant; the egress-MTU proof does network round trips, so publish the
	// sysctl-only report immediately and swap in the full one when it lands.
	var selftestPtr atomic.Pointer[selftest.Report]
	initial := selftest.Report{Sysctls: selftest.Sysctls()}
	selftestPtr.Store(&initial)
	for _, c := range initial.Sysctls {
		if c.Severity == selftest.Warn {
			slog.Warn("sysctl not measurement-clean", "sysctl", c.Name, "got", c.Got, "want", c.Want, "why", c.Why)
		}
	}
	go func() {
		r := selftest.Run(config.Addrs(cfg.MTUProbeTargets))
		selftestPtr.Store(&r)
		for _, m := range r.EgressMTU {
			if !m.FullMTU {
				slog.Warn("egress MTU below 1500 — client MTU results measure THIS server, not the client",
					"target", m.Target, "discovered_mtu", m.DiscoveredMTU, "err", m.Err)
			}
		}
		slog.Info("self-test complete", "sysctl_ok", r.SysctlOK, "mtu_ok", r.MTUOK)
	}()
	ctl.ProvenGood = func() (mtuOK, sysctlOK bool) {
		r := selftestPtr.Load()
		return r.MTUOK, r.SysctlOK
	}
	// The smallest egress MTU we measured is the ceiling for DF-mode big_send: above it our own
	// kernel refuses the datagram, which would otherwise look like a downstream path limit.
	ctl.EgressMTU = func() int {
		best := 0
		for _, m := range selftestPtr.Load().EgressMTU {
			if m.DiscoveredMTU > 0 && (best == 0 || m.DiscoveredMTU < best) {
				best = m.DiscoveredMTU
			}
		}
		return best
	}

	// The admin interface. Every route except /healthz requires a session — the old arrangement
	// (no auth, kept safe by binding to loopback) failed the moment the address changed, and a
	// binding address is a deployment detail rather than an access control.
	secret, err := st.SessionSecret()
	if err != nil {
		return fmt.Errorf("admin session secret: %w", err)
	}
	adminSecure := cfg.AdminTLSCert != ""
	ui := &adminui.Server{
		Store:        st,
		Runs:         runStore,
		OIDC:         adminIdP,
		Sessions:     adminauth.NewSessions(secret, 12*time.Hour),
		Throttle:     adminauth.NewThrottle(),
		AdminUser:    cfg.AdminUser,
		BaseURL:      cfg.AdminBaseURL,
		ClientSecret: cfg.OIDCClientSecret,
		Secure:       adminSecure,
		EnrollLink:   ctl.EnrollmentLink,
		// Where devices should connect, for /v1/discover. Derived from the control hostname so it
		// cannot drift from the name that actually selects the pinned certificate.
		ControlURL:   controlURL(cfg),
		ServerName:   cfg.Name,
		SelfTest:     func() any { return selftestPtr.Load() },
		Version:      Version,
	}
	if st.LocalAdmin() == nil && adminIdP == nil {
		slog.Warn("nobody can sign in to the admin UI: no break-glass password is set " +
			"(--set-admin-password) and no identity provider is configured")
	}
	admin := ui.Handler()

	// One listener per configured address, all serving the same handler.
	//
	// Multi-address rather than a wildcard because this host reserves addresses for measurement:
	// binding 0.0.0.0 would put the admin UI on port 443 of the reserved pair, and their value
	// comes precisely from nothing answering there. Explicit addresses are also what let the
	// service and management addresses differ without a second process.
	adminAddrs := config.Addrs(cfg.AdminListen)
	if len(adminAddrs) == 0 {
		return fmt.Errorf("admin: no listen address configured")
	}
	var adminTLS *tls.Config
	if cfg.AdminTLSCert != "" {
		// Terminated here rather than behind a reverse proxy: this binary already serves TLS for
		// the control plane, so it is reuse rather than new machinery, and one process with one
		// config file is the property that makes this pleasant to run. A proxy would also invite
		// someone to later front the control plane too, which would break SPKI pinning.
		reloader, err := certreload.New(cfg.AdminTLSCert, cfg.AdminTLSKey)
		if err != nil {
			return fmt.Errorf("admin TLS: %w", err)
		}
		adminTLS = reloader.TLSConfig()
		if exp := reloader.NotAfter(); !exp.IsZero() {
			slog.Info("admin UI TLS", "listen", adminAddrs, "cert_expires", exp.Format(time.RFC3339))
			if time.Until(exp) < 14*24*time.Hour {
				slog.Warn("admin certificate expires soon", "expires", exp.Format(time.RFC3339))
			}
		}
	}
	// Kept for shutdown: each listener gets its own server, and a graceful stop has to reach all
	// of them or an in-flight admin request is cut off mid-response on every address but one.
	var adminSrvs []*http.Server
	// Sharing port 443 between two services that cannot share a certificate. The name in the TLS
	// handshake picks the certificate, and the name in the request picks the handler; both have to
	// agree or a client would get the pinned certificate and the admin UI behind it.
	//
	// The control plane keeps its own listener as well. Devices enrolled before this carry the old
	// URL in their settings, and taking that away would strand every one of them for the sake of a
	// port number.
	ctlHandler := ctl.Handler()
	sharedCert := cert
	// Which side of the port a request belongs to.
	//
	// The control hostname is the obvious case. A bare IP is the other one, and it matters: a
	// client whose DNS has failed can still reach the server by an address it cached from the
	// profile, and a measurement tool that cannot report from a broken network is useless
	// precisely when it is needed. That client authenticates by pin, so the name it used to get
	// here is not part of the trust decision.
	//
	// Safe to route that way because the admin UI is only ever reached by name: browsers always
	// send SNI and nobody bookmarks an IP for a site with a Let's Encrypt certificate. Anything
	// addressing this server numerically is a pinned client.
	isControl := func(host string) bool {
		if h, _, err := net.SplitHostPort(host); err == nil {
			host = h
		}
		host = strings.Trim(host, "[]")
		if cfg.ControlHostname != "" && strings.EqualFold(host, cfg.ControlHostname) {
			return true
		}
		return net.ParseIP(host) != nil
	}
	pickCert := func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
		// No SNI at all also means a numeric client: every browser sends it.
		if hi.ServerName == "" || isControl(hi.ServerName) {
			return &sharedCert, nil
		}
		if adminTLS != nil && adminTLS.GetCertificate != nil {
			return adminTLS.GetCertificate(hi)
		}
		return &sharedCert, nil
	}
	route := func(w http.ResponseWriter, r *http.Request) {
		if isControl(r.Host) {
			ctlHandler.ServeHTTP(w, r)
			return
		}
		admin.ServeHTTP(w, r)
	}
	sharedTLS := &tls.Config{GetCertificate: pickCert, MinVersion: tls.VersionTLS12}
	if cfg.ControlHostname != "" {
		slog.Info("control plane shares the admin port",
			"hostname", cfg.ControlHostname, "listen", adminAddrs)
	}

	for _, addr := range adminAddrs {
		// Bound before the goroutine starts, so a bad address fails startup rather than being
		// reported asynchronously after the process has already declared itself healthy.
		ln, err := net.Listen("tcp", addr)
		if err != nil {
			return fmt.Errorf("admin listen %s: %w", addr, err)
		}
		srv := &http.Server{
			Handler:           http.HandlerFunc(route),
			ReadHeaderTimeout: 10 * time.Second,
			TLSConfig:         sharedTLS,
		}
		adminSrvs = append(adminSrvs, srv)
		go func(ln net.Listener, addr string) {
			// Plaintext only where there is no certificate at all — checkAdminExposure has
			// already refused that anywhere but loopback.
			if adminTLS == nil && cfg.ControlHostname == "" {
				errCh <- fmt.Errorf("admin %s: %w", addr, srv.Serve(ln))
				return
			}
			errCh <- fmt.Errorf("admin %s: %w", addr, srv.ServeTLS(ln, "", ""))
		}(ln, addr)
	}

	// ACME HTTP-01 responder. Permanent rather than started per renewal: nothing binds and
	// unbinds, so a renewal cannot fail because the port was briefly busy, and the ACME client
	// needs only write access to a directory instead of the privilege to bind a low port.
	if cfg.ACMEHTTPListen != "" {
		webroot := cfg.ACMEWebroot
		if webroot == "" {
			webroot = filepath.Join(cfg.StateDir, "acme")
		}
		if err := acmehttp.EnsureWebroot(webroot); err != nil {
			return fmt.Errorf("acme webroot: %w", err)
		}
		acmeHandler := acmehttp.Handler(webroot, cfg.AdminBaseURL)
		for _, addr := range config.Addrs(cfg.ACMEHTTPListen) {
			ln, err := net.Listen("tcp", addr)
			if err != nil {
				return fmt.Errorf("acme-http listen %s: %w", addr, err)
			}
			srv := &http.Server{Handler: acmeHandler, ReadHeaderTimeout: 10 * time.Second}
			go func(ln net.Listener, addr string) {
				errCh <- fmt.Errorf("acme-http %s: %w", addr, srv.Serve(ln))
			}(ln, addr)
		}
		// Every address the name may resolve to needs the responder: the CA picks one, and a
		// challenge that lands on an unbound address fails a renewal rather than a request.
		slog.Info("acme http-01 responder", "listen", config.Addrs(cfg.ACMEHTTPListen),
			"webroot", webroot, "redirects_to", cfg.AdminBaseURL)
	}

	// UDP data plane — one socket per configured address. Distinct sockets
	// (not wildcard) also guarantee responses leave from the address the
	// request arrived on, which stun-5780 will rely on.
	var udpConns []*net.UDPConn
	for _, addr := range udpAddrs {
		udpAddr, err := net.ResolveUDPAddr("udp", addr)
		if err != nil {
			return fmt.Errorf("udp addr %s: %w", addr, err)
		}
		conn, err := net.ListenUDP("udp", udpAddr)
		if err != nil {
			return fmt.Errorf("udp listen %s: %w", addr, err)
		}
		udpConns = append(udpConns, conn)
		go func(a string, c *net.UDPConn) {
			errCh <- fmt.Errorf("udp %s: %w", a, dp.Serve(c))
		}(addr, conn)
	}

	// TCP echo (spec §4)
	var tcpLns []net.Listener
	for _, addr := range config.Addrs(cfg.TCPListen) {
		ln, err := net.Listen("tcp", addr)
		if err != nil {
			return fmt.Errorf("tcp listen %s: %w", addr, err)
		}
		tcpLns = append(tcpLns, ln)
		go func(a string, l net.Listener) {
			errCh <- fmt.Errorf("tcp %s: %w", a, tcpSrv.Serve(l))
		}(addr, ln)
	}

	// Optional cleartext HTTP-echo (spec §4 plaintext-path test) — only
	// POST /v1/echo, no auth, no secrets. Off unless configured.
	var httpEchoSrvs []*http.Server
	for _, addr := range config.Addrs(cfg.HTTPEchoListen) {
		hs := &http.Server{Addr: addr, Handler: ctl.EchoHandler(), ReadHeaderTimeout: 10 * time.Second}
		httpEchoSrvs = append(httpEchoSrvs, hs)
		go func(a string, srv *http.Server) { errCh <- fmt.Errorf("http-echo %s: %w", a, srv.ListenAndServe()) }(addr, hs)
	}

	// STUN (spec §4) — advertises stun-5780 only with ≥2 same-family addrs.
	var stunSrv *stun.Server
	if stunAddrs := config.Addrs(cfg.StunListen); len(stunAddrs) > 0 {
		stunSrv, err = stun.Listen(stunAddrs)
		if err != nil {
			return fmt.Errorf("stun listen: %w", err)
		}
		go func() { errCh <- fmt.Errorf("stun: %w", stunSrv.Serve()) }()
		if stunSrv.Has5780() {
			ctl.Capabilities = append(caps, "stun-5780")
		} else {
			ctl.Capabilities = append(caps, "stun-basic")
		}
	} else {
		ctl.Capabilities = caps
	}

	// Canary DNS (spec §6.1) — authoritative for CanaryZone, udp+tcp per addr.
	var dnsUDP []*net.UDPConn
	var dnsTCP []net.Listener
	if dnsAddrs := config.Addrs(cfg.DNSListen); len(dnsAddrs) > 0 && cfg.CanaryZone != "" {
		v4, v6 := firstByFamily(dnsAddrs)
		cd := canarydns.New(cfg.CanaryZone, cfg.Name, v4, v6)
		for _, addr := range dnsAddrs {
			ua, err := net.ResolveUDPAddr("udp", addr)
			if err != nil {
				return fmt.Errorf("dns udp addr %s: %w", addr, err)
			}
			uc, err := net.ListenUDP("udp", ua)
			if err != nil {
				return fmt.Errorf("dns udp listen %s: %w", addr, err)
			}
			dnsUDP = append(dnsUDP, uc)
			go func(a string, c *net.UDPConn) { errCh <- fmt.Errorf("dns-udp %s: %w", a, cd.ServeUDP(c)) }(addr, uc)

			tl, err := net.Listen("tcp", addr)
			if err != nil {
				return fmt.Errorf("dns tcp listen %s: %w", addr, err)
			}
			dnsTCP = append(dnsTCP, tl)
			go func(a string, l net.Listener) { errCh <- fmt.Errorf("dns-tcp %s: %w", a, cd.ServeTCP(l)) }(addr, tl)
		}
		ctl.CanaryZone = cfg.CanaryZone
		ctl.CanaryQueries = func(prefix string) any { return cd.RecentForPrefix(prefix) }
		ctl.Capabilities = append(ctl.Capabilities, "canary-dns")
	}

	slog.Info("listening",
		"control", ctlAddrs, "admin", adminAddrs, "udp", udpAddrs,
		"tcp", config.Addrs(cfg.TCPListen), "stun", config.Addrs(cfg.StunListen),
		"dns", config.Addrs(cfg.DNSListen), "capabilities", ctl.Capabilities)

	select {
	case <-ctx.Done():
		slog.Info("shutting down")
		shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		_ = ctlSrv.Shutdown(shutCtx)
		for _, srv := range adminSrvs {
			_ = srv.Shutdown(shutCtx)
		}
		for _, c := range udpConns {
			_ = c.Close()
		}
		for _, l := range tcpLns {
			_ = l.Close()
		}
		if stunSrv != nil {
			stunSrv.Close()
		}
		for _, c := range dnsUDP {
			_ = c.Close()
		}
		for _, l := range dnsTCP {
			_ = l.Close()
		}
		for _, hs := range httpEchoSrvs {
			_ = hs.Shutdown(shutCtx)
		}
		return nil
	case err := <-errCh:
		return err
	}
}

// firstByFamily returns the first v4 and first v6 address from a list of
// "ip:port" specs — used for the canary zone's apex/NS answers.
func firstByFamily(addrs []string) (v4, v6 netip.Addr) {
	for _, a := range addrs {
		if ap, err := netip.ParseAddrPort(a); err == nil {
			if ap.Addr().Unmap().Is4() && !v4.IsValid() {
				v4 = ap.Addr().Unmap()
			} else if ap.Addr().Is6() && !ap.Addr().Is4In6() && !v6.IsValid() {
				v6 = ap.Addr()
			}
		}
	}
	return
}

func firstAddr(spec string) string {
	if a := config.Addrs(spec); len(a) > 0 {
		return a[0]
	}
	return ""
}

func mustPort(listen string) int {
	_, p, err := net.SplitHostPort(listen)
	if err != nil {
		return 0
	}
	n, _ := strconv.Atoi(p)
	return n
}

// loadOrCreateCert loads the configured cert/key, or generates a self-signed
// ECDSA P-256 cert into the state dir on first start. Self-signed is
// first-class per spec §1 — clients pin the SPKI, they don't chase CAs.
func loadOrCreateCert(cfg *config.Config) (tls.Certificate, error) {
	certPath, keyPath := cfg.TLSCert, cfg.TLSKey
	if certPath == "" {
		certPath = filepath.Join(cfg.StateDir, "tls-cert.pem")
		keyPath = filepath.Join(cfg.StateDir, "tls-key.pem")
	}
	if c, err := tls.LoadX509KeyPair(certPath, keyPath); err == nil {
		return c, nil
	} else if cfg.TLSCert != "" {
		// Explicitly configured paths must exist — do not silently overwrite.
		return tls.Certificate{}, fmt.Errorf("loading configured cert %s: %w", certPath, err)
	}

	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		return tls.Certificate{}, err
	}
	serial, _ := rand.Int(rand.Reader, big.NewInt(1).Lsh(big.NewInt(1), 62))
	tmpl := x509.Certificate{
		SerialNumber: serial,
		Subject:      pkix.Name{CommonName: "echolot-server"},
		NotBefore:    time.Now().Add(-time.Hour),
		// Long-lived on purpose: trust is the SPKI pin from enrollment, not
		// cert validity; rotation happens via next_pins in the profile.
		NotAfter:              time.Now().AddDate(10, 0, 0),
		KeyUsage:              x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key)
	if err != nil {
		return tls.Certificate{}, err
	}
	keyDer, err := x509.MarshalECPrivateKey(key)
	if err != nil {
		return tls.Certificate{}, err
	}
	if err := os.MkdirAll(cfg.StateDir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
		return tls.Certificate{}, err
	}
	certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
	keyPem := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer})
	if err := os.WriteFile(certPath, certPem, 0o644); err != nil {
		return tls.Certificate{}, err
	}
	if err := os.WriteFile(keyPath, keyPem, 0o600); err != nil {
		return tls.Certificate{}, err
	}
	slog.Info("generated self-signed certificate", "cert", certPath)
	return tls.X509KeyPair(certPem, keyPem)
}

// publicControlURL is where clients should reach this server's control plane.
//
// Configured wins; otherwise the first control listen address is used, which is correct for the
// plain case (bind an address, hand out that address). A wildcard bind has no single right answer,
// so it is left to the operator rather than guessed — a link pointing at 0.0.0.0 is worse than a
// link the operator was told to configure.
func publicControlURL(cfg *config.Config) string {
	if cfg.PublicControlURL != "" {
		return strings.TrimRight(cfg.PublicControlURL, "/")
	}
	addr := firstAddr(cfg.ControlListen)
	if addr == "" {
		return ""
	}
	if strings.HasPrefix(addr, ":") || strings.HasPrefix(addr, "0.0.0.0:") || strings.HasPrefix(addr, "[::]:") {
		slog.Warn("control plane is bound to a wildcard address; set ECHOLOT_PUBLIC_URL "+
			"so enrollment links point somewhere reachable", "listen", addr)
	}
	return "https://" + addr
}

// setAdminPassword stores the break-glass admin credential.
//
// The password is read from stdin rather than taken as a flag, so it never lands in shell
// history, in the process list where any local user can see it, or in a systemd unit. Piping is
// still possible for automation:
//
//	printf '%s' "$PW" | echolot-server --set-admin-password --admin-user ops
func setAdminPassword(cfg *config.Config) error {
	st, err := store.Open(cfg.StateDir)
	if err != nil {
		return fmt.Errorf("state store: %w", err)
	}
	fmt.Fprintf(os.Stderr, "New password for %q (input is not echoed if this is a terminal): ", cfg.AdminUser)
	pw, err := readSecret()
	if err != nil {
		return err
	}
	fmt.Fprintln(os.Stderr)

	cred, err := adminauth.NewCredential(cfg.AdminUser, pw)
	if err != nil {
		return err
	}
	if err := st.SetLocalAdmin(cred); err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "Break-glass admin %q set. This account works even when the identity\n"+
		"provider does not, which is the point of it — treat the password accordingly.\n", cfg.AdminUser)
	return nil
}

// readSecret reads one line from stdin, without echo where the terminal allows it.
func readSecret() (string, error) {
	restore, _ := system.DisableEcho(os.Stdin)
	if restore != nil {
		defer restore()
	}
	r := bufio.NewReader(os.Stdin)
	line, err := r.ReadString('\n')
	if err != nil && line == "" {
		return "", err
	}
	return strings.TrimSpace(line), nil
}