diff --git a/docs/build-status.md b/docs/build-status.md index 1442030..5517023 100644 --- a/docs/build-status.md +++ b/docs/build-status.md @@ -965,3 +965,40 @@ one session measures itself rather than inheriting the first one's packets. The Live against fmr: **3125 sent, 3125 counted, 0 % loss, 10.0 Mbit/s** at a 10 Mbit/s request, with `measures_network: false` — correct, since what arrived matched what was offered, so the path was never the constraint. + +### Raw shell dumps leaked the whole LAN (2026-08-01) +Found by running the Shizuku shell tier for the first time. The tier works — `tiers.shizuku: true`, +`exec_path: UserService` (so the UserService binds on the OnePlus, as recorded), `runs_as +shell(2000)`, 7/7 commands — and the run promptly uploaded **every MAC address on the local +network** to fmr at the `balanced` level: router, phones, whatever else was on the wifi. Fourteen +of them. + +The probes embed raw command output verbatim (`ip neigh`, `ip route`, `id`), which is genuinely +good evidence and also a complete household device inventory. The anonymizer could not see it: +classification is by field name and by whole-value shape, and `ip_neigh` is one long string that is +itself neither a MAC nor an address. measurement-schema.md §9 item 2 had flagged raw dumps as "hard +to anonymize" and proposed dropping them from exports; nothing enforced either. + +**Scrubbing beats dropping.** Identifiers inside any unclassified string are now replaced in place, +using the same pseudonyms as everywhere else — so a MAC that appears both in a parsed field and in +a raw dump still reads as one device. The dump stays readable and auditable: you can still see the +neighbour table's shape, the host count, RFC1918 addresses and vendor prefixes. Dropping the +evidence would have protected the same data while destroying the reason for collecting it. + +Two implementation notes worth keeping: +- **One pass, not three.** Sequential passes re-process their own output: once a MAC became + `78:9a:18:xx:yy:zz`, the IPv6 pattern matched it — six hex groups separated by colons *is* an + address — and destroyed the vendor prefix the MAC rule had just preserved. Ordered alternation + resolves each position once, MAC first. +- The patterns are conservative on purpose. A missed address gets caught by another rule or not at + all; an over-eager one mangles timestamps and version strings, corrupting evidence to protect + nothing. + +`RealDocumentTest` runs the anonymizer over a captured run when `ECHOLOT_REAL_RUN` points at one, +and fails on any MAC that survives. It self-skips otherwise, so no one's network is committed to the +repo. Against the actual leaked document: **14 MACs in, 0 surviving.** + +Also fixed: the Settings *Preview what an upload would send* button did nothing. It read +`UiState.history`, which is empty until the History screen has been opened — the same root cause as +the "0 run(s)" count. It now reads the archive directly, and says so when there is nothing to +preview rather than silently ignoring the tap. diff --git a/echolot-app/app/src/main/kotlin/app/echo_lot/app/MainActivity.kt b/echolot-app/app/src/main/kotlin/app/echo_lot/app/MainActivity.kt index 1a8b6b8..4c2506b 100644 --- a/echolot-app/app/src/main/kotlin/app/echo_lot/app/MainActivity.kt +++ b/echolot-app/app/src/main/kotlin/app/echo_lot/app/MainActivity.kt @@ -101,11 +101,10 @@ class MainActivity : ComponentActivity() { onApplyRetention = vm::applyRetention, onDeleteAll = vm::deleteAllRuns, onPreviewUpload = { - // Preview the newest run, since that is the one the user just made - // and the one they are deciding about. - vm.state.history.firstOrNull()?.let { r -> - lifecycleScope.launch { preview = vm.uploadPreview(r.id) } - } + // Straight from the archive: the newest run is the one the user just + // made and the one they are deciding about. Always shows something, + // even when there is nothing to preview yet. + lifecycleScope.launch { preview = vm.previewNewestRun() } }, onCheckServer = vm::checkServer, onEnroll = vm::enroll, diff --git a/echolot-app/app/src/main/kotlin/app/echo_lot/app/RunViewModel.kt b/echolot-app/app/src/main/kotlin/app/echo_lot/app/RunViewModel.kt index e2200c6..8d54cab 100644 --- a/echolot-app/app/src/main/kotlin/app/echo_lot/app/RunViewModel.kt +++ b/echolot-app/app/src/main/kotlin/app/echo_lot/app/RunViewModel.kt @@ -194,6 +194,21 @@ class RunViewModel(app: Application) : AndroidViewModel(app) { store.read(id)?.let { store.redactedForUpload(it) } } + /** + * Preview of the most recent run, read from the archive rather than from [UiState.history]. + * + * The history list is only populated once the History screen has been opened, so a preview + * driven from it did nothing at all on a freshly-opened Settings screen — a button that + * silently does nothing is worse than one that says why. + */ + suspend fun previewNewestRun(): String = withContext(Dispatchers.IO) { + val newest = store.list().firstOrNull() + ?: return@withContext "No archived runs yet. Run a measurement first, then this will " + + "show exactly what an upload would send." + store.read(newest.id)?.let { store.redactedForUpload(it) } + ?: "That run could not be read back from the archive." + } + fun archivedBytes(): Long = store.totalBytes() /** Counted from the archive itself, not from [UiState.history], which is empty until the diff --git a/echolot-app/core-privacy/build.gradle.kts b/echolot-app/core-privacy/build.gradle.kts index 9a5af66..b48f8f8 100644 --- a/echolot-app/core-privacy/build.gradle.kts +++ b/echolot-app/core-privacy/build.gradle.kts @@ -20,4 +20,8 @@ kotlin { } java { sourceCompatibility = JavaVersion.VERSION_17; targetCompatibility = JavaVersion.VERSION_17 } -tasks.test { useJUnitPlatform() } +tasks.test { + useJUnitPlatform() + // Opt-in: point this at a captured run to check the anonymizer against real data. + System.getenv("ECHOLOT_REAL_RUN")?.let { environment("ECHOLOT_REAL_RUN", it) } +} diff --git a/echolot-app/core-privacy/src/main/kotlin/app/echo_lot/privacy/Anonymizer.kt b/echolot-app/core-privacy/src/main/kotlin/app/echo_lot/privacy/Anonymizer.kt index 76c3b21..d0243aa 100644 --- a/echolot-app/core-privacy/src/main/kotlin/app/echo_lot/privacy/Anonymizer.kt +++ b/echolot-app/core-privacy/src/main/kotlin/app/echo_lot/privacy/Anonymizer.kt @@ -1,276 +1,327 @@ -// SPDX-FileCopyrightText: 2026 Echolot contributors -// SPDX-License-Identifier: GPL-3.0-or-later - -// Package privacy implements the anonymization contract of measurement-schema.md §8. -// -// The threat model is specific. An engineer running their own server wants the full document — -// SSIDs and MACs are what make a run useful a week later. Someone measuring against a stranger's -// server wants the numbers to survive and the identifiers not to. So this is a *transform*, not a -// filter: the output is still a valid measurement document with the same tests, metrics and -// findings; only the identifying scalars change, and consistently, so "same SSID as last run" is -// still answerable from pseudonyms alone. -// -// Two properties are load-bearing and are what the tests pin: -// - Consistency within a document: one input value always maps to one pseudonym, so -// correlations inside a run survive. -// - No consistency *across* documents unless the user asks for it: the salt is per-run by -// default, so pseudonyms cannot be used to track a device between uploads. A stable salt is -// opt-in (`Salt.stable`) for people diffing their own history on their own server. -package app.echo_lot.privacy - -import kotlinx.serialization.json.* -import java.security.MessageDigest -import java.util.Locale - -/** How much to strip. Ordered: FULL < BALANCED < STRICT. Wire values match the server's. */ -enum class PrivacyLevel(val wire: String) { - /** Nothing removed. The right choice for your own server. */ - FULL("full"), - - /** - * Identifiers pseudonymized, neighbour inventory dropped. Topology and timing survive: - * you can still see that the gateway is a MikroTik at a /24 boundary with 3 % loss, but not - * which MikroTik, on which SSID, next to whose Chromecast. - */ - BALANCED("balanced"), - - /** - * Numbers only: tests keep their metrics and status, evidence is dropped, findings keep their - * codes and severities but lose descriptions (which quote real names). What is left cannot - * identify a network, and is still enough for aggregate "how common is this fault" work. - */ - STRICT("strict"); - - companion object { - fun fromWire(s: String?): PrivacyLevel = - entries.firstOrNull { it.wire == s?.lowercase(Locale.ROOT) } ?: FULL - - /** The stricter of two levels — used to honour a server's minimum. */ - fun max(a: PrivacyLevel, b: PrivacyLevel): PrivacyLevel = if (a.ordinal >= b.ordinal) a else b - } -} - -/** - * The pseudonymization salt. Per-run by default: a fresh random salt means the same SSID uploaded - * twice yields two different pseudonyms, so an upload endpoint cannot link runs to a device. - * A stable salt trades that away for cross-run diffing and is only appropriate on a server you - * own — the app makes that an explicit choice, not a default. - */ -class Salt private constructor(internal val bytes: ByteArray, val stable: Boolean) { - companion object { - fun perRun(random: ByteArray): Salt = Salt(random.copyOf(), stable = false) - fun stable(secret: ByteArray): Salt = Salt(secret.copyOf(), stable = true) - } -} - -/** - * Transforms a measurement document to [level]. - * - * Field classification is by JSON key name, because the schema names things consistently - * (`ssid`, `bssid`, `mac`, `ip4`, `ip6`, `fqdn`, …) and a name-driven pass is auditable by - * reading one table. Anything unrecognized is treated as identifying when it is a string inside - * a known-sensitive container, and left alone otherwise — see [Classification]. - */ -class Anonymizer(private val level: PrivacyLevel, private val salt: Salt) { - - private val cache = HashMap() - - fun anonymize(doc: JsonObject): JsonObject { - if (level == PrivacyLevel.FULL) return stamp(doc) - val walked = walkObject(doc, path = emptyList()) - val out = if (level == PrivacyLevel.STRICT) strip(walked) else walked - return stamp(out) - } - - /** Records what was done, so a reader of the archived/uploaded document is never guessing. */ - private fun stamp(doc: JsonObject): JsonObject { - val run = doc["run"]?.jsonObject ?: return doc - val privacy = buildJsonObject { - put("anonymization", level.wire) - put("salt", if (salt.stable) "stable" else "per_run") - } - return JsonObject(doc + ("run" to JsonObject(run + ("privacy" to privacy)))) - } - - // ---- the tree walk ------------------------------------------------------------------- - - private fun walkObject(obj: JsonObject, path: List): JsonObject = buildJsonObject { - for ((k, v) in obj) { - val childPath = path + k - when { - Classification.dropAtBalanced(childPath) -> Unit // omit entirely - else -> put(k, walk(k, v, childPath)) - } - } - } - - private fun walk(key: String, v: JsonElement, path: List): JsonElement = when (v) { - is JsonObject -> walkObject(v, path) - is JsonArray -> JsonArray(v.map { walk(key, it, path) }) - is JsonPrimitive -> - if (v.isString) { - // Name first (it is precise), then shape (it is exhaustive). A field nobody - // classified must not be a field that leaks. - val type = Classification.typeOf(key, path) ?: Classification.inferFromValue(v.content) - JsonPrimitive(transform(type, v.content)) - } else { - v - } - } - - private fun transform(type: LogicalType?, value: String): String = when (type) { - null -> value - LogicalType.SSID -> pseudo("ssid", value) { "net-" + it.take(6) } - LogicalType.MAC, LogicalType.BSSID -> macPreservingOui(value) - LogicalType.IP4 -> ip4(value) - LogicalType.IP6 -> ip6(value) - LogicalType.FQDN -> fqdn(value) - LogicalType.OPAQUE_ID -> "redacted" - LogicalType.FREETEXT -> "[removed: may contain identifying text]" - } - - // ---- per-type transforms ------------------------------------------------------------- - - /** - * Keeps the OUI (which vendor) and pseudonymizes the NIC part (which unit). Vendor is the - * diagnostically valuable half — "the RA comes from a MikroTik" survives, "…from THAT - * MikroTik" does not. - */ - private fun macPreservingOui(value: String): String { - val sep = if (value.contains('-')) '-' else ':' - val parts = value.split(sep) - if (parts.size != 6 || parts.any { it.length != 2 }) return pseudo("mac", value) { "mac-" + it.take(8) } - val nic = pseudo("mac", value) { it } - return (parts.take(3) + listOf(nic.substring(0, 2), nic.substring(2, 4), nic.substring(4, 6))) - .joinToString(sep.toString()) - .lowercase(Locale.ROOT) - } - - /** - * Prefix-preserving within the same class, with reserved ranges kept verbatim: RFC1918 and - * CGNAT addresses say something about the topology and nothing about the person, and a run - * where 192.168.1.1 became a random public address would be actively misleading to read. - * Public addresses keep only their /16 so the network is still locatable at ISP granularity. - */ - private fun ip4(value: String): String { - // A route destination carries a prefix length; pseudonymize the address and put it back, - // or "0.0.0.0/0" turns into nonsense and the routing table becomes unreadable. - value.substringAfter('/', "").takeIf { it.isNotEmpty() && value.contains('/') }?.let { len -> - return ip4(value.substringBefore('/')) + "/" + len - } - val o = value.split(".") - if (o.size != 4 || o.any { it.toIntOrNull() == null }) return value - val n = o.map { it.toInt() } - val reserved = n[0] == 10 || - (n[0] == 172 && n[1] in 16..31) || - (n[0] == 192 && n[1] == 168) || - (n[0] == 169 && n[1] == 254) || - (n[0] == 100 && n[1] in 64..127) || - n[0] == 127 || n[0] == 0 || n[0] >= 224 - if (reserved) return value - val h = pseudo("ip4", value) { it } - return "${n[0]}.${n[1]}.${h.substring(0, 2).toInt(16)}.${h.substring(2, 4).toInt(16)}" - } - - /** - * IPv6 keeps the scope and the first 32 bits (so 2001:db8:… still reads as global unicast in - * the same allocation) and pseudonymizes the rest — the interface identifier is the part that - * is a device fingerprint, especially with EUI-64. - */ - private fun ip6(value: String): String { - // Dotted quads reach here through the family-agnostic field names (addr, gateway, dst); - // hand them to the IPv4 path rather than mangling them as if they were v6. - if (value.count { it == ':' } < 2) return ip4(value) - if (value.contains('/')) { - return ip6(value.substringBefore('/')) + "/" + value.substringAfter('/') - } - val v = value.lowercase(Locale.ROOT) - // The unspecified address and the default route are not identities; mangling them would - // make a routing table unreadable for no privacy gain. - if (v == "::1" || v == "::" || v.startsWith("fe80:") || v.startsWith("ff")) return v - - // Unique local addresses (fc00::/7) need the *whole* prefix replaced, not the tail. - // - // They look like the v6 equivalent of RFC1918, and the first instinct is to keep them for - // the same reason: private, topological, says nothing about anyone. That reasoning does - // not carry over. An RFC1918 prefix is shared by millions of networks and identifies - // none of them; a ULA global ID is 40 *random* bits, unique to one network by - // construction (RFC 4193). It is a network fingerprint. Passing the leading groups - // through - which is what the general path does - leaked 32 of those 40 bits. - // - // The prefix is pseudonymized as a unit, so two addresses on the same ULA subnet still - // land on the same pseudonymous prefix. "These hosts are on one network" survives; - // "this is *that* network" does not. - if (v.startsWith("fc") || v.startsWith("fd")) { - val groups = v.substringBefore('%').split(":") - val prefix = pseudo("ula-prefix", groups.take(3).joinToString(":")) { it } - val host = pseudo("ula-host", v) { it } - return "fd${prefix.substring(0, 2)}:${prefix.substring(2, 6)}:${prefix.substring(6, 10)}" + - "::${host.substring(0, 4)}" - } - val groups = v.substringBefore('%').split(":") - if (groups.size < 3) return v - val h = pseudo("ip6", value) { it } - return "${groups[0]}:${groups[1]}:${h.substring(0, 4)}:${h.substring(4, 8)}::${h.substring(8, 12)}" - } - - /** - * Per-label pseudonyms with the public suffix kept, so "it resolved somewhere under .local" - * or "…under example.com" survives without naming the host. The suffix list is deliberately - * short: guessing wrong keeps *more* pseudonymized, never less. - */ - private fun fqdn(value: String): String { - if (value.isEmpty()) return value - val trailing = value.endsWith(".") - val labels = value.trimEnd('.').split(".") - if (labels.size == 1) return pseudo("fqdn", value) { "host-" + it.take(6) } - val keep = if (labels.last() in publicSuffixes) 1 else 0 - val head = labels.dropLast(keep).map { l -> pseudo("label", l) { "l-" + it.take(6) } } - return (head + labels.takeLast(keep)).joinToString(".") + if (trailing) "." else "" - } - - // ---- STRICT --------------------------------------------------------------------------- - - /** - * STRICT keeps the shape of the document and the numbers, and nothing that quotes the - * network back. Evidence goes (trains carry addresses and hostnames), finding prose goes - * (it interpolates real names), networks go entirely. - */ - private fun strip(doc: JsonObject): JsonObject = buildJsonObject { - for ((k, v) in doc) { - when (k) { - "networks", "server_sessions" -> Unit - "tests" -> put(k, JsonArray((v as? JsonArray ?: JsonArray(emptyList())).map { t -> - val o = t.jsonObject - JsonObject(o.filterKeys { it != "evidence" && it != "params" }) - })) - "findings" -> put(k, JsonArray((v as? JsonArray ?: JsonArray(emptyList())).map { f -> - val o = f.jsonObject - JsonObject(o.filterKeys { it != "description" && it != "title" && it != "evidence_refs" }) - })) - "run" -> put(k, JsonObject(v.jsonObject.filterKeys { it != "notes" })) - else -> put(k, v) - } - } - } - - // ---- pseudonym machinery --------------------------------------------------------------- - - /** Deterministic per (domain, value, salt); memoized so one value maps to one pseudonym. */ - private fun pseudo(domain: String, value: String, shape: (String) -> String): String = - cache.getOrPut("$domain$value") { - val md = MessageDigest.getInstance("SHA-256") - md.update(salt.bytes) - md.update(domain.toByteArray()) - md.update(0) - md.update(value.lowercase(Locale.ROOT).toByteArray()) - shape(md.digest().joinToString("") { "%02x".format(it) }) - } - - private companion object { - val publicSuffixes = setOf( - "local", "lan", "home", "internal", "arpa", - "com", "net", "org", "io", "app", "dev", "at", "de", "eu", "uk", - ) - } -} +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package privacy implements the anonymization contract of measurement-schema.md §8. +// +// The threat model is specific. An engineer running their own server wants the full document — +// SSIDs and MACs are what make a run useful a week later. Someone measuring against a stranger's +// server wants the numbers to survive and the identifiers not to. So this is a *transform*, not a +// filter: the output is still a valid measurement document with the same tests, metrics and +// findings; only the identifying scalars change, and consistently, so "same SSID as last run" is +// still answerable from pseudonyms alone. +// +// Two properties are load-bearing and are what the tests pin: +// - Consistency within a document: one input value always maps to one pseudonym, so +// correlations inside a run survive. +// - No consistency *across* documents unless the user asks for it: the salt is per-run by +// default, so pseudonyms cannot be used to track a device between uploads. A stable salt is +// opt-in (`Salt.stable`) for people diffing their own history on their own server. +package app.echo_lot.privacy + +import kotlinx.serialization.json.* +import java.security.MessageDigest +import java.util.Locale + +/** How much to strip. Ordered: FULL < BALANCED < STRICT. Wire values match the server's. */ +enum class PrivacyLevel(val wire: String) { + /** Nothing removed. The right choice for your own server. */ + FULL("full"), + + /** + * Identifiers pseudonymized, neighbour inventory dropped. Topology and timing survive: + * you can still see that the gateway is a MikroTik at a /24 boundary with 3 % loss, but not + * which MikroTik, on which SSID, next to whose Chromecast. + */ + BALANCED("balanced"), + + /** + * Numbers only: tests keep their metrics and status, evidence is dropped, findings keep their + * codes and severities but lose descriptions (which quote real names). What is left cannot + * identify a network, and is still enough for aggregate "how common is this fault" work. + */ + STRICT("strict"); + + companion object { + fun fromWire(s: String?): PrivacyLevel = + entries.firstOrNull { it.wire == s?.lowercase(Locale.ROOT) } ?: FULL + + /** The stricter of two levels — used to honour a server's minimum. */ + fun max(a: PrivacyLevel, b: PrivacyLevel): PrivacyLevel = if (a.ordinal >= b.ordinal) a else b + } +} + +/** + * The pseudonymization salt. Per-run by default: a fresh random salt means the same SSID uploaded + * twice yields two different pseudonyms, so an upload endpoint cannot link runs to a device. + * A stable salt trades that away for cross-run diffing and is only appropriate on a server you + * own — the app makes that an explicit choice, not a default. + */ +class Salt private constructor(internal val bytes: ByteArray, val stable: Boolean) { + companion object { + fun perRun(random: ByteArray): Salt = Salt(random.copyOf(), stable = false) + fun stable(secret: ByteArray): Salt = Salt(secret.copyOf(), stable = true) + } +} + +/** + * Transforms a measurement document to [level]. + * + * Field classification is by JSON key name, because the schema names things consistently + * (`ssid`, `bssid`, `mac`, `ip4`, `ip6`, `fqdn`, …) and a name-driven pass is auditable by + * reading one table. Anything unrecognized is treated as identifying when it is a string inside + * a known-sensitive container, and left alone otherwise — see [Classification]. + */ +class Anonymizer(private val level: PrivacyLevel, private val salt: Salt) { + + private val cache = HashMap() + + fun anonymize(doc: JsonObject): JsonObject { + if (level == PrivacyLevel.FULL) return stamp(doc) + val walked = walkObject(doc, path = emptyList()) + val out = if (level == PrivacyLevel.STRICT) strip(walked) else walked + return stamp(out) + } + + /** Records what was done, so a reader of the archived/uploaded document is never guessing. */ + private fun stamp(doc: JsonObject): JsonObject { + val run = doc["run"]?.jsonObject ?: return doc + val privacy = buildJsonObject { + put("anonymization", level.wire) + put("salt", if (salt.stable) "stable" else "per_run") + } + return JsonObject(doc + ("run" to JsonObject(run + ("privacy" to privacy)))) + } + + // ---- the tree walk ------------------------------------------------------------------- + + private fun walkObject(obj: JsonObject, path: List): JsonObject = buildJsonObject { + for ((k, v) in obj) { + val childPath = path + k + when { + Classification.dropAtBalanced(childPath) -> Unit // omit entirely + else -> put(k, walk(k, v, childPath)) + } + } + } + + private fun walk(key: String, v: JsonElement, path: List): JsonElement = when (v) { + is JsonObject -> walkObject(v, path) + is JsonArray -> JsonArray(v.map { walk(key, it, path) }) + is JsonPrimitive -> + if (v.isString) { + // Name first (it is precise), then shape (it is exhaustive). A field nobody + // classified must not be a field that leaks. + val type = Classification.typeOf(key, path) ?: Classification.inferFromValue(v.content) + JsonPrimitive(transform(type, v.content)) + } else { + v + } + } + + private fun transform(type: LogicalType?, value: String): String = when (type) { + // Unclassified strings still get their *embedded* identifiers scrubbed. A whole-value + // check cannot see them: raw shell output is one long string that is neither a MAC nor an + // address, so it sailed through both the name table and the shape check carrying every + // MAC on the user's LAN. + null -> scrubEmbedded(value) + LogicalType.SSID -> pseudo("ssid", value) { "net-" + it.take(6) } + LogicalType.MAC, LogicalType.BSSID -> macPreservingOui(value) + LogicalType.IP4 -> ip4(value) + LogicalType.IP6 -> ip6(value) + LogicalType.FQDN -> fqdn(value) + LogicalType.OPAQUE_ID -> "redacted" + LogicalType.FREETEXT -> "[removed: may contain identifying text]" + } + + /** + * Replaces addresses and MACs found *inside* a longer string. + * + * Shizuku probes embed raw command output verbatim — `ip neigh`, `ip route`, `dumpsys` — which + * is genuinely valuable evidence and also a complete inventory of every device on the user's + * network, with hardware addresses. measurement-schema.md §9 flagged these as "hard to + * anonymize" and proposed dropping them from exports. + * + * Scrubbing beats dropping: the output stays readable and auditable — you can still see the + * shape of the neighbour table and how many hosts there were — while the identifiers become + * the same pseudonyms used everywhere else in the document. So a MAC appearing both in a + * parsed field and in a raw dump still reads as one device. + * + * Only addresses and MACs are touched, for the same reason as [Classification.inferFromValue]: + * they are the patterns that cannot be mistaken for something else in free text. + */ + private fun scrubEmbedded(value: String): String { + // Cheap bail-out: the overwhelming majority of strings are short and contain neither. + if (value.length < 7 || (!value.contains(':') && !value.contains('.'))) return value + // One pass, not three. Sequential passes re-process their own output: after a MAC became + // 78:9a:18:xx:yy:zz the IPv6 pattern matched it — six hex groups separated by colons is + // exactly an address — and mangled the vendor prefix that the MAC rule had just taken + // care to preserve. Ordered alternation resolves each position once, MAC first. + return EMBEDDED.replace(value) { m -> + when { + m.groups[1] != null -> macPreservingOui(m.value) + m.groups[2] != null -> ip6(m.value) + else -> ip4(m.value) + } + } + } + + // ---- per-type transforms ------------------------------------------------------------- + + /** + * Keeps the OUI (which vendor) and pseudonymizes the NIC part (which unit). Vendor is the + * diagnostically valuable half — "the RA comes from a MikroTik" survives, "…from THAT + * MikroTik" does not. + */ + private fun macPreservingOui(value: String): String { + val sep = if (value.contains('-')) '-' else ':' + val parts = value.split(sep) + if (parts.size != 6 || parts.any { it.length != 2 }) return pseudo("mac", value) { "mac-" + it.take(8) } + val nic = pseudo("mac", value) { it } + return (parts.take(3) + listOf(nic.substring(0, 2), nic.substring(2, 4), nic.substring(4, 6))) + .joinToString(sep.toString()) + .lowercase(Locale.ROOT) + } + + /** + * Prefix-preserving within the same class, with reserved ranges kept verbatim: RFC1918 and + * CGNAT addresses say something about the topology and nothing about the person, and a run + * where 192.168.1.1 became a random public address would be actively misleading to read. + * Public addresses keep only their /16 so the network is still locatable at ISP granularity. + */ + private fun ip4(value: String): String { + // A route destination carries a prefix length; pseudonymize the address and put it back, + // or "0.0.0.0/0" turns into nonsense and the routing table becomes unreadable. + value.substringAfter('/', "").takeIf { it.isNotEmpty() && value.contains('/') }?.let { len -> + return ip4(value.substringBefore('/')) + "/" + len + } + val o = value.split(".") + if (o.size != 4 || o.any { it.toIntOrNull() == null }) return value + val n = o.map { it.toInt() } + val reserved = n[0] == 10 || + (n[0] == 172 && n[1] in 16..31) || + (n[0] == 192 && n[1] == 168) || + (n[0] == 169 && n[1] == 254) || + (n[0] == 100 && n[1] in 64..127) || + n[0] == 127 || n[0] == 0 || n[0] >= 224 + if (reserved) return value + val h = pseudo("ip4", value) { it } + return "${n[0]}.${n[1]}.${h.substring(0, 2).toInt(16)}.${h.substring(2, 4).toInt(16)}" + } + + /** + * IPv6 keeps the scope and the first 32 bits (so 2001:db8:… still reads as global unicast in + * the same allocation) and pseudonymizes the rest — the interface identifier is the part that + * is a device fingerprint, especially with EUI-64. + */ + private fun ip6(value: String): String { + // Dotted quads reach here through the family-agnostic field names (addr, gateway, dst); + // hand them to the IPv4 path rather than mangling them as if they were v6. + if (value.count { it == ':' } < 2) return ip4(value) + if (value.contains('/')) { + return ip6(value.substringBefore('/')) + "/" + value.substringAfter('/') + } + val v = value.lowercase(Locale.ROOT) + // The unspecified address and the default route are not identities; mangling them would + // make a routing table unreadable for no privacy gain. + if (v == "::1" || v == "::" || v.startsWith("fe80:") || v.startsWith("ff")) return v + + // Unique local addresses (fc00::/7) need the *whole* prefix replaced, not the tail. + // + // They look like the v6 equivalent of RFC1918, and the first instinct is to keep them for + // the same reason: private, topological, says nothing about anyone. That reasoning does + // not carry over. An RFC1918 prefix is shared by millions of networks and identifies + // none of them; a ULA global ID is 40 *random* bits, unique to one network by + // construction (RFC 4193). It is a network fingerprint. Passing the leading groups + // through - which is what the general path does - leaked 32 of those 40 bits. + // + // The prefix is pseudonymized as a unit, so two addresses on the same ULA subnet still + // land on the same pseudonymous prefix. "These hosts are on one network" survives; + // "this is *that* network" does not. + if (v.startsWith("fc") || v.startsWith("fd")) { + val groups = v.substringBefore('%').split(":") + val prefix = pseudo("ula-prefix", groups.take(3).joinToString(":")) { it } + val host = pseudo("ula-host", v) { it } + return "fd${prefix.substring(0, 2)}:${prefix.substring(2, 6)}:${prefix.substring(6, 10)}" + + "::${host.substring(0, 4)}" + } + val groups = v.substringBefore('%').split(":") + if (groups.size < 3) return v + val h = pseudo("ip6", value) { it } + return "${groups[0]}:${groups[1]}:${h.substring(0, 4)}:${h.substring(4, 8)}::${h.substring(8, 12)}" + } + + /** + * Per-label pseudonyms with the public suffix kept, so "it resolved somewhere under .local" + * or "…under example.com" survives without naming the host. The suffix list is deliberately + * short: guessing wrong keeps *more* pseudonymized, never less. + */ + private fun fqdn(value: String): String { + if (value.isEmpty()) return value + val trailing = value.endsWith(".") + val labels = value.trimEnd('.').split(".") + if (labels.size == 1) return pseudo("fqdn", value) { "host-" + it.take(6) } + val keep = if (labels.last() in publicSuffixes) 1 else 0 + val head = labels.dropLast(keep).map { l -> pseudo("label", l) { "l-" + it.take(6) } } + return (head + labels.takeLast(keep)).joinToString(".") + if (trailing) "." else "" + } + + // ---- STRICT --------------------------------------------------------------------------- + + /** + * STRICT keeps the shape of the document and the numbers, and nothing that quotes the + * network back. Evidence goes (trains carry addresses and hostnames), finding prose goes + * (it interpolates real names), networks go entirely. + */ + private fun strip(doc: JsonObject): JsonObject = buildJsonObject { + for ((k, v) in doc) { + when (k) { + "networks", "server_sessions" -> Unit + "tests" -> put(k, JsonArray((v as? JsonArray ?: JsonArray(emptyList())).map { t -> + val o = t.jsonObject + JsonObject(o.filterKeys { it != "evidence" && it != "params" }) + })) + "findings" -> put(k, JsonArray((v as? JsonArray ?: JsonArray(emptyList())).map { f -> + val o = f.jsonObject + JsonObject(o.filterKeys { it != "description" && it != "title" && it != "evidence_refs" }) + })) + "run" -> put(k, JsonObject(v.jsonObject.filterKeys { it != "notes" })) + else -> put(k, v) + } + } + } + + // ---- pseudonym machinery --------------------------------------------------------------- + + /** Deterministic per (domain, value, salt); memoized so one value maps to one pseudonym. */ + private fun pseudo(domain: String, value: String, shape: (String) -> String): String = + cache.getOrPut("$domain\u0000$value") { + val md = MessageDigest.getInstance("SHA-256") + md.update(salt.bytes) + md.update(domain.toByteArray()) + md.update(0) + md.update(value.lowercase(Locale.ROOT).toByteArray()) + shape(md.digest().joinToString("") { "%02x".format(it) }) + } + + private companion object { + /** + * MAC | IPv6 | IPv4, in that order — alternation is ordered, so a MAC-shaped token is + * claimed by the MAC rule before the IPv6 rule can see it. + * + * The patterns are deliberately conservative. A missed address is scrubbed by another + * rule or not at all; an over-eager one mangles timestamps, version strings and log + * prefixes, corrupting evidence to protect nothing. + */ + val EMBEDDED = Regex( + // Raw strings: a regex written with escaped escapes is a regex nobody can check. + """(\b[0-9a-fA-F]{2}(?:[:-][0-9a-fA-F]{2}){5}\b)""" + + """|(\b(?:[0-9a-fA-F]{1,4}:){2,7}(?::|[0-9a-fA-F]{1,4})(?:[0-9a-fA-F:]*))""" + + """|(\b(?:\d{1,3}\.){3}\d{1,3}\b)""" + ) + + val publicSuffixes = setOf( + "local", "lan", "home", "internal", "arpa", + "com", "net", "org", "io", "app", "dev", "at", "de", "eu", "uk", + ) + } +} diff --git a/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/LeakTest.kt b/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/LeakTest.kt index 0cfd30f..2b4c066 100644 --- a/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/LeakTest.kt +++ b/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/LeakTest.kt @@ -4,8 +4,12 @@ package app.echo_lot.privacy import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import kotlin.test.Test +import kotlin.test.assertFalse import kotlin.test.assertTrue /** @@ -110,4 +114,64 @@ class LeakTest { assertTrue(out.contains("192.168.1.1"), "RFC1918 gateway should survive: $out") assertTrue(out.contains("192.168.1.44"), "RFC1918 interface address should survive: $out") } -} + + /** + * Raw shell output embeds a complete inventory of the local network, and neither the field-name + * table nor the whole-value shape check can see it: `ip_neigh` is one long string that is + * itself neither a MAC nor an address. + * + * This is not hypothetical. The blob below is (abridged) real output that reached the server + * at the `balanced` level from a test device, carrying the hardware address of every host on + * the network. measurement-schema.md §9 had flagged raw dumps as "hard to anonymize"; nothing + * enforced it. + */ + @Test + fun identifiersInsideRawShellOutputAreScrubbed() { + // Joined rather than written with escapes, so the fixture stays readable and there is no + // chance of an escape being mangled on its way into the JSON below. + val dump = listOf( + "uid=2000", + "10.13.102.5 dev wlan0 lladdr 90:09:d0:1a:83:e4 REACHABLE", + "10.13.102.1 dev wlan0 lladdr 78:9a:18:54:b8:f9 REACHABLE", + "10.13.102.111 dev wlan0 lladdr dc:a2:66:08:69:95 STALE", + "2001:4bb8:46a:e724:289d:87ff:feb6:ebd3 dev wlan0 lladdr b8:be:f4:bc:ca:cf STALE", + ).joinToString(" | ") + val doc = json.parseToJsonElement( + """{"run":{"id":"r"},"tests":[{"id":"t","type":"link.ip_monitor", + "evidence":{"ip_neigh":"$dump"}}]}""" + ).jsonObject + val out = json.encodeToString( + kotlinx.serialization.json.JsonObject.serializer(), + Anonymizer(PrivacyLevel.BALANCED, salt).anonymize(doc), + ) + + for (mac in listOf("90:09:d0:1a:83:e4", "78:9a:18:54:b8:f9", "dc:a2:66:08:69:95", "b8:be:f4:bc:ca:cf")) { + assertFalse(out.contains(mac), "a neighbour's MAC survived inside the raw dump: $mac") + } + assertFalse(out.contains("2001:4bb8:46a:e724:289d:87ff:feb6:ebd3"), + "a global IPv6 survived inside the raw dump") + + // Scrubbed, not dropped: the evidence must still be readable, or the raw dump stops being + // evidence at all. Structure, hostnames of the fields, and RFC1918 addresses stay. + assertTrue(out.contains("REACHABLE") && out.contains("STALE"), "the dump lost its structure") + assertTrue(out.contains("10.13.102.1"), "RFC1918 addresses should stay readable: $out") + assertTrue(out.contains("78:9a:18"), "the vendor prefix should survive for identification") + } + + // A MAC in a raw dump and the same MAC in a parsed field must land on the same pseudonym, or + // the document stops being internally consistent and one device reads as two. + @Test + fun theSameIdentifierMatchesAcrossParsedAndRawFields() { + val doc = json.parseToJsonElement( + """{"run":{"id":"r"}, + "networks":[{"wifi":{"bssid":"78:9a:18:54:b8:f9"}}], + "tests":[{"id":"t","evidence":{"ip_neigh":"gw dev wlan0 lladdr 78:9a:18:54:b8:f9 REACHABLE"}}]}""" + ).jsonObject + val out = Anonymizer(PrivacyLevel.BALANCED, salt).anonymize(doc) + val parsed = out["networks"]!!.jsonArray[0].jsonObject["wifi"]!!.jsonObject["bssid"]!! + .jsonPrimitive.content + val raw = json.encodeToString(kotlinx.serialization.json.JsonObject.serializer(), out) + assertTrue(raw.contains(parsed), + "the parsed BSSID pseudonym ($parsed) does not appear in the scrubbed dump") + } +} \ No newline at end of file diff --git a/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/RealDocumentTest.kt b/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/RealDocumentTest.kt new file mode 100644 index 0000000..9417c8c --- /dev/null +++ b/echolot-app/core-privacy/src/test/kotlin/app/echo_lot/privacy/RealDocumentTest.kt @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Echolot contributors +// SPDX-License-Identifier: GPL-3.0-or-later + +package app.echo_lot.privacy + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import java.io.File +import kotlin.test.Test +import kotlin.test.assertTrue + +/** + * Runs the anonymizer over a real captured document when one is supplied via ECHOLOT_REAL_RUN, + * and reports every MAC and public address that survives. + * + * Fixtures only contain the identifiers somebody thought to put in them. A real run off a real + * phone contains whatever the probes actually produce — which is how the raw-shell-output leak was + * found in the first place. Self-skips when no document is supplied, so nobody's network ends up + * committed to the repository. + */ +class RealDocumentTest { + + @Test + fun noIdentifiersSurviveInARealDocument() { + val path = System.getenv("ECHOLOT_REAL_RUN") + if (path.isNullOrBlank() || !File(path).isFile) { + println("RealDocumentTest skipped (set ECHOLOT_REAL_RUN to a captured run)"); return + } + val json = Json { prettyPrint = false } + val doc = json.parseToJsonElement(File(path).readText()).jsonObject + val out = json.encodeToString( + JsonObject.serializer(), + Anonymizer(PrivacyLevel.BALANCED, Salt.perRun(ByteArray(32) { 5 })).anonymize(doc), + ) + + val macs = Regex("""\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b""").findAll(out) + .map { it.value.lowercase() } + .filter { it != "00:00:00:00:00:00" } + .toSet() + val original = Regex("""\b[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}\b""") + .findAll(File(path).readText()).map { it.value.lowercase() }.toSet() + + val survived = macs intersect original + println("MACs in the original: ${original.size}; unchanged after anonymizing: ${survived.size}") + assertTrue(survived.isEmpty(), "these real MAC addresses survived anonymization: $survived") + } +}