server: send granted traffic from the address the session actually used
fmr binds two IPv4 addresses. connFor picked whichever socket of the right family came first in the bind list, so a downtrain for a session established on .150 went out from .151 — and every packet was dropped by the client's NAT, which has no mapping for that pair. tcpdump on the server showed all 50 leaving; the client saw none. Read as "100% downstream loss", which is the worst kind of wrong: a confident measurement of something that never happened. Sessions now record which of our own bound addresses received their traffic, and granted sends (and delayed echo) go back out through that socket. The fallback to a family match is kept for the case where nothing has been received yet, and the test pins both paths — a single-homed lab can never reproduce this. Also: the client-side halves of the same work — anonymizer (core-privacy), local run archive with retention (core-archive), upload client, and the app's settings and history screens. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
7a94c9a3d7
commit
ce1aaa332a
@@ -0,0 +1,209 @@
|
||||
// SPDX-FileCopyrightText: 2026 Echolot contributors
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Package archive keeps completed measurement runs on the device.
|
||||
//
|
||||
// The point of an archive is the second run: "this network was fine on Tuesday" is only
|
||||
// answerable if Tuesday was kept. But an app that silently accumulates network dumps forever is
|
||||
// its own privacy problem, so retention is a first-class part of the type rather than a cleanup
|
||||
// job somebody remembers to write — every save enforces it.
|
||||
//
|
||||
// Storage is one JSON file per run plus a small index entry, in a plain directory. Nothing here
|
||||
// needs a database, and a plain directory is something a user can inspect, copy off, or delete
|
||||
// with a file manager. Files are written to a temp name and renamed, so a run interrupted mid-
|
||||
// write never leaves a half-document that reads as real.
|
||||
package app.echo_lot.archive
|
||||
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import java.io.File
|
||||
|
||||
/** Index entry for one archived run — enough for a history list without opening the documents. */
|
||||
@Serializable
|
||||
data class ArchivedRun(
|
||||
val id: String,
|
||||
@SerialName("saved_at_epoch_ms") val savedAtEpochMs: Long,
|
||||
@SerialName("started_at") val startedAt: String? = null,
|
||||
val verdict: String? = null,
|
||||
@SerialName("finding_count") val findingCount: Int = 0,
|
||||
@SerialName("size_bytes") val sizeBytes: Long = 0,
|
||||
val anonymization: String = "full",
|
||||
/** Whether this run has been accepted by a server, so history can show what is backed up. */
|
||||
val uploaded: Boolean = false,
|
||||
@SerialName("uploaded_to") val uploadedTo: String? = null,
|
||||
)
|
||||
|
||||
/**
|
||||
* Retention limits. All three are independent ceilings; a run is dropped when it violates any of
|
||||
* them. Zero disables that limit.
|
||||
*
|
||||
* The default keeps a hundred runs or three months, whichever comes first. That is enough to see
|
||||
* a pattern ("it degrades every evening") without turning the phone into an archive of every
|
||||
* network its owner ever walked past.
|
||||
*/
|
||||
@Serializable
|
||||
data class RetentionPolicy(
|
||||
/**
|
||||
* Whether to archive at all. Separate from the limits because "no limits" (every limit zero)
|
||||
* and "keep nothing" are opposite intentions, and collapsing them onto the same value is how
|
||||
* a user who turns all the caps off ends up with an empty history.
|
||||
*/
|
||||
val enabled: Boolean = true,
|
||||
@SerialName("max_runs") val maxRuns: Int = 100,
|
||||
@SerialName("max_age_days") val maxAgeDays: Int = 90,
|
||||
@SerialName("max_total_bytes") val maxTotalBytes: Long = 64L * 1024 * 1024,
|
||||
) {
|
||||
companion object {
|
||||
/** Archiving off: runs are shown once and never written. */
|
||||
val KeepNothing = RetentionPolicy(enabled = false)
|
||||
|
||||
/** Archiving on with no ceilings. Every run is kept until the user deletes it. */
|
||||
val Unlimited = RetentionPolicy(maxRuns = 0, maxAgeDays = 0, maxTotalBytes = 0)
|
||||
|
||||
val Default = RetentionPolicy()
|
||||
}
|
||||
|
||||
val keepsAnything: Boolean get() = enabled
|
||||
}
|
||||
|
||||
/** What a purge removed, so the UI can say "dropped 3 old runs" instead of silently deleting. */
|
||||
data class PurgeResult(val removed: List<String>, val freedBytes: Long) {
|
||||
val isEmpty: Boolean get() = removed.isEmpty()
|
||||
}
|
||||
|
||||
class RunArchive(private val dir: File, private val now: () -> Long = System::currentTimeMillis) {
|
||||
|
||||
private val json = Json { ignoreUnknownKeys = true; encodeDefaults = true }
|
||||
|
||||
init {
|
||||
dir.mkdirs()
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes one run and applies retention. Returns the index entry, or null when the policy
|
||||
* keeps nothing at all — in which case nothing is written, rather than written and instantly
|
||||
* deleted (the difference matters on flash storage and to anyone watching the filesystem).
|
||||
*/
|
||||
fun save(runJson: String, policy: RetentionPolicy = RetentionPolicy.Default): ArchivedRun? {
|
||||
if (!policy.keepsAnything) return null
|
||||
val doc = runCatching { json.parseToJsonElement(runJson).jsonObject }.getOrNull() ?: return null
|
||||
val meta = indexOf(doc, runJson.toByteArray().size.toLong()) ?: return null
|
||||
|
||||
writeAtomically(File(dir, meta.id + EXT), runJson)
|
||||
writeAtomically(File(dir, meta.id + META_EXT), json.encodeToString(ArchivedRun.serializer(), meta))
|
||||
purge(policy)
|
||||
return meta
|
||||
}
|
||||
|
||||
/** History, newest first. */
|
||||
fun list(): List<ArchivedRun> =
|
||||
(dir.listFiles { f -> f.name.endsWith(META_EXT) } ?: emptyArray())
|
||||
.mapNotNull { f ->
|
||||
runCatching { json.decodeFromString(ArchivedRun.serializer(), f.readText()) }.getOrNull()
|
||||
}
|
||||
.sortedByDescending { it.savedAtEpochMs }
|
||||
|
||||
fun read(id: String): String? = File(dir, safe(id) + EXT).takeIf { it.isFile }?.readText()
|
||||
|
||||
fun delete(id: String): Boolean {
|
||||
val s = safe(id)
|
||||
val doc = File(dir, s + EXT).delete()
|
||||
File(dir, s + META_EXT).delete()
|
||||
return doc
|
||||
}
|
||||
|
||||
fun deleteAll(): Int = list().count { delete(it.id) }
|
||||
|
||||
/** Records that a server accepted this run, so history can distinguish backed-up from local. */
|
||||
fun markUploaded(id: String, serverName: String) {
|
||||
val f = File(dir, safe(id) + META_EXT)
|
||||
val meta = runCatching { json.decodeFromString(ArchivedRun.serializer(), f.readText()) }.getOrNull()
|
||||
?: return
|
||||
writeAtomically(
|
||||
f,
|
||||
json.encodeToString(
|
||||
ArchivedRun.serializer(),
|
||||
meta.copy(uploaded = true, uploadedTo = serverName),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fun totalBytes(): Long = list().sumOf { it.sizeBytes }
|
||||
|
||||
/**
|
||||
* Enforces the policy. Age first, then total size, then count: dropping stale runs may already
|
||||
* satisfy the other two, and it is the limit a user reasons about ("keep three months"), so it
|
||||
* should not be pre-empted by a size sweep deleting last week instead.
|
||||
*/
|
||||
fun purge(policy: RetentionPolicy): PurgeResult {
|
||||
val removed = ArrayList<String>()
|
||||
var freed = 0L
|
||||
fun drop(r: ArchivedRun) {
|
||||
if (delete(r.id)) {
|
||||
removed.add(r.id)
|
||||
freed += r.sizeBytes
|
||||
}
|
||||
}
|
||||
|
||||
var kept = list()
|
||||
if (policy.maxAgeDays > 0) {
|
||||
val cutoff = now() - policy.maxAgeDays * 24L * 60 * 60 * 1000
|
||||
val (fresh, stale) = kept.partition { it.savedAtEpochMs >= cutoff }
|
||||
stale.forEach(::drop)
|
||||
kept = fresh
|
||||
}
|
||||
if (policy.maxTotalBytes > 0) {
|
||||
var total = kept.sumOf { it.sizeBytes }
|
||||
// Oldest first until we are under the ceiling.
|
||||
for (r in kept.reversed()) {
|
||||
if (total <= policy.maxTotalBytes) break
|
||||
drop(r)
|
||||
total -= r.sizeBytes
|
||||
}
|
||||
kept = kept.filter { it.id !in removed }
|
||||
}
|
||||
if (policy.maxRuns > 0 && kept.size > policy.maxRuns) {
|
||||
kept.drop(policy.maxRuns).forEach(::drop) // list() is newest-first
|
||||
}
|
||||
return PurgeResult(removed, freed)
|
||||
}
|
||||
|
||||
// ---- internals ---------------------------------------------------------------------
|
||||
|
||||
private fun indexOf(doc: JsonObject, size: Long): ArchivedRun? {
|
||||
val run = doc["run"]?.jsonObject ?: return null
|
||||
val id = run["id"]?.jsonPrimitive?.content?.let(::safe)?.takeIf { it.isNotEmpty() } ?: return null
|
||||
return ArchivedRun(
|
||||
id = id,
|
||||
savedAtEpochMs = now(),
|
||||
startedAt = run["started_at"]?.jsonPrimitive?.content,
|
||||
verdict = doc["summary"]?.jsonObject?.get("verdict")?.jsonPrimitive?.content,
|
||||
findingCount = (doc["findings"] as? kotlinx.serialization.json.JsonArray)?.size ?: 0,
|
||||
sizeBytes = size,
|
||||
anonymization = run["privacy"]?.jsonObject?.get("anonymization")?.jsonPrimitive?.content ?: "full",
|
||||
)
|
||||
}
|
||||
|
||||
private fun writeAtomically(target: File, content: String) {
|
||||
val tmp = File(target.parentFile, target.name + ".tmp")
|
||||
tmp.writeText(content)
|
||||
if (!tmp.renameTo(target)) {
|
||||
target.delete()
|
||||
tmp.renameTo(target)
|
||||
}
|
||||
}
|
||||
|
||||
/** Run ids reach the filesystem; keep them to characters that cannot climb out of [dir]. */
|
||||
private fun safe(id: String): String = buildString {
|
||||
for (c in id) if (c.isLetterOrDigit() || c == '-' || c == '_') append(c)
|
||||
}.take(64)
|
||||
|
||||
private companion object {
|
||||
const val EXT = ".json"
|
||||
const val META_EXT = ".meta.json"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// SPDX-FileCopyrightText: 2026 Echolot contributors
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package app.echo_lot.archive
|
||||
|
||||
import java.io.File
|
||||
import java.nio.file.Files
|
||||
import kotlin.test.AfterTest
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertNotNull
|
||||
import kotlin.test.assertNull
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class RunArchiveTest {
|
||||
|
||||
private val dir: File = Files.createTempDirectory("echolot-archive").toFile()
|
||||
private var clock = 1_000_000_000_000L // fixed: retention is time arithmetic, not wall time
|
||||
|
||||
private fun archive() = RunArchive(dir) { clock }
|
||||
|
||||
@AfterTest fun cleanup() { dir.deleteRecursively() }
|
||||
|
||||
private fun doc(id: String, findings: Int = 1, pad: Int = 0): String {
|
||||
val f = (1..findings).joinToString(",") { """{"id":"f$it"}""" }
|
||||
return """{"run":{"id":"$id","started_at":"2026-08-01T10:00:00Z","privacy":{"anonymization":"balanced"}},""" +
|
||||
""""findings":[$f],"summary":{"verdict":"warn"},"pad":"${"x".repeat(pad)}"}"""
|
||||
}
|
||||
|
||||
@Test
|
||||
fun savedRunsComeBackNewestFirst() {
|
||||
val a = archive()
|
||||
for (i in 1..3) {
|
||||
a.save(doc("run-$i"))
|
||||
clock += 60_000
|
||||
}
|
||||
assertEquals(listOf("run-3", "run-2", "run-1"), a.list().map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun theIndexSummarisesTheDocument() {
|
||||
val meta = assertNotNull(archive().save(doc("run-1", findings = 4)))
|
||||
assertEquals("warn", meta.verdict)
|
||||
assertEquals(4, meta.findingCount)
|
||||
assertEquals("balanced", meta.anonymization)
|
||||
assertEquals("2026-08-01T10:00:00Z", meta.startedAt)
|
||||
assertFalse(meta.uploaded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun theDocumentComesBackByteForByte() {
|
||||
val a = archive()
|
||||
val original = doc("run-1")
|
||||
a.save(original)
|
||||
assertEquals(original, a.read("run-1"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun countLimitKeepsTheNewest() {
|
||||
val a = archive()
|
||||
val policy = RetentionPolicy(maxRuns = 3, maxAgeDays = 0, maxTotalBytes = 0)
|
||||
for (i in 1..7) {
|
||||
a.save(doc("run-$i"), policy)
|
||||
clock += 60_000
|
||||
}
|
||||
assertEquals(listOf("run-7", "run-6", "run-5"), a.list().map { it.id })
|
||||
assertNull(a.read("run-1"), "purged run's document should be gone, not just its index entry")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun ageLimitDropsRunsPastTheWindow() {
|
||||
val a = archive()
|
||||
val policy = RetentionPolicy(maxRuns = 0, maxAgeDays = 7, maxTotalBytes = 0)
|
||||
a.save(doc("old"), policy)
|
||||
clock += 30L * 24 * 60 * 60 * 1000 // a month later
|
||||
a.save(doc("new"), policy)
|
||||
assertEquals(listOf("new"), a.list().map { it.id })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun sizeLimitDropsOldestUntilUnderTheCeiling() {
|
||||
val a = archive()
|
||||
val one = doc("x", pad = 900).toByteArray().size.toLong()
|
||||
val policy = RetentionPolicy(maxRuns = 0, maxAgeDays = 0, maxTotalBytes = one * 2 + 10)
|
||||
for (i in 1..5) {
|
||||
a.save(doc("run-$i", pad = 900), policy)
|
||||
clock += 60_000
|
||||
}
|
||||
val kept = a.list()
|
||||
assertTrue(kept.size <= 2, "size ceiling not enforced: kept ${kept.size}")
|
||||
assertEquals("run-5", kept.first().id, "the newest run must always survive")
|
||||
assertTrue(a.totalBytes() <= policy.maxTotalBytes)
|
||||
}
|
||||
|
||||
// A policy that keeps nothing must not write-then-delete: the run should never touch storage.
|
||||
@Test
|
||||
fun keepNothingWritesNothing() {
|
||||
val a = archive()
|
||||
assertNull(a.save(doc("run-1"), RetentionPolicy.KeepNothing))
|
||||
assertTrue(a.list().isEmpty())
|
||||
assertEquals(0, dir.listFiles()?.size ?: 0, "files were written for a keep-nothing policy")
|
||||
}
|
||||
|
||||
// "no ceilings" and "keep nothing" must not be the same policy, however a user arrives at
|
||||
// one: turning every limit off should keep everything, not wipe the history.
|
||||
@Test
|
||||
fun unlimitedKeepsEverythingWhileKeepNothingKeepsNone() {
|
||||
val a = archive()
|
||||
for (i in 1..5) {
|
||||
a.save(doc("run-$i"), RetentionPolicy.Unlimited)
|
||||
clock += 60_000
|
||||
}
|
||||
assertEquals(5, a.list().size)
|
||||
assertNull(a.save(doc("run-6"), RetentionPolicy.KeepNothing))
|
||||
assertEquals(5, a.list().size, "keep-nothing must not touch what is already archived")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun uploadStateIsRecorded() {
|
||||
val a = archive()
|
||||
a.save(doc("run-1"))
|
||||
a.markUploaded("run-1", "fmr")
|
||||
val meta = a.list().single()
|
||||
assertTrue(meta.uploaded)
|
||||
assertEquals("fmr", meta.uploadedTo)
|
||||
assertEquals("run-1", meta.id, "marking upload must not disturb the rest of the entry")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun deleteRemovesBothFiles() {
|
||||
val a = archive()
|
||||
a.save(doc("run-1"))
|
||||
assertTrue(a.delete("run-1"))
|
||||
assertTrue(a.list().isEmpty())
|
||||
assertNull(a.read("run-1"))
|
||||
assertEquals(0, dir.listFiles()?.size ?: 0)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun malformedInputIsRejectedRatherThanArchived() {
|
||||
val a = archive()
|
||||
assertNull(a.save("not json"))
|
||||
assertNull(a.save("""{"summary":{"verdict":"ok"}}"""), "a document with no run id has no identity")
|
||||
assertTrue(a.list().isEmpty())
|
||||
}
|
||||
|
||||
// Run ids come from a document that may have been produced elsewhere; they must not be able
|
||||
// to write outside the archive directory.
|
||||
@Test
|
||||
fun runIdsCannotEscapeTheArchiveDirectory() {
|
||||
val a = archive()
|
||||
a.save(doc("../../evil"))
|
||||
val strays = dir.parentFile.listFiles { f -> f.name.contains("evil") } ?: emptyArray()
|
||||
assertTrue(strays.isEmpty(), "wrote outside the archive: ${strays.toList()}")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun purgeReportsWhatItRemoved() {
|
||||
val a = archive()
|
||||
for (i in 1..5) {
|
||||
a.save(doc("run-$i"), RetentionPolicy.Unlimited)
|
||||
clock += 60_000
|
||||
}
|
||||
val result = a.purge(RetentionPolicy(maxRuns = 2, maxAgeDays = 0, maxTotalBytes = 0))
|
||||
assertEquals(3, result.removed.size)
|
||||
assertTrue(result.freedBytes > 0)
|
||||
assertEquals(2, a.list().size)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user