commit a6168221d11e9be41ff25ef9483eefba68e313fc Author: mrambossek Date: Sun May 24 22:48:39 2026 +0200 initial: fdroidserver-ipfs image Downstream of registry.gitlab.com/fdroid/docker-executable-fdroidserver:master, patched for publishing F-Droid repos over IPFS: - install Debian ipfs-cid so fdroidserver populates ipfsCIDv1 on every package file (APKs, icons, screenshots), which fdroidclient with an IPFS-gateway mirror routes via CID natively - rehash.py wrapper that adds ipfsCIDv1 to entry["index"], renames the index to a content-addressed filename, and re-signs entry.jar after `fdroid update` — closes the cache-busting gap on the index file itself - entrypoint.sh chains rehash automatically after any `fdroid update`, so existing fdroid-via-docker wrappers need no changes See SPEC.md for the rationale and Change 2's algorithm; README.md for end-user usage. Co-Authored-By: Claude Opus 4.7 (1M context) diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..d634e61 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "permissions": { + "allow": [ + "WebFetch(domain:gitlab.com)", + "WebFetch(domain:f-droid.org)", + "Read(research/**)", + "Glob", + "Grep", + "Bash(git clone *)", + "Bash(git:*)", + "Bash(ls:*)", + "Bash(wc:*)" + ] + } +} diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..291d10b --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,66 @@ +name: Build fdroidserver-ipfs image + +# Triggers: +# - push to main → immediate rebuild (any change in the repo) +# - weekly cron → picks up upstream :master refreshes in +# docker-executable-fdroidserver without a +# source change here +# - manual dispatch → on-demand +on: + push: + branches: [main] + schedule: + # Sun 04:00 UTC. + - cron: "0 4 * * 0" + workflow_dispatch: + +jobs: + build: + # Skip the job entirely (no runner provisioning) when the registry host + # isn't configured — shows as "skipped" in the UI rather than burning + # minutes on a build that can't be pushed. Secrets can't be referenced + # in job-level if:, so REGISTRY_TOKEN is still checked below. + if: vars.REGISTRY_HOST != '' + runs-on: ubuntu-docker + permissions: + packages: write + steps: + - name: Validate registry token + run: | + if [ -z "${{ secrets.REGISTRY_TOKEN }}" ]; then + echo "::error::Missing required config: secrets.REGISTRY_TOKEN" + exit 1 + fi + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Compute date tag + id: date + run: echo "date=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Log in to Gitea registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.REGISTRY_HOST }} + username: ${{ gitea.actor }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: true + # Two tags: + # :stable — moving, what callers pull by default + # :YYYYMMDD — immutable, for pinning / rollback + tags: | + ${{ vars.REGISTRY_HOST }}/ci-images/fdroidserver-ipfs:stable + ${{ vars.REGISTRY_HOST }}/ci-images/fdroidserver-ipfs:${{ steps.date.outputs.date }} + build-args: | + IMAGE_SOURCE=https://${{ vars.REGISTRY_HOST }}/ci-images/fdroidserver-ipfs + cache-from: type=registry,ref=${{ vars.REGISTRY_HOST }}/ci-images/fdroidserver-ipfs:stable + cache-to: type=inline diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..feff2bd --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +research/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab48e72 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +# fdroidserver-ipfs: docker-executable-fdroidserver patched for IPFS publishing. +# +# Two changes vs. upstream (see SPEC.md): +# 1. Install ipfs-cid so fdroidserver populates ipfsCIDv1 on all package +# files (APKs, icons, screenshots). fdroidclient with an IPFS-gateway +# mirror configured then fetches those by CID. +# 2. Install rehash.py as `fdroidserver-ipfs-rehash` — a post-`fdroid update` +# wrapper that adds ipfsCIDv1 to entry["index"], renames the index to a +# content-addressed filename, and re-signs entry.jar. Run it once after +# every `fdroid update`. +# +# Base is pinned via build-arg. The weekly cron in .gitea/workflows/build.yaml +# picks up upstream :master refreshes without a source change. + +ARG FDROIDSERVER_IMAGE=registry.gitlab.com/fdroid/docker-executable-fdroidserver:master +ARG IMAGE_SOURCE= + +FROM ${FDROIDSERVER_IMAGE} +ARG IMAGE_SOURCE + +# Links the published OCI package to its source repo so Gitea inherits +# visibility (public repo → publicly pullable image, no auth needed). +LABEL org.opencontainers.image.source="${IMAGE_SOURCE}" +LABEL org.opencontainers.image.description="fdroidserver with ipfs-cid + post-update CID/cache-bust wrapper for IPFS publishing." +LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later" + +# Change 1. Debian's ipfs-cid package installs the `ipfs_cid` *binary* that +# fdroidserver finds via shutil.which(). The PyPI package of the same name +# is a Python library only and ships no binary — using pip would silently +# leave CID generation disabled. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ipfs-cid \ + && rm -rf /var/lib/apt/lists/* + +# Change 2. +COPY rehash.py /usr/local/bin/fdroidserver-ipfs-rehash +RUN chmod +x /usr/local/bin/fdroidserver-ipfs-rehash + +# Override the upstream entrypoint so `fdroid update` automatically chains +# into `fdroidserver-ipfs-rehash`. Drop-in for the wrapper script — callers +# don't change their invocation. +COPY entrypoint.sh /usr/local/bin/fdroidserver-ipfs-entrypoint +RUN chmod +x /usr/local/bin/fdroidserver-ipfs-entrypoint +ENTRYPOINT ["/usr/local/bin/fdroidserver-ipfs-entrypoint"] +CMD ["--help"] + +# Sanity gate: source bsenv.sh (the upstream entrypoint does the same) so +# ${fdroidserver} resolves, then verify every component our additions touch. +RUN . /etc/profile.d/bsenv.sh \ + && command -v ipfs_cid \ + && test -x "${fdroidserver}/fdroid" \ + && command -v fdroidserver-ipfs-rehash \ + && command -v fdroidserver-ipfs-entrypoint \ + && PYTHONPATH="${fdroidserver}" python3 -c "from fdroidserver import common, signindex" diff --git a/README.md b/README.md new file mode 100644 index 0000000..f91645f --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# fdroidserver-ipfs + +Downstream of `registry.gitlab.com/fdroid/docker-executable-fdroidserver:master`, +patched for publishing F-Droid repos over IPFS. See [SPEC.md](SPEC.md) for the why. + +## Usage + +Drop-in replacement for the upstream image. Run `fdroid update` exactly as +before — the image's entrypoint chains `fdroidserver-ipfs-rehash` after any +successful `update`, so you never have to remember the second step: + +```bash +docker run --rm -v "$PWD:/repo" your-image-tag update +# ↑ runs `fdroid update`, then automatically renames index, adds ipfsCIDv1, +# re-signs entry.jar (and regenerates entry.json.asc if GPG was in the loop). +``` + +Every other subcommand (`build`, `publish`, `gpgsign`, …) passes straight +through to `fdroid` unchanged. + +### Running the rehash standalone + +If you want to invoke the rehash directly (e.g. on an already-published repo): + +```bash +docker run --rm -v "$PWD:/repo" \ + --entrypoint fdroidserver-ipfs-rehash \ + your-image-tag +``` + +`fdroidserver-ipfs-rehash` discovers `repo/` (and `archive/` if configured) +from `config.yml`. If GPG signing was already part of your pipeline (i.e. +`entry.json.asc` exists), it invokes `fdroid gpgsign` itself to regenerate +the invalidated signatures. Pass repodirs explicitly to override discovery: + +```bash +fdroidserver-ipfs-rehash repo archive +``` diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..5ba5d2a --- /dev/null +++ b/SPEC.md @@ -0,0 +1,124 @@ +# fdroidserver-ipfs + +A drop-in replacement for `registry.gitlab.com/fdroid/docker-executable-fdroidserver:master` +tailored for publishing F-Droid repositories over IPFS. + +The upstream image works for HTTP-served repos but has two gaps that break the +IPFS workflow. This image patches both. + +## Problem + +Pinning an F-Droid repo to IPFS and serving it through public HTTP gateways +(ipfs.io, dweb.link, cloudflare-ipfs.com, …) exposes two upstream limitations: + +1. The `ipfs-cid` Python module isn't installed, so any fdroidserver code path + that computes a CIDv1 for repo artifacts fails on the stock image. +2. `index-v2.json` is published under a stable filename. Public IPFS gateways + cache aggressively by URL path, so clients keep getting the previous index + for hours after a republish even though the underlying CID has changed. + +## Scope + +This repo produces one OCI image — call it `fdroidserver-ipfs` — built `FROM` +the upstream `docker-executable-fdroidserver:master` and applying the minimum +set of changes needed for the IPFS workflow. **Everything not listed below is +intentionally inherited unchanged from upstream.** + +### Change 1 — install `ipfs-cid` + +Install the **Debian** `ipfs-cid` package (available in Trixie, which the +upstream image is based on): + +```dockerfile +RUN apt-get update \ + && apt-get install -y --no-install-recommends ipfs-cid \ + && rm -rf /var/lib/apt/lists/* +``` + +Note: the **PyPI** package of the same name is a Python library only and +ships no binary, so `pip install ipfs-cid` silently leaves CID generation +disabled. fdroidserver auto-detects the binary on PATH +(`fdroidserver/common.py::shutil.which('ipfs_cid')`) and, when present, +calls `calculate_IPFS_cid()` for every published file — APKs, source +tarballs, icons, screenshots, etc. — writing the resulting CIDv1 into +each file's `ipfsCIDv1` JSON field (`fdroidserver/update.py:1661-1663`, +`:1717-1719`). On the stock upstream image the binary is missing, so the +field is silently omitted and the IPFS-routing code path on the client is +never exercised. + +**Why this matters for the client.** fdroidclient has native IPFS support: +`org.fdroid.download.MirrorChooser` (in `libs/download/`) reads the +`ipfsCIDv1` field on each file and, when a configured mirror is flagged +`isIpfsGateway`, fetches the file by CID via that gateway instead of by +URL. So once `ipfs-cid` is installed, every per-package asset (APK, icon, +…) is automatically reachable via any IPFS gateway by any client with one +configured — no patching required. + +### Change 2 — add `ipfsCIDv1` to the index entry + cache-busted filename + +Change 1 fixes per-package files but **does not** fix the index itself. +The `index` block inside `entry.json` is built by +`fdroidserver/common.py::file_entry()`, which only emits `name` / `sha256` +/ `size` and never adds `ipfsCIDv1` — even when one could be computed. So +fdroidclient has no CID for `index-v2.json` to feed into `MirrorChooser`, +and falls back to the stable HTTP path → public IPFS gateways serve a +stale cached copy for hours after a republish. + +The post-process wrapper closes both halves of this gap in one pass: + +1. Computes the IPFS CIDv1 of `repo/index-v2.json` (using `calculate_IPFS_cid` + from `fdroidserver.common`, which is the same code path fdroidserver + itself uses for APKs). +2. Renames `repo/index-v2.json` → `repo/index-v2-.json`, where `` is + the **last 12 hex characters of the SHA-256** of the file's content. + This image is **IPFS-only** — no stable copy is kept alongside. +3. Rewrites `repo/entry.json` so that `entry["index"]`: + - Updates `name` to `/index-v2-.json` + - Adds `ipfsCIDv1` with the value computed in step 1 + - Leaves `sha256` and `size` unchanged (the bytes didn't move) +4. Re-signs `entry.jar` by calling + `fdroidserver.signindex.sign_index(repodir, "entry.json")` — reuses the + existing `config.yml` keystore + `FDROID_KEY_STORE_PASS` / + `FDROID_KEY_PASS` env vars; no extra config surface. +5. If GPG signing is enabled in `config.yml`, regenerates `entry.json.asc`. +6. Applies the same CID/rename/rewrite to any `diff/*.json` files published + alongside (each is referenced by its own `name` field in entry.json). + +Two-layer cache strategy, on purpose: + +- **IPFS-aware clients** (any user with an IPFS gateway configured as a + mirror) follow the `ipfsCIDv1` and fetch the index by CID. Always fresh, + never gateway-cached by URL. +- **Plain-HTTP clients** still hit a new URL on every publish thanks to + the `` suffix → public gateway path caches naturally miss → fresh + index without waiting for TTLs. + +`entry.jar` itself stays at the fixed `/entry.jar` URL — gateways will +still cache it, but it's tiny and clients fetch it first to discover both +the current index filename and its CID, which are the cache-busted +references we actually care about. + +`entry.json` does not currently carry an `ipfsCIDv1` for the index entry, +but fdroidclient's parser is configured with `ignoreUnknownKeys = true` +(`libs/index/.../IndexParser.kt`) and `EntryFileV2` already declares the +field (`libs/index/.../IndexV2.kt`), so adding it is both backwards-safe +and natively consumed by current client versions. + +Delivery: a small Python script (`fdroidserver-ipfs-rehash`, or similar) +installed into the image's PATH that imports `fdroidserver.signindex` / +`fdroidserver.common` directly. Either invoked manually after `fdroid update`, +or wired into a container entrypoint that runs both. Entrypoint design is +deferred until we know how the image gets driven in production. + +## Out of scope + +- Changes to `index-v1.*` or the legacy `index.jar` flow. +- Pinning to IPFS, gateway selection, DNSLink updates — handled by whatever + drives the container, not by the image itself. +- Anything unrelated to the two issues above. + +## Non-goals for this spec + +This document captures *what* the image needs to do and *why*. The concrete +Dockerfile layering, the exact rehash/re-sign script, and the workflow YAML +are implementation details and land in follow-up changes. diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..bc9a02d --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# fdroidserver-ipfs entrypoint. +# +# Mirrors the upstream image's behavior — sources bsenv.sh, runs +# ${fdroidserver}/fdroid with the given args — then automatically invokes +# fdroidserver-ipfs-rehash after a successful `fdroid update` so callers +# don't have to remember the second step. + +set -eu +. /etc/profile.d/bsenv.sh + +GRADLE_USER_HOME="${home_vagrant}/.gradle" "${fdroidserver}/fdroid" "$@" + +# Auto-rehash after a successful update. We check every arg (not just $1) +# so global flags before the subcommand (e.g. `-v update`) still trigger it. +for arg in "$@"; do + if [ "$arg" = "update" ]; then + fdroidserver-ipfs-rehash + break + fi +done diff --git a/rehash.py b/rehash.py new file mode 100644 index 0000000..a5d1654 --- /dev/null +++ b/rehash.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Post-process `fdroid update` output for IPFS publishing. + +For each repodir (repo, archive) discovered from config.yml: + - rename index-v2.json to index-v2-.json (cache-busts gateways) + - rename diff/.json to diff/-.json + - rewrite entry.json so name/ipfsCIDv1 point at the renamed files + - re-sign entry.jar via fdroidserver.signindex.sign_index + - regenerate entry.json.asc if GPG signing is configured + +Run in the same working directory as `fdroid update` (where config.yml lives). +""" + +import argparse +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +# Make fdroidserver importable + invocable when this script is run outside the +# upstream image's ENTRYPOINT (which sources /etc/profile.d/bsenv.sh). The +# buildserver-style image places fdroidserver at $fdroidserver — typically +# /home/vagrant/fdroidserver. Both sys.path (for the import below) and PATH +# (for the `fdroid gpgsign` subprocess call) need it. +_FDS = os.environ.get('fdroidserver') or '/home/vagrant/fdroidserver' +if os.path.isdir(_FDS): + if _FDS not in sys.path: + sys.path.insert(0, _FDS) + if _FDS not in os.environ.get('PATH', '').split(os.pathsep): + os.environ['PATH'] = _FDS + os.pathsep + os.environ.get('PATH', '') + +from fdroidserver import common, signindex + + +HASH_SUFFIX_LEN = 12 + + +def sha256_last_chars(path: Path) -> str: + h = hashlib.sha256() + with path.open('rb') as fp: + for chunk in iter(lambda: fp.read(65536), b''): + h.update(chunk) + return h.hexdigest()[-HASH_SUFFIX_LEN:] + + +def write_entry_json(entry: dict, path: Path) -> None: + with path.open('w', encoding='utf-8') as fp: + json.dump(entry, fp, ensure_ascii=False, sort_keys=True) + + +def rehash_entry_file(repodir: Path, file_entry: dict) -> None: + """Rename the file referenced by file_entry["name"] and update name/ipfsCIDv1 in place.""" + old_rel = file_entry['name'].lstrip('/') + old_path = repodir / old_rel + suffix = sha256_last_chars(old_path) + new_path = old_path.with_name(f"{old_path.stem}-{suffix}{old_path.suffix}") + + cid = common.calculate_IPFS_cid(str(old_path)) + + if new_path != old_path: + old_path.rename(new_path) + + file_entry['name'] = '/' + new_path.relative_to(repodir).as_posix() + if cid: + file_entry['ipfsCIDv1'] = cid + + +def rehash_repodir(repodir_name: str) -> bool: + repodir = Path(repodir_name) + entry_json = repodir / 'entry.json' + if not entry_json.exists(): + return False + + with entry_json.open(encoding='utf-8') as fp: + entry = json.load(fp) + + rehash_entry_file(repodir, entry['index']) + for diff_entry in entry.get('diffs', {}).values(): + rehash_entry_file(repodir, diff_entry) + + write_entry_json(entry, entry_json) + signindex.sign_index(repodir_name, 'entry.json') + return True + + +def regenerate_gpg_sigs(repodirs: list) -> None: + """Drop stale .asc files we just invalidated, then let `fdroid gpgsign` recreate them. + + We detect "GPG is part of this pipeline" by the prior existence of entry.json.asc + rather than from config keys — matches whatever the user's pipeline actually does. + gpgsign only signs files missing a sibling .asc, so deleting the stale ones is + enough to trigger a fresh signature on the new filenames. + """ + pipeline_uses_gpg = any( + (Path(r) / 'entry.json.asc').exists() for r in repodirs + ) + if not pipeline_uses_gpg: + return + + for repodir_name in repodirs: + repodir = Path(repodir_name) + stale = [repodir / 'entry.json.asc'] + stale += list(repodir.glob('index-v2*.json.asc')) + if (repodir / 'diff').is_dir(): + stale += list((repodir / 'diff').glob('*.json.asc')) + for path in stale: + if path.exists(): + path.unlink() + + subprocess.run(['fdroid', 'gpgsign'], check=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + 'repodirs', nargs='*', + help='Repo directories to process. Default: auto-discover from config.yml.', + ) + args = parser.parse_args() + + config = common.read_config() + signindex.config = config + + repodirs = args.repodirs + if not repodirs: + repodirs = ['repo'] + if config.get('archive_older', 0) != 0: + repodirs.append('archive') + + processed = [] + for repodir in repodirs: + if rehash_repodir(repodir): + processed.append(repodir) + + regenerate_gpg_sigs(processed) + return 0 + + +if __name__ == '__main__': + sys.exit(main())