diff --git a/.github/workflows/rootfs-2-bake.yml b/.github/workflows/rootfs-2-bake.yml index 562cb9215..efba88bb3 100644 --- a/.github/workflows/rootfs-2-bake.yml +++ b/.github/workflows/rootfs-2-bake.yml @@ -307,8 +307,8 @@ jobs: source "$GITHUB_WORKSPACE/tools/rootfs-builder/r2-cp.sh" # s3cp: aws s3 cp with retries cd dist - # Big artifacts first, then their per-artifact sidecars (.meta4/.sha256/.installed/ - # .torrent). Pointers, logs and the queue stay for the publish job. + # Big artifacts first, then their per-artifact sidecars (.meta4/.sha256/.installed). + # No .torrent since K2GO-588. Pointers, logs and the queue stay for the publish job. shopt -s nullglob for f in k2go_*.tar.gz; do echo ">> upload artifact $f" @@ -412,6 +412,7 @@ jobs: if (( n > KEEP )); then for key in "${keys[@]:0:n-KEEP}"; do echo ">> prune $key (+ sidecars)" + # .torrent: older bakes published one (before K2GO-588); prune it with its tarball. for ext in '' .meta4 .sha256 .installed .torrent .minisig; do aws s3 rm "s3://${BUCKET_NAME}/${key}${ext}" --endpoint-url "$ENDPOINT" || true done diff --git a/controller/app/src/main/java/org/appdevforall/k2go/Aria2Manager.java b/controller/app/src/main/java/org/appdevforall/k2go/Aria2Manager.java index 552763ee8..c6d92fd05 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/Aria2Manager.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/Aria2Manager.java @@ -188,11 +188,8 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig File downloadDir = new File(SystemStateEvaluator.rootfsBaseDir(context), "downloads"); if (!downloadDir.exists()) downloadDir.mkdirs(); - // Secure DHT file within our app (avoids the com.termux crash) - File dhtFile = new File(context.getFilesDir(), "dht.dat"); - if (!dhtFile.exists()) { - try { dhtFile.createNewFile(); } catch (Exception e) { Log.w(TAG, "Could not create dht.dat"); } - } + // K2GO-588: no BitTorrent any more; remove the DHT state that older builds kept. + new File(context.getFilesDir(), "dht.dat").delete(); // --- Extract SSL Certificate from assets --- File caCertFile = new File(context.getFilesDir(), "cacert.pem"); @@ -213,7 +210,25 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig // --split mirror count, and reconciling whatever is already on disk so that // stopping and restarting a download is safe (no wasted re-download, and the // network profiler never trips over a leftover completed file). - MetalinkFile mf = MetalinkSplit.isMetalinkUrl(url) ? fetchMetalink(url) : null; + // K2GO-588 (ADR-K2GO-554 A4): aria2 gets a local copy of the .meta4 without its + // , so the download is HTTP only. Without that copy aria2 would read the URL + // and turn the torrent metaurl into a BitTorrent session, so a failed read stops here. + File httpOnlyMeta = new File(context.getCacheDir(), "rootfs-http-only.meta4"); + MetalinkFile mf = null; + String source = url; + if (MetalinkSplit.isMetalinkUrl(url)) { + mf = fetchMetalink(url, httpOnlyMeta); + if (mf == null) { + if (postLatchedStop(mainHandler, listener)) return; + // The same answer aria2 gives for a network problem (exit 6), so the caller + // offers a retry as before. + mainHandler.post(() -> listener.onError( + org.appdevforall.k2go.download.domain.Aria2Exit.label(6), + org.appdevforall.k2go.download.domain.Aria2Exit.Kind.TRANSIENT)); + return; + } + source = "--metalink-file=" + httpOnlyMeta.getAbsolutePath(); + } // resumeOrRepair: an existing (partial or complete-but-corrupt) file is on // disk. Keep it and let the main aria2 run below salvage it via // --check-integrity against the .meta4 piece hashes (re-downloading only the @@ -274,7 +289,7 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig // right, only the preamble was wasted. boolean resuming = resumeOrRepair || hasControlFile(downloadDir); boolean forceIpv4 = resuming ? false - : Aria2NetworkProfiler.shouldForceIpv4(aria2Bin, downloadDir, dhtFile, url, mainHandler, listener); + : Aria2NetworkProfiler.shouldForceIpv4(aria2Bin, downloadDir, source, mainHandler, listener); // ---------------------------------------------------- // 3. Build the command dynamically. @@ -283,9 +298,9 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig // "Get more" downloads. Keep both aligned so behaviour matches; if you change flags // here, mirror them there (and note any intentional divergence). Known divergences: // the server uses Debian's system CA (no bundled cacert), the container resolver - // (--async-dns=false) instead of ApplyDnsUseCase + the IPv4 profiler, and has no - // DHT/BitTorrent yet; only this app path pre-reconciles/resumes via MetalinkSplit + - // DownloadVerifier before invoking aria2. + // (--async-dns=false) instead of ApplyDnsUseCase + the IPv4 profiler. Neither path + // uses BitTorrent (K2GO-588). Only this app path pre-reconciles/resumes via + // MetalinkSplit + DownloadVerifier before invoking aria2. java.util.List command = new java.util.ArrayList<>(); command.add(aria2Bin.getAbsolutePath()); command.add("--dir=" + downloadDir.getAbsolutePath()); @@ -305,10 +320,12 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig // while downloading. On mismatch aria2 exits non-zero, so onError fires // and the archive is never extracted/executed. command.add("--check-integrity=true"); - command.add("--enable-dht=true"); - command.add("--dht-file-path=" + dhtFile.getAbsolutePath()); - command.add("--bt-enable-lpd=true"); - command.add("--seed-time=0"); + // K2GO-588: HTTP only. The metaurl is gone from the .meta4 above; these keep aria2 + // from any peer traffic even so (its default is DHT on). + command.add("--enable-dht=false"); + command.add("--enable-dht6=false"); + command.add("--bt-enable-lpd=false"); + command.add("--enable-peer-exchange=false"); // --- Apply SSL Certificate Validation (D6: always strict; cacert // presence was already enforced above, so there is no insecure path) --- @@ -375,7 +392,7 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig command.add("--disable-ipv6=true"); } - command.add(url); + command.add(source); ProcessBuilder pb = new ProcessBuilder(command); @@ -384,16 +401,7 @@ public void startDownload(Context context, String url, RootfsManifest.Target sig // null, so a stop arriving in that window latches the reason and kills nothing. // Without this the caller's cleanup could delete the download directory and aria2 // would then start, recreate it and write fresh residue behind it. - if (stopRequest == Stop.CANCEL) { - Log.d(TAG, "Download cancelled before aria2 was started."); - mainHandler.post(listener::onCancelled); - return; - } - if (stopRequest == Stop.PAUSE) { - Log.d(TAG, "Download paused before aria2 was started."); - mainHandler.post(listener::onPaused); - return; - } + if (postLatchedStop(mainHandler, listener)) return; // Redirect errors to the same input stream pb.redirectErrorStream(true); @@ -613,8 +621,23 @@ private void requestStop(Stop reason) { /** * Helper method to copy files from the APK assets folder to internal storage. */ + /** ADFA-5119: a stop latched before aria2 started; reports it and returns true. */ + private boolean postLatchedStop(Handler mainHandler, DownloadListener listener) { + if (stopRequest == Stop.CANCEL) { + Log.d(TAG, "Download cancelled before aria2 was started."); + mainHandler.post(listener::onCancelled); + return true; + } + if (stopRequest == Stop.PAUSE) { + Log.d(TAG, "Download paused before aria2 was started."); + mainHandler.post(listener::onPaused); + return true; + } + return false; + } + /** ADFA-4676: fetch + parse the .meta4 into a verifiable MetalinkFile, or null on any error. */ - private MetalinkFile fetchMetalink(String url) { + private MetalinkFile fetchMetalink(String url, File httpOnlyCopy) { HttpURLConnection conn = null; try { conn = (HttpURLConnection) new URL(url).openConnection(); @@ -624,9 +647,22 @@ private MetalinkFile fetchMetalink(String url) { if (conn.getResponseCode() != 200) { return null; } + byte[] raw; try (InputStream in = conn.getInputStream()) { - return MetalinkFile.parse(in); + java.io.ByteArrayOutputStream buf = new java.io.ByteArrayOutputStream(); + byte[] chunk = new byte[16384]; + int n; + while ((n = in.read(chunk)) > 0) buf.write(chunk, 0, n); + raw = buf.toByteArray(); + } + MetalinkFile mf = MetalinkFile.parse(new java.io.ByteArrayInputStream(raw)); + // K2GO-588: the copy that aria2 reads, without the BitTorrent metaurl. + try (java.io.Writer w = new java.io.OutputStreamWriter( + new java.io.FileOutputStream(httpOnlyCopy), java.nio.charset.StandardCharsets.UTF_8)) { + w.write(org.appdevforall.k2go.download.domain.MetalinkHttpOnly.of( + new String(raw, java.nio.charset.StandardCharsets.UTF_8))); } + return mf; } catch (Exception e) { Log.w(TAG, "fetchMetalink fallback (" + e.getMessage() + ")"); return null; diff --git a/controller/app/src/main/java/org/appdevforall/k2go/Aria2NetworkProfiler.java b/controller/app/src/main/java/org/appdevforall/k2go/Aria2NetworkProfiler.java index b89d4da2a..c5a720c00 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/Aria2NetworkProfiler.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/Aria2NetworkProfiler.java @@ -29,18 +29,18 @@ public class Aria2NetworkProfiler { * Runs a real-world speed test by downloading the target file for a few seconds. * Returns TRUE if IPv4 proves to be more stable and faster. */ - public static boolean shouldForceIpv4(File aria2Bin, File downloadDir, File dhtFile, String targetUrl, Handler mainHandler, Aria2Manager.DownloadListener listener) { + public static boolean shouldForceIpv4(File aria2Bin, File downloadDir, String source, Handler mainHandler, Aria2Manager.DownloadListener listener) { Log.i(TAG, "Starting Network Profiling (Time-Boxed Race)..."); // 1. Test IPv4 (Forced) mainHandler.post(() -> listener.onProgress(0, "Test IPv4", "Wait")); Log.i(TAG, "=== TEST 1: FORCED IPv4 ROUTE ==="); - double speedIpv4 = runTimeBoxedTest(aria2Bin, downloadDir, dhtFile, targetUrl, true); + double speedIpv4 = runTimeBoxedTest(aria2Bin, downloadDir, source, true); // 2. Test Default (Usually IPv6 preferred) mainHandler.post(() -> listener.onProgress(0, "Test IPv6", "Wait")); Log.i(TAG, "=== TEST 2: DEFAULT ROUTE (IPv6 Preferred) ==="); - double speedDefault = runTimeBoxedTest(aria2Bin, downloadDir, dhtFile, targetUrl, false); + double speedDefault = runTimeBoxedTest(aria2Bin, downloadDir, source, false); Log.i(TAG, "=== FINAL PROFILING RESULTS ==="); Log.i(TAG, String.format("Sustained Speed IPv4: %.2f KB/s", speedIpv4)); @@ -66,7 +66,7 @@ public static boolean shouldForceIpv4(File aria2Bin, File downloadDir, File dhtF return forceIpv4; } - private static double runTimeBoxedTest(File aria2Bin, File downloadDir, File dhtFile, String url, boolean forceIpv4) { + private static double runTimeBoxedTest(File aria2Bin, File downloadDir, String source, boolean forceIpv4) { Process process = null; double lastRecordedSpeedKB = 0.0; @@ -80,8 +80,9 @@ private static double runTimeBoxedTest(File aria2Bin, File downloadDir, File dht // Match your baseline configuration command.add("--max-connection-per-server=4"); command.add("--split=4"); - command.add("--enable-dht=true"); - command.add("--dht-file-path=" + dhtFile.getAbsolutePath()); + // K2GO-588: HTTP only, like the real download (Aria2Manager). + command.add("--enable-dht=false"); + command.add("--enable-dht6=false"); command.add("--check-certificate=false"); if (forceIpv4) { @@ -91,7 +92,7 @@ private static double runTimeBoxedTest(File aria2Bin, File downloadDir, File dht command.add("--console-log-level=warn"); command.add("--summary-interval=1"); // Need speed output every second command.add("--download-result=hide"); - command.add(url); + command.add(source); ProcessBuilder pb = new ProcessBuilder(command); pb.redirectErrorStream(true); diff --git a/controller/app/src/main/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnly.java b/controller/app/src/main/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnly.java new file mode 100644 index 000000000..8713f11f2 --- /dev/null +++ b/controller/app/src/main/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnly.java @@ -0,0 +1,34 @@ +/* + * ============================================================================ + * Name : MetalinkHttpOnly.java + * Author : AppDevForAll + * Copyright : Copyright (c) 2026 AppDevForAll + * Description : K2GO-588 (ADR-K2GO-554 A4). A Metalink with its elements + * removed, so aria2 downloads over HTTP only, never BitTorrent. + * ============================================================================ + */ +package org.appdevforall.k2go.download.domain; + +import java.util.regex.Pattern; + +/** + * aria2 turns a Metalink that carries a {@code } into a BitTorrent + * session, and {@code --follow-torrent=false} does not stop it (measured, ADR-K2GO-554 A4). So the + * app gives aria2 a copy of the Metalink without the metaurl elements. The {@code } mirrors, + * size and hashes stay unchanged. + */ +public final class MetalinkHttpOnly { + + // A ... element or a self-closing , with its leading space. + private static final Pattern METAURL = Pattern.compile( + "[ \\t]*]*?(?:/>|>.*?)[ \\t]*\\r?\\n?", + Pattern.DOTALL | Pattern.CASE_INSENSITIVE); + + private MetalinkHttpOnly() {} + + /** The same Metalink text without any {@code } element. */ + public static String of(String metalinkXml) { + if (metalinkXml == null) return null; + return METAURL.matcher(metalinkXml).replaceAll(""); + } +} diff --git a/controller/app/src/test/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnlyTest.java b/controller/app/src/test/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnlyTest.java new file mode 100644 index 000000000..f56103ff4 --- /dev/null +++ b/controller/app/src/test/java/org/appdevforall/k2go/download/domain/MetalinkHttpOnlyTest.java @@ -0,0 +1,64 @@ +package org.appdevforall.k2go.download.domain; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +public class MetalinkHttpOnlyTest { + + private static final String PUBLISHED = + "\n" + + "\n" + + " \n" + + " 1590750919\n" + + " 28173c20010e4bf54e3b276cb0dd68201e2e473422704bfae85b8ed7e24d3560\n" + + " https://example.org/k2go_2026.321_basic_c746edd_arm64-v8a.tar.gz\n" + + " k2go_2026.321_basic_c746edd_arm64-v8a.tar.gz.torrent\n" + + " \n" + + "\n"; + + @Test + public void removesTheTorrentMetaurlAndKeepsEverythingElse() { + String out = MetalinkHttpOnly.of(PUBLISHED); + assertFalse(out.contains("metaurl")); + assertFalse(out.contains(".torrent")); + assertEquals(PUBLISHED.replaceAll(" \n" + + "\n x.torrent\n\n\n"; + assertEquals("\nhttps://a/x\n\n", MetalinkHttpOnly.of(in)); + } + + @Test + public void aMetalinkWithoutMetaurlIsUnchanged() { + String in = PUBLISHED.replaceAll(" ")); + } + + @Test + public void nullStaysNull() { + assertNull(MetalinkHttpOnly.of(null)); + } +} diff --git a/controller/docs/ADR-K2GO-554-signed-rootfs-skip-listing.md b/controller/docs/ADR-K2GO-554-signed-rootfs-skip-listing.md index 953ed0b55..fc58a59bd 100644 --- a/controller/docs/ADR-K2GO-554-signed-rootfs-skip-listing.md +++ b/controller/docs/ADR-K2GO-554-signed-rootfs-skip-listing.md @@ -1,6 +1,6 @@ # ADR-K2GO-554 : Signed official rootfs skips the second archive pass -- Status: Accepted (2026-11-12, with Amendment 1) +- Status: Accepted (2026-11-12, with Amendment 1; Amendment 3 on 2026-11-20) - Date: 2026-11-07 - Ticket: K2GO-554 (relates to K2GO-555, K2GO-372, K2GO-511, K2GO-457, ADFA-5110) - Author: AppDevForAll @@ -262,6 +262,7 @@ Instrumented tests in the app process (same ART and BouncyCastle as production). A3 this is not an identity gap. To use only HTTP, the app must give aria2 a meta4 without the metaurl (the same local-meta4 handoff as A6), or the builder must stop writing the metaurl. DHT and LPD were enabled in 2026-05 with no recorded reason. + Superseded by C1 (Amendment 3, K2GO-588): BitTorrent is removed. - A5. Trusted import (amends D7). An import needs the tarball and its `.minisig` (2 files). Identity: A2. Integrity: one SHA-256 pass of the tarball against the manifest `sha256` and `size`. Then the import skips the listing and the treehash. @@ -309,6 +310,39 @@ manifest (only the `-t` value changes); (3) new bakes carry the manifest. An old A restore picks one file; the app finds `X.tar.json` or `X.tar.gz.minisig` next to it by name (`ArchivePair`). +## Amendment 3 (2026-11-20, K2GO-588): BitTorrent removed (supersedes A4) + +Context: K2GO-588 makes "Keep offline" mean real network silence, and the app sends +nothing to third parties that the user did not start (privacy by default). A +BitTorrent session announces the phone to a tracker and to peers, and DHT and LPD +add more traffic. On 2026-11-20 a OnePlus 7T download of +`latest_basic_arm64-v8a.meta4` (build 2026.321) ended in the aria2 state `SEED`, +which exists only for BitTorrent: the published meta4 still carries the torrent +metaurl. + +- C1. The rootfs download is HTTP only. Both measures apply, so neither alone is + load-bearing: + - The app (`Aria2Manager`) gives aria2 a local copy of the meta4 without the + `` elements (`--metalink-file`, `MetalinkHttpOnly`). This is the + local-meta4 handoff of A4/A6. It covers the meta4 files that are already + published. The IPv4 profiler uses the same copy. If the app cannot read the + meta4, it reports a network problem (as aria2 exit 6) and does not fall back to + the URL. + - aria2 runs with `--enable-dht=false --enable-dht6=false --bt-enable-lpd=false + --enable-peer-exchange=false` (the aria2 default is DHT on). `--seed-time=0` and + the `dht.dat` file are removed; the app deletes a leftover `dht.dat`. + - The builder deletes the `.torrent` and removes the `` line from the + meta4 before the bake signs it, so new meta4 files are HTTP only and their + sha-256 in the signed manifest matches the copy that aria2 reads (A6 stays + possible). +- C2. Integrity does not change: aria2 checks the meta4 SHA-256 pieces + (`--check-integrity`), then DownloadVerifier checks the file (A3). +- C3. Cost: no peer download. The HTTP mirrors (R2 and its mirror) carry all the + traffic, which is what the measured downloads already used. +- C4. The box (dash-node) aria2 for ZIM, maps and other content has no BitTorrent + flags, and the Kiwix meta4 files have no metaurl (checked 2026-11-20). No change. +- C5. Old `.torrent` files on R2 are pruned with their tarball (`rootfs-2-bake.yml`). + ## Out of scope - Replacing the meta4 or the aria2 download path. The signature is additive. diff --git a/tools/rootfs-builder/build-iiab-rootfs.sh b/tools/rootfs-builder/build-iiab-rootfs.sh index a75ba74f9..94e3921ed 100755 --- a/tools/rootfs-builder/build-iiab-rootfs.sh +++ b/tools/rootfs-builder/build-iiab-rootfs.sh @@ -23,7 +23,7 @@ # # Output: # - k2go____.tar.gz (top-level = installed-rootfs/iiab/) -# - k2go_<...>.tar.gz.meta4 / .torrent (per-artifact, mkmetalink) +# - k2go_<...>.tar.gz.meta4 (per-artifact, mkmetalink; HTTP only) # - latest__.meta4 (STABLE pointer the APK requests; a copy of the # per-artifact .meta4. Inside it, mirrors point at the full-named tarball, # so this stable name always resolves to the newest build.) @@ -922,6 +922,8 @@ cp -f "${OUTDIR}/${ARTIFACT}.installed" "${OUTDIR}/${INSTALLED}" # ----------------------------- 5) Generate .meta4 (Metalink, via mkmetalink) -- # mkmetalink (Go) builds a valid Metalink v4 (.meta4) + matching .torrent. # https://github.com/chapmanjacobd/mkmetalink +# K2GO-588 (ADR-K2GO-554 A4): the downloads are HTTP only. The .torrent is deleted and the +# line is removed from the .meta4, else aria2 turns the download into BitTorrent. # We pass the publish + mirror base URLs so the .meta4 carries the same # entries the APK/aria2 expect (priority 10 = primary, 11 = mirror). It names its # output after the artifact; the APK requests a STABLE name, so we copy it to @@ -942,10 +944,12 @@ done || die "mkmetalink failed to generate the metalink." GEN_META4="${OUTDIR}/${ARTIFACT}.meta4" [[ -f "$GEN_META4" ]] || die "mkmetalink did not produce ${GEN_META4}" +sed -i '/]/d' "$GEN_META4" +grep -q ' is still in ${GEN_META4}" +rm -f "${OUTDIR}/${ARTIFACT}.torrent" cp -f "$GEN_META4" "${OUTDIR}/${META4}" ok "Metalink ready: ${OUTDIR}/${META4} (copy of ${ARTIFACT}.meta4)" ok "Sources in .meta4: primary ${PUBLISH_URL%/}/ + ${#MIRRORS[@]} mirror(s)." -ok "(also produced: ${ARTIFACT}.torrent in ${OUTDIR})" # ----------------------------- Summary ---------------------------------------- echo