diff --git a/app/build.gradle b/app/build.gradle index 1e6adf2..df91b42 100755 --- a/app/build.gradle +++ b/app/build.gradle @@ -58,6 +58,7 @@ dependencies { implementation 'androidx.core:core:1.13.1' implementation 'androidx.fragment:fragment:1.8.2' + implementation 'androidx.documentfile:documentfile:1.0.1' testImplementation 'junit:junit:4.13.2' } diff --git a/app/src/main/java/com/httrack/android/HTTrackActivity.java b/app/src/main/java/com/httrack/android/HTTrackActivity.java index b5375bd..3e138ec 100755 --- a/app/src/main/java/com/httrack/android/HTTrackActivity.java +++ b/app/src/main/java/com/httrack/android/HTTrackActivity.java @@ -53,6 +53,7 @@ import android.app.NotificationManager; import android.app.PendingIntent; import android.content.ActivityNotFoundException; +import android.content.ContentResolver; import android.content.Context; import android.content.Intent; import android.content.SharedPreferences; @@ -66,6 +67,7 @@ import android.os.Bundle; import android.os.Environment; import android.os.Handler; +import android.os.Looper; import android.os.Parcelable; import android.os.StrictMode; import android.text.Editable; @@ -91,6 +93,7 @@ import androidx.activity.OnBackPressedCallback; import androidx.core.app.ActivityCompat; +import androidx.documentfile.provider.DocumentFile; import androidx.core.app.NotificationChannelCompat; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationManagerCompat; @@ -124,6 +127,8 @@ public class HTTrackActivity extends FragmentActivity { // Whether POST_NOTIFICATIONS was ever asked for. Has to outlive the activity: pane_id is // restored on rotation, and a second refusal is the one that sticks for good. protected static final String NOTIFY_ASKED_NAME = "NotificationPermissionAsked"; + // Whether the one-time "import your old mirrors" offer has been shown and dismissed for good. + protected static final String IMPORT_OFFERED_NAME = "LegacyImportOffered"; //
Pattern protected static final Pattern brHtmlPattern = Pattern.compile(Pattern @@ -161,6 +166,7 @@ protected static class VERSION_CODES { protected static final int ACTIVITY_FILE_CHOOSER = 1; protected static final int ACTIVITY_PROJECT_NAME_CHOOSER = 2; protected static final int ACTIVITY_CLEANUP = 3; + protected static final int ACTIVITY_IMPORT_TREE = 4; // Process unique session ID for the fragment identifier protected String sessionID = "runner_task" + "_" @@ -551,6 +557,12 @@ protected void onCreate(final Bundle savedInstanceState) { if (extras != null) { restoreInstanceState(extras); } + + // First launch only, so a rotation does not bring the offer back; and not stacked on top + // of the native-load failure dialog. + if (savedInstanceState == null && HTTrackLib.loadedSuccessfully()) { + offerLegacyMirrorImportOnce(); + } } /* Install/Update time. */ @@ -2222,6 +2234,123 @@ public void onClickPrevious(final View view) { } } + /** + * Let the user pick a legacy mirror folder to copy in. The storage-access framework is the + * only way left to reach the old public location once we no longer hold storage permissions. + */ + private void startLegacyMirrorImport() { + final Intent intent = new Intent(Intent.ACTION_OPEN_DOCUMENT_TREE); + intent.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION); + try { + showNotification(getString(R.string.import_mirrors_prompt)); + startActivityForResult(intent, ACTIVITY_IMPORT_TREE); + } catch (final ActivityNotFoundException e) { + Log.w(getClass().getSimpleName(), "no document-tree picker", e); + showNotification(getString(R.string.import_mirrors_none)); + } + } + + private volatile boolean importInProgress; + + /** + * Copy the picked tree into our Websites directory, entirely off the UI thread: even walking + * the tree to find "Websites" is one IPC per entry, too much for the main thread. The source + * is only read, never changed. Guarded against a second run while one is going, whose two + * threads would race on the same temp files. + */ + private void importMirrorsFrom(final Uri treeUri) { + if (importInProgress) { + showNotification(getString(R.string.import_mirrors_running)); + return; + } + final ContentResolver resolver = getContentResolver(); + resolver.takePersistableUriPermission(treeUri, Intent.FLAG_GRANT_READ_URI_PERMISSION); + final Context appContext = getApplicationContext(); + final File dest = getProjectRootFile(); + importInProgress = true; + showNotification(getString(R.string.import_mirrors_running)); + new Thread(new Runnable() { + @Override + public void run() { + String outcome; + try { + outcome = runImport(appContext, resolver, treeUri, dest); + } finally { + importInProgress = false; + } + deliverImportOutcome(appContext, outcome); + } + }, "legacy-import").start(); + } + + /** The copy itself, on the worker thread; returns the message to show. **/ + private String runImport(final Context context, final ContentResolver resolver, + final Uri treeUri, final File dest) { + DocumentFile root = DocumentFile.fromTreeUri(context, treeUri); + if (root == null) { + return getString(R.string.import_mirrors_none); + } + // A picked "HTTrack" folder holds "Websites"; descend so project dirs keep their depth. + final DocumentFile websites = root.findFile("Websites"); + if (websites != null && websites.isDirectory()) { + root = websites; + } + final LegacyMirrorImport.Source source = + new LegacyMirrorImport.DocumentFileSource(resolver, root); + if (LegacyMirrorImport.totalSize(source) > dest.getUsableSpace()) { + return getString(R.string.import_mirrors_no_space); + } + final LegacyMirrorImport.Result result = LegacyMirrorImport.copyTree(source, dest); + if (result.firstError() != null) { + Log.w(getClass().getSimpleName(), "import: " + result.firstError()); + } + if (!result.isComplete()) { + return getString(R.string.import_mirrors_partial, result.copied, result.failed); + } + if (result.copied == 0 && result.skipped == 0) { + return getString(R.string.import_mirrors_none); + } + return getString(R.string.import_mirrors_done, result.copied, result.skipped); + } + + /** + * Report the outcome on the main thread. Uses the application context and its own handler so + * the toast still shows if the activity was rotated away during a long copy; the project list + * is refreshed only when an activity is still around to hold it. + */ + private void deliverImportOutcome(final Context appContext, final String message) { + new Handler(Looper.getMainLooper()).post(new Runnable() { + @Override + public void run() { + Toast.makeText(appContext, message, Toast.LENGTH_LONG).show(); + if (!isFinishing() && !isDestroyed()) { + refreshprojectNameSuggests(); + } + } + }); + } + + /** + * Once per install, point users at the import. No storage permission remains to detect the + * old folder, so this offers rather than asserts; "Not now" leaves it to ask again. + */ + private void offerLegacyMirrorImportOnce() { + final SharedPreferences settings = getSharedPreferences(PREFS_NAME, 0); + if (settings.getBoolean(IMPORT_OFFERED_NAME, false)) { + return; + } + new AlertDialog.Builder(this) + .setMessage(R.string.import_mirrors_offer) + .setPositiveButton(R.string.import_mirrors_offer_yes, (dialog, which) -> { + settings.edit().putBoolean(IMPORT_OFFERED_NAME, true).apply(); + startLegacyMirrorImport(); + }) + .setNegativeButton(R.string.import_mirrors_offer_later, null) + .setNeutralButton(R.string.import_mirrors_offer_never, (dialog, which) -> + settings.edit().putBoolean(IMPORT_OFFERED_NAME, true).apply()) + .show(); + } + /** * Change base path */ @@ -2318,6 +2447,11 @@ protected void onActivityResult(final int requestCode, final int resultCode, setBasePath(path); } break; + case ACTIVITY_IMPORT_TREE: + if (resultCode == Activity.RESULT_OK && data != null && data.getData() != null) { + importMirrorsFrom(data.getData()); + } + break; case ACTIVITY_PROJECT_NAME_CHOOSER: if (resultCode == Activity.RESULT_OK) { final String projectName = data @@ -2577,6 +2711,9 @@ public boolean onOptionsItemSelected(final MenuItem item) { case R.id.action_help: browse(new File(new File(getResourceFile(), "html"), "index.html")); break; + case R.id.action_import_mirrors: + startLegacyMirrorImport(); + break; default: return super.onOptionsItemSelected(item); } diff --git a/app/src/main/java/com/httrack/android/LegacyMirrorImport.java b/app/src/main/java/com/httrack/android/LegacyMirrorImport.java new file mode 100644 index 0000000..5ef1f36 --- /dev/null +++ b/app/src/main/java/com/httrack/android/LegacyMirrorImport.java @@ -0,0 +1,189 @@ +package com.httrack.android; + +import android.content.ContentResolver; +import android.net.Uri; + +import androidx.documentfile.provider.DocumentFile; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * One-time copy of mirrors from a user-picked legacy location into the app-specific directory. + * + * Scoped storage put the old public HTTrack/Websites tree out of reach, so the user grants it + * once through the storage-access framework and its contents are copied here. The copy never + * touches the source, so a failed or interrupted run loses nothing and can simply be re-run: + * existing destination files are skipped, which also makes it idempotent. + */ +final class LegacyMirrorImport { + private static final int BUFFER = 64 * 1024; + + private LegacyMirrorImport() { + } + + /** A node of the tree being copied, so the walk can be tested without a real DocumentFile. */ + interface Source { + String getName(); + + boolean isDirectory(); + + long length(); + + List children(); + + InputStream openStream() throws IOException; + } + + // Cap the retained messages: a broken tree could otherwise produce one per file. + private static final int MAX_ERRORS = 20; + + /** Tally of a copy run; a per-file failure is recorded, not thrown, so the rest still copies. */ + static final class Result { + int copied; + int skipped; + int failed; + // Set by the caller when the tree does not fit; nothing is copied in that case. + boolean outOfSpace; + final List errors = new ArrayList(); + + boolean isComplete() { + return failed == 0 && !outOfSpace; + } + + String firstError() { + return errors.isEmpty() ? null : errors.get(0); + } + } + + /** + * Copy {@code src}'s contents into {@code destDir}, recursively. Files already present at the + * destination are left as they are. Never modifies the source. + * + * @return the tally; {@link Result#isComplete()} is false if any file could not be copied + */ + static Result copyTree(final Source src, final File destDir) { + final Result result = new Result(); + copyInto(src, destDir, result); + return result; + } + + private static void copyInto(final Source dir, final File destDir, final Result result) { + if (!destDir.exists() && !destDir.mkdirs()) { + record(result, "could not create " + destDir); + return; + } + for (final Source child : dir.children()) { + final String name = child.getName(); + if (name == null || name.isEmpty() || name.equals(".") || name.equals("..") + || name.indexOf('/') >= 0) { + record(result, "unsafe entry name '" + name + "'"); + continue; + } + final File dest = new File(destDir, name); + if (child.isDirectory()) { + copyInto(child, dest, result); + } else if (dest.isDirectory()) { + // A directory already sits where this file would go; skipping would drop it silently. + record(result, "name clash at " + dest); + } else if (dest.exists()) { + result.skipped++; + } else { + copyFile(child, dest, result); + } + } + } + + private static void copyFile(final Source src, final File dest, final Result result) { + // A partial file from an interrupted run must not look complete on the next pass; write to a + // temp sibling and rename only once whole, so only finished files ever satisfy dest.exists(). + final File tmp = new File(dest.getParentFile(), dest.getName() + ".part"); + try (final InputStream in = src.openStream(); + final OutputStream out = new FileOutputStream(tmp)) { + final byte[] buffer = new byte[BUFFER]; + int n; + while ((n = in.read(buffer)) != -1) { + out.write(buffer, 0, n); + } + } catch (final IOException e) { + tmp.delete(); + record(result, "could not copy " + src.getName() + ": " + e.getMessage()); + return; + } + if (tmp.renameTo(dest)) { + result.copied++; + } else { + tmp.delete(); + record(result, "could not finalize " + dest); + } + } + + private static void record(final Result result, final String message) { + result.failed++; + if (result.errors.size() < MAX_ERRORS) { + result.errors.add(message); + } + } + + /** Bytes the tree holds, for a free-space check before starting. Skipped files still count. */ + static long totalSize(final Source src) { + if (!src.isDirectory()) { + return src.length(); + } + long total = 0; + for (final Source child : src.children()) { + total += totalSize(child); + } + return total; + } + + /** A DocumentFile-backed source; the adapter the app uses over a picked tree. */ + static final class DocumentFileSource implements Source { + private final ContentResolver resolver; + private final DocumentFile file; + + DocumentFileSource(final ContentResolver resolver, final DocumentFile file) { + this.resolver = resolver; + this.file = file; + } + + @Override + public String getName() { + return file.getName(); + } + + @Override + public boolean isDirectory() { + return file.isDirectory(); + } + + @Override + public long length() { + return file.length(); + } + + @Override + public List children() { + final List out = new ArrayList(); + for (final DocumentFile child : file.listFiles()) { + out.add(new DocumentFileSource(resolver, child)); + } + return out; + } + + @Override + public InputStream openStream() throws IOException { + final Uri uri = file.getUri(); + final InputStream in = resolver.openInputStream(uri); + if (in == null) { + throw new IOException("no stream for " + uri); + } + return in; + } + } +} diff --git a/app/src/main/res/menu/menu.xml b/app/src/main/res/menu/menu.xml index f5a7ca6..593d0e0 100644 --- a/app/src/main/res/menu/menu.xml +++ b/app/src/main/res/menu/menu.xml @@ -4,6 +4,10 @@ android:id="@+id/action_help" android:orderInCategory="100" android:title="@string/action_help"/> + Help View Documentation About HTTrack + Import existing mirrors + Pick the folder that holds your older HTTrack mirrors (for example HTTrack on the internal storage). Their contents will be copied into this app\'s storage; the originals are left untouched. + Importing mirrors… + Imported %1$d files (%2$d already present). + Imported %1$d files; %2$d could not be copied. You can run the import again. + Nothing to import from that folder. + Not enough free space to import these mirrors. + Mirrors from an older version can no longer be reached directly. Import them into this app now? + Import + Not now + Don\'t ask again Go To HTTrack Website Go To HTTrack Forum View License diff --git a/app/src/test/java/com/httrack/android/LegacyMirrorImportTest.java b/app/src/test/java/com/httrack/android/LegacyMirrorImportTest.java new file mode 100644 index 0000000..5f105e1 --- /dev/null +++ b/app/src/test/java/com/httrack/android/LegacyMirrorImportTest.java @@ -0,0 +1,358 @@ +package com.httrack.android; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** Exercises the copy walk over an in-memory tree: fidelity, idempotency, and error tolerance. */ +public class LegacyMirrorImportTest { + @Rule + public final TemporaryFolder tmp = new TemporaryFolder(); + + /** In-memory Source: a directory holds children; a file holds bytes, or throws if poisoned. */ + private static final class FakeNode implements LegacyMirrorImport.Source { + final String name; + final boolean dir; + final byte[] data; + final boolean poisoned; + final List kids = new ArrayList(); + + static FakeNode dir(final String name) { + return new FakeNode(name, true, null, false); + } + + static FakeNode file(final String name, final String body) { + return new FakeNode(name, false, body.getBytes(StandardCharsets.UTF_8), false); + } + + static FakeNode poison(final String name) { + return new FakeNode(name, false, null, true); + } + + private FakeNode(final String name, final boolean dir, final byte[] data, final boolean poisoned) { + this.name = name; + this.dir = dir; + this.data = data; + this.poisoned = poisoned; + } + + FakeNode with(final LegacyMirrorImport.Source... children) { + for (final LegacyMirrorImport.Source c : children) { + kids.add(c); + } + return this; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isDirectory() { + return dir; + } + + @Override + public long length() { + return data != null ? data.length : 0; + } + + @Override + public List children() { + return kids; + } + + @Override + public InputStream openStream() throws IOException { + if (poisoned) { + throw new IOException("poisoned " + name); + } + return new ByteArrayInputStream(data); + } + } + + private String read(final File f) throws IOException { + return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8); + } + + @Test + public void copiesTheWholeTreeFaithfully() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with( + FakeNode.file("index.html", "top"), + FakeNode.dir("site").with( + FakeNode.file("page.html", "body"), + FakeNode.dir("img").with(FakeNode.file("a.png", "PNGDATA")))); + final File dest = tmp.newFolder("dest"); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + assertTrue(r.isComplete()); + assertEquals(3, r.copied); + assertEquals(0, r.skipped); + assertEquals("top", read(new File(dest, "index.html"))); + assertEquals("body", read(new File(dest, "site/page.html"))); + assertEquals("PNGDATA", read(new File(dest, "site/img/a.png"))); + } + + @Test + public void reRunSkipsWhatIsAlreadyThere() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with(FakeNode.file("index.html", "first")); + final File dest = tmp.newFolder("dest"); + + LegacyMirrorImport.copyTree(root, dest); + final LegacyMirrorImport.Result second = LegacyMirrorImport.copyTree(root, dest); + + assertEquals(0, second.copied); + assertEquals(1, second.skipped); + assertEquals("first", read(new File(dest, "index.html"))); + } + + @Test + public void oneUnreadableFileDoesNotAbortTheRest() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with( + FakeNode.file("good1.html", "ok"), + FakeNode.poison("bad.html"), + FakeNode.file("good2.html", "ok")); + final File dest = tmp.newFolder("dest"); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + assertFalse(r.isComplete()); + assertEquals(2, r.copied); + assertEquals(1, r.failed); + assertTrue(new File(dest, "good1.html").exists()); + assertTrue(new File(dest, "good2.html").exists()); + // The poisoned file leaves nothing half-written that a re-run would mistake for complete. + assertFalse(new File(dest, "bad.html").exists()); + assertFalse(new File(dest, "bad.html.part").exists()); + } + + @Test + public void refusesAnEntryThatWouldEscapeTheDestination() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with( + FakeNode.file("../escape.html", "evil"), + FakeNode.file("ok.html", "ok")); + final File dest = tmp.newFolder("dest"); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + assertEquals(1, r.copied); + assertEquals(1, r.failed); + assertFalse(new File(dest.getParentFile(), "escape.html").exists()); + } + + /** A file whose stream yields some bytes and then throws, to model a copy killed mid-write. */ + private static final class HalfThenThrow implements LegacyMirrorImport.Source { + private final String name; + + HalfThenThrow(final String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public long length() { + return 100; + } + + @Override + public List children() { + return new ArrayList(); + } + + @Override + public InputStream openStream() { + return new InputStream() { + private int served; + + @Override + public int read() throws IOException { + if (served++ < 50) { + return 'x'; + } + throw new IOException("stream died mid-file"); + } + }; + } + } + + @Test + public void aMidWriteFailureLeavesNoFileThatLooksComplete() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with(new HalfThenThrow("truncated.html")); + final File dest = tmp.newFolder("dest"); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + assertEquals(1, r.failed); + assertEquals(0, r.copied); + // Neither the final name nor the temp may survive: a re-run must see it as missing, not done. + assertFalse(new File(dest, "truncated.html").exists()); + assertFalse(new File(dest, "truncated.html.part").exists()); + } + + /** + * The real point of the temp-then-rename: a file only ever appears under its final name once + * whole, so a run killed mid-copy cannot leave a truncated file that a re-run trusts. We cannot + * kill the process here, but we can observe that while bytes are flowing they are NOT at the + * final name. A direct-to-destination copy fails this. + */ + @Test + public void bytesAreNotVisibleUnderTheFinalNameUntilComplete() throws Exception { + final File dest = tmp.newFolder("dest"); + final boolean[] finalNameSeenMidWrite = { false }; + final LegacyMirrorImport.Source watcher = new LegacyMirrorImport.Source() { + @Override + public String getName() { + return "HTTrack"; + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public long length() { + return 0; + } + + @Override + public List children() { + final List kids = new ArrayList(); + kids.add(new LegacyMirrorImport.Source() { + @Override + public String getName() { + return "page.html"; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public long length() { + return 10; + } + + @Override + public List children() { + return new ArrayList(); + } + + @Override + public InputStream openStream() { + return new InputStream() { + private int served; + + @Override + public int read() { + if (new File(dest, "page.html").exists()) { + finalNameSeenMidWrite[0] = true; + } + return served++ < 10 ? 'x' : -1; + } + }; + } + }); + return kids; + } + + @Override + public InputStream openStream() { + throw new UnsupportedOperationException(); + } + }; + + LegacyMirrorImport.copyTree(watcher, dest); + + assertFalse("bytes appeared under the final name before the copy finished", + finalNameSeenMidWrite[0]); + assertEquals("xxxxxxxxxx", read(new File(dest, "page.html"))); + } + + /** + * DocumentFile.getName() may return null, and "."/".." are meaningless as entries. The null is + * the load-bearing case: without the guard, new File(dir, null) throws and takes the whole + * import down mid-run. The good sibling must still arrive, and copyTree must not throw. + */ + @Test + public void aNullOrDotEntryIsRefusedWithoutDeraillingTheRest() throws Exception { + final FakeNode root = FakeNode.dir("HTTrack").with( + FakeNode.file(null, "no-name"), + FakeNode.file("..", "parent"), + FakeNode.file(".", "self"), + FakeNode.file("", "empty"), + FakeNode.file("keep.html", "ok")); + final File dest = tmp.newFolder("dest"); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + assertEquals(1, r.copied); + assertEquals(4, r.failed); + assertTrue(new File(dest, "keep.html").exists()); + // Nothing escaped to the destination's parent via "..". + assertFalse(new File(dest.getParentFile(), "parent").exists()); + } + + @Test + public void doesNotDropAFileWhereADirectoryAlreadyStands() throws Exception { + final File dest = tmp.newFolder("dest"); + assertTrue(new File(dest, "clash").mkdir()); + final FakeNode root = FakeNode.dir("HTTrack").with(FakeNode.file("clash", "would-be-lost")); + + final LegacyMirrorImport.Result r = LegacyMirrorImport.copyTree(root, dest); + + // The name collision is reported, not silently counted as skipped. + assertEquals(1, r.failed); + assertEquals(0, r.skipped); + assertTrue(new File(dest, "clash").isDirectory()); + } + + @Test + public void totalSizeSumsEveryFile() { + final FakeNode root = FakeNode.dir("HTTrack").with( + FakeNode.file("a", "12345"), + FakeNode.dir("d").with(FakeNode.file("b", "123"))); + assertEquals(8, LegacyMirrorImport.totalSize(root)); + } + + @Test + public void aByteForByteCopyMatchesBinaryContent() throws Exception { + final byte[] raw = new byte[1000]; + for (int i = 0; i < raw.length; i++) { + raw[i] = (byte) (i * 7); + } + final FakeNode root = FakeNode.dir("HTTrack").with( + new FakeNode("blob.bin", false, raw, false)); + final File dest = tmp.newFolder("dest"); + + LegacyMirrorImport.copyTree(root, dest); + + assertArrayEquals(raw, Files.readAllBytes(new File(dest, "blob.bin").toPath())); + } +}