fix: come back to the listing page an import stopped in

.github/workflows/integration-test.yml

@@ -342,6 +342,86 @@ jobs:
           fi
           exit $fail
 
+      - name: Import again in batches that end in the middle of a listing page
+        working-directory: server/
+        run: |
+          # 20 bytes runs out on the first file of the second listing page, so the import
+          # has to come back to that page in a later batch
+          php occ config:app:set ${{ env.APP_NAME }} import_batch_size --value 20
+          php occ user:setting admin ${{ env.APP_NAME }} onedrive_output_dir "/OneDrive resume"
+          # drop the notification of the first import, the one checked below has to be the
+          # one this import sends
+          curl -s -X DELETE -u admin:admin -H 'OCS-APIRequest: true' \
+            'http://localhost:8080/ocs/v2.php/apps/notifications/api/v2/notifications' >/dev/null
+          mkdir -p "data/admin/files/OneDrive resume"
+          printf 'already there' > "data/admin/files/OneDrive resume/already.txt"
+          php occ files:scan admin -q
+          echo "STUB_LOG_LINES=$(wc -l < stub.log)" >> "$GITHUB_ENV"
+          php occ ${{ env.APP_NAME }}:start-import admin
+          importing=1
+          for run in {1..8}; do
+            php cron.php
+            importing=$(php occ user:setting admin ${{ env.APP_NAME }} importing_onedrive 2>/dev/null | tail -1 || echo "1")
+            imported=$(php occ user:setting admin ${{ env.APP_NAME }} nb_imported_files 2>/dev/null | tail -1 || echo "?")
+            echo "batch $run: importing=$importing imported=$imported"
+            if [ "$importing" = "0" ]; then break; fi
+          done
+          if [ "$importing" != "0" ]; then
+            echo "The import did not finish"
+            exit 1
+          fi
+
+      - name: Verify the import came back for the page it stopped in
+        working-directory: server/
+        run: |
+          tail -n +$((STUB_LOG_LINES + 1)) stub.log > resume.log
+          grep -oE 'stub (GET|POST) [^ ]+ -> [0-9]+' resume.log | sort | uniq -c
+          fail=0
+          pages=$(grep -c 'skiptoken=page2' resume.log || true)
+          if [ "$pages" -lt 2 ]; then
+            echo "the second listing page was fetched $pages time(s), the import never came back to it"
+            fail=1
+          fi
+          # coming back to the page is the point: walking the whole folder again would
+          # fetch the first page a second time as well, and would pass the check above
+          first=$(grep -cE 'root/children -> 200' resume.log || true)
+          if [ "$first" != "1" ]; then
+            echo "the first listing page was fetched $first time(s), the import started the folder over"
+            fail=1
+          fi
+          target="data/admin/files/OneDrive resume"
+          find "$target" -type f -printf '%P %s bytes\n' | sort
+          check() { # path, expected size
+            if [ ! -f "$target/$1" ]; then echo "missing: $1"; fail=1; return; fi
+            size=$(stat -c%s "$target/$1")
+            if [ "$size" != "$2" ]; then echo "$1 has $size bytes, expected $2"; fail=1; fi
+          }
+          check normal.txt 12
+          check empty.txt 0
+          check flaky.txt 9
+          check sub/nested.txt 11
+          if [ -f "$target/broken.txt" ]; then
+            echo "broken.txt should not have been created, its download failed twice"
+            fail=1
+          fi
+          # flaky.txt was brought by the first batch and sits in the page the second batch
+          # walked again: it must not be reported among the files that were already there,
+          # and already.txt shares that page, so no file is
+          notification=$(curl -s -u admin:admin -H 'OCS-APIRequest: true' -H 'Accept: application/json' \
+            'http://localhost:8080/ocs/v2.php/apps/notifications/api/v2/notifications')
+          expected='4 files were imported from OneDrive storage. 1 file could not be downloaded, check the server logs for details.'
+          subjects=$(php -r 'foreach (json_decode(stream_get_contents(STDIN), true)["ocs"]["data"] as $n) { if (($n["app"] ?? "") === "integration_onedrive") { echo $n["subject"], "\n"; } }' <<< "$notification")
+          echo "$subjects"
+          if [ "$(printf '%s\n' "$subjects" | grep -c .)" != "1" ]; then
+            echo "expected exactly one notification of this import"
+            fail=1
+          fi
+          if [ "$subjects" != "$expected" ]; then
+            echo "expected: $expected"
+            fail=1
+          fi
+          exit $fail
+
       - name: Show the logs on failure
         working-directory: server/
         if: failure()

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - Added support for Nextcloud 36
+- Added the import_batch_size app setting, how much a single run of the import job downloads before it lets the next one continue
 
 ### Fixed
 
@@ -25,6 +26,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Count imported empty files as imported
 - Report the number of files the current import brought, a counter left over from an interrupted import is no longer added to it
 - Say in the log which file of an import could not be looked up in the target folder
+- Come back to the folder listing page an import stopped in, the files behind it were left out and the import still reported itself as finished
+- Stop counting the files an import downloaded itself among the files that were already there
+- Try a folder whose listing failed again in a later batch instead of leaving it out of the import
+- List a folder again from its first page when the page an import stopped in cannot be listed any more
 - Test the import against a stubbed Graph API, including failed downloads, empty files and paging
 
 ## [3.5.2] - 2026-07-28

README.md

@@ -18,6 +18,12 @@ The account configuration and data migration happen in the "Data migration" user
 
 There also is a "Connected accounts" **admin** settings section that you must visit to configure a Microsoft Azure OAuth app to allow your Nextcloud users to authenticate to Microsoft services.
 
+A single run of the file import job downloads 500 MB before it lets the next run continue. On a server where that is too much for one job, set another size in bytes:
+
+```
+occ config:app:set integration_onedrive import_batch_size --value 100000000
+```
+
 ## **🛠️ State of maintenance**
 
 While there are many things that could be done to further improve this app, the app is currently maintained with **limited effort**. This means:

lib/AppInfo/Application.php

@@ -27,6 +27,12 @@ class Application extends App implements IBootstrap {
 	public const APP_ID = 'integration_onedrive';
 	public const IMPORT_JOB_TIMEOUT = 3600;
 
+	/**
+	 * How much a single run of the import job downloads before it stops and lets the next
+	 * one continue, in bytes. Overridable with the import_batch_size app setting.
+	 */
+	public const IMPORT_BATCH_SIZE = 500000000;
+
 	public function __construct(array $urlParams = []) {
 		parent::__construct(self::APP_ID, $urlParams);
 	}

lib/Service/OnedriveStorageAPIService.php

@@ -40,6 +40,12 @@
 
 class OnedriveStorageAPIService {
 
+	/** A directory of the import tree that has not been listed at all yet. */
+	private const DIR_NOT_STARTED = 'todo';
+
+	/** A directory whose first listing page a batch has begun but not finished. */
+	private const DIR_STARTED = 'started';
+
 	private const FILE_DOWNLOADED = 'downloaded';
 	private const FILE_ALREADY_THERE = 'already there';
 	private const FILE_FAILED = 'failed';
@@ -199,11 +205,15 @@ public function importOnedriveJob(string $userId): void {
 		$importTreeStr = $this->config->getUserValue($userId, Application::APP_ID, 'import_tree', '[]');
 		/** @var array $importTree */
 		$importTree = ($importTreeStr === '[]' || $importTreeStr === '') ? [] : json_decode($importTreeStr, true);
-		// import by batch of 500 MB
+		// import by batches, 500 MB each unless the admin asked for another size
+		$batchSize = (int)$this->config->getAppValue(Application::APP_ID, 'import_batch_size', (string)Application::IMPORT_BATCH_SIZE);
+		if ($batchSize <= 0) {
+			$batchSize = Application::IMPORT_BATCH_SIZE;
+		}
 		$alreadyImportedSize = (float)$this->config->getUserValue($userId, Application::APP_ID, 'imported_size', '0');
 		$alreadyImportedNumber = (int)$this->config->getUserValue($userId, Application::APP_ID, 'nb_imported_files', '0');
 		try {
-			$result = $this->importFiles($userId, $targetPath, 500000000, $alreadyImportedSize, $alreadyImportedNumber, $importTree);
+			$result = $this->importFiles($userId, $targetPath, $batchSize, $alreadyImportedSize, $alreadyImportedNumber, $importTree);
 		} catch (Exception|Throwable $e) {
 			$result = [
 				'error' => 'Unknow job failure. ' . $e->getMessage(),
@@ -282,11 +292,19 @@ public function importFiles(string $userId, string $targetPath,
 				);
 			} else {
 				foreach ($importTree as $path => $state) {
-					if ($state === 'todo') {
-						$downloadResult = $this->downloadDir(
-							$userId, $topFolder, $maxDownloadSize, 0, 0, 0, (string)$path, $alreadyImportedSize, $alreadyImportedNumber, $importTree
-						);
+					if (!isset($importTree[$path])) {
+						// a directory this batch already finished on its way through a parent
+						continue;
 					}
+					// an unfinished directory is remembered as not started at all, as begun
+					// on its first page, or with the listing page the last batch stopped in
+					$startedBefore = $state !== self::DIR_NOT_STARTED;
+					$resumeToken = (!$startedBefore || $state === self::DIR_STARTED || !is_string($state) || $state === '')
+						? null
+						: $state;
+					$downloadResult = $this->downloadDir(
+						$userId, $topFolder, $maxDownloadSize, 0, 0, 0, (string)$path, $alreadyImportedSize, $alreadyImportedNumber, $importTree, $resumeToken, $startedBefore
+					);
 				}
 			}
 		} catch (MaxDownloadSizeReachedException $e) {
@@ -314,6 +332,8 @@ private function downloadDir(
 		float $alreadyImportedSize,
 		int $alreadyImportedNumber,
 		array &$importTree,
+		?string $resumeToken = null,
+		bool $resuming = false,
 	): array {
 		$newDownloadedSize = (float)$downloadedSize;
 		$newTotalSeenNumber = $totalSeenNumber;
@@ -335,9 +355,40 @@ private function downloadDir(
 		/** @var string[] $subDirs */
 		$subDirs = [];
 		$params = [];
+		if ($resumeToken !== null) {
+			$params['$skiptoken'] = $resumeToken;
+		}
+		// Remember this directory as unfinished for as long as its listing is not
+		// exhausted. A batch that stops in the middle of it, because it reached its
+		// download size, has to come back to it, and to the page it stopped in.
+		$importTree[$path] = $resumeToken ?? self::DIR_NOT_STARTED;
+		$listingStartedOver = false;
+		// the files of a page an earlier batch had already started are not files that
+		// were "already there": this import downloaded them itself
+		$walkingThePageAgain = $resuming;
 		do {
 			$result = $this->onedriveApiService->request($userId, $endPoint, $params);
 			if (isset($result['error']) || !isset($result['value']) || !is_array($result['value'])) {
+				if (isset($params['$skiptoken']) && !$listingStartedOver) {
+					// the page cannot be listed any more, its token may simply have expired:
+					// start the directory over once, the files it already brought are skipped
+					// as existing ones
+					$this->logger->info(
+						'OneDrive could not list a page of ' . ($path === '' ? 'the import folder' : $path) . ', starting the folder over',
+						['app' => Application::APP_ID]
+					);
+					$listingStartedOver = true;
+					unset($params['$skiptoken']);
+					$importTree[$path] = self::DIR_NOT_STARTED;
+					$subDirs = [];
+					continue;
+				}
+				// the directory stays in the tree: a later batch tries it again, and an
+				// import that ends before that at least says in the log what it missed
+				$this->logger->warning(
+					'OneDrive error listing ' . ($path === '' ? 'the import folder' : $path) . ': ' . ($result['error'] ?? 'no file list in the answer'),
+					['app' => Application::APP_ID]
+				);
 				return [
 					'downloadedSize' => $newDownloadedSize,
 					'totalSeenNumber' => $newTotalSeenNumber,
@@ -346,6 +397,11 @@ private function downloadDir(
 			}
 
 			$pageSkipped = 0;
+			if (!isset($params['$skiptoken'])) {
+				// the first page has begun: a batch that stops inside it has to come back
+				// to it, and must not count its files as files that were already there
+				$importTree[$path] = self::DIR_STARTED;
+			}
 			/** @var OneDriveItem $item */
 			foreach ($result['value'] as $item) {
 				if (isset($item['file'])) {
@@ -380,27 +436,28 @@ private function downloadDir(
 					$subDirs[] = $item['name'];
 					// mark for progress tracking
 					$subPath = ltrim($path . '/' . $item['name']);
-					$importTree[$subPath] = 'todo';
+					$importTree[$subPath] = self::DIR_NOT_STARTED;
 				}
 			}
-			if ($pageSkipped > 0) {
+			if ($pageSkipped > 0 && !$walkingThePageAgain) {
 				// one write per listing page, skipped files are frequent on re-imports
 				$nbSkipped = (int)$this->config->getUserValue($userId, Application::APP_ID, 'nb_skipped_files', '0');
 				$this->config->setUserValue($userId, Application::APP_ID, 'nb_skipped_files', (string)($nbSkipped + $pageSkipped));
 			}
-
-			// if this directory was marked unfinished, remove it now
-			if (isset($importTree[$path])) {
-				unset($importTree[$path]);
-			}
+			// only the first page of a resumed directory is one an earlier batch had started
+			$walkingThePageAgain = false;
 
 			// prepare next page if any
 			if (isset($result['@odata.nextLink'])
 				&& $result['@odata.nextLink']
 				&& preg_match('/\$skiptoken=/i', $result['@odata.nextLink'])
 			) {
 				$params['$skiptoken'] = preg_replace('/.*\$skiptoken=/', '', $result['@odata.nextLink']);
+				// come back to this page, not to the first one, if the import stops here
+				$importTree[$path] = $params['$skiptoken'];
 			} else {
+				// the whole directory has been listed
+				unset($importTree[$path]);
 				break;
 			}
 		} while (true);