feat: import books from watched device folders

.github/workflows/linux-packaging.yml

@@ -1,12 +1,11 @@
 name: Linux packaging check
 
 on:
+  workflow_dispatch:
   pull_request:
     paths:
-      - 'app/linux/**'
-      - 'app/pubspec.yaml'
-      - 'app/pubspec.lock'
-      - 'app/integration_test/desktop_package_*.dart'
+      - 'app/**'
+      - 'packages/**'
       - 'tools/run_packaged_smoke.py'
       - 'tools/check_packaged_reader.*'
       - 'tools/install_linux_build_dependencies.sh'
@@ -45,6 +44,8 @@ jobs:
           cache: true
       - name: Install locked dependencies
         run: flutter pub get --enforce-lockfile
+      - name: Validate watched folders on Linux
+        run: flutter test test/folders
       - name: Compile Linux release
         run: flutter build linux --release
       - name: Package Linux downloads

.github/workflows/windows-packaging.yml

@@ -41,6 +41,8 @@ jobs:
           flutter-version: '3.41.2'
           cache: true
       - run: flutter pub get --enforce-lockfile
+      - name: Validate watched folders on Windows
+        run: flutter test test/folders
       - name: Prepare preview version and endpoints
         run: |
           python ../tools/release_gate.py client --endpoints

app/android/app/src/main/kotlin/com/papyrus/reader/MainActivity.kt

@@ -1,5 +1,186 @@
 package com.papyrus.reader
 
+import android.app.Activity
+import android.content.Intent
+import android.database.Cursor
+import android.net.Uri
+import android.os.Handler
+import android.os.Looper
+import android.provider.DocumentsContract
 import io.flutter.embedding.android.FlutterActivity
+import io.flutter.embedding.engine.FlutterEngine
+import io.flutter.plugin.common.MethodChannel
+import java.util.ArrayDeque
+import java.util.UUID
+import java.util.concurrent.Executors
 
-class MainActivity : FlutterActivity()
+class MainActivity : FlutterActivity() {
+    private val folderRequest = 7142
+    private var folderResult: MethodChannel.Result? = null
+    private val worker = Executors.newSingleThreadExecutor()
+    private val main = Handler(Looper.getMainLooper())
+    private val cursors = mutableMapOf<String, FolderCursor>()
+    private val grants by lazy { getSharedPreferences("papyrus-folder-grants", MODE_PRIVATE) }
+
+    override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
+        super.configureFlutterEngine(flutterEngine)
+        MethodChannel(flutterEngine.dartExecutor.binaryMessenger, "papyrus/folders").setMethodCallHandler { call, result ->
+            if (call.method == "pick") {
+                if (folderResult != null) {
+                    result.error("busy", "A folder picker is already open.", null)
+                } else {
+                    folderResult = result
+                    try {
+                        startActivityForResult(Intent(Intent.ACTION_OPEN_DOCUMENT_TREE).apply {
+                            addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION)
+                        }, folderRequest)
+                    } catch (error: Exception) {
+                        folderResult = null
+                        result.error("unavailable", "The system folder picker is unavailable.", null)
+                    }
+                }
+                return@setMethodCallHandler
+            }
+            worker.execute {
+                try {
+                    val key = call.argument<String>("grant")
+                    val tree = key?.let { grants.getString(it, null)?.let(Uri::parse) }
+                    fun requireTree(): Uri = tree ?: throw SecurityException("Reconnect this folder to allow access.")
+                    val value: Any? = when (call.method) {
+                        "restore" -> {
+                            val uri = requireTree()
+                            contentResolver.persistedUriPermissions.any { it.uri == uri && it.isReadPermission }
+                        }
+                        "start" -> {
+                            val uri = requireTree()
+                            val cursor = UUID.randomUUID().toString()
+                            cursors[cursor] = FolderCursor(uri, call.argument<Boolean>("recursive") == true)
+                            cursor
+                        }
+                        "next" -> cursors[call.argument<String>("cursor")]?.next() ?: emptyList<Any>()
+                        "close" -> { cursors.remove(call.argument<String>("cursor"))?.close(); null }
+                        "stat" -> {
+                            val uri = DocumentsContract.buildDocumentUriUsingTree(requireTree(), call.argument<String>("key")!!)
+                            contentResolver.query(uri, projection, null, null, null)?.use { cursor ->
+                                if (!cursor.moveToFirst()) throw java.io.FileNotFoundException("Source file unavailable offline.")
+                                metadata(cursor)
+                            } ?: throw java.io.FileNotFoundException("Source file unavailable offline.")
+                        }
+                        "read" -> {
+                            val uri = DocumentsContract.buildDocumentUriUsingTree(requireTree(), call.argument<String>("key")!!)
+                            contentResolver.openInputStream(uri)?.use { it.readBytes() }
+                                ?: throw java.io.FileNotFoundException("Source file unavailable offline.")
+                        }
+                        "release" -> {
+                            if (key != null && !grants.edit().remove(key).commit()) {
+                                throw java.io.IOException("Could not release folder permission.")
+                            }
+                            if (tree != null && grants.all.values.none { it == tree.toString() }) {
+                                try {
+                                    contentResolver.releasePersistableUriPermission(tree, Intent.FLAG_GRANT_READ_URI_PERMISSION)
+                                } catch (_: SecurityException) { }
+                            }
+                            null
+                        }
+                        else -> throw IllegalArgumentException("Unknown folder operation.")
+                    }
+                    main.post { result.success(value) }
+                } catch (error: Exception) {
+                    val code = if (error is SecurityException) "access" else "unavailable"
+                    main.post { result.error(code, error.message ?: "Could not access this folder.", null) }
+                }
+            }
+        }
+    }
+
+    @Deprecated("Activity result bridge for Flutter folder access")
+    override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
+        if (requestCode != folderRequest) {
+            super.onActivityResult(requestCode, resultCode, data)
+            return
+        }
+        val result = folderResult
+        folderResult = null
+        val uri = data?.data
+        if (resultCode != Activity.RESULT_OK || uri == null) {
+            result?.success(null)
+            return
+        }
+        try {
+            contentResolver.takePersistableUriPermission(uri, Intent.FLAG_GRANT_READ_URI_PERMISSION)
+            val document = DocumentsContract.buildDocumentUriUsingTree(uri, DocumentsContract.getTreeDocumentId(uri))
+            val name = contentResolver.query(document, arrayOf(DocumentsContract.Document.COLUMN_DISPLAY_NAME), null, null, null)
+                ?.use { if (it.moveToFirst()) it.getString(0) else null } ?: "Book folder"
+            val key = UUID.randomUUID().toString()
+            if (!grants.edit().putString(key, uri.toString()).commit()) throw java.io.IOException("Could not save folder permission.")
+            result?.success(mapOf("key" to key, "identity" to uri.toString(), "name" to name, "location" to name, "persistent" to true))
+        } catch (error: Exception) {
+            if (grants.all.values.none { it == uri.toString() }) {
+                try {
+                    contentResolver.releasePersistableUriPermission(uri, Intent.FLAG_GRANT_READ_URI_PERMISSION)
+                } catch (_: SecurityException) { }
+            }
+            result?.error("access", "Could not keep access to this folder. Select a permitted subfolder.", null)
+        }
+    }
+
+    override fun onDestroy() {
+        worker.execute {
+            cursors.values.forEach { it.close() }
+            cursors.clear()
+        }
+        worker.shutdown()
+        super.onDestroy()
+    }
+
+    private val projection = arrayOf(
+        DocumentsContract.Document.COLUMN_DOCUMENT_ID,
+        DocumentsContract.Document.COLUMN_DISPLAY_NAME,
+        DocumentsContract.Document.COLUMN_MIME_TYPE,
+        DocumentsContract.Document.COLUMN_SIZE,
+        DocumentsContract.Document.COLUMN_LAST_MODIFIED
+    )
+
+    private fun metadata(cursor: Cursor): Map<String, Any> = mapOf(
+        "key" to cursor.getString(0), "name" to cursor.getString(1),
+        "size" to (if (cursor.isNull(3)) -1L else cursor.getLong(3)),
+        "modified" to (if (cursor.isNull(4)) 0L else cursor.getLong(4))
+    )
+
+    private inner class FolderCursor(private val tree: Uri, private val recursive: Boolean) {
+        private val directories = ArrayDeque<String>().apply { add(DocumentsContract.getTreeDocumentId(tree)) }
+        private var current: Cursor? = null
+
+        fun next(): List<Map<String, Any>> {
+            val batch = mutableListOf<Map<String, Any>>()
+            while (batch.size < 128) {
+                if (current == null) {
+                    if (directories.isEmpty()) break
+                    val directory = directories.removeFirst()
+                    val uri = DocumentsContract.buildChildDocumentsUriUsingTree(tree, directory)
+                    try {
+                        current = contentResolver.query(uri, projection, null, null, null)
+                            ?: throw java.io.FileNotFoundException("Folder unavailable. Reconnect the drive or folder.")
+                    } catch (error: Exception) {
+                        if (directory == DocumentsContract.getTreeDocumentId(tree)) throw error
+                        batch.add(mapOf("key" to directory, "name" to directory.substringAfterLast('/'),
+                            "size" to 0L, "modified" to 0L, "error" to (error.message ?: "Could not read this subfolder.")))
+                        continue
+                    }
+                }
+                val cursor = current!!
+                if (!cursor.moveToNext()) { cursor.close(); current = null; continue }
+                val name = cursor.getString(1)
+                if (name.startsWith(".") || name.startsWith("~") || Regex("\\.(tmp|part|partial|crdownload|download)$", RegexOption.IGNORE_CASE).containsMatchIn(name)) continue
+                if (cursor.getString(2) == DocumentsContract.Document.MIME_TYPE_DIR) {
+                    if (recursive) directories.add(cursor.getString(0))
+                } else {
+                    batch.add(metadata(cursor))
+                }
+            }
+            return batch
+        }
+
+        fun close() { current?.close(); current = null; directories.clear() }
+    }
+}

app/integration_test/powersync_books_integration_test.dart

@@ -1,19 +1,139 @@
 import 'dart:io';
+import 'dart:typed_data';
 
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
 import 'package:papyrus/auth/auth_api_client.dart';
 import 'package:papyrus/auth/auth_repository.dart';
 import 'package:papyrus/auth/papyrus_api_config.dart';
 import 'package:papyrus/auth/token_store.dart';
+import 'package:papyrus/data/data_store.dart';
+import 'package:papyrus/folders/directory_access.dart';
+import 'package:papyrus/folders/folder_import_target.dart';
+import 'package:papyrus/folders/folder_repository.dart';
+import 'package:papyrus/folders/watched_folders.dart';
+import 'package:papyrus/media/media_storage_scope.dart';
+import 'package:papyrus/media/media_upload_queue.dart';
 import 'package:papyrus/models/book.dart';
 import 'package:papyrus/powersync/papyrus_powersync_connector.dart';
 import 'package:papyrus/powersync/powersync_service.dart';
+import 'package:papyrus/providers/auth_provider.dart';
+import 'package:papyrus/services/book_import_service_stub.dart';
 import 'package:path/path.dart' as path;
+import 'package:shared_preferences/shared_preferences.dart';
 import 'package:uuid/uuid.dart';
 
+import 'desktop_package_fixtures.dart' show createEpub;
+
 const runIntegration = bool.fromEnvironment('RUN_POWERSYNC_INTEGRATION');
 
+/// Keeps OS grant acceptance separate from the live import and sync contract.
+class IntegrationFolderAccess implements DirectoryAccess {
+  IntegrationFolderAccess(this.bytes);
+  final Uint8List bytes;
+  static const grant = FolderGrant(key: 'fixture', identity: 'fixture', name: 'Fixture', location: 'Test books');
+  FolderFile get file => FolderFile(key: 'book.epub', name: 'book.epub', size: bytes.length, modified: 1);
+  @override
+  bool get supportsWatching => true;
+  @override
+  Future<FolderGrant?> pick() async => grant;
+  @override
+  Future<bool> reconnect(FolderGrant grant) async => true;
+  @override
+  Stream<FolderFile> enumerate(FolderGrant grant, {required bool recursive}) => Stream.value(file);
+  @override
+  Future<FolderFile> stat(FolderGrant grant, FolderFile file) async => this.file;
+  @override
+  Future<Uint8List> read(FolderGrant grant, FolderFile file) async => bytes;
+  @override
+  Stream<void> changes(FolderGrant grant) => const Stream.empty();
+  @override
+  Future<void> release(FolderGrant grant) async {}
+}
+
+Future<void> verifyFolderImport(
+  PapyrusPowerSyncService first,
+  PapyrusPowerSyncService second,
+  AuthRepository repository,
+  Directory root,
+) async {
+  SharedPreferences.setMockInitialValues({});
+  final preferences = await SharedPreferences.getInstance();
+  final auth = AuthProvider(preferences, repository: repository, bootstrapOnCreate: false);
+  await auth.bootstrap();
+  final queue = MediaUploadQueue(preferences);
+  await queue.activateScope(MediaStorageScope(profileKey: 'official', userId: auth.user!.userId));
+  final store = DataStore();
+  await store.attachBookRepository(first);
+  final importer = BookImportService();
+  final source = File(path.join(root.path, 'source.epub'));
+  final original = createEpub();
+  await source.writeAsBytes(original);
+  final access = IntegrationFolderAccess(original);
+  final folders = WatchedFolders(
+    access: access,
+    libraryReady: () => true,
+    captureTarget: () => LibraryFolderImportTarget(
+      store: store,
+      queue: queue,
+      importer: importer,
+      auth: auth,
+      database: first,
+    ),
+    openRepository: (_) => FolderRepository.open('integration', databasePath: path.join(root.path, 'folders.db')),
+  );
+  String? importedId;
+
+  try {
+    await first.setOnline(false);
+    await folders.activate('integration');
+    await folders.connect(IntegrationFolderAccess.grant);
+    await folders.waitUntilIdle();
+    expect(folders.folders.single.added, 1);
+    final imported = store.books.singleWhere((book) => book.fileHash != null);
+    importedId = imported.id;
+    expect(await importer.getBookFile(imported.id), original);
+    expect(queue.pendingTasks.where((task) => task.bookId == imported.id), isNotEmpty);
+    expect(await second.getById(imported.id), isNull);
+    await folders.activate(null);
+    await folders.activate('integration');
+    await folders.waitUntilIdle();
+    expect(store.books.where((book) => book.fileHash == imported.fileHash), hasLength(1));
+    await first.setOnline(true);
+    await waitForBook(second, imported.id);
+    await queue.processPending(
+      dataStore: store,
+      readBookFile: importer.getBookFile,
+      readPendingCover: importer.getPendingCoverFile,
+      uploadMedia: repository.uploadMedia,
+    );
+    expect(queue.pendingTasks, isEmpty);
+    final deadline = DateTime.now().add(const Duration(seconds: 20));
+
+    while ((await second.getById(imported.id))?.fileMediaId == null && DateTime.now().isBefore(deadline)) {
+      await Future<void>.delayed(const Duration(milliseconds: 200));
+    }
+
+    expect((await second.getById(imported.id))?.fileMediaId, isNotNull);
+    expect(await source.readAsBytes(), original);
+  } finally {
+    await folders.activate(null);
+    folders.dispose();
+
+    if (importedId != null) {
+      await first.delete(importedId);
+      await importer.deleteBookFile(importedId);
+      await importer.deletePendingCoverFile(queue.activeScope!, importedId);
+    }
+
+    await store.disposeBookRepository();
+    store.dispose();
+    queue.dispose();
+    auth.dispose();
+    importer.dispose();
+  }
+}
+
 class MemoryRefreshTokenStorage implements RefreshTokenStorage {
   String? value;
 
@@ -119,11 +239,14 @@ void main() {
       await first.setOnline(true);
       await waitForBook(second, book.id, title: 'Offline edit');
       expect((await second.getById(book.id))?.title, 'Offline edit');
+      await verifyFolderImport(first, second, firstAuth, root);
       await second.delete(book.id);
       await waitForBook(first, book.id, present: false);
     } finally {
       await Future.wait([first.close(), second.close(), other.close()]);
       await root.delete(recursive: true);
+      await firstAuth.deleteAccount();
+      await otherAuth.deleteAccount();
     }
   }, skip: !runIntegration);
 }

app/ios/Flutter/Debug.xcconfig

@@ -1,2 +1,9 @@
 #include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"
 #include "Generated.xcconfig"
+
+// Flutter 3.41 invokes lipo with multiple architectures, which Xcode 27 rejects.
+// Debug simulators run on the host architecture. Device and release builds keep their defaults.
+PAPYRUS_SIMULATOR_ARCH_arm64 = arm64
+PAPYRUS_SIMULATOR_ARCH_arm64e = arm64
+PAPYRUS_SIMULATOR_ARCH_x86_64 = x86_64
+ARCHS[sdk=iphonesimulator*] = $(PAPYRUS_SIMULATOR_ARCH_$(NATIVE_ARCH_ACTUAL))