fix(sidebar): stream and cache folders with bounded GUI batches

src/core/async_fs.py

@@ -8,12 +8,25 @@
 
 import os
 import threading
+from itertools import chain, islice
 
 from PySide6.QtCore import QObject, Signal
 
 
 class AsyncFs(QObject):
     finished = Signal(object, object, object)  # token, result, exception
+    preview = Signal(object, object)          # token, first bounded entries
+
+    def submit_preview(self, token, fn, *args):
+        """Emit one early page, then a complete sorted snapshot; no signal flood."""
+        def collect():
+            remaining = iter(fn(*args))
+            first = list(islice(remaining, 128))
+            first.sort(key=lambda entry: entry[0].lower())
+            if first:
+                self.preview.emit(token, first)
+            return sorted(chain(first, remaining), key=lambda entry: entry[0].lower())
+        self.submit(token, collect)
 
     def submit(self, token, fn, *args):
         # ponytail: one thread per request; a pool if request storms appear
@@ -45,3 +58,16 @@ def list_subfolders(path):
         except OSError:
             continue
     return entries
+
+
+def iter_subfolders(path):
+    """Stream visible subfolders; metadata stays in the filesystem worker."""
+    with os.scandir(path) as entries:
+        for entry in entries:
+            if entry.name.startswith('.'):
+                continue
+            try:
+                if entry.is_dir():
+                    yield entry.name, entry.path
+            except OSError:
+                continue

src/gui/sidebar/sidebar_main.py

@@ -6,6 +6,7 @@
 """
 
 import time
+from itertools import islice
 
 from PySide6.QtWidgets import (
     QWidget, QVBoxLayout, QHBoxLayout, QStackedWidget,
@@ -17,7 +18,7 @@
 
 # Module importieren - absolute Imports
 from core.cloud_locations import find_cloud_locations
-from core.async_fs import AsyncFs, list_subfolders
+from core.async_fs import AsyncFs, iter_subfolders
 from core.file_icon_helper import generic_icon
 from gui.sidebar.search_panel import SearchPanel as AdvancedSearchPanel
 from modules.launcher import AppsPanel
@@ -43,6 +44,7 @@ class TreePanel(QWidget):
     _usage_ready = Signal(str, object)
 
     LIST_TIMEOUT_MS = 15000        # give up on one folder listing
+    TREE_BATCH_SIZE = 128          # bound widget creation per GUI event
     USAGE_MIN_INTERVAL_S = 60      # automatic capacity refresh per drive
     _dir_cache = {}                # path -> [(name, path)], shared, last known listing
 
@@ -55,6 +57,7 @@ def __init__(self, parent=None):
         self._usage_ready.connect(self._on_drive_usage)
         self._fs = AsyncFs(self)
         self._fs.finished.connect(self._on_fs_result)
+        self._fs.preview.connect(self._on_fs_preview)
         self._pending = {}         # token -> (kind, item)
         self._inflight = {}        # token -> path, until the worker really returned
         self._token = 0
@@ -229,7 +232,10 @@ def _submit(self, kind, target, fn, *args):
         self._pending[token] = (kind, target)
         if kind == "list":
             self._inflight[token] = args[0]
-        self._fs.submit(token, fn, *args)
+        if kind == "list":
+            self._fs.submit_preview(token, fn, *args)
+        else:
+            self._fs.submit(token, fn, *args)
         if kind == "list":
             QTimer.singleShot(self.LIST_TIMEOUT_MS, lambda: self._on_list_timeout(token))
 
@@ -250,13 +256,15 @@ def _on_item_expanded(self, item: QTreeWidgetItem):
         if not path:
             return
 
+        cached = self._dir_cache.get(path)
         if item.childCount() > 0 and self._is_real(item.child(0)):
+            if cached is not None and getattr(item, "_shown_entries", None) is not cached:
+                self._fill(item, cached)
             return
         if any(kind == "list" and target is item for kind, target in self._pending.values()):
             return
 
         item.takeChildren()  # drops any earlier placeholder, so there is never a second one
-        cached = self._dir_cache.get(path)
         if cached is not None:
             self._fill(item, cached)
         else:
@@ -268,23 +276,60 @@ def _on_item_expanded(self, item: QTreeWidgetItem):
             if cached is None:
                 placeholder.setText(0, t("Keine Antwort – später erneut aufklappen"))
             return
-        self._submit("list", item, list_subfolders, path)
+        self._submit("list", item, iter_subfolders, path)
 
     def _fill(self, item, entries):
-        """Swap the children in one go, so the user never sees a half-built list."""
-        children = [self._folder_item(name, full) for name, full in entries]
+        """Show the first bounded batch now and yield before creating more nodes."""
+        generation = object()
+        item._fill_generation = generation
+        item._shown_entries = None  # a collapsed partial fill must resume on return
+        remaining = iter(entries)
         item.takeChildren()
-        item.addChildren(children)
-        item.setChildIndicatorPolicy(
-            QTreeWidgetItem.ChildIndicatorPolicy.ShowIndicator if children
-            else QTreeWidgetItem.ChildIndicatorPolicy.DontShowIndicatorWhenChildless
-        )
+
+        def add_batch(first=False):
+            if not self._alive(item) or item._fill_generation is not generation:
+                return
+            if not first and not item.isExpanded():
+                return
+            batch = list(islice(remaining, self.TREE_BATCH_SIZE))
+            item.addChildren([self._folder_item(name, full) for name, full in batch])
+            item.setChildIndicatorPolicy(
+                QTreeWidgetItem.ChildIndicatorPolicy.ShowIndicator if item.childCount()
+                else QTreeWidgetItem.ChildIndicatorPolicy.DontShowIndicatorWhenChildless
+            )
+            if len(batch) == self.TREE_BATCH_SIZE:
+                if item.isExpanded():
+                    QTimer.singleShot(0, add_batch)
+            else:
+                item._shown_entries = entries
+
+        add_batch(first=True)
+
+    @Slot(object, object)
+    def _on_fs_preview(self, token, entries):
+        request = self._pending.get(token)
+        path = self._inflight.get(token)
+        if request is None or path in self._dir_cache:
+            return  # a completed cached snapshot is preferable to a partial one
+        item = request[1]
+        if not self._alive(item) or not item.isExpanded():
+            return
+        self._fill(item, entries)
+        loading = QTreeWidgetItem([t("wird geladen …")])
+        loading.setFlags(Qt.ItemFlag.NoItemFlags)
+        item.addChild(loading)
 
     @Slot(object, object, object)
     def _on_fs_result(self, token, result, error):
-        self._inflight.pop(token, None)  # the worker is done, whatever we do with the answer
+        listed_path = self._inflight.pop(token, None)
+        previous = self._dir_cache.get(listed_path)
+        # Completion is useful even if the user collapsed the node or its
+        # display deadline elapsed. Keep the snapshot for the next visit,
+        # without applying a late result to the current tree view.
+        if listed_path is not None and error is None and previous != result:
+            self._dir_cache[listed_path] = result
         entry = self._pending.pop(token, None)
-        if entry is None:  # timed out: stale answer
+        if entry is None:  # timed out: cache only, never change the current view
             return
         kind, item = entry
         if kind == "cloud":
@@ -297,19 +342,19 @@ def _on_fs_result(self, token, result, error):
             if path not in self._dir_cache:  # nothing stale worth keeping
                 self._fill(item, [])
             return
-        previous = self._dir_cache.get(path)
-        self._dir_cache[path] = result
         showing = item.childCount() > 0 and self._is_real(item.child(0))
         if previous != result or not showing:
             self._fill(item, result)
 
     def _on_list_timeout(self, token):
-        entry = self._pending.pop(token, None)  # a late answer is dropped
+        entry = self._pending.pop(token, None)  # late answers may cache, never update this view
         if entry is None:
             return
         item = entry[1]
-        if self._alive(item) and item.childCount() == 1 and not self._is_real(item.child(0)):
-            item.child(0).setText(0, t("Keine Antwort – später erneut aufklappen"))
+        if self._alive(item) and item.childCount():
+            last = item.child(item.childCount() - 1)
+            if not self._is_real(last):
+                last.setText(0, t("Keine Antwort – später erneut aufklappen"))
 
     def _show_cloud(self, cloud_locations):
         if not cloud_locations:

tests/test_gui_freeze.py

@@ -7,6 +7,8 @@
 import os
 import time
 from pathlib import Path
+from contextlib import nullcontext
+from types import SimpleNamespace
 
 import pytest
 from PySide6.QtCore import Qt, QTimer
@@ -22,6 +24,19 @@
 _real_isdir = os.path.isdir
 
 
+def mock_marker_scandir(monkeypatch, listdir):
+    real = os.scandir
+
+    def scan(path):
+        if isinstance(path, (str, os.PathLike)) and os.fspath(path) == SLOW:
+            entries = [SimpleNamespace(name=name, path=os.path.join(path, name),
+                                       is_dir=lambda: True) for name in listdir(path)]
+            return nullcontext(iter(entries))
+        return real(path)
+
+    monkeypatch.setattr(os, "scandir", scan)
+
+
 @pytest.fixture(autouse=True)
 def isolated_state(tmp_path, monkeypatch):
     monkeypatch.setattr("core.drive_usage.cache_path", lambda: tmp_path / "usage.json", raising=False)
@@ -72,6 +87,7 @@ def fake_isdir(path):
 
     monkeypatch.setattr(os, "listdir", fake_listdir)
     monkeypatch.setattr(os.path, "isdir", fake_isdir)
+    mock_marker_scandir(monkeypatch, fake_listdir)
 
 
 @pytest.fixture
@@ -97,6 +113,98 @@ def names(item):
     return [item.child(i).text(0) for i in range(item.childCount())]
 
 
+@pytest.mark.parametrize("timeout", [False, True])
+def test_first_folder_preview_arrives_before_enumeration_finishes(panel, monkeypatch, timeout):
+    import threading
+
+    entered, release = threading.Event(), threading.Event()
+    entries = [(f"folder-{n:03}", os.path.join(SLOW, str(n))) for n in range(128)]
+
+    def stream(path):
+        yield from entries
+        entered.set()
+        assert release.wait(5)
+        yield ("aaa-late", os.path.join(path, "late"))
+
+    monkeypatch.setattr("gui.sidebar.sidebar_main.iter_subfolders", stream, raising=False)
+    previews = []
+    panel._fs.preview.connect(lambda token, entries: previews.append(entries))
+    if timeout:
+        panel.LIST_TIMEOUT_MS = 1000
+    item = slow_node(panel)
+    item.setExpanded(True)
+    try:
+        assert pump(lambda: entered.is_set() and names(item)[:128] == [n for n, _ in entries], timeout=2)
+        assert panel._inflight, "the enumeration should still be blocked after its preview"
+        assert "wird geladen" in item.child(item.childCount() - 1).text(0)
+        if timeout:
+            assert pump(lambda: "Keine Antwort" in item.child(item.childCount() - 1).text(0), timeout=2)
+            assert names(item)[:128] == [n for n, _ in entries]
+        item.setExpanded(False)
+    finally:
+        release.set()
+    assert pump(lambda: not panel._inflight)
+    item.setExpanded(True)
+    assert pump(lambda: item.childCount() == 129)
+    assert names(item)[0] == "aaa-late"  # final snapshot is globally sorted
+    assert len(previews) == 1  # bounded notification count even for a long scan
+
+
+def test_streamed_folder_scan_keeps_visible_directory_filter(tmp_path):
+    from core.async_fs import iter_subfolders
+
+    for name in ("alpha", "beta", ".hidden"):
+        (tmp_path / name).mkdir()
+    (tmp_path / "file.txt").write_text("x", encoding="utf-8")
+    assert sorted(iter_subfolders(str(tmp_path))) == [
+        (name, str(tmp_path / name)) for name in ("alpha", "beta")
+    ]
+
+
+def test_large_tree_fill_yields_between_bounded_batches(panel):
+    item = QTreeWidgetItem(["large"])
+    item.setChildIndicatorPolicy(QTreeWidgetItem.ChildIndicatorPolicy.ShowIndicator)
+    panel.tree.addTopLevelItem(item)
+    item.setExpanded(True)
+    entries = [(f"folder-{n:05}", os.path.join(SLOW, str(n))) for n in range(1000)]
+    observed = []
+    QTimer.singleShot(0, lambda: observed.append(item.childCount()))
+    panel._fill(item, entries)
+    assert 0 < item.childCount() <= 128
+    assert pump(lambda: item.childCount() == len(entries))
+    assert observed and 0 < observed[0] < len(entries), "GUI never ran between insertion batches"
+    assert names(item) == [name for name, _ in entries]
+
+
+def test_new_tree_fill_invalidates_older_queued_batches(panel):
+    item = QTreeWidgetItem(["replace"])
+    item.setChildIndicatorPolicy(QTreeWidgetItem.ChildIndicatorPolicy.ShowIndicator)
+    panel.tree.addTopLevelItem(item)
+    item.setExpanded(True)
+    entries = [(str(n), os.path.join(SLOW, str(n))) for n in range(1000)]
+    panel._fill(item, entries)
+    panel._fill(item, [("new", os.path.join(SLOW, "new"))])
+    pump(lambda: False, timeout=0.1)
+    assert names(item) == ["new"]
+
+
+def test_collapsed_partial_tree_fill_resumes_from_cache_on_return(panel):
+    item = QTreeWidgetItem(["pause"])
+    item.setChildIndicatorPolicy(QTreeWidgetItem.ChildIndicatorPolicy.ShowIndicator)
+    panel.tree.addTopLevelItem(item)
+    item.setExpanded(True)
+    item.setData(0, Qt.ItemDataRole.UserRole, SLOW)
+    entries = [(str(n), os.path.join(SLOW, str(n))) for n in range(1000)]
+    TreePanel._dir_cache[SLOW] = entries
+    panel._fill(item, entries)
+    item.setExpanded(False)
+    pump(lambda: False, timeout=0.1)
+    assert item.childCount() == panel.TREE_BATCH_SIZE
+    item.setExpanded(True)
+    assert pump(lambda: item.childCount() == len(entries))
+    assert names(item) == [name for name, _ in entries]
+
+
 def test_expanding_slow_folder_does_not_block_event_loop(panel, slow_listdir):
     item = slow_node(panel)
     meter = TickMeter()
@@ -123,12 +231,20 @@ def test_last_known_children_shown_until_fresh_state_arrives(panel, slow_listdir
     assert pump(lambda: names(item) == ["alpha", "beta"])
 
 
-def test_answer_for_collapsed_node_is_dropped(panel, slow_listdir):
+@pytest.mark.parametrize("cached", [False, True])
+def test_answer_for_collapsed_node_is_cached_without_changing_hidden_children(panel, slow_listdir, cached):
+    if cached:
+        TreePanel._dir_cache[SLOW] = [("old", os.path.join(SLOW, "old"))]
     item = slow_node(panel)
     item.setExpanded(True)
     item.setExpanded(False)
     pump(lambda: not panel._pending)
     assert names(item) != ["alpha", "beta"]
+    assert TreePanel._dir_cache[SLOW] == [("alpha", os.path.join(SLOW, "alpha")),
+                                         ("beta", os.path.join(SLOW, "beta"))]
+    item.setExpanded(True)
+    assert names(item) == ["alpha", "beta"]  # cached before the refresh completes
+    assert pump(lambda: not panel._inflight)
 
 
 def test_hanging_listing_times_out_and_late_answer_is_ignored(panel, slow_listdir):
@@ -139,6 +255,11 @@ def test_hanging_listing_times_out_and_late_answer_is_ignored(panel, slow_listdi
     time.sleep(DELAY)
     pump(lambda: False, timeout=0.2)
     assert "Keine Antwort" in item.child(0).text(0)  # late result did not overwrite
+    assert SLOW in TreePanel._dir_cache
+    item.setExpanded(False)
+    item.setExpanded(True)
+    assert names(item) == ["alpha", "beta"]
+    assert pump(lambda: not panel._inflight)
 
 
 def test_failed_capacity_read_releases_request_and_shows_unavailable(monkeypatch):
@@ -239,6 +360,7 @@ def fake_listdir(path="."):
 
     monkeypatch.setattr(os, "listdir", fake_listdir)
     monkeypatch.setattr(os.path, "isdir", lambda p: os.fspath(p).startswith(SLOW) or _real_isdir(p))
+    mock_marker_scandir(monkeypatch, fake_listdir)
     return calls
 
 
@@ -258,9 +380,10 @@ def test_reexpanding_after_timeout_keeps_exactly_one_placeholder_and_no_new_thre
     assert pump(lambda: not panel._inflight, timeout=DELAY + 2)
     item.setExpanded(False)
     item.setExpanded(True)
-    assert item.childCount() == 1 and "wird geladen" in item.child(0).text(0)
+    assert names(item) == ["alpha"]  # completed late snapshot appears immediately
     assert len(counted_slow_listdir) == 2
-    assert pump(lambda: "Keine Antwort" in item.child(0).text(0), timeout=2)
+    assert pump(lambda: not panel._pending, timeout=2)
+    assert names(item) == ["alpha"]  # refresh timeout does not replace cached entries
     assert item.childCount() == 1
     pump(lambda: not panel._inflight, timeout=DELAY + 2)