fix: download cases

progression_backend/app.py

@@ -146,7 +146,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     async def mods(
         offset: int = Query(0, ge=0),
         limit: int = Query(100, ge=1, le=500),
-        state: str | None = Query(None, pattern="^(ready|pending|failed)$"),
+        state: str | None = Query(None, pattern="^(ready|pending|failed|skipped)$"),
     ) -> dict:
         clauses = [
             "workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)"
@@ -188,7 +188,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
 
     def ready_workshop_rows(allow_partial: bool) -> list[dict]:
         rows = database.current_mods()
-        failed = [row for row in rows if row.get("status") != "ready"]
+        failed = [
+            row for row in rows if row.get("status") not in {"ready", "skipped"}
+        ]
         if failed and not allow_partial:
             raise HTTPException(
                 status_code=409,

progression_backend/database.py

@@ -265,6 +265,7 @@ class Database:
             SELECT COUNT(*) AS total_mods,
                    SUM(CASE WHEN status = 'ready' THEN 1 ELSE 0 END) AS ready_mods,
                    SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed_mods,
+                   SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped_mods,
                    COALESCE(SUM(disk_size), 0) AS cache_bytes
             FROM mods
             WHERE workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)

progression_backend/refresh.py

@@ -48,22 +48,26 @@ class RefreshService:
         rows = self.database.current_mods()
         if not rows:
             return True
-        return any(
+        for row in rows:
-            row.get("status") != "ready"
+            if row.get("status") == "skipped":
-            or not (self.settings.workshop_content_dir / row["workshop_id"]).is_dir()
+                continue
-            for row in rows
+            if row.get("status") != "ready" or not (
-        )
+                self.settings.workshop_content_dir / row["workshop_id"]
+            ).is_dir():
+                return True
+        return False
 
     def _store_metadata(
         self,
         detail: WorkshopDetails,
         previous: dict[str, Any] | None,
         stale: bool,
+        skip_reason: str | None = None,
     ) -> dict[str, Any]:
-        invalid_reason = None
+        invalid_reason = skip_reason
-        if detail.result != 1:
+        if invalid_reason is None and detail.result != 1:
             invalid_reason = f"Steam returned result {detail.result}"
-        elif detail.consumer_app_id not in (0, 294100):
+        elif invalid_reason is None and detail.consumer_app_id not in (0, 294100):
             invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}"
         return {
             "workshop_id": detail.workshop_id,
@@ -75,7 +79,7 @@ class RefreshService:
             "metadata_file_size": detail.file_size,
             "disk_size": None,
             "local_path": None,
-            "status": "failed"
+            "status": "skipped"
             if invalid_reason
             else ("pending" if stale else (previous or {}).get("status", "pending")),
             "error": invalid_reason,
@@ -176,7 +180,7 @@ class RefreshService:
             report(stage="collections", completed_mods=0, total_mods=0, percent=0.0)
             remaining_time()
             collection_ids = list(self.settings.collections.values())
-            response = self.steam.get_collections(collection_ids)
+            response, collection_reference_ids = self.steam.get_collections(collection_ids)
             named_collections: dict[str, tuple[str, list[str]]] = {}
             all_ids: list[str] = []
             for name, collection_id in self.settings.collections.items():
@@ -203,7 +207,7 @@ class RefreshService:
                     self.database.upsert_mod(
                         {
                             "workshop_id": workshop_id,
-                            "status": "failed",
+                            "status": "skipped",
                             "error": "Steam returned no item metadata",
                             "last_checked_at": utc_now(),
                         }
@@ -217,10 +221,18 @@ class RefreshService:
                     or old.get("status") != "ready"
                     or not local_path.is_dir()
                 )
-                metadata_rows.append(self._store_metadata(detail, old, stale))
+                skip_reason = (
+                    "Workshop entry is a collection reference"
+                    if workshop_id in collection_reference_ids
+                    else None
+                )
+                metadata_rows.append(
+                    self._store_metadata(detail, old, stale, skip_reason=skip_reason)
+                )
                 if (
                     detail.result == 1
                     and detail.consumer_app_id in (0, 294100)
+                    and workshop_id not in collection_reference_ids
                     and stale
                 ):
                     targets.append(workshop_id)
@@ -230,6 +242,14 @@ class RefreshService:
             metrics["downloads_requested"] = len(targets)
             total_targets = len(targets)
             completed_targets = 0
+            current_after_metadata = self.database.mod_map(unique_ids)
+            resolved_before = sum(
+                1
+                for row in current_after_metadata.values()
+                if row.get("status") in {"ready", "skipped"}
+            )
+            progress_total = len(unique_ids) if initial_download else total_targets
+            progress_completed = resolved_before if initial_download else 0
             title_by_id = {
                 workshop_id: details[workshop_id].title
                 for workshop_id in targets
@@ -237,14 +257,21 @@ class RefreshService:
             }
             report(
                 stage="downloading" if targets else "inspecting",
-                completed_mods=0,
+                completed_mods=progress_completed,
-                total_mods=total_targets,
+                total_mods=progress_total,
-                percent=100.0 if not targets else 0.0,
+                percent=round(
+                    (progress_completed / max(progress_total, 1)) * 100, 1
+                )
+                if progress_total
+                else 100.0,
             )
             if initial_download:
                 logger.info(
-                    "Initial Workshop download started: %d mods, %.1f-hour hard limit",
+                    "Initial Workshop download started: %d mods need download, "
+                    "%d/%d collection entries already resolved, %.1f-hour hard limit",
                     total_targets,
+                    resolved_before,
+                    len(unique_ids),
                     (overall_timeout_seconds or 0) / 3600,
                 )
             if targets:
@@ -291,8 +318,20 @@ class RefreshService:
                 batch_number = 0
                 pass_number = 1
                 while download_queue:
-                    batches = _batches(
+                    # SteamCMD stops processing the rest of a command when one
-                        download_queue, self.settings.steamcmd_batch_size
+                    # Workshop item times out. Keep the first pass efficient,
+                    # then isolate every remaining item so one bad mod cannot
+                    # permanently block the items that followed it in a batch.
+                    pass_batch_size = (
+                        self.settings.steamcmd_batch_size if pass_number == 1 else 1
+                    )
+                    batches = _batches(download_queue, pass_batch_size)
+                    if pass_number > 1:
+                        logger.info(
+                            "Initial Workshop download retry pass %d is "
+                            "isolating %d remaining mods individually",
+                            pass_number,
+                            len(download_queue),
                         )
                     for pass_batch_number, batch in enumerate(batches, start=1):
                         remaining = remaining_time()
@@ -353,7 +392,12 @@ class RefreshService:
                         inspection_rows = []
                         for workshop_id in batch:
                             failure = (
-                                "SteamCMD reported a download failure"
+                                (
+                                    "SteamCMD did not complete this item; "
+                                    "queued for retry"
+                                    if initial_download
+                                    else "SteamCMD reported a download failure"
+                                )
                                 if workshop_id in result.failed_ids
                                 else None
                             )
@@ -367,16 +411,25 @@ class RefreshService:
                             else:
                                 metrics["downloads_failed"] += 1
                             completed_targets = len(ready_target_ids)
+                            progress_completed = (
+                                resolved_before + completed_targets
+                                if initial_download
+                                else completed_targets
+                            )
                             percent = round(
-                                (completed_targets / max(total_targets, 1)) * 100,
+                                (progress_completed / max(progress_total, 1)) * 100,
                                 1,
                             )
                             title = title_by_id.get(workshop_id, "Unknown mod")
-                            outcome = "ready" if success else "failed"
+                            outcome = (
+                                "ready"
+                                if success
+                                else ("retrying" if initial_download else "failed")
+                            )
                             report(
                                 stage="downloading",
-                                completed_mods=completed_targets,
+                                completed_mods=progress_completed,
-                                total_mods=total_targets,
+                                total_mods=progress_total,
                                 percent=percent,
                                 current_workshop_id=workshop_id,
                                 current_title=title,
@@ -387,8 +440,8 @@ class RefreshService:
                                 "Initial mod download"
                                 if initial_download
                                 else "Mod refresh",
-                                completed_targets,
+                                progress_completed,
-                                total_targets,
+                                progress_total,
                                 percent,
                                 outcome,
                                 title,
@@ -438,23 +491,28 @@ class RefreshService:
             metrics["ready_items"] = sum(
                 1 for row in final_rows.values() if row.get("status") == "ready"
             )
+            resolved_items = sum(
+                1
+                for row in final_rows.values()
+                if row.get("status") in {"ready", "skipped"}
+            )
             metrics["status"] = (
                 "completed"
-                if metrics["ready_items"] == metrics["unique_items"]
+                if resolved_items == metrics["unique_items"]
                 else "partial"
             )
             if metrics["status"] == "partial" and not metrics["error"]:
                 metrics["error"] = (
-                    f"{metrics['unique_items'] - metrics['ready_items']} items are not ready"
+                    f"{metrics['unique_items'] - resolved_items} items are not resolved"
                 )
             report(
                 stage=metrics["status"],
-                completed_mods=completed_targets,
+                completed_mods=resolved_items,
-                total_mods=total_targets,
+                total_mods=metrics["unique_items"],
                 percent=round(
-                    (completed_targets / max(total_targets, 1)) * 100, 1
+                    (resolved_items / max(metrics["unique_items"], 1)) * 100, 1
                 )
-                if total_targets
+                if metrics["unique_items"]
                 else 100.0,
             )
         except Exception as error:  # noqa: BLE001 - the run must always persist a terminal state

progression_backend/static/app.js

@@ -106,6 +106,8 @@ function renderServiceState(data) {
   const label = element.querySelector("span:last-child");
   const total = Number(data.totals?.total_mods || 0);
   const ready = Number(data.totals?.ready_mods || 0);
+  const skipped = Number(data.totals?.skipped_mods || 0);
+  const resolved = ready + skipped;
   element.className = "service-state";
 
   const progress = data.scheduler?.progress || {};
@@ -127,7 +129,7 @@ function renderServiceState(data) {
   } else if (!data.steamcmd_available) {
     element.classList.add("failed");
     label.textContent = "Downloads unavailable";
-  } else if (ready < total) {
+  } else if (resolved < total) {
     element.classList.add("working");
     label.textContent = "Preparing RML";
   } else {
@@ -164,6 +166,8 @@ function renderDashboard(data) {
   const totals = data.totals || {};
   const total = Number(totals.total_mods || 0);
   const ready = Number(totals.ready_mods || 0);
+  const skipped = Number(totals.skipped_mods || 0);
+  const resolved = ready + skipped;
   const counts = Object.fromEntries(
     (data.collections || []).map(collection => [String(collection.name).toLowerCase(), Number(collection.item_count || 0)])
   );
@@ -183,13 +187,13 @@ function renderDashboard(data) {
     : (data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt));
   $("#last-updated").title = formatDate(updatedAt);
 
-  state.rmlReady = total > 0 && ready === total && !data.scheduler?.running;
+  state.rmlReady = total > 0 && resolved === total && !data.scheduler?.running;
   const nextRefresh = data.scheduler?.next_refresh_at;
   if (initialDownload) {
     const completed = Number(progress.completed_mods || 0).toLocaleString();
     const target = Number(progress.total_mods || 0).toLocaleString();
     $("#update-detail").textContent = progress.total_mods
-      ? `First-time download: ${completed} of ${target} mods. Hourly checks begin after it finishes.`
+      ? `First-time setup: ${completed} of ${target} entries resolved · ${ready.toLocaleString()} mods ready. Hourly checks begin after it finishes.`
       : "Preparing the first-time Workshop download. Hourly checks are paused.";
   } else if (data.scheduler?.running) {
     $("#update-detail").textContent = "The hourly Workshop refresh is currently running.";

progression_backend/steam_api.py

@@ -43,7 +43,9 @@ class SteamWorkshopClient:
         self.session.mount("https://", HTTPAdapter(max_retries=retries))
         self.session.headers.update({"User-Agent": "ProgressionMods/1.0"})
 
-    def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]:
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
         payload: dict[str, str | int] = {"collectioncount": len(collection_ids)}
         for index, collection_id in enumerate(collection_ids):
             payload[f"publishedfileids[{index}]"] = collection_id
@@ -55,6 +57,7 @@ class SteamWorkshopClient:
         details = response.json().get("response", {}).get("collectiondetails", [])
 
         result: dict[str, list[str]] = {}
+        collection_references: set[str] = set()
         for detail in details:
             collection_id = str(detail.get("publishedfileid", ""))
             if int(detail.get("result", 0)) != 1:
@@ -63,10 +66,21 @@ class SteamWorkshopClient:
                 detail.get("children", []),
                 key=lambda child: int(child.get("sortorder", 0)),
             )
-            result[collection_id] = [
+            child_ids = [str(child["publishedfileid"]) for child in children]
-                str(child["publishedfileid"]) for child in children
+            result[collection_id] = child_ids
-            ]
+            collection_references.update(
-        return result
+                str(child["publishedfileid"])
+                for child in children
+                if int(child.get("filetype", 0) or 0) == 2
+            )
+
+        # Steam currently reports filetype=2 for nested collections. Also
+        # recognize configured collections by id in case that field is absent.
+        all_children = {
+            workshop_id for workshop_ids in result.values() for workshop_id in workshop_ids
+        }
+        collection_references.update(set(collection_ids) & all_children)
+        return result, collection_references
 
     def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
         result: dict[str, WorkshopDetails] = {}

tests/test_backend.py

@@ -22,7 +22,7 @@ from progression_backend.modlist import (
     workshop_items,
 )
 from progression_backend.refresh import RefreshCoordinator, RefreshService
-from progression_backend.steam_api import WorkshopDetails
+from progression_backend.steam_api import SteamWorkshopClient, WorkshopDetails
 from progression_backend.workshop import (
     SteamCmdBatchResult,
     SteamCmdRunner,
@@ -32,8 +32,13 @@ from progression_backend.workshop import (
 
 
 class FakeSteam:
-    def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]:
+    def get_collections(
-        return {collection_id: ["123456"] for collection_id in collection_ids}
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        return (
+            {collection_id: ["123456"] for collection_id in collection_ids},
+            set(),
+        )
 
     def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
         return {
@@ -109,7 +114,154 @@ class FailingRunner:
         )
 
 
+class MultiModSteam:
+    workshop_ids = ["100", "200", "300"]
+
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        return (
+            {
+                collection_id: list(self.workshop_ids)
+                for collection_id in collection_ids
+            },
+            set(),
+        )
+
+    def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
+        return {
+            workshop_id: WorkshopDetails(
+                workshop_id=workshop_id,
+                title=f"Test mod {workshop_id}",
+                time_updated=123,
+                file_size=456,
+                consumer_app_id=294100,
+                result=1,
+            )
+            for workshop_id in workshop_ids
+        }
+
+
+class CollectionReferenceSteam:
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        core, content, cosmetics = collection_ids
+        return (
+            {
+                core: ["100", content, cosmetics, "999"],
+                content: [core, "100", cosmetics],
+                cosmetics: [core, content, "100"],
+            },
+            set(collection_ids),
+        )
+
+    def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
+        return {
+            workshop_id: WorkshopDetails(
+                workshop_id=workshop_id,
+                title=(
+                    "Unavailable mod"
+                    if workshop_id == "999"
+                    else f"Test entry {workshop_id}"
+                ),
+                time_updated=123,
+                file_size=456,
+                consumer_app_id=294100 if workshop_id != "999" else 0,
+                result=9 if workshop_id == "999" else 1,
+            )
+            for workshop_id in workshop_ids
+        }
+
+
+class TimeoutBlockingRunner:
+    def __init__(self, content_dir: Path):
+        self.content_dir = content_dir
+        self.calls: list[list[str]] = []
+        self.blocker_attempts = 0
+
+    def available(self) -> bool:
+        return True
+
+    def _install(self, workshop_id: str) -> None:
+        about = self.content_dir / workshop_id / "About"
+        about.mkdir(parents=True)
+        (about / "About.xml").write_text(
+            "<ModMetaData><name>Test Mod</name>"
+            "<packageId>Author.Test</packageId>"
+            "<steamAppId>294100</steamAppId></ModMetaData>",
+            encoding="utf-8",
+        )
+
+    def run_batch(
+        self, workshop_ids: list[str], timeout_seconds: float | None = None
+    ) -> SteamCmdBatchResult:
+        del timeout_seconds
+        self.calls.append(list(workshop_ids))
+        failed_ids: set[str] = set()
+        output = "Success"
+
+        if workshop_ids == ["100", "200", "300"]:
+            self._install("100")
+            failed_ids = {"200", "300"}
+            output = "ERROR! Timeout downloading item 200"
+        elif workshop_ids == ["200"]:
+            self.blocker_attempts += 1
+            if self.blocker_attempts == 1:
+                failed_ids = {"200"}
+                output = "ERROR! Timeout downloading item 200"
+            else:
+                self._install("200")
+        else:
+            for workshop_id in workshop_ids:
+                self._install(workshop_id)
+
+        timestamp = utc_now()
+        return SteamCmdBatchResult(
+            workshop_ids=workshop_ids,
+            started_at=timestamp,
+            finished_at=timestamp,
+            elapsed_seconds=0.001,
+            exit_code=0,
+            status="failed" if failed_ids else "completed",
+            failed_ids=failed_ids,
+            output_tail=output,
+        )
+
+
 class BackendTests(unittest.TestCase):
+    def test_collection_api_tracks_nested_collection_references(self) -> None:
+        client = SteamWorkshopClient()
+        response = Mock()
+        response.json.return_value = {
+            "response": {
+                "collectiondetails": [
+                    {
+                        "publishedfileid": "1",
+                        "result": 1,
+                        "children": [
+                            {"publishedfileid": "100", "sortorder": 0, "filetype": 0},
+                            {"publishedfileid": "2", "sortorder": 1, "filetype": 2},
+                        ],
+                    },
+                    {
+                        "publishedfileid": "2",
+                        "result": 1,
+                        "children": [
+                            {"publishedfileid": "1", "sortorder": 0},
+                        ],
+                    },
+                ]
+            }
+        }
+        client.session.post = Mock(return_value=response)
+
+        collections, references = client.get_collections(["1", "2"])
+
+        self.assertEqual(collections, {"1": ["100", "2"], "2": ["1"]})
+        self.assertEqual(references, {"1", "2"})
+        response.raise_for_status.assert_called_once_with()
+
     @patch("progression_backend.app.requests.get")
     def test_latest_windows_release_includes_prereleases(self, get: Mock) -> None:
         response = Mock()
@@ -378,6 +530,126 @@ class BackendTests(unittest.TestCase):
             self.assertEqual(runner.calls, 1)
             self.assertLessEqual(runner.last_timeout_seconds or 0, 0.25)
 
+    def test_collection_references_and_unavailable_items_are_skipped(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            settings = Settings(
+                host="127.0.0.1",
+                port=8000,
+                database_path=root / "progression.db",
+                steamcmd_path="steamcmd",
+                workshop_content_dir=root / "workshop" / "294100",
+                steamcmd_batch_size=25,
+                steamcmd_timeout_seconds=60,
+                refresh_interval_seconds=3600,
+                refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
+                game_version="1.6",
+                admin_token=None,
+                collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
+                static_dir=root,
+            )
+            database = Database(settings.database_path)
+            database.initialize()
+            service = RefreshService(
+                settings,
+                database,
+                steam=CollectionReferenceSteam(),
+                runner=FakeRunner(settings.workshop_content_dir),
+            )
+            progress_events: list[dict] = []
+
+            service.refresh(
+                "collection-reference-run",
+                "startup",
+                initial_download=True,
+                overall_timeout_seconds=60,
+                progress=progress_events.append,
+            )
+
+            run = database.fetch_one(
+                "SELECT * FROM refresh_runs WHERE id = ?",
+                ("collection-reference-run",),
+            )
+            self.assertIsNotNone(run)
+            self.assertEqual(run["status"], "completed", run)
+            self.assertEqual(run["unique_items"], 5)
+            self.assertEqual(run["downloads_requested"], 1)
+            self.assertEqual(run["ready_items"], 1)
+            dashboard = database.dashboard()
+            self.assertEqual(dashboard["totals"]["total_mods"], 5)
+            self.assertEqual(dashboard["totals"]["ready_mods"], 1)
+            self.assertEqual(dashboard["totals"]["skipped_mods"], 4)
+            skipped = database.fetch_all(
+                "SELECT workshop_id FROM mods WHERE status = 'skipped' "
+                "ORDER BY CAST(workshop_id AS INTEGER)"
+            )
+            self.assertEqual(
+                [row["workshop_id"] for row in skipped], ["1", "2", "3", "999"]
+            )
+            self.assertFalse(service.needs_initial_download())
+            self.assertEqual(progress_events[-1]["completed_mods"], 5)
+            self.assertEqual(progress_events[-1]["total_mods"], 5)
+            self.assertEqual(progress_events[-1]["percent"], 100.0)
+
+            with TestClient(create_app(settings)) as client:
+                response = client.get("/api/modlists/vanilla")
+                self.assertEqual(response.status_code, 200, response.text)
+                self.assertIn(b"100", response.content)
+                self.assertNotIn(b"999", response.content)
+
+    @patch("progression_backend.refresh.time.sleep", return_value=None)
+    def test_initial_download_isolates_items_after_batch_timeout(
+        self, sleep: Mock
+    ) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            settings = Settings(
+                host="127.0.0.1",
+                port=8000,
+                database_path=root / "progression.db",
+                steamcmd_path="steamcmd",
+                workshop_content_dir=root / "workshop" / "294100",
+                steamcmd_batch_size=25,
+                steamcmd_timeout_seconds=60,
+                refresh_interval_seconds=3600,
+                refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
+                game_version="1.6",
+                admin_token=None,
+                collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
+                static_dir=root,
+            )
+            database = Database(settings.database_path)
+            database.initialize()
+            runner = TimeoutBlockingRunner(settings.workshop_content_dir)
+            service = RefreshService(
+                settings,
+                database,
+                steam=MultiModSteam(),
+                runner=runner,
+            )
+
+            service.refresh(
+                "blocking-timeout-run",
+                "startup",
+                initial_download=True,
+                overall_timeout_seconds=60,
+            )
+
+            run = database.fetch_one(
+                "SELECT * FROM refresh_runs WHERE id = ?",
+                ("blocking-timeout-run",),
+            )
+            self.assertIsNotNone(run)
+            self.assertEqual(run["status"], "completed", run)
+            self.assertEqual(run["ready_items"], 3)
+            self.assertEqual(
+                runner.calls,
+                [["100", "200", "300"], ["200"], ["300"], ["200"]],
+            )
+            sleep.assert_called()
+
     def test_api_serves_dashboard_and_reports_empty_modlist(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)