fix: download cases

progression_backend/app.py

@@ -146,7 +146,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     async def mods(
         offset: int = Query(0, ge=0),
         limit: int = Query(100, ge=1, le=500),
-        state: str | None = Query(None, pattern="^(ready|pending|failed)$"),
+        state: str | None = Query(None, pattern="^(ready|pending|failed|skipped)$"),
     ) -> dict:
         clauses = [
             "workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)"
@@ -188,7 +188,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
 
     def ready_workshop_rows(allow_partial: bool) -> list[dict]:
         rows = database.current_mods()
-        failed = [row for row in rows if row.get("status") != "ready"]
+        failed = [
+            row for row in rows if row.get("status") not in {"ready", "skipped"}
+        ]
         if failed and not allow_partial:
             raise HTTPException(
                 status_code=409,

progression_backend/database.py

@@ -265,6 +265,7 @@ class Database:
             SELECT COUNT(*) AS total_mods,
                    SUM(CASE WHEN status = 'ready' THEN 1 ELSE 0 END) AS ready_mods,
                    SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed_mods,
+                   SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped_mods,
                    COALESCE(SUM(disk_size), 0) AS cache_bytes
             FROM mods
             WHERE workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)

progression_backend/refresh.py

@@ -48,22 +48,26 @@ class RefreshService:
         rows = self.database.current_mods()
         if not rows:
             return True
-        return any(
-            row.get("status") != "ready"
-            or not (self.settings.workshop_content_dir / row["workshop_id"]).is_dir()
-            for row in rows
-        )
+        for row in rows:
+            if row.get("status") == "skipped":
+                continue
+            if row.get("status") != "ready" or not (
+                self.settings.workshop_content_dir / row["workshop_id"]
+            ).is_dir():
+                return True
+        return False
 
     def _store_metadata(
         self,
         detail: WorkshopDetails,
         previous: dict[str, Any] | None,
         stale: bool,
+        skip_reason: str | None = None,
     ) -> dict[str, Any]:
-        invalid_reason = None
-        if detail.result != 1:
+        invalid_reason = skip_reason
+        if invalid_reason is None and detail.result != 1:
             invalid_reason = f"Steam returned result {detail.result}"
-        elif detail.consumer_app_id not in (0, 294100):
+        elif invalid_reason is None and detail.consumer_app_id not in (0, 294100):
             invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}"
         return {
             "workshop_id": detail.workshop_id,
@@ -75,7 +79,7 @@ class RefreshService:
             "metadata_file_size": detail.file_size,
             "disk_size": None,
             "local_path": None,
-            "status": "failed"
+            "status": "skipped"
             if invalid_reason
             else ("pending" if stale else (previous or {}).get("status", "pending")),
             "error": invalid_reason,
@@ -176,7 +180,7 @@ class RefreshService:
             report(stage="collections", completed_mods=0, total_mods=0, percent=0.0)
             remaining_time()
             collection_ids = list(self.settings.collections.values())
-            response = self.steam.get_collections(collection_ids)
+            response, collection_reference_ids = self.steam.get_collections(collection_ids)
             named_collections: dict[str, tuple[str, list[str]]] = {}
             all_ids: list[str] = []
             for name, collection_id in self.settings.collections.items():
@@ -203,7 +207,7 @@ class RefreshService:
                     self.database.upsert_mod(
                         {
                             "workshop_id": workshop_id,
-                            "status": "failed",
+                            "status": "skipped",
                             "error": "Steam returned no item metadata",
                             "last_checked_at": utc_now(),
                         }
@@ -217,10 +221,18 @@ class RefreshService:
                     or old.get("status") != "ready"
                     or not local_path.is_dir()
                 )
-                metadata_rows.append(self._store_metadata(detail, old, stale))
+                skip_reason = (
+                    "Workshop entry is a collection reference"
+                    if workshop_id in collection_reference_ids
+                    else None
+                )
+                metadata_rows.append(
+                    self._store_metadata(detail, old, stale, skip_reason=skip_reason)
+                )
                 if (
                     detail.result == 1
                     and detail.consumer_app_id in (0, 294100)
+                    and workshop_id not in collection_reference_ids
                     and stale
                 ):
                     targets.append(workshop_id)
@@ -230,6 +242,14 @@ class RefreshService:
             metrics["downloads_requested"] = len(targets)
             total_targets = len(targets)
             completed_targets = 0
+            current_after_metadata = self.database.mod_map(unique_ids)
+            resolved_before = sum(
+                1
+                for row in current_after_metadata.values()
+                if row.get("status") in {"ready", "skipped"}
+            )
+            progress_total = len(unique_ids) if initial_download else total_targets
+            progress_completed = resolved_before if initial_download else 0
             title_by_id = {
                 workshop_id: details[workshop_id].title
                 for workshop_id in targets
@@ -237,14 +257,21 @@ class RefreshService:
             }
             report(
                 stage="downloading" if targets else "inspecting",
-                completed_mods=0,
-                total_mods=total_targets,
-                percent=100.0 if not targets else 0.0,
+                completed_mods=progress_completed,
+                total_mods=progress_total,
+                percent=round(
+                    (progress_completed / max(progress_total, 1)) * 100, 1
+                )
+                if progress_total
+                else 100.0,
             )
             if initial_download:
                 logger.info(
-                    "Initial Workshop download started: %d mods, %.1f-hour hard limit",
+                    "Initial Workshop download started: %d mods need download, "
+                    "%d/%d collection entries already resolved, %.1f-hour hard limit",
                     total_targets,
+                    resolved_before,
+                    len(unique_ids),
                     (overall_timeout_seconds or 0) / 3600,
                 )
             if targets:
@@ -291,9 +318,21 @@ class RefreshService:
                 batch_number = 0
                 pass_number = 1
                 while download_queue:
-                    batches = _batches(
-                        download_queue, self.settings.steamcmd_batch_size
+                    # SteamCMD stops processing the rest of a command when one
+                    # Workshop item times out. Keep the first pass efficient,
+                    # then isolate every remaining item so one bad mod cannot
+                    # permanently block the items that followed it in a batch.
+                    pass_batch_size = (
+                        self.settings.steamcmd_batch_size if pass_number == 1 else 1
                     )
+                    batches = _batches(download_queue, pass_batch_size)
+                    if pass_number > 1:
+                        logger.info(
+                            "Initial Workshop download retry pass %d is "
+                            "isolating %d remaining mods individually",
+                            pass_number,
+                            len(download_queue),
+                        )
                     for pass_batch_number, batch in enumerate(batches, start=1):
                         remaining = remaining_time()
                         batch_number += 1
@@ -353,7 +392,12 @@ class RefreshService:
                         inspection_rows = []
                         for workshop_id in batch:
                             failure = (
-                                "SteamCMD reported a download failure"
+                                (
+                                    "SteamCMD did not complete this item; "
+                                    "queued for retry"
+                                    if initial_download
+                                    else "SteamCMD reported a download failure"
+                                )
                                 if workshop_id in result.failed_ids
                                 else None
                             )
@@ -367,16 +411,25 @@ class RefreshService:
                             else:
                                 metrics["downloads_failed"] += 1
                             completed_targets = len(ready_target_ids)
+                            progress_completed = (
+                                resolved_before + completed_targets
+                                if initial_download
+                                else completed_targets
+                            )
                             percent = round(
-                                (completed_targets / max(total_targets, 1)) * 100,
+                                (progress_completed / max(progress_total, 1)) * 100,
                                 1,
                             )
                             title = title_by_id.get(workshop_id, "Unknown mod")
-                            outcome = "ready" if success else "failed"
+                            outcome = (
+                                "ready"
+                                if success
+                                else ("retrying" if initial_download else "failed")
+                            )
                             report(
                                 stage="downloading",
-                                completed_mods=completed_targets,
-                                total_mods=total_targets,
+                                completed_mods=progress_completed,
+                                total_mods=progress_total,
                                 percent=percent,
                                 current_workshop_id=workshop_id,
                                 current_title=title,
@@ -387,8 +440,8 @@ class RefreshService:
                                 "Initial mod download"
                                 if initial_download
                                 else "Mod refresh",
-                                completed_targets,
-                                total_targets,
+                                progress_completed,
+                                progress_total,
                                 percent,
                                 outcome,
                                 title,
@@ -438,23 +491,28 @@ class RefreshService:
             metrics["ready_items"] = sum(
                 1 for row in final_rows.values() if row.get("status") == "ready"
             )
+            resolved_items = sum(
+                1
+                for row in final_rows.values()
+                if row.get("status") in {"ready", "skipped"}
+            )
             metrics["status"] = (
                 "completed"
-                if metrics["ready_items"] == metrics["unique_items"]
+                if resolved_items == metrics["unique_items"]
                 else "partial"
             )
             if metrics["status"] == "partial" and not metrics["error"]:
                 metrics["error"] = (
-                    f"{metrics['unique_items'] - metrics['ready_items']} items are not ready"
+                    f"{metrics['unique_items'] - resolved_items} items are not resolved"
                 )
             report(
                 stage=metrics["status"],
-                completed_mods=completed_targets,
-                total_mods=total_targets,
+                completed_mods=resolved_items,
+                total_mods=metrics["unique_items"],
                 percent=round(
-                    (completed_targets / max(total_targets, 1)) * 100, 1
+                    (resolved_items / max(metrics["unique_items"], 1)) * 100, 1
                 )
-                if total_targets
+                if metrics["unique_items"]
                 else 100.0,
             )
         except Exception as error:  # noqa: BLE001 - the run must always persist a terminal state

progression_backend/static/app.js

@@ -106,6 +106,8 @@ function renderServiceState(data) {
   const label = element.querySelector("span:last-child");
   const total = Number(data.totals?.total_mods || 0);
   const ready = Number(data.totals?.ready_mods || 0);
+  const skipped = Number(data.totals?.skipped_mods || 0);
+  const resolved = ready + skipped;
   element.className = "service-state";
 
   const progress = data.scheduler?.progress || {};
@@ -127,7 +129,7 @@ function renderServiceState(data) {
   } else if (!data.steamcmd_available) {
     element.classList.add("failed");
     label.textContent = "Downloads unavailable";
-  } else if (ready < total) {
+  } else if (resolved < total) {
     element.classList.add("working");
     label.textContent = "Preparing RML";
   } else {
@@ -164,6 +166,8 @@ function renderDashboard(data) {
   const totals = data.totals || {};
   const total = Number(totals.total_mods || 0);
   const ready = Number(totals.ready_mods || 0);
+  const skipped = Number(totals.skipped_mods || 0);
+  const resolved = ready + skipped;
   const counts = Object.fromEntries(
     (data.collections || []).map(collection => [String(collection.name).toLowerCase(), Number(collection.item_count || 0)])
   );
@@ -183,13 +187,13 @@ function renderDashboard(data) {
     : (data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt));
   $("#last-updated").title = formatDate(updatedAt);
 
-  state.rmlReady = total > 0 && ready === total && !data.scheduler?.running;
+  state.rmlReady = total > 0 && resolved === total && !data.scheduler?.running;
   const nextRefresh = data.scheduler?.next_refresh_at;
   if (initialDownload) {
     const completed = Number(progress.completed_mods || 0).toLocaleString();
     const target = Number(progress.total_mods || 0).toLocaleString();
     $("#update-detail").textContent = progress.total_mods
-      ? `First-time download: ${completed} of ${target} mods. Hourly checks begin after it finishes.`
+      ? `First-time setup: ${completed} of ${target} entries resolved · ${ready.toLocaleString()} mods ready. Hourly checks begin after it finishes.`
       : "Preparing the first-time Workshop download. Hourly checks are paused.";
   } else if (data.scheduler?.running) {
     $("#update-detail").textContent = "The hourly Workshop refresh is currently running.";

progression_backend/steam_api.py

@@ -43,7 +43,9 @@ class SteamWorkshopClient:
         self.session.mount("https://", HTTPAdapter(max_retries=retries))
         self.session.headers.update({"User-Agent": "ProgressionMods/1.0"})
 
-    def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]:
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
         payload: dict[str, str | int] = {"collectioncount": len(collection_ids)}
         for index, collection_id in enumerate(collection_ids):
             payload[f"publishedfileids[{index}]"] = collection_id
@@ -55,6 +57,7 @@ class SteamWorkshopClient:
         details = response.json().get("response", {}).get("collectiondetails", [])
 
         result: dict[str, list[str]] = {}
+        collection_references: set[str] = set()
         for detail in details:
             collection_id = str(detail.get("publishedfileid", ""))
             if int(detail.get("result", 0)) != 1:
@@ -63,10 +66,21 @@ class SteamWorkshopClient:
                 detail.get("children", []),
                 key=lambda child: int(child.get("sortorder", 0)),
             )
-            result[collection_id] = [
-                str(child["publishedfileid"]) for child in children
-            ]
-        return result
+            child_ids = [str(child["publishedfileid"]) for child in children]
+            result[collection_id] = child_ids
+            collection_references.update(
+                str(child["publishedfileid"])
+                for child in children
+                if int(child.get("filetype", 0) or 0) == 2
+            )
+
+        # Steam currently reports filetype=2 for nested collections. Also
+        # recognize configured collections by id in case that field is absent.
+        all_children = {
+            workshop_id for workshop_ids in result.values() for workshop_id in workshop_ids
+        }
+        collection_references.update(set(collection_ids) & all_children)
+        return result, collection_references
 
     def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
         result: dict[str, WorkshopDetails] = {}