From 208ae62ca63b20baad97cc22930aa27a43011052 Mon Sep 17 00:00:00 2001 From: HRiggs Date: Tue, 6 Oct 2026 01:04:23 -0400 Subject: [PATCH] fix: download cases --- progression_backend/app.py | 6 +- progression_backend/database.py | 1 + progression_backend/refresh.py | 120 +++++++++---- progression_backend/static/app.js | 10 +- progression_backend/steam_api.py | 24 ++- tests/test_backend.py | 278 +++++++++++++++++++++++++++++- 6 files changed, 395 insertions(+), 44 deletions(-) diff --git a/progression_backend/app.py b/progression_backend/app.py index ee9df05..caf4506 100644 --- a/progression_backend/app.py +++ b/progression_backend/app.py @@ -146,7 +146,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: async def mods( offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=500), - state: str | None = Query(None, pattern="^(ready|pending|failed)$"), + state: str | None = Query(None, pattern="^(ready|pending|failed|skipped)$"), ) -> dict: clauses = [ "workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)" @@ -188,7 +188,9 @@ def create_app(settings: Settings | None = None) -> FastAPI: def ready_workshop_rows(allow_partial: bool) -> list[dict]: rows = database.current_mods() - failed = [row for row in rows if row.get("status") != "ready"] + failed = [ + row for row in rows if row.get("status") not in {"ready", "skipped"} + ] if failed and not allow_partial: raise HTTPException( status_code=409, diff --git a/progression_backend/database.py b/progression_backend/database.py index 687b8dc..aaba961 100644 --- a/progression_backend/database.py +++ b/progression_backend/database.py @@ -265,6 +265,7 @@ class Database: SELECT COUNT(*) AS total_mods, SUM(CASE WHEN status = 'ready' THEN 1 ELSE 0 END) AS ready_mods, SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed_mods, + SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped_mods, COALESCE(SUM(disk_size), 0) AS cache_bytes FROM mods WHERE workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members) diff --git a/progression_backend/refresh.py b/progression_backend/refresh.py index ccdbaf5..2afb008 100644 --- a/progression_backend/refresh.py +++ b/progression_backend/refresh.py @@ -48,22 +48,26 @@ class RefreshService: rows = self.database.current_mods() if not rows: return True - return any( - row.get("status") != "ready" - or not (self.settings.workshop_content_dir / row["workshop_id"]).is_dir() - for row in rows - ) + for row in rows: + if row.get("status") == "skipped": + continue + if row.get("status") != "ready" or not ( + self.settings.workshop_content_dir / row["workshop_id"] + ).is_dir(): + return True + return False def _store_metadata( self, detail: WorkshopDetails, previous: dict[str, Any] | None, stale: bool, + skip_reason: str | None = None, ) -> dict[str, Any]: - invalid_reason = None - if detail.result != 1: + invalid_reason = skip_reason + if invalid_reason is None and detail.result != 1: invalid_reason = f"Steam returned result {detail.result}" - elif detail.consumer_app_id not in (0, 294100): + elif invalid_reason is None and detail.consumer_app_id not in (0, 294100): invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}" return { "workshop_id": detail.workshop_id, @@ -75,7 +79,7 @@ class RefreshService: "metadata_file_size": detail.file_size, "disk_size": None, "local_path": None, - "status": "failed" + "status": "skipped" if invalid_reason else ("pending" if stale else (previous or {}).get("status", "pending")), "error": invalid_reason, @@ -176,7 +180,7 @@ class RefreshService: report(stage="collections", completed_mods=0, total_mods=0, percent=0.0) remaining_time() collection_ids = list(self.settings.collections.values()) - response = self.steam.get_collections(collection_ids) + response, collection_reference_ids = self.steam.get_collections(collection_ids) named_collections: dict[str, tuple[str, list[str]]] = {} all_ids: list[str] = [] for name, collection_id in self.settings.collections.items(): @@ -203,7 +207,7 @@ class RefreshService: self.database.upsert_mod( { "workshop_id": workshop_id, - "status": "failed", + "status": "skipped", "error": "Steam returned no item metadata", "last_checked_at": utc_now(), } @@ -217,10 +221,18 @@ class RefreshService: or old.get("status") != "ready" or not local_path.is_dir() ) - metadata_rows.append(self._store_metadata(detail, old, stale)) + skip_reason = ( + "Workshop entry is a collection reference" + if workshop_id in collection_reference_ids + else None + ) + metadata_rows.append( + self._store_metadata(detail, old, stale, skip_reason=skip_reason) + ) if ( detail.result == 1 and detail.consumer_app_id in (0, 294100) + and workshop_id not in collection_reference_ids and stale ): targets.append(workshop_id) @@ -230,6 +242,14 @@ class RefreshService: metrics["downloads_requested"] = len(targets) total_targets = len(targets) completed_targets = 0 + current_after_metadata = self.database.mod_map(unique_ids) + resolved_before = sum( + 1 + for row in current_after_metadata.values() + if row.get("status") in {"ready", "skipped"} + ) + progress_total = len(unique_ids) if initial_download else total_targets + progress_completed = resolved_before if initial_download else 0 title_by_id = { workshop_id: details[workshop_id].title for workshop_id in targets @@ -237,14 +257,21 @@ class RefreshService: } report( stage="downloading" if targets else "inspecting", - completed_mods=0, - total_mods=total_targets, - percent=100.0 if not targets else 0.0, + completed_mods=progress_completed, + total_mods=progress_total, + percent=round( + (progress_completed / max(progress_total, 1)) * 100, 1 + ) + if progress_total + else 100.0, ) if initial_download: logger.info( - "Initial Workshop download started: %d mods, %.1f-hour hard limit", + "Initial Workshop download started: %d mods need download, " + "%d/%d collection entries already resolved, %.1f-hour hard limit", total_targets, + resolved_before, + len(unique_ids), (overall_timeout_seconds or 0) / 3600, ) if targets: @@ -291,9 +318,21 @@ class RefreshService: batch_number = 0 pass_number = 1 while download_queue: - batches = _batches( - download_queue, self.settings.steamcmd_batch_size + # SteamCMD stops processing the rest of a command when one + # Workshop item times out. Keep the first pass efficient, + # then isolate every remaining item so one bad mod cannot + # permanently block the items that followed it in a batch. + pass_batch_size = ( + self.settings.steamcmd_batch_size if pass_number == 1 else 1 ) + batches = _batches(download_queue, pass_batch_size) + if pass_number > 1: + logger.info( + "Initial Workshop download retry pass %d is " + "isolating %d remaining mods individually", + pass_number, + len(download_queue), + ) for pass_batch_number, batch in enumerate(batches, start=1): remaining = remaining_time() batch_number += 1 @@ -353,7 +392,12 @@ class RefreshService: inspection_rows = [] for workshop_id in batch: failure = ( - "SteamCMD reported a download failure" + ( + "SteamCMD did not complete this item; " + "queued for retry" + if initial_download + else "SteamCMD reported a download failure" + ) if workshop_id in result.failed_ids else None ) @@ -367,16 +411,25 @@ class RefreshService: else: metrics["downloads_failed"] += 1 completed_targets = len(ready_target_ids) + progress_completed = ( + resolved_before + completed_targets + if initial_download + else completed_targets + ) percent = round( - (completed_targets / max(total_targets, 1)) * 100, + (progress_completed / max(progress_total, 1)) * 100, 1, ) title = title_by_id.get(workshop_id, "Unknown mod") - outcome = "ready" if success else "failed" + outcome = ( + "ready" + if success + else ("retrying" if initial_download else "failed") + ) report( stage="downloading", - completed_mods=completed_targets, - total_mods=total_targets, + completed_mods=progress_completed, + total_mods=progress_total, percent=percent, current_workshop_id=workshop_id, current_title=title, @@ -387,8 +440,8 @@ class RefreshService: "Initial mod download" if initial_download else "Mod refresh", - completed_targets, - total_targets, + progress_completed, + progress_total, percent, outcome, title, @@ -438,23 +491,28 @@ class RefreshService: metrics["ready_items"] = sum( 1 for row in final_rows.values() if row.get("status") == "ready" ) + resolved_items = sum( + 1 + for row in final_rows.values() + if row.get("status") in {"ready", "skipped"} + ) metrics["status"] = ( "completed" - if metrics["ready_items"] == metrics["unique_items"] + if resolved_items == metrics["unique_items"] else "partial" ) if metrics["status"] == "partial" and not metrics["error"]: metrics["error"] = ( - f"{metrics['unique_items'] - metrics['ready_items']} items are not ready" + f"{metrics['unique_items'] - resolved_items} items are not resolved" ) report( stage=metrics["status"], - completed_mods=completed_targets, - total_mods=total_targets, + completed_mods=resolved_items, + total_mods=metrics["unique_items"], percent=round( - (completed_targets / max(total_targets, 1)) * 100, 1 + (resolved_items / max(metrics["unique_items"], 1)) * 100, 1 ) - if total_targets + if metrics["unique_items"] else 100.0, ) except Exception as error: # noqa: BLE001 - the run must always persist a terminal state diff --git a/progression_backend/static/app.js b/progression_backend/static/app.js index e30e915..aec0c5c 100644 --- a/progression_backend/static/app.js +++ b/progression_backend/static/app.js @@ -106,6 +106,8 @@ function renderServiceState(data) { const label = element.querySelector("span:last-child"); const total = Number(data.totals?.total_mods || 0); const ready = Number(data.totals?.ready_mods || 0); + const skipped = Number(data.totals?.skipped_mods || 0); + const resolved = ready + skipped; element.className = "service-state"; const progress = data.scheduler?.progress || {}; @@ -127,7 +129,7 @@ function renderServiceState(data) { } else if (!data.steamcmd_available) { element.classList.add("failed"); label.textContent = "Downloads unavailable"; - } else if (ready < total) { + } else if (resolved < total) { element.classList.add("working"); label.textContent = "Preparing RML"; } else { @@ -164,6 +166,8 @@ function renderDashboard(data) { const totals = data.totals || {}; const total = Number(totals.total_mods || 0); const ready = Number(totals.ready_mods || 0); + const skipped = Number(totals.skipped_mods || 0); + const resolved = ready + skipped; const counts = Object.fromEntries( (data.collections || []).map(collection => [String(collection.name).toLowerCase(), Number(collection.item_count || 0)]) ); @@ -183,13 +187,13 @@ function renderDashboard(data) { : (data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt)); $("#last-updated").title = formatDate(updatedAt); - state.rmlReady = total > 0 && ready === total && !data.scheduler?.running; + state.rmlReady = total > 0 && resolved === total && !data.scheduler?.running; const nextRefresh = data.scheduler?.next_refresh_at; if (initialDownload) { const completed = Number(progress.completed_mods || 0).toLocaleString(); const target = Number(progress.total_mods || 0).toLocaleString(); $("#update-detail").textContent = progress.total_mods - ? `First-time download: ${completed} of ${target} mods. Hourly checks begin after it finishes.` + ? `First-time setup: ${completed} of ${target} entries resolved ยท ${ready.toLocaleString()} mods ready. Hourly checks begin after it finishes.` : "Preparing the first-time Workshop download. Hourly checks are paused."; } else if (data.scheduler?.running) { $("#update-detail").textContent = "The hourly Workshop refresh is currently running."; diff --git a/progression_backend/steam_api.py b/progression_backend/steam_api.py index 8916efc..29fd3d8 100644 --- a/progression_backend/steam_api.py +++ b/progression_backend/steam_api.py @@ -43,7 +43,9 @@ class SteamWorkshopClient: self.session.mount("https://", HTTPAdapter(max_retries=retries)) self.session.headers.update({"User-Agent": "ProgressionMods/1.0"}) - def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]: + def get_collections( + self, collection_ids: list[str] + ) -> tuple[dict[str, list[str]], set[str]]: payload: dict[str, str | int] = {"collectioncount": len(collection_ids)} for index, collection_id in enumerate(collection_ids): payload[f"publishedfileids[{index}]"] = collection_id @@ -55,6 +57,7 @@ class SteamWorkshopClient: details = response.json().get("response", {}).get("collectiondetails", []) result: dict[str, list[str]] = {} + collection_references: set[str] = set() for detail in details: collection_id = str(detail.get("publishedfileid", "")) if int(detail.get("result", 0)) != 1: @@ -63,10 +66,21 @@ class SteamWorkshopClient: detail.get("children", []), key=lambda child: int(child.get("sortorder", 0)), ) - result[collection_id] = [ - str(child["publishedfileid"]) for child in children - ] - return result + child_ids = [str(child["publishedfileid"]) for child in children] + result[collection_id] = child_ids + collection_references.update( + str(child["publishedfileid"]) + for child in children + if int(child.get("filetype", 0) or 0) == 2 + ) + + # Steam currently reports filetype=2 for nested collections. Also + # recognize configured collections by id in case that field is absent. + all_children = { + workshop_id for workshop_ids in result.values() for workshop_id in workshop_ids + } + collection_references.update(set(collection_ids) & all_children) + return result, collection_references def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: result: dict[str, WorkshopDetails] = {} diff --git a/tests/test_backend.py b/tests/test_backend.py index fc1ce60..ec3355a 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -22,7 +22,7 @@ from progression_backend.modlist import ( workshop_items, ) from progression_backend.refresh import RefreshCoordinator, RefreshService -from progression_backend.steam_api import WorkshopDetails +from progression_backend.steam_api import SteamWorkshopClient, WorkshopDetails from progression_backend.workshop import ( SteamCmdBatchResult, SteamCmdRunner, @@ -32,8 +32,13 @@ from progression_backend.workshop import ( class FakeSteam: - def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]: - return {collection_id: ["123456"] for collection_id in collection_ids} + def get_collections( + self, collection_ids: list[str] + ) -> tuple[dict[str, list[str]], set[str]]: + return ( + {collection_id: ["123456"] for collection_id in collection_ids}, + set(), + ) def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: return { @@ -109,7 +114,154 @@ class FailingRunner: ) +class MultiModSteam: + workshop_ids = ["100", "200", "300"] + + def get_collections( + self, collection_ids: list[str] + ) -> tuple[dict[str, list[str]], set[str]]: + return ( + { + collection_id: list(self.workshop_ids) + for collection_id in collection_ids + }, + set(), + ) + + def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: + return { + workshop_id: WorkshopDetails( + workshop_id=workshop_id, + title=f"Test mod {workshop_id}", + time_updated=123, + file_size=456, + consumer_app_id=294100, + result=1, + ) + for workshop_id in workshop_ids + } + + +class CollectionReferenceSteam: + def get_collections( + self, collection_ids: list[str] + ) -> tuple[dict[str, list[str]], set[str]]: + core, content, cosmetics = collection_ids + return ( + { + core: ["100", content, cosmetics, "999"], + content: [core, "100", cosmetics], + cosmetics: [core, content, "100"], + }, + set(collection_ids), + ) + + def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: + return { + workshop_id: WorkshopDetails( + workshop_id=workshop_id, + title=( + "Unavailable mod" + if workshop_id == "999" + else f"Test entry {workshop_id}" + ), + time_updated=123, + file_size=456, + consumer_app_id=294100 if workshop_id != "999" else 0, + result=9 if workshop_id == "999" else 1, + ) + for workshop_id in workshop_ids + } + + +class TimeoutBlockingRunner: + def __init__(self, content_dir: Path): + self.content_dir = content_dir + self.calls: list[list[str]] = [] + self.blocker_attempts = 0 + + def available(self) -> bool: + return True + + def _install(self, workshop_id: str) -> None: + about = self.content_dir / workshop_id / "About" + about.mkdir(parents=True) + (about / "About.xml").write_text( + "Test Mod" + "Author.Test" + "294100", + encoding="utf-8", + ) + + def run_batch( + self, workshop_ids: list[str], timeout_seconds: float | None = None + ) -> SteamCmdBatchResult: + del timeout_seconds + self.calls.append(list(workshop_ids)) + failed_ids: set[str] = set() + output = "Success" + + if workshop_ids == ["100", "200", "300"]: + self._install("100") + failed_ids = {"200", "300"} + output = "ERROR! Timeout downloading item 200" + elif workshop_ids == ["200"]: + self.blocker_attempts += 1 + if self.blocker_attempts == 1: + failed_ids = {"200"} + output = "ERROR! Timeout downloading item 200" + else: + self._install("200") + else: + for workshop_id in workshop_ids: + self._install(workshop_id) + + timestamp = utc_now() + return SteamCmdBatchResult( + workshop_ids=workshop_ids, + started_at=timestamp, + finished_at=timestamp, + elapsed_seconds=0.001, + exit_code=0, + status="failed" if failed_ids else "completed", + failed_ids=failed_ids, + output_tail=output, + ) + + class BackendTests(unittest.TestCase): + def test_collection_api_tracks_nested_collection_references(self) -> None: + client = SteamWorkshopClient() + response = Mock() + response.json.return_value = { + "response": { + "collectiondetails": [ + { + "publishedfileid": "1", + "result": 1, + "children": [ + {"publishedfileid": "100", "sortorder": 0, "filetype": 0}, + {"publishedfileid": "2", "sortorder": 1, "filetype": 2}, + ], + }, + { + "publishedfileid": "2", + "result": 1, + "children": [ + {"publishedfileid": "1", "sortorder": 0}, + ], + }, + ] + } + } + client.session.post = Mock(return_value=response) + + collections, references = client.get_collections(["1", "2"]) + + self.assertEqual(collections, {"1": ["100", "2"], "2": ["1"]}) + self.assertEqual(references, {"1", "2"}) + response.raise_for_status.assert_called_once_with() + @patch("progression_backend.app.requests.get") def test_latest_windows_release_includes_prereleases(self, get: Mock) -> None: response = Mock() @@ -378,6 +530,126 @@ class BackendTests(unittest.TestCase): self.assertEqual(runner.calls, 1) self.assertLessEqual(runner.last_timeout_seconds or 0, 0.25) + def test_collection_references_and_unavailable_items_are_skipped(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + settings = Settings( + host="127.0.0.1", + port=8000, + database_path=root / "progression.db", + steamcmd_path="steamcmd", + workshop_content_dir=root / "workshop" / "294100", + steamcmd_batch_size=25, + steamcmd_timeout_seconds=60, + refresh_interval_seconds=3600, + refresh_on_startup=False, + initial_download_timeout_seconds=43200, + game_version="1.6", + admin_token=None, + collections={"Core": "1", "Content": "2", "Cosmetics": "3"}, + static_dir=root, + ) + database = Database(settings.database_path) + database.initialize() + service = RefreshService( + settings, + database, + steam=CollectionReferenceSteam(), + runner=FakeRunner(settings.workshop_content_dir), + ) + progress_events: list[dict] = [] + + service.refresh( + "collection-reference-run", + "startup", + initial_download=True, + overall_timeout_seconds=60, + progress=progress_events.append, + ) + + run = database.fetch_one( + "SELECT * FROM refresh_runs WHERE id = ?", + ("collection-reference-run",), + ) + self.assertIsNotNone(run) + self.assertEqual(run["status"], "completed", run) + self.assertEqual(run["unique_items"], 5) + self.assertEqual(run["downloads_requested"], 1) + self.assertEqual(run["ready_items"], 1) + dashboard = database.dashboard() + self.assertEqual(dashboard["totals"]["total_mods"], 5) + self.assertEqual(dashboard["totals"]["ready_mods"], 1) + self.assertEqual(dashboard["totals"]["skipped_mods"], 4) + skipped = database.fetch_all( + "SELECT workshop_id FROM mods WHERE status = 'skipped' " + "ORDER BY CAST(workshop_id AS INTEGER)" + ) + self.assertEqual( + [row["workshop_id"] for row in skipped], ["1", "2", "3", "999"] + ) + self.assertFalse(service.needs_initial_download()) + self.assertEqual(progress_events[-1]["completed_mods"], 5) + self.assertEqual(progress_events[-1]["total_mods"], 5) + self.assertEqual(progress_events[-1]["percent"], 100.0) + + with TestClient(create_app(settings)) as client: + response = client.get("/api/modlists/vanilla") + self.assertEqual(response.status_code, 200, response.text) + self.assertIn(b"100", response.content) + self.assertNotIn(b"999", response.content) + + @patch("progression_backend.refresh.time.sleep", return_value=None) + def test_initial_download_isolates_items_after_batch_timeout( + self, sleep: Mock + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + settings = Settings( + host="127.0.0.1", + port=8000, + database_path=root / "progression.db", + steamcmd_path="steamcmd", + workshop_content_dir=root / "workshop" / "294100", + steamcmd_batch_size=25, + steamcmd_timeout_seconds=60, + refresh_interval_seconds=3600, + refresh_on_startup=False, + initial_download_timeout_seconds=43200, + game_version="1.6", + admin_token=None, + collections={"Core": "1", "Content": "2", "Cosmetics": "3"}, + static_dir=root, + ) + database = Database(settings.database_path) + database.initialize() + runner = TimeoutBlockingRunner(settings.workshop_content_dir) + service = RefreshService( + settings, + database, + steam=MultiModSteam(), + runner=runner, + ) + + service.refresh( + "blocking-timeout-run", + "startup", + initial_download=True, + overall_timeout_seconds=60, + ) + + run = database.fetch_one( + "SELECT * FROM refresh_runs WHERE id = ?", + ("blocking-timeout-run",), + ) + self.assertIsNotNone(run) + self.assertEqual(run["status"], "completed", run) + self.assertEqual(run["ready_items"], 3) + self.assertEqual( + runner.calls, + [["100", "200", "300"], ["200"], ["300"], ["200"]], + ) + sleep.assert_called() + def test_api_serves_dashboard_and_reports_empty_modlist(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory)