fix: download cases

tests/test_backend.py

@@ -22,7 +22,7 @@ from progression_backend.modlist import (
     workshop_items,
 )
 from progression_backend.refresh import RefreshCoordinator, RefreshService
-from progression_backend.steam_api import WorkshopDetails
+from progression_backend.steam_api import SteamWorkshopClient, WorkshopDetails
 from progression_backend.workshop import (
     SteamCmdBatchResult,
     SteamCmdRunner,
@@ -32,8 +32,13 @@ from progression_backend.workshop import (
 
 
 class FakeSteam:
-    def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]:
-        return {collection_id: ["123456"] for collection_id in collection_ids}
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        return (
+            {collection_id: ["123456"] for collection_id in collection_ids},
+            set(),
+        )
 
     def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
         return {
@@ -109,7 +114,154 @@ class FailingRunner:
         )
 
 
+class MultiModSteam:
+    workshop_ids = ["100", "200", "300"]
+
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        return (
+            {
+                collection_id: list(self.workshop_ids)
+                for collection_id in collection_ids
+            },
+            set(),
+        )
+
+    def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
+        return {
+            workshop_id: WorkshopDetails(
+                workshop_id=workshop_id,
+                title=f"Test mod {workshop_id}",
+                time_updated=123,
+                file_size=456,
+                consumer_app_id=294100,
+                result=1,
+            )
+            for workshop_id in workshop_ids
+        }
+
+
+class CollectionReferenceSteam:
+    def get_collections(
+        self, collection_ids: list[str]
+    ) -> tuple[dict[str, list[str]], set[str]]:
+        core, content, cosmetics = collection_ids
+        return (
+            {
+                core: ["100", content, cosmetics, "999"],
+                content: [core, "100", cosmetics],
+                cosmetics: [core, content, "100"],
+            },
+            set(collection_ids),
+        )
+
+    def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]:
+        return {
+            workshop_id: WorkshopDetails(
+                workshop_id=workshop_id,
+                title=(
+                    "Unavailable mod"
+                    if workshop_id == "999"
+                    else f"Test entry {workshop_id}"
+                ),
+                time_updated=123,
+                file_size=456,
+                consumer_app_id=294100 if workshop_id != "999" else 0,
+                result=9 if workshop_id == "999" else 1,
+            )
+            for workshop_id in workshop_ids
+        }
+
+
+class TimeoutBlockingRunner:
+    def __init__(self, content_dir: Path):
+        self.content_dir = content_dir
+        self.calls: list[list[str]] = []
+        self.blocker_attempts = 0
+
+    def available(self) -> bool:
+        return True
+
+    def _install(self, workshop_id: str) -> None:
+        about = self.content_dir / workshop_id / "About"
+        about.mkdir(parents=True)
+        (about / "About.xml").write_text(
+            "<ModMetaData><name>Test Mod</name>"
+            "<packageId>Author.Test</packageId>"
+            "<steamAppId>294100</steamAppId></ModMetaData>",
+            encoding="utf-8",
+        )
+
+    def run_batch(
+        self, workshop_ids: list[str], timeout_seconds: float | None = None
+    ) -> SteamCmdBatchResult:
+        del timeout_seconds
+        self.calls.append(list(workshop_ids))
+        failed_ids: set[str] = set()
+        output = "Success"
+
+        if workshop_ids == ["100", "200", "300"]:
+            self._install("100")
+            failed_ids = {"200", "300"}
+            output = "ERROR! Timeout downloading item 200"
+        elif workshop_ids == ["200"]:
+            self.blocker_attempts += 1
+            if self.blocker_attempts == 1:
+                failed_ids = {"200"}
+                output = "ERROR! Timeout downloading item 200"
+            else:
+                self._install("200")
+        else:
+            for workshop_id in workshop_ids:
+                self._install(workshop_id)
+
+        timestamp = utc_now()
+        return SteamCmdBatchResult(
+            workshop_ids=workshop_ids,
+            started_at=timestamp,
+            finished_at=timestamp,
+            elapsed_seconds=0.001,
+            exit_code=0,
+            status="failed" if failed_ids else "completed",
+            failed_ids=failed_ids,
+            output_tail=output,
+        )
+
+
 class BackendTests(unittest.TestCase):
+    def test_collection_api_tracks_nested_collection_references(self) -> None:
+        client = SteamWorkshopClient()
+        response = Mock()
+        response.json.return_value = {
+            "response": {
+                "collectiondetails": [
+                    {
+                        "publishedfileid": "1",
+                        "result": 1,
+                        "children": [
+                            {"publishedfileid": "100", "sortorder": 0, "filetype": 0},
+                            {"publishedfileid": "2", "sortorder": 1, "filetype": 2},
+                        ],
+                    },
+                    {
+                        "publishedfileid": "2",
+                        "result": 1,
+                        "children": [
+                            {"publishedfileid": "1", "sortorder": 0},
+                        ],
+                    },
+                ]
+            }
+        }
+        client.session.post = Mock(return_value=response)
+
+        collections, references = client.get_collections(["1", "2"])
+
+        self.assertEqual(collections, {"1": ["100", "2"], "2": ["1"]})
+        self.assertEqual(references, {"1", "2"})
+        response.raise_for_status.assert_called_once_with()
+
     @patch("progression_backend.app.requests.get")
     def test_latest_windows_release_includes_prereleases(self, get: Mock) -> None:
         response = Mock()
@@ -378,6 +530,126 @@ class BackendTests(unittest.TestCase):
             self.assertEqual(runner.calls, 1)
             self.assertLessEqual(runner.last_timeout_seconds or 0, 0.25)
 
+    def test_collection_references_and_unavailable_items_are_skipped(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            settings = Settings(
+                host="127.0.0.1",
+                port=8000,
+                database_path=root / "progression.db",
+                steamcmd_path="steamcmd",
+                workshop_content_dir=root / "workshop" / "294100",
+                steamcmd_batch_size=25,
+                steamcmd_timeout_seconds=60,
+                refresh_interval_seconds=3600,
+                refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
+                game_version="1.6",
+                admin_token=None,
+                collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
+                static_dir=root,
+            )
+            database = Database(settings.database_path)
+            database.initialize()
+            service = RefreshService(
+                settings,
+                database,
+                steam=CollectionReferenceSteam(),
+                runner=FakeRunner(settings.workshop_content_dir),
+            )
+            progress_events: list[dict] = []
+
+            service.refresh(
+                "collection-reference-run",
+                "startup",
+                initial_download=True,
+                overall_timeout_seconds=60,
+                progress=progress_events.append,
+            )
+
+            run = database.fetch_one(
+                "SELECT * FROM refresh_runs WHERE id = ?",
+                ("collection-reference-run",),
+            )
+            self.assertIsNotNone(run)
+            self.assertEqual(run["status"], "completed", run)
+            self.assertEqual(run["unique_items"], 5)
+            self.assertEqual(run["downloads_requested"], 1)
+            self.assertEqual(run["ready_items"], 1)
+            dashboard = database.dashboard()
+            self.assertEqual(dashboard["totals"]["total_mods"], 5)
+            self.assertEqual(dashboard["totals"]["ready_mods"], 1)
+            self.assertEqual(dashboard["totals"]["skipped_mods"], 4)
+            skipped = database.fetch_all(
+                "SELECT workshop_id FROM mods WHERE status = 'skipped' "
+                "ORDER BY CAST(workshop_id AS INTEGER)"
+            )
+            self.assertEqual(
+                [row["workshop_id"] for row in skipped], ["1", "2", "3", "999"]
+            )
+            self.assertFalse(service.needs_initial_download())
+            self.assertEqual(progress_events[-1]["completed_mods"], 5)
+            self.assertEqual(progress_events[-1]["total_mods"], 5)
+            self.assertEqual(progress_events[-1]["percent"], 100.0)
+
+            with TestClient(create_app(settings)) as client:
+                response = client.get("/api/modlists/vanilla")
+                self.assertEqual(response.status_code, 200, response.text)
+                self.assertIn(b"100", response.content)
+                self.assertNotIn(b"999", response.content)
+
+    @patch("progression_backend.refresh.time.sleep", return_value=None)
+    def test_initial_download_isolates_items_after_batch_timeout(
+        self, sleep: Mock
+    ) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            settings = Settings(
+                host="127.0.0.1",
+                port=8000,
+                database_path=root / "progression.db",
+                steamcmd_path="steamcmd",
+                workshop_content_dir=root / "workshop" / "294100",
+                steamcmd_batch_size=25,
+                steamcmd_timeout_seconds=60,
+                refresh_interval_seconds=3600,
+                refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
+                game_version="1.6",
+                admin_token=None,
+                collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
+                static_dir=root,
+            )
+            database = Database(settings.database_path)
+            database.initialize()
+            runner = TimeoutBlockingRunner(settings.workshop_content_dir)
+            service = RefreshService(
+                settings,
+                database,
+                steam=MultiModSteam(),
+                runner=runner,
+            )
+
+            service.refresh(
+                "blocking-timeout-run",
+                "startup",
+                initial_download=True,
+                overall_timeout_seconds=60,
+            )
+
+            run = database.fetch_one(
+                "SELECT * FROM refresh_runs WHERE id = ?",
+                ("blocking-timeout-run",),
+            )
+            self.assertIsNotNone(run)
+            self.assertEqual(run["status"], "completed", run)
+            self.assertEqual(run["ready_items"], 3)
+            self.assertEqual(
+                runner.calls,
+                [["100", "200", "300"], ["200"], ["300"], ["200"]],
+            )
+            sleep.assert_called()
+
     def test_api_serves_dashboard_and_reports_empty_modlist(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)