feat: better first time downloading, feat: more logging

tests/test_backend.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import asyncio
 import tempfile
+import threading
 import unittest
 import xml.etree.ElementTree as ET
 from pathlib import Path
@@ -19,7 +21,7 @@ from progression_backend.modlist import (
     parse_active_mods,
     workshop_items,
 )
-from progression_backend.refresh import RefreshService
+from progression_backend.refresh import RefreshCoordinator, RefreshService
 from progression_backend.steam_api import WorkshopDetails
 from progression_backend.workshop import SteamCmdBatchResult, parse_about_xml, utc_now
 
@@ -45,11 +47,15 @@ class FakeSteam:
 class FakeRunner:
     def __init__(self, content_dir: Path):
         self.content_dir = content_dir
+        self.last_timeout_seconds: float | None = None
 
     def available(self) -> bool:
         return True
 
-    def run_batch(self, workshop_ids: list[str]) -> SteamCmdBatchResult:
+    def run_batch(
+        self, workshop_ids: list[str], timeout_seconds: float | None = None
+    ) -> SteamCmdBatchResult:
+        self.last_timeout_seconds = timeout_seconds
         for workshop_id in workshop_ids:
             about = self.content_dir / workshop_id / "About"
             about.mkdir(parents=True)
@@ -72,6 +78,32 @@ class FakeRunner:
         )
 
 
+class FailingRunner:
+    def __init__(self) -> None:
+        self.calls = 0
+        self.last_timeout_seconds: float | None = None
+
+    def available(self) -> bool:
+        return True
+
+    def run_batch(
+        self, workshop_ids: list[str], timeout_seconds: float | None = None
+    ) -> SteamCmdBatchResult:
+        self.calls += 1
+        self.last_timeout_seconds = timeout_seconds
+        timestamp = utc_now()
+        return SteamCmdBatchResult(
+            workshop_ids=workshop_ids,
+            started_at=timestamp,
+            finished_at=timestamp,
+            elapsed_seconds=0.001,
+            exit_code=1,
+            status="failed",
+            failed_ids=set(workshop_ids),
+            output_tail="Download failed",
+        )
+
+
 class BackendTests(unittest.TestCase):
     @patch("progression_backend.app.requests.get")
     def test_latest_windows_release_includes_prereleases(self, get: Mock) -> None:
@@ -182,6 +214,7 @@ class BackendTests(unittest.TestCase):
                 steamcmd_timeout_seconds=60,
                 refresh_interval_seconds=3600,
                 refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
                 game_version="1.6",
                 admin_token=None,
                 collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
@@ -189,13 +222,21 @@ class BackendTests(unittest.TestCase):
             )
             database = Database(settings.database_path)
             database.initialize()
+            runner = FakeRunner(settings.workshop_content_dir)
             service = RefreshService(
                 settings,
                 database,
                 steam=FakeSteam(),
-                runner=FakeRunner(settings.workshop_content_dir),
+                runner=runner,
+            )
+            progress_events: list[dict] = []
+            service.refresh(
+                "test-run",
+                "test",
+                initial_download=True,
+                overall_timeout_seconds=10,
+                progress=progress_events.append,
             )
-            service.refresh("test-run", "test")
 
             run = database.fetch_one(
                 "SELECT * FROM refresh_runs WHERE id = ?", ("test-run",)
@@ -209,6 +250,52 @@ class BackendTests(unittest.TestCase):
             self.assertEqual(mod["steam_app_id"], "294100")
             generated = workshop_items([mod])
             self.assertEqual(generated[0].workshop_id, "123456")
+            self.assertIsNotNone(runner.last_timeout_seconds)
+            self.assertLessEqual(runner.last_timeout_seconds or 0, 10)
+            self.assertEqual(progress_events[-1]["stage"], "completed")
+            self.assertEqual(progress_events[-1]["percent"], 100.0)
+
+    def test_initial_download_stops_at_overall_timeout(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            settings = Settings(
+                host="127.0.0.1",
+                port=8000,
+                database_path=root / "progression.db",
+                steamcmd_path="steamcmd",
+                workshop_content_dir=root / "workshop" / "294100",
+                steamcmd_batch_size=25,
+                steamcmd_timeout_seconds=60,
+                refresh_interval_seconds=3600,
+                refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
+                game_version="1.6",
+                admin_token=None,
+                collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
+                static_dir=root,
+            )
+            database = Database(settings.database_path)
+            database.initialize()
+            runner = FailingRunner()
+            service = RefreshService(
+                settings, database, steam=FakeSteam(), runner=runner  # type: ignore[arg-type]
+            )
+
+            service.refresh(
+                "timeout-run",
+                "startup",
+                initial_download=True,
+                overall_timeout_seconds=0.25,  # type: ignore[arg-type]
+            )
+
+            run = database.fetch_one(
+                "SELECT * FROM refresh_runs WHERE id = ?", ("timeout-run",)
+            )
+            self.assertIsNotNone(run)
+            self.assertEqual(run["status"], "failed")
+            self.assertIn("initial Workshop download reached", run["error"])
+            self.assertEqual(runner.calls, 1)
+            self.assertLessEqual(runner.last_timeout_seconds or 0, 0.25)
 
     def test_api_serves_dashboard_and_reports_empty_modlist(self) -> None:
         with tempfile.TemporaryDirectory() as directory:
@@ -224,6 +311,7 @@ class BackendTests(unittest.TestCase):
                 steamcmd_timeout_seconds=60,
                 refresh_interval_seconds=3600,
                 refresh_on_startup=False,
+                initial_download_timeout_seconds=43200,
                 game_version="1.6",
                 admin_token=None,
                 collections={"Core": "1", "Content": "2", "Cosmetics": "3"},
@@ -233,7 +321,13 @@ class BackendTests(unittest.TestCase):
                 self.assertEqual(client.get("/").status_code, 200)
                 self.assertEqual(client.get("/art/GameTitle.png").status_code, 200)
                 self.assertEqual(client.get("/api/health").json()["status"], "ok")
-                self.assertIn("rml_updated_at", client.get("/api/status").json())
+                dashboard = client.get("/api/status").json()
+                self.assertIn("rml_updated_at", dashboard)
+                self.assertIn("initial_download", dashboard["scheduler"])
+                self.assertEqual(
+                    dashboard["scheduler"]["initial_download_timeout_seconds"],
+                    43200,
+                )
                 response = client.get("/api/modlists/vanilla")
                 self.assertEqual(response.status_code, 409)
 
@@ -252,5 +346,58 @@ class BackendTests(unittest.TestCase):
                 self.assertEqual(response.headers["location"], release_url)
 
 
+class SlowInitialService:
+    def __init__(self) -> None:
+        self.started = threading.Event()
+        self.release = threading.Event()
+        self.needs_download = True
+        self.received_timeout: int | None = None
+
+    def needs_initial_download(self) -> bool:
+        return self.needs_download
+
+    def refresh(self, run_id: str, trigger: str, **options: object) -> None:
+        self.received_timeout = options.get("overall_timeout_seconds")  # type: ignore[assignment]
+        progress = options.get("progress")
+        if callable(progress):
+            progress(
+                {
+                    "stage": "downloading",
+                    "completed_mods": 1,
+                    "total_mods": 4,
+                    "percent": 25.0,
+                }
+            )
+        self.started.set()
+        self.release.wait(timeout=2)
+        self.needs_download = False
+
+
+class CoordinatorTests(unittest.IsolatedAsyncioTestCase):
+    async def test_initial_download_pauses_hourly_scheduler(self) -> None:
+        service = SlowInitialService()
+        coordinator = RefreshCoordinator(
+            service, 3600, True, initial_download_timeout_seconds=43200  # type: ignore[arg-type]
+        )
+        await coordinator.start()
+        started = await asyncio.to_thread(service.started.wait, 1)
+        self.assertTrue(started)
+
+        active = coordinator.state()
+        self.assertTrue(active["initial_download"])
+        self.assertIsNone(active["next_refresh_at"])
+        self.assertEqual(active["progress"]["percent"], 25.0)
+        self.assertEqual(service.received_timeout, 43200)
+
+        service.release.set()
+        if coordinator._refresh_task:
+            await asyncio.wait_for(coordinator._refresh_task, timeout=1)
+        await asyncio.sleep(0.01)
+        completed = coordinator.state()
+        self.assertFalse(completed["initial_download"])
+        self.assertIsNotNone(completed["next_refresh_at"])
+        await coordinator.stop()
+
+
 if __name__ == "__main__":
     unittest.main()