feat: better first time downloading, feat: more logging

progression_backend/app.py

@@ -83,7 +83,10 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     )
     service = RefreshService(settings, database, runner=runner)
     coordinator = RefreshCoordinator(
-        service, settings.refresh_interval_seconds, settings.refresh_on_startup
+        service,
+        settings.refresh_interval_seconds,
+        settings.refresh_on_startup,
+        settings.initial_download_timeout_seconds,
     )
 
     @asynccontextmanager

progression_backend/config.py

@@ -38,6 +38,7 @@ class Settings:
     steamcmd_timeout_seconds: int
     refresh_interval_seconds: int
     refresh_on_startup: bool
+    initial_download_timeout_seconds: int
     game_version: str
     admin_token: str | None
     collections: dict[str, str]
@@ -71,6 +72,9 @@ class Settings:
                 60, int(os.getenv("REFRESH_INTERVAL_SECONDS", "3600"))
             ),
             refresh_on_startup=_as_bool(os.getenv("REFRESH_ON_STARTUP"), True),
+            initial_download_timeout_seconds=max(
+                60, int(os.getenv("INITIAL_DOWNLOAD_TIMEOUT_SECONDS", "43200"))
+            ),
             game_version=os.getenv("MODLIST_GAME_VERSION", "1.6").strip() or "1.6",
             admin_token=os.getenv("PROGRESSION_ADMIN_TOKEN") or None,
             collections={

progression_backend/refresh.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import time
 import uuid
+from collections.abc import Callable
 from datetime import datetime, timedelta, timezone
 from typing import Any
 
@@ -12,6 +14,14 @@ from .steam_api import SteamWorkshopClient, WorkshopDetails
 from .workshop import SteamCmdRunner, directory_size, parse_about_xml, utc_now
 
 
+logger = logging.getLogger("uvicorn.error")
+ProgressCallback = Callable[[dict[str, Any]], None]
+
+
+class InitialDownloadTimeout(RuntimeError):
+    pass
+
+
 def _batches(values: list[str], size: int) -> list[list[str]]:
     return [values[index : index + size] for index in range(0, len(values), size)]
 
@@ -33,6 +43,16 @@ class RefreshService:
             settings.steamcmd_timeout_seconds,
         )
 
+    def needs_initial_download(self) -> bool:
+        rows = self.database.current_mods()
+        if not rows:
+            return True
+        return any(
+            row.get("status") != "ready"
+            or not (self.settings.workshop_content_dir / row["workshop_id"]).is_dir()
+            for row in rows
+        )
+
     def _store_metadata(
         self,
         detail: WorkshopDetails,
@@ -104,9 +124,22 @@ class RefreshService:
                 "last_downloaded_at": utc_now() if downloaded else None,
             }
 
-    def refresh(self, run_id: str, trigger: str) -> None:
+    def refresh(
+        self,
+        run_id: str,
+        trigger: str,
+        *,
+        initial_download: bool = False,
+        overall_timeout_seconds: int | None = None,
+        progress: ProgressCallback | None = None,
+    ) -> None:
         started_at = utc_now()
         started = time.monotonic()
+        deadline = (
+            started + overall_timeout_seconds
+            if initial_download and overall_timeout_seconds is not None
+            else None
+        )
         self.database.create_run(run_id, trigger, started_at)
         metrics: dict[str, Any] = {
             "collection_items": 0,
@@ -123,7 +156,24 @@ class RefreshService:
             "error": None,
         }
 
+        def report(**values: Any) -> None:
+            if progress:
+                progress(values)
+
+        def remaining_time() -> float | None:
+            if deadline is None:
+                return None
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise InitialDownloadTimeout(
+                    "The initial Workshop download reached its "
+                    f"{overall_timeout_seconds}-second limit"
+                )
+            return remaining
+
         try:
+            report(stage="collections", completed_mods=0, total_mods=0, percent=0.0)
+            remaining_time()
             collection_ids = list(self.settings.collections.values())
             response = self.steam.get_collections(collection_ids)
             named_collections: dict[str, tuple[str, list[str]]] = {}
@@ -177,6 +227,25 @@ class RefreshService:
             self.database.upsert_mods(metadata_rows)
 
             metrics["downloads_requested"] = len(targets)
+            total_targets = len(targets)
+            completed_targets = 0
+            title_by_id = {
+                workshop_id: details[workshop_id].title
+                for workshop_id in targets
+                if workshop_id in details
+            }
+            report(
+                stage="downloading" if targets else "inspecting",
+                completed_mods=0,
+                total_mods=total_targets,
+                percent=100.0 if not targets else 0.0,
+            )
+            if initial_download:
+                logger.info(
+                    "Initial Workshop download started: %d mods, %.1f-hour hard limit",
+                    total_targets,
+                    (overall_timeout_seconds or 0) / 3600,
+                )
             if targets and not self.runner.available():
                 reason = (
                     f"SteamCMD was not found at '{self.settings.steamcmd_path}'. "
@@ -195,40 +264,116 @@ class RefreshService:
                         metrics["downloads_failed"] += 1
                 self.database.upsert_mods(inspection_rows)
             else:
-                for batch_number, batch in enumerate(
-                    _batches(targets, self.settings.steamcmd_batch_size), start=1
-                ):
-                    result = self.runner.run_batch(batch)
-                    self.database.add_download_attempt(
-                        {
-                            "run_id": run_id,
-                            "batch_number": batch_number,
-                            "item_count": len(batch),
-                            "workshop_ids": batch,
-                            "started_at": result.started_at,
-                            "finished_at": result.finished_at,
-                            "elapsed_seconds": result.elapsed_seconds,
-                            "exit_code": result.exit_code,
-                            "status": result.status,
-                            "output_tail": result.output_tail,
-                        }
+                download_queue = list(targets)
+                ready_target_ids: set[str] = set()
+                batch_number = 0
+                pass_number = 1
+                while download_queue:
+                    batches = _batches(
+                        download_queue, self.settings.steamcmd_batch_size
                     )
-                    inspection_rows = []
-                    for workshop_id in batch:
-                        failure = (
-                            "SteamCMD reported a download failure"
-                            if workshop_id in result.failed_ids
-                            else None
+                    for pass_batch_number, batch in enumerate(batches, start=1):
+                        remaining = remaining_time()
+                        batch_number += 1
+                        logger.info(
+                            "%s pass %d batch %d/%d starting (%d mods): %s",
+                            "Initial Workshop download"
+                            if initial_download
+                            else "Workshop refresh",
+                            pass_number,
+                            pass_batch_number,
+                            len(batches),
+                            len(batch),
+                            ", ".join(
+                                f"{title_by_id.get(workshop_id, 'Unknown mod')} "
+                                f"[{workshop_id}]"
+                                for workshop_id in batch
+                            ),
                         )
-                        success, values = self._inspection_values(
-                            workshop_id, downloaded=True, forced_error=failure
+                        result = self.runner.run_batch(
+                            batch, timeout_seconds=remaining
                         )
-                        inspection_rows.append(values)
-                        if success:
-                            metrics["downloads_succeeded"] += 1
-                        else:
-                            metrics["downloads_failed"] += 1
-                    self.database.upsert_mods(inspection_rows)
+                        self.database.add_download_attempt(
+                            {
+                                "run_id": run_id,
+                                "batch_number": batch_number,
+                                "item_count": len(batch),
+                                "workshop_ids": batch,
+                                "started_at": result.started_at,
+                                "finished_at": result.finished_at,
+                                "elapsed_seconds": result.elapsed_seconds,
+                                "exit_code": result.exit_code,
+                                "status": result.status,
+                                "output_tail": result.output_tail,
+                            }
+                        )
+                        inspection_rows = []
+                        for workshop_id in batch:
+                            failure = (
+                                "SteamCMD reported a download failure"
+                                if workshop_id in result.failed_ids
+                                else None
+                            )
+                            success, values = self._inspection_values(
+                                workshop_id, downloaded=True, forced_error=failure
+                            )
+                            inspection_rows.append(values)
+                            if success:
+                                metrics["downloads_succeeded"] += 1
+                                ready_target_ids.add(workshop_id)
+                            else:
+                                metrics["downloads_failed"] += 1
+                            completed_targets = len(ready_target_ids)
+                            percent = round(
+                                (completed_targets / max(total_targets, 1)) * 100,
+                                1,
+                            )
+                            title = title_by_id.get(workshop_id, "Unknown mod")
+                            outcome = "ready" if success else "failed"
+                            report(
+                                stage="downloading",
+                                completed_mods=completed_targets,
+                                total_mods=total_targets,
+                                percent=percent,
+                                current_workshop_id=workshop_id,
+                                current_title=title,
+                                current_status=outcome,
+                            )
+                            logger.info(
+                                "%s %d/%d (%.1f%%) %s: %s [%s]",
+                                "Initial mod download"
+                                if initial_download
+                                else "Mod refresh",
+                                completed_targets,
+                                total_targets,
+                                percent,
+                                outcome,
+                                title,
+                                workshop_id,
+                            )
+                        self.database.upsert_mods(inspection_rows)
+                        remaining_time()
+
+                    if not initial_download:
+                        break
+                    download_queue = [
+                        workshop_id
+                        for workshop_id in download_queue
+                        if workshop_id not in ready_target_ids
+                    ]
+                    if download_queue:
+                        remaining = remaining_time()
+                        retry_delay = min(30.0, remaining or 30.0)
+                        logger.warning(
+                            "Initial Workshop download pass %d left %d mods; "
+                            "retrying in %.0f seconds",
+                            pass_number,
+                            len(download_queue),
+                            retry_delay,
+                        )
+                        time.sleep(retry_delay)
+                        remaining_time()
+                        pass_number += 1
 
             # Re-inspect ready cached items whose metadata has never been parsed by this backend.
             current = self.database.mod_map(unique_ids)
@@ -259,12 +404,37 @@ class RefreshService:
                 metrics["error"] = (
                     f"{metrics['unique_items'] - metrics['ready_items']} items are not ready"
                 )
+            report(
+                stage=metrics["status"],
+                completed_mods=completed_targets,
+                total_mods=total_targets,
+                percent=round(
+                    (completed_targets / max(total_targets, 1)) * 100, 1
+                )
+                if total_targets
+                else 100.0,
+            )
         except Exception as error:  # noqa: BLE001 - the run must always persist a terminal state
             metrics["status"] = "failed"
             metrics["error"] = str(error)
             metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir)
             metrics["cache_bytes"] = metrics["bytes_after"]
+            metrics["bytes_downloaded"] = max(
+                metrics["bytes_after"] - metrics["bytes_before"], 0
+            )
+            report(stage="failed", error=str(error))
+            logger.error(
+                "%s failed: %s",
+                "Initial Workshop download" if initial_download else "Workshop refresh",
+                error,
+            )
         finally:
+            if metrics["unique_items"]:
+                metrics["ready_items"] = sum(
+                    1
+                    for row in self.database.current_mods()
+                    if row.get("status") == "ready"
+                )
             elapsed = max(time.monotonic() - started, 0.001)
             metrics["elapsed_seconds"] = round(elapsed, 3)
             metrics["average_mbps"] = round(
@@ -272,20 +442,37 @@ class RefreshService:
             )
             metrics["finished_at"] = utc_now()
             self.database.finish_run(run_id, metrics)
+            logger.info(
+                "%s finished: status=%s, ready=%d/%d, elapsed=%.1fs, "
+                "cache=%d bytes",
+                "Initial Workshop download" if initial_download else "Workshop refresh",
+                metrics["status"],
+                metrics["ready_items"],
+                metrics["unique_items"],
+                elapsed,
+                metrics["cache_bytes"],
+            )
 
 
 class RefreshCoordinator:
     def __init__(
-        self, service: RefreshService, interval_seconds: int, refresh_on_startup: bool
+        self,
+        service: RefreshService,
+        interval_seconds: int,
+        refresh_on_startup: bool,
+        initial_download_timeout_seconds: int = 43200,
     ):
         self.service = service
         self.interval_seconds = interval_seconds
         self.refresh_on_startup = refresh_on_startup
+        self.initial_download_timeout_seconds = initial_download_timeout_seconds
         self.current_run_id: str | None = None
         self.next_refresh_at: str | None = None
         self._refresh_task: asyncio.Task[None] | None = None
         self._scheduler_task: asyncio.Task[None] | None = None
         self._stop = asyncio.Event()
+        self._active_initial_download = False
+        self._progress: dict[str, Any] | None = None
 
     async def start(self) -> None:
         if self._scheduler_task:
@@ -303,19 +490,48 @@ class RefreshCoordinator:
             return self.current_run_id or "", False
         run_id = uuid.uuid4().hex
         self.current_run_id = run_id
+        initial_download = self.service.needs_initial_download()
+        self._active_initial_download = initial_download
+        self._progress = {
+            "stage": "starting",
+            "completed_mods": 0,
+            "total_mods": 0,
+            "percent": 0.0,
+        }
+
+        def update_progress(values: dict[str, Any]) -> None:
+            self._progress = {**(self._progress or {}), **values}
 
         async def run() -> None:
             try:
-                await asyncio.to_thread(self.service.refresh, run_id, trigger)
+                await asyncio.to_thread(
+                    self.service.refresh,
+                    run_id,
+                    trigger,
+                    initial_download=initial_download,
+                    overall_timeout_seconds=(
+                        self.initial_download_timeout_seconds
+                        if initial_download
+                        else None
+                    ),
+                    progress=update_progress,
+                )
             finally:
                 self.current_run_id = None
+                self._active_initial_download = False
 
         self._refresh_task = asyncio.create_task(run())
         return run_id, True
 
+    async def _wait_for_initial_download(self, started: bool) -> None:
+        if started and self._active_initial_download and self._refresh_task:
+            self.next_refresh_at = None
+            await asyncio.shield(self._refresh_task)
+
     async def _scheduler_loop(self) -> None:
         if self.refresh_on_startup:
-            await self.trigger("startup")
+            _, started = await self.trigger("startup")
+            await self._wait_for_initial_download(started)
         while not self._stop.is_set():
             next_time = datetime.now(timezone.utc) + timedelta(
                 seconds=self.interval_seconds
@@ -324,7 +540,8 @@ class RefreshCoordinator:
             try:
                 await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds)
             except asyncio.TimeoutError:
-                await self.trigger("scheduled")
+                _, started = await self.trigger("scheduled")
+                await self._wait_for_initial_download(started)
 
     def state(self) -> dict[str, Any]:
         return {
@@ -332,4 +549,11 @@ class RefreshCoordinator:
             "current_run_id": self.current_run_id,
             "next_refresh_at": self.next_refresh_at,
             "interval_seconds": self.interval_seconds,
+            "initial_download": bool(
+                self._active_initial_download
+                and self._refresh_task
+                and not self._refresh_task.done()
+            ),
+            "initial_download_timeout_seconds": self.initial_download_timeout_seconds,
+            "progress": dict(self._progress) if self._progress else None,
         }

progression_backend/static/app.js

@@ -108,9 +108,19 @@ function renderServiceState(data) {
   const ready = Number(data.totals?.ready_mods || 0);
   element.className = "service-state";
 
-  if (data.scheduler?.running) {
+  const progress = data.scheduler?.progress || {};
+  const progressPercent = Number(progress.percent || 0);
+
+  if (data.scheduler?.initial_download) {
     element.classList.add("working");
-    label.textContent = "Updating now";
+    label.textContent = progress.total_mods
+      ? `Downloading first-time mods · ${Math.round(progressPercent)}%`
+      : "Downloading first-time mods";
+  } else if (data.scheduler?.running) {
+    element.classList.add("working");
+    label.textContent = progress.total_mods
+      ? `Updating now · ${Math.round(progressPercent)}%`
+      : "Updating now";
   } else if (data.latest_run?.status === "failed") {
     element.classList.add("failed");
     label.textContent = "Update needs attention";
@@ -166,12 +176,22 @@ function renderDashboard(data) {
   }
 
   const updatedAt = data.rml_updated_at;
-  $("#last-updated").textContent = data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt);
+  const progress = data.scheduler?.progress || {};
+  const initialDownload = Boolean(data.scheduler?.initial_download);
+  $("#last-updated").textContent = initialDownload
+    ? `Downloading ${Math.round(Number(progress.percent || 0))}%`
+    : (data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt));
   $("#last-updated").title = formatDate(updatedAt);
 
   state.rmlReady = total > 0 && ready === total && !data.scheduler?.running;
   const nextRefresh = data.scheduler?.next_refresh_at;
-  if (data.scheduler?.running) {
+  if (initialDownload) {
+    const completed = Number(progress.completed_mods || 0).toLocaleString();
+    const target = Number(progress.total_mods || 0).toLocaleString();
+    $("#update-detail").textContent = progress.total_mods
+      ? `First-time download: ${completed} of ${target} mods. Hourly checks begin after it finishes.`
+      : "Preparing the first-time Workshop download. Hourly checks are paused.";
+  } else if (data.scheduler?.running) {
     $("#update-detail").textContent = "The hourly Workshop refresh is currently running.";
   } else if (!state.rmlReady) {
     $("#update-detail").textContent = "The current RML is being prepared from the latest Workshop data.";

progression_backend/static/index.html

@@ -90,14 +90,13 @@
     </main>
 
     <footer>
-      <div class="footer-meta">
-        <span id="footer-status">Workshop data refreshed every hour</span>
+      <span id="footer-status">Workshop data refreshed every hour</span>
+      <div class="footer-credits">
         <span class="art-credit">Art by Ferny</span>
+        <a class="hr-link" href="https://hudsonriggs.systems" target="_blank" rel="noopener noreferrer" aria-label="Visit Hudson Riggs Systems">
+          <img src="/art/hudsonriggssystems.png" alt="Hudson Riggs Systems">
+        </a>
       </div>
-      <a class="hr-link" href="https://hudsonriggs.systems" target="_blank" rel="noopener noreferrer" aria-label="Visit Hudson Riggs Systems">
-        <span>Hudson Riggs Systems</span>
-        <img src="/art/hudsonriggssystems.png" alt="">
-      </a>
     </footer>
   </div>
 

progression_backend/static/styles.css

@@ -48,18 +48,28 @@
 html { min-width: 320px; background: var(--bg); }
 
 body {
+  position: relative;
   min-width: 320px;
   min-height: 100vh;
   margin: 0;
-  background:
-    radial-gradient(circle at 85% 8%, rgba(84, 114, 79, .12), transparent 27rem),
-    linear-gradient(180deg, #171717 0, #101214 100%);
+  background: #0b0b0c url("/art/Desolation.png") center / cover fixed no-repeat;
   color: var(--text);
   font-family: var(--font-ui);
   font-size: 16px;
   line-height: 1.5;
 }
 
+body::before {
+  content: "";
+  position: fixed;
+  z-index: 0;
+  inset: 0;
+  pointer-events: none;
+  background:
+    radial-gradient(circle at center, rgba(21, 21, 21, .08), rgba(21, 21, 21, .76) 72%),
+    linear-gradient(rgba(21, 21, 21, .36), rgba(21, 21, 21, .66));
+}
+
 button, input { font: inherit; }
 button { color: inherit; }
 img { display: block; max-width: 100%; }
@@ -117,6 +127,8 @@ h1, h2, p { margin: 0; }
 .is-ready .intro { opacity: 0; visibility: hidden; }
 
 .page-shell {
+  position: relative;
+  z-index: 1;
   width: min(1220px, calc(100% - 48px));
   margin: 0 auto;
   padding-bottom: 26px;
@@ -146,9 +158,13 @@ h1, h2, p { margin: 0; }
   font-size: .78rem;
   font-weight: 800;
   letter-spacing: .08em;
+  line-height: 1.3;
+  text-align: right;
   text-transform: uppercase;
 }
 
+.service-state span:last-child { max-width: 300px; }
+
 .state-dot {
   width: 8px;
   height: 8px;
@@ -352,20 +368,22 @@ footer {
   font-size: .76rem;
 }
 
-.footer-meta { display: flex; align-items: center; gap: 10px; }
+.footer-credits { margin-left: auto; display: flex; align-items: center; gap: 10px; }
 .art-credit { padding: 4px 8px; border: 1px solid #484848; border-radius: 999px; color: #999; font-family: var(--font-serif); font-style: italic; }
 
 .hr-link {
-  display: inline-flex;
+  width: 42px;
+  height: 42px;
+  display: block;
   align-items: center;
-  gap: 10px;
-  color: #f16365;
+  flex: 0 0 auto;
   text-decoration: none;
-  transition: color .16s ease, transform .16s ease;
+  transition: transform .16s ease, filter .16s ease;
 }
 
-.hr-link:hover { color: #ff7b7d; transform: translateY(-1px); }
-.hr-link img { width: 42px; height: 42px; object-fit: contain; }
+.hr-link:hover { transform: translateY(-1px); filter: brightness(1.12); }
+.hr-link:focus-visible { outline: 3px solid rgba(241, 99, 101, .28); outline-offset: 3px; border-radius: 5px; }
+.hr-link img { width: 100%; height: 100%; object-fit: contain; }
 
 .wizard {
   width: min(820px, calc(100% - 30px));
@@ -578,9 +596,9 @@ footer {
 
 @media (max-width: 700px) {
   .page-shell { width: min(calc(100% - 24px), 1220px); }
-  .site-header { min-height: 78px; }
-  .brand { width: min(67vw, 330px); }
-  .service-state { font-size: 0; }
+  .site-header { min-height: 78px; gap: 12px; }
+  .brand { width: min(59vw, 330px); }
+  .service-state { max-width: 36vw; font-size: .62rem; }
   .hero { grid-template-columns: 1fr; }
   .hero-copy { padding: 30px 24px; }
   .hero-art { min-height: 180px; border-top: 1px solid rgba(255,255,255,.09); border-left: 0; }
@@ -588,7 +606,6 @@ footer {
   .freshness { grid-template-columns: 1fr; }
   .freshness small { grid-column: auto; }
   footer { align-items: flex-start; gap: 10px; }
-  .hr-link span { display: none; }
   .wizard { width: min(100% - 18px, 820px); max-height: 94vh; }
   .wizard-header { min-height: 66px; padding-left: 18px; }
   .wizard-progress { padding: 16px 16px 14px; }
@@ -611,7 +628,7 @@ footer {
   .count-grid { grid-template-columns: 1fr; }
   .count-card { min-height: 100px; }
   .start-button { min-height: 78px; padding-left: 16px; }
-  .footer-meta { align-items: flex-start; flex-direction: column; gap: 5px; }
+  .footer-credits { gap: 6px; }
   .wizard-actions { display: grid; grid-template-columns: 1fr 1.5fr; }
   .button, .button-download { min-width: 0; }
 }

progression_backend/workshop.py

@@ -4,6 +4,7 @@ import re
 import shutil
 import subprocess
 import time
+from math import ceil
 import xml.etree.ElementTree as ET
 from dataclasses import dataclass
 from datetime import datetime, timezone
@@ -115,7 +116,11 @@ class SteamCmdRunner:
     def available(self) -> bool:
         return self.executable_path() is not None
 
-    def run_batch(self, workshop_ids: list[str]) -> SteamCmdBatchResult:
+    def run_batch(
+        self,
+        workshop_ids: list[str],
+        timeout_seconds: float | None = None,
+    ) -> SteamCmdBatchResult:
         executable = self.executable_path()
         if not executable:
             raise FileNotFoundError(
@@ -142,6 +147,9 @@ class SteamCmdRunner:
         exit_code: int | None = None
         status = "completed"
         output = ""
+        timeout = self.timeout_seconds
+        if timeout_seconds is not None:
+            timeout = max(1, min(timeout, ceil(timeout_seconds)))
         try:
             completed = subprocess.run(
                 command,
@@ -150,7 +158,7 @@ class SteamCmdRunner:
                 text=True,
                 encoding="utf-8",
                 errors="replace",
-                timeout=self.timeout_seconds,
+                timeout=timeout,
                 check=False,
                 shell=False,
             )