from __future__ import annotations import asyncio import time import uuid from datetime import datetime, timedelta, timezone from typing import Any from .config import Settings from .database import Database from .steam_api import SteamWorkshopClient, WorkshopDetails from .workshop import SteamCmdRunner, directory_size, parse_about_xml, utc_now def _batches(values: list[str], size: int) -> list[list[str]]: return [values[index : index + size] for index in range(0, len(values), size)] class RefreshService: def __init__( self, settings: Settings, database: Database, steam: SteamWorkshopClient | None = None, runner: SteamCmdRunner | None = None, ): self.settings = settings self.database = database self.steam = steam or SteamWorkshopClient() self.runner = runner or SteamCmdRunner( settings.steamcmd_path, settings.workshop_content_dir, settings.steamcmd_timeout_seconds, ) def _store_metadata( self, detail: WorkshopDetails, previous: dict[str, Any] | None, stale: bool, ) -> dict[str, Any]: invalid_reason = None if detail.result != 1: invalid_reason = f"Steam returned result {detail.result}" elif detail.consumer_app_id not in (0, 294100): invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}" return { "workshop_id": detail.workshop_id, "title": detail.title, "package_id": None, "steam_app_id": None, "display_name": None, "time_updated": detail.time_updated, "metadata_file_size": detail.file_size, "disk_size": None, "local_path": None, "status": "failed" if invalid_reason else ("pending" if stale else (previous or {}).get("status", "pending")), "error": invalid_reason, "last_checked_at": utc_now(), "last_downloaded_at": None, } def _inspection_values( self, workshop_id: str, downloaded: bool, forced_error: str | None = None, ) -> tuple[bool, dict[str, Any]]: mod_path = self.settings.workshop_content_dir / workshop_id if forced_error: return False, { "workshop_id": workshop_id, "status": "failed", "error": forced_error, "disk_size": directory_size(mod_path), "local_path": str(mod_path) if mod_path.exists() else None, "last_checked_at": utc_now(), "last_downloaded_at": utc_now() if downloaded else None, } try: parsed = parse_about_xml(mod_path) return True, { "workshop_id": workshop_id, "package_id": parsed.package_id, "steam_app_id": parsed.declared_steam_app_id, "display_name": parsed.display_name, "disk_size": directory_size(mod_path), "local_path": str(mod_path), "status": "ready", "error": "", "last_checked_at": utc_now(), "last_downloaded_at": utc_now() if downloaded else None, } except (OSError, ValueError) as error: return False, { "workshop_id": workshop_id, "status": "failed", "error": str(error), "disk_size": directory_size(mod_path), "local_path": str(mod_path) if mod_path.exists() else None, "last_checked_at": utc_now(), "last_downloaded_at": utc_now() if downloaded else None, } def refresh(self, run_id: str, trigger: str) -> None: started_at = utc_now() started = time.monotonic() self.database.create_run(run_id, trigger, started_at) metrics: dict[str, Any] = { "collection_items": 0, "unique_items": 0, "ready_items": 0, "downloads_requested": 0, "downloads_succeeded": 0, "downloads_failed": 0, "bytes_before": directory_size(self.settings.workshop_content_dir), "bytes_after": 0, "bytes_downloaded": 0, "cache_bytes": 0, "status": "failed", "error": None, } try: collection_ids = list(self.settings.collections.values()) response = self.steam.get_collections(collection_ids) named_collections: dict[str, tuple[str, list[str]]] = {} all_ids: list[str] = [] for name, collection_id in self.settings.collections.items(): workshop_ids = response.get(collection_id) if workshop_ids is None: raise RuntimeError( f"Steam did not return collection {collection_id}" ) named_collections[name] = (collection_id, workshop_ids) all_ids.extend(workshop_ids) self.database.replace_collection_members(named_collections) unique_ids = sorted(set(all_ids), key=int) metrics["collection_items"] = len(all_ids) metrics["unique_items"] = len(unique_ids) details = self.steam.get_item_details(unique_ids) previous = self.database.mod_map(unique_ids) targets: list[str] = [] metadata_rows: list[dict[str, Any]] = [] for workshop_id in unique_ids: detail = details.get(workshop_id) if not detail: self.database.upsert_mod( { "workshop_id": workshop_id, "status": "failed", "error": "Steam returned no item metadata", "last_checked_at": utc_now(), } ) continue old = previous.get(workshop_id) local_path = self.settings.workshop_content_dir / workshop_id stale = ( old is None or int(old.get("time_updated") or 0) != detail.time_updated or old.get("status") != "ready" or not local_path.is_dir() ) metadata_rows.append(self._store_metadata(detail, old, stale)) if ( detail.result == 1 and detail.consumer_app_id in (0, 294100) and stale ): targets.append(workshop_id) self.database.upsert_mods(metadata_rows) metrics["downloads_requested"] = len(targets) if targets and not self.runner.available(): reason = ( f"SteamCMD was not found at '{self.settings.steamcmd_path}'. " "Set STEAMCMD_PATH to enable anonymous downloads." ) inspection_rows: list[dict[str, Any]] = [] for workshop_id in targets: mod_path = self.settings.workshop_content_dir / workshop_id success, values = self._inspection_values( workshop_id, downloaded=False, forced_error=None if mod_path.is_dir() else reason, ) inspection_rows.append(values) if not success: metrics["downloads_failed"] += 1 self.database.upsert_mods(inspection_rows) else: for batch_number, batch in enumerate( _batches(targets, self.settings.steamcmd_batch_size), start=1 ): result = self.runner.run_batch(batch) self.database.add_download_attempt( { "run_id": run_id, "batch_number": batch_number, "item_count": len(batch), "workshop_ids": batch, "started_at": result.started_at, "finished_at": result.finished_at, "elapsed_seconds": result.elapsed_seconds, "exit_code": result.exit_code, "status": result.status, "output_tail": result.output_tail, } ) inspection_rows = [] for workshop_id in batch: failure = ( "SteamCMD reported a download failure" if workshop_id in result.failed_ids else None ) success, values = self._inspection_values( workshop_id, downloaded=True, forced_error=failure ) inspection_rows.append(values) if success: metrics["downloads_succeeded"] += 1 else: metrics["downloads_failed"] += 1 self.database.upsert_mods(inspection_rows) # Re-inspect ready cached items whose metadata has never been parsed by this backend. current = self.database.mod_map(unique_ids) pending_rows = [] for workshop_id in unique_ids: row = current.get(workshop_id) if row and row.get("status") == "pending": _, values = self._inspection_values(workshop_id, downloaded=False) pending_rows.append(values) if pending_rows: self.database.upsert_mods(pending_rows) metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir) metrics["cache_bytes"] = metrics["bytes_after"] metrics["bytes_downloaded"] = max( metrics["bytes_after"] - metrics["bytes_before"], 0 ) final_rows = self.database.mod_map(unique_ids) metrics["ready_items"] = sum( 1 for row in final_rows.values() if row.get("status") == "ready" ) metrics["status"] = ( "completed" if metrics["ready_items"] == metrics["unique_items"] else "partial" ) if metrics["status"] == "partial" and not metrics["error"]: metrics["error"] = ( f"{metrics['unique_items'] - metrics['ready_items']} items are not ready" ) except Exception as error: # noqa: BLE001 - the run must always persist a terminal state metrics["status"] = "failed" metrics["error"] = str(error) metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir) metrics["cache_bytes"] = metrics["bytes_after"] finally: elapsed = max(time.monotonic() - started, 0.001) metrics["elapsed_seconds"] = round(elapsed, 3) metrics["average_mbps"] = round( (metrics["bytes_downloaded"] * 8 / 1_000_000) / elapsed, 3 ) metrics["finished_at"] = utc_now() self.database.finish_run(run_id, metrics) class RefreshCoordinator: def __init__( self, service: RefreshService, interval_seconds: int, refresh_on_startup: bool ): self.service = service self.interval_seconds = interval_seconds self.refresh_on_startup = refresh_on_startup self.current_run_id: str | None = None self.next_refresh_at: str | None = None self._refresh_task: asyncio.Task[None] | None = None self._scheduler_task: asyncio.Task[None] | None = None self._stop = asyncio.Event() async def start(self) -> None: if self._scheduler_task: return self._scheduler_task = asyncio.create_task(self._scheduler_loop()) async def stop(self) -> None: self._stop.set() if self._scheduler_task: self._scheduler_task.cancel() await asyncio.gather(self._scheduler_task, return_exceptions=True) async def trigger(self, trigger: str) -> tuple[str, bool]: if self._refresh_task and not self._refresh_task.done(): return self.current_run_id or "", False run_id = uuid.uuid4().hex self.current_run_id = run_id async def run() -> None: try: await asyncio.to_thread(self.service.refresh, run_id, trigger) finally: self.current_run_id = None self._refresh_task = asyncio.create_task(run()) return run_id, True async def _scheduler_loop(self) -> None: if self.refresh_on_startup: await self.trigger("startup") while not self._stop.is_set(): next_time = datetime.now(timezone.utc) + timedelta( seconds=self.interval_seconds ) self.next_refresh_at = next_time.isoformat() try: await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds) except asyncio.TimeoutError: await self.trigger("scheduled") def state(self) -> dict[str, Any]: return { "running": bool(self._refresh_task and not self._refresh_task.done()), "current_run_id": self.current_run_id, "next_refresh_at": self.next_refresh_at, "interval_seconds": self.interval_seconds, }