336 lines
14 KiB
Python
336 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any
|
|
|
|
from .config import Settings
|
|
from .database import Database
|
|
from .steam_api import SteamWorkshopClient, WorkshopDetails
|
|
from .workshop import SteamCmdRunner, directory_size, parse_about_xml, utc_now
|
|
|
|
|
|
def _batches(values: list[str], size: int) -> list[list[str]]:
|
|
return [values[index : index + size] for index in range(0, len(values), size)]
|
|
|
|
|
|
class RefreshService:
|
|
def __init__(
|
|
self,
|
|
settings: Settings,
|
|
database: Database,
|
|
steam: SteamWorkshopClient | None = None,
|
|
runner: SteamCmdRunner | None = None,
|
|
):
|
|
self.settings = settings
|
|
self.database = database
|
|
self.steam = steam or SteamWorkshopClient()
|
|
self.runner = runner or SteamCmdRunner(
|
|
settings.steamcmd_path,
|
|
settings.workshop_content_dir,
|
|
settings.steamcmd_timeout_seconds,
|
|
)
|
|
|
|
def _store_metadata(
|
|
self,
|
|
detail: WorkshopDetails,
|
|
previous: dict[str, Any] | None,
|
|
stale: bool,
|
|
) -> dict[str, Any]:
|
|
invalid_reason = None
|
|
if detail.result != 1:
|
|
invalid_reason = f"Steam returned result {detail.result}"
|
|
elif detail.consumer_app_id not in (0, 294100):
|
|
invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}"
|
|
return {
|
|
"workshop_id": detail.workshop_id,
|
|
"title": detail.title,
|
|
"package_id": None,
|
|
"steam_app_id": None,
|
|
"display_name": None,
|
|
"time_updated": detail.time_updated,
|
|
"metadata_file_size": detail.file_size,
|
|
"disk_size": None,
|
|
"local_path": None,
|
|
"status": "failed"
|
|
if invalid_reason
|
|
else ("pending" if stale else (previous or {}).get("status", "pending")),
|
|
"error": invalid_reason,
|
|
"last_checked_at": utc_now(),
|
|
"last_downloaded_at": None,
|
|
}
|
|
|
|
def _inspection_values(
|
|
self,
|
|
workshop_id: str,
|
|
downloaded: bool,
|
|
forced_error: str | None = None,
|
|
) -> tuple[bool, dict[str, Any]]:
|
|
mod_path = self.settings.workshop_content_dir / workshop_id
|
|
if forced_error:
|
|
return False, {
|
|
"workshop_id": workshop_id,
|
|
"status": "failed",
|
|
"error": forced_error,
|
|
"disk_size": directory_size(mod_path),
|
|
"local_path": str(mod_path) if mod_path.exists() else None,
|
|
"last_checked_at": utc_now(),
|
|
"last_downloaded_at": utc_now() if downloaded else None,
|
|
}
|
|
try:
|
|
parsed = parse_about_xml(mod_path)
|
|
return True, {
|
|
"workshop_id": workshop_id,
|
|
"package_id": parsed.package_id,
|
|
"steam_app_id": parsed.declared_steam_app_id,
|
|
"display_name": parsed.display_name,
|
|
"disk_size": directory_size(mod_path),
|
|
"local_path": str(mod_path),
|
|
"status": "ready",
|
|
"error": "",
|
|
"last_checked_at": utc_now(),
|
|
"last_downloaded_at": utc_now() if downloaded else None,
|
|
}
|
|
except (OSError, ValueError) as error:
|
|
return False, {
|
|
"workshop_id": workshop_id,
|
|
"status": "failed",
|
|
"error": str(error),
|
|
"disk_size": directory_size(mod_path),
|
|
"local_path": str(mod_path) if mod_path.exists() else None,
|
|
"last_checked_at": utc_now(),
|
|
"last_downloaded_at": utc_now() if downloaded else None,
|
|
}
|
|
|
|
def refresh(self, run_id: str, trigger: str) -> None:
|
|
started_at = utc_now()
|
|
started = time.monotonic()
|
|
self.database.create_run(run_id, trigger, started_at)
|
|
metrics: dict[str, Any] = {
|
|
"collection_items": 0,
|
|
"unique_items": 0,
|
|
"ready_items": 0,
|
|
"downloads_requested": 0,
|
|
"downloads_succeeded": 0,
|
|
"downloads_failed": 0,
|
|
"bytes_before": directory_size(self.settings.workshop_content_dir),
|
|
"bytes_after": 0,
|
|
"bytes_downloaded": 0,
|
|
"cache_bytes": 0,
|
|
"status": "failed",
|
|
"error": None,
|
|
}
|
|
|
|
try:
|
|
collection_ids = list(self.settings.collections.values())
|
|
response = self.steam.get_collections(collection_ids)
|
|
named_collections: dict[str, tuple[str, list[str]]] = {}
|
|
all_ids: list[str] = []
|
|
for name, collection_id in self.settings.collections.items():
|
|
workshop_ids = response.get(collection_id)
|
|
if workshop_ids is None:
|
|
raise RuntimeError(
|
|
f"Steam did not return collection {collection_id}"
|
|
)
|
|
named_collections[name] = (collection_id, workshop_ids)
|
|
all_ids.extend(workshop_ids)
|
|
self.database.replace_collection_members(named_collections)
|
|
|
|
unique_ids = sorted(set(all_ids), key=int)
|
|
metrics["collection_items"] = len(all_ids)
|
|
metrics["unique_items"] = len(unique_ids)
|
|
details = self.steam.get_item_details(unique_ids)
|
|
previous = self.database.mod_map(unique_ids)
|
|
|
|
targets: list[str] = []
|
|
metadata_rows: list[dict[str, Any]] = []
|
|
for workshop_id in unique_ids:
|
|
detail = details.get(workshop_id)
|
|
if not detail:
|
|
self.database.upsert_mod(
|
|
{
|
|
"workshop_id": workshop_id,
|
|
"status": "failed",
|
|
"error": "Steam returned no item metadata",
|
|
"last_checked_at": utc_now(),
|
|
}
|
|
)
|
|
continue
|
|
old = previous.get(workshop_id)
|
|
local_path = self.settings.workshop_content_dir / workshop_id
|
|
stale = (
|
|
old is None
|
|
or int(old.get("time_updated") or 0) != detail.time_updated
|
|
or old.get("status") != "ready"
|
|
or not local_path.is_dir()
|
|
)
|
|
metadata_rows.append(self._store_metadata(detail, old, stale))
|
|
if (
|
|
detail.result == 1
|
|
and detail.consumer_app_id in (0, 294100)
|
|
and stale
|
|
):
|
|
targets.append(workshop_id)
|
|
|
|
self.database.upsert_mods(metadata_rows)
|
|
|
|
metrics["downloads_requested"] = len(targets)
|
|
if targets and not self.runner.available():
|
|
reason = (
|
|
f"SteamCMD was not found at '{self.settings.steamcmd_path}'. "
|
|
"Set STEAMCMD_PATH to enable anonymous downloads."
|
|
)
|
|
inspection_rows: list[dict[str, Any]] = []
|
|
for workshop_id in targets:
|
|
mod_path = self.settings.workshop_content_dir / workshop_id
|
|
success, values = self._inspection_values(
|
|
workshop_id,
|
|
downloaded=False,
|
|
forced_error=None if mod_path.is_dir() else reason,
|
|
)
|
|
inspection_rows.append(values)
|
|
if not success:
|
|
metrics["downloads_failed"] += 1
|
|
self.database.upsert_mods(inspection_rows)
|
|
else:
|
|
for batch_number, batch in enumerate(
|
|
_batches(targets, self.settings.steamcmd_batch_size), start=1
|
|
):
|
|
result = self.runner.run_batch(batch)
|
|
self.database.add_download_attempt(
|
|
{
|
|
"run_id": run_id,
|
|
"batch_number": batch_number,
|
|
"item_count": len(batch),
|
|
"workshop_ids": batch,
|
|
"started_at": result.started_at,
|
|
"finished_at": result.finished_at,
|
|
"elapsed_seconds": result.elapsed_seconds,
|
|
"exit_code": result.exit_code,
|
|
"status": result.status,
|
|
"output_tail": result.output_tail,
|
|
}
|
|
)
|
|
inspection_rows = []
|
|
for workshop_id in batch:
|
|
failure = (
|
|
"SteamCMD reported a download failure"
|
|
if workshop_id in result.failed_ids
|
|
else None
|
|
)
|
|
success, values = self._inspection_values(
|
|
workshop_id, downloaded=True, forced_error=failure
|
|
)
|
|
inspection_rows.append(values)
|
|
if success:
|
|
metrics["downloads_succeeded"] += 1
|
|
else:
|
|
metrics["downloads_failed"] += 1
|
|
self.database.upsert_mods(inspection_rows)
|
|
|
|
# Re-inspect ready cached items whose metadata has never been parsed by this backend.
|
|
current = self.database.mod_map(unique_ids)
|
|
pending_rows = []
|
|
for workshop_id in unique_ids:
|
|
row = current.get(workshop_id)
|
|
if row and row.get("status") == "pending":
|
|
_, values = self._inspection_values(workshop_id, downloaded=False)
|
|
pending_rows.append(values)
|
|
if pending_rows:
|
|
self.database.upsert_mods(pending_rows)
|
|
|
|
metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir)
|
|
metrics["cache_bytes"] = metrics["bytes_after"]
|
|
metrics["bytes_downloaded"] = max(
|
|
metrics["bytes_after"] - metrics["bytes_before"], 0
|
|
)
|
|
final_rows = self.database.mod_map(unique_ids)
|
|
metrics["ready_items"] = sum(
|
|
1 for row in final_rows.values() if row.get("status") == "ready"
|
|
)
|
|
metrics["status"] = (
|
|
"completed"
|
|
if metrics["ready_items"] == metrics["unique_items"]
|
|
else "partial"
|
|
)
|
|
if metrics["status"] == "partial" and not metrics["error"]:
|
|
metrics["error"] = (
|
|
f"{metrics['unique_items'] - metrics['ready_items']} items are not ready"
|
|
)
|
|
except Exception as error: # noqa: BLE001 - the run must always persist a terminal state
|
|
metrics["status"] = "failed"
|
|
metrics["error"] = str(error)
|
|
metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir)
|
|
metrics["cache_bytes"] = metrics["bytes_after"]
|
|
finally:
|
|
elapsed = max(time.monotonic() - started, 0.001)
|
|
metrics["elapsed_seconds"] = round(elapsed, 3)
|
|
metrics["average_mbps"] = round(
|
|
(metrics["bytes_downloaded"] * 8 / 1_000_000) / elapsed, 3
|
|
)
|
|
metrics["finished_at"] = utc_now()
|
|
self.database.finish_run(run_id, metrics)
|
|
|
|
|
|
class RefreshCoordinator:
|
|
def __init__(
|
|
self, service: RefreshService, interval_seconds: int, refresh_on_startup: bool
|
|
):
|
|
self.service = service
|
|
self.interval_seconds = interval_seconds
|
|
self.refresh_on_startup = refresh_on_startup
|
|
self.current_run_id: str | None = None
|
|
self.next_refresh_at: str | None = None
|
|
self._refresh_task: asyncio.Task[None] | None = None
|
|
self._scheduler_task: asyncio.Task[None] | None = None
|
|
self._stop = asyncio.Event()
|
|
|
|
async def start(self) -> None:
|
|
if self._scheduler_task:
|
|
return
|
|
self._scheduler_task = asyncio.create_task(self._scheduler_loop())
|
|
|
|
async def stop(self) -> None:
|
|
self._stop.set()
|
|
if self._scheduler_task:
|
|
self._scheduler_task.cancel()
|
|
await asyncio.gather(self._scheduler_task, return_exceptions=True)
|
|
|
|
async def trigger(self, trigger: str) -> tuple[str, bool]:
|
|
if self._refresh_task and not self._refresh_task.done():
|
|
return self.current_run_id or "", False
|
|
run_id = uuid.uuid4().hex
|
|
self.current_run_id = run_id
|
|
|
|
async def run() -> None:
|
|
try:
|
|
await asyncio.to_thread(self.service.refresh, run_id, trigger)
|
|
finally:
|
|
self.current_run_id = None
|
|
|
|
self._refresh_task = asyncio.create_task(run())
|
|
return run_id, True
|
|
|
|
async def _scheduler_loop(self) -> None:
|
|
if self.refresh_on_startup:
|
|
await self.trigger("startup")
|
|
while not self._stop.is_set():
|
|
next_time = datetime.now(timezone.utc) + timedelta(
|
|
seconds=self.interval_seconds
|
|
)
|
|
self.next_refresh_at = next_time.isoformat()
|
|
try:
|
|
await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds)
|
|
except asyncio.TimeoutError:
|
|
await self.trigger("scheduled")
|
|
|
|
def state(self) -> dict[str, Any]:
|
|
return {
|
|
"running": bool(self._refresh_task and not self._refresh_task.done()),
|
|
"current_run_id": self.current_run_id,
|
|
"next_refresh_at": self.next_refresh_at,
|
|
"interval_seconds": self.interval_seconds,
|
|
}
|