From 7ac541def5203566306f0d1544e429e0070fe2fe Mon Sep 17 00:00:00 2001 From: HRiggs Date: Fri, 2 Oct 2026 12:57:22 -0400 Subject: [PATCH] public site and containers for that. --- .env.example | 17 +- .gitea/workflows/container.yml | 79 ++++ .gitignore | 6 +- Dockerfile | 37 ++ README.md | 70 ++- compose.yaml | 17 + deploy/.env.example | 23 + deploy/README.md | 17 + deploy/compose.yaml | 33 ++ progression_backend/__init__.py | 3 + progression_backend/__main__.py | 18 + progression_backend/app.py | 284 ++++++++++++ progression_backend/config.py | 89 ++++ progression_backend/database.py | 335 ++++++++++++++ progression_backend/modlist.py | 110 +++++ progression_backend/refresh.py | 335 ++++++++++++++ progression_backend/static/app.js | 360 +++++++++++++++ progression_backend/static/index.html | 254 +++++++++++ progression_backend/static/styles.css | 623 ++++++++++++++++++++++++++ progression_backend/steam_api.py | 94 ++++ progression_backend/workshop.py | 197 ++++++++ requirements-dev.txt | 2 + requirements-web.txt | 5 + requirements.txt | 5 +- tests/test_backend.py | 256 +++++++++++ 25 files changed, 3265 insertions(+), 4 deletions(-) create mode 100644 .gitea/workflows/container.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 deploy/.env.example create mode 100644 deploy/README.md create mode 100644 deploy/compose.yaml create mode 100644 progression_backend/__init__.py create mode 100644 progression_backend/__main__.py create mode 100644 progression_backend/app.py create mode 100644 progression_backend/config.py create mode 100644 progression_backend/database.py create mode 100644 progression_backend/modlist.py create mode 100644 progression_backend/refresh.py create mode 100644 progression_backend/static/app.js create mode 100644 progression_backend/static/index.html create mode 100644 progression_backend/static/styles.css create mode 100644 progression_backend/steam_api.py create mode 100644 progression_backend/workshop.py create mode 100644 requirements-dev.txt create mode 100644 requirements-web.txt create mode 100644 tests/test_backend.py diff --git a/.env.example b/.env.example index 381495d..638a5a3 100644 --- a/.env.example +++ b/.env.example @@ -4,4 +4,19 @@ CONTENT_COLLECTION_URL=steam://openurl/https://steamcommunity.com/sharedfiles/fi COSMETICS_COLLECTION_URL=steam://openurl/https://steamcommunity.com/sharedfiles/filedetails/?id=3637541646 # ModsConfig.xml Path Template -MODSCONFIG_PATH_TEMPLATE=%USERPROFILE%\AppData\LocalLow\Ludeon Studios\RimWorld by Ludeon Studios\Config\ModsConfig.xml \ No newline at end of file +MODSCONFIG_PATH_TEMPLATE=%USERPROFILE%\AppData\LocalLow\Ludeon Studios\RimWorld by Ludeon Studios\Config\ModsConfig.xml + +# Web backend +PROGRESSION_HOST=127.0.0.1 +PROGRESSION_PORT=8000 +PROGRESSION_DB_PATH=.data/progression.db +# Windows default; use `steamcmd` when it is installed on PATH on Linux. +STEAMCMD_PATH=.steamcmd/steamcmd.exe +STEAMCMD_WORKSHOP_DIR=.steamcmd/steamapps/workshop/content/294100 +STEAMCMD_BATCH_SIZE=25 +STEAMCMD_TIMEOUT_SECONDS=1800 +REFRESH_INTERVAL_SECONDS=3600 +REFRESH_ON_STARTUP=true +MODLIST_GAME_VERSION=1.6 +# Optional bearer token required by POST /api/refresh when set. +PROGRESSION_ADMIN_TOKEN= diff --git a/.gitea/workflows/container.yml b/.gitea/workflows/container.yml new file mode 100644 index 0000000..06f69c5 --- /dev/null +++ b/.gitea/workflows/container.yml @@ -0,0 +1,79 @@ +name: Build and Publish Container + +on: + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: {} + +permissions: + contents: read + packages: write + +concurrency: + group: container-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish: + name: Publish Linux container + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve registry image name + id: image + shell: bash + env: + SERVER_URL: ${{ github.server_url }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + registry="${SERVER_URL#http://}" + registry="${registry#https://}" + registry="${registry%%/*}" + repository="$(printf '%s' "$REPOSITORY" | tr '[:upper:]' '[:lower:]')" + echo "registry=$registry" >> "$GITHUB_OUTPUT" + echo "image=$registry/$repository" >> "$GITHUB_OUTPUT" + + - name: Log in to Gitea Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ steps.image.outputs.registry }} + username: ${{ secrets.REGISTRY_USERNAME || github.actor }} + password: ${{ secrets.REGISTRY_TOKEN || secrets.GITHUB_TOKEN }} + + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ steps.image.outputs.image }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix=sha- + labels: | + org.opencontainers.image.title=Progression Mods + org.opencontainers.image.description=Progression RimWorld mod-list service + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and publish + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: false + diff --git a/.gitignore b/.gitignore index c8a2187..8aacb2f 100644 --- a/.gitignore +++ b/.gitignore @@ -200,4 +200,8 @@ art/thumbnails/ # Steam Workshop related (if any cache files are generated) workshop_cache/ -*.cache \ No newline at end of file +*.cache + +# Progression web backend runtime data +.data/ +.steamcmd/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f0cb456 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN dpkg --add-architecture i386 \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl lib32gcc-s1 python3 python3-pip python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --shell /bin/bash progression \ + && mkdir -p /opt/steamcmd/steamapps /srv/progression/.data \ + && curl -fsSL https://steamcdn-a.akamaihd.net/client/installer/steamcmd_linux.tar.gz \ + | tar -xz -C /opt/steamcmd \ + && chown -R progression:progression /opt/steamcmd /srv/progression + +WORKDIR /srv/progression +COPY --chown=progression:progression requirements-web.txt ./ +RUN python3 -m venv /srv/venv \ + && /srv/venv/bin/pip install --no-cache-dir -r requirements-web.txt + +COPY --chown=progression:progression progression_backend ./progression_backend +COPY --chown=progression:progression art ./art +USER progression + +ENV PROGRESSION_HOST=0.0.0.0 \ + PROGRESSION_PORT=8000 \ + PROGRESSION_DB_PATH=.data/progression.db \ + STEAMCMD_PATH=/opt/steamcmd/steamcmd.sh \ + STEAMCMD_WORKSHOP_DIR=/opt/steamcmd/steamapps/workshop/content/294100 \ + REFRESH_INTERVAL_SECONDS=3600 \ + REFRESH_ON_STARTUP=true + +VOLUME ["/srv/progression/.data", "/opt/steamcmd/steamapps"] +EXPOSE 8000 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8000/api/health || exit 1 + +CMD ["/srv/venv/bin/python", "-m", "progression_backend"] diff --git a/README.md b/README.md index 7f85469..c5c5970 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,72 @@ python version_manager.py 1.0.1 build -pyinstaller --clean --onefile --windowed --icon=art/Progression.ico --add-data "art;art" --name ProgressionLoader steam_workshop_gui.py \ No newline at end of file +pyinstaller --clean --onefile --windowed --icon=art/Progression.ico --add-data "art;art" --name ProgressionLoader steam_workshop_gui.py + +## Web dashboard and backend + +The repository now includes a FastAPI backend and responsive dashboard in +`progression_backend`. It resolves the three Workshop collections through +Steam's `GetCollectionDetails` API, downloads missing or updated RimWorld mods +with anonymous SteamCMD, reads `About/About.xml`, and generates the same RimWorld +`.rml` format as the desktop loader. + +The backend refreshes once per hour and records each run in SQLite, including: + +- complete refresh time and status +- requested, successful, and failed downloads +- Workshop cache size on disk +- size added during the run and average throughput +- per-mod size, validation status, timestamps, and SteamCMD batch diagnostics + +### Run locally + +Install SteamCMD, copy `.env.example` to `.env`, and set `STEAMCMD_PATH` plus +`STEAMCMD_WORKSHOP_DIR` to the matching SteamCMD Workshop directory. Then run: + +```powershell +python -m pip install -r requirements-web.txt +python -m progression_backend +``` + +Open `http://127.0.0.1:8000`. API documentation is available at +`http://127.0.0.1:8000/api/docs`. + +Use one application process/worker because that process owns the hourly +scheduler. For an externally scheduled deployment, set `REFRESH_ON_STARTUP=false` +and call `POST /api/refresh` once per hour. + +### Run with Docker + +Docker provisions the SteamCMD binary and persists both analytics and downloaded +Workshop content: + +```powershell +docker compose up --build +``` + +The SteamCMD Linux runtime requires an amd64 host or amd64 emulation. Set +`PROGRESSION_ADMIN_TOKEN` before exposing the manual refresh endpoint publicly. + +### Published container + +Gitea Actions builds and publishes the `linux/amd64` image on every push to +`main` and every `v*` version tag: + +```text +git.hudsonriggs.systems/hriggs/progressionmods:latest +``` + +Published tags include `latest`, the branch or semantic version, and +`sha-`. The workflow uses the Gitea action token by default. If the +server does not grant that token package-write access, configure repository +secrets named `REGISTRY_USERNAME` and `REGISTRY_TOKEN`. + +### Mod-list endpoints + +- `GET /api/modlists/vanilla?dlc=royalty&dlc=biotech` +- `POST /api/modlists/homebrew` with a `mods_config` XML upload +- `GET /api/status`, `/api/analytics`, and `/api/mods` + +Generation fails closed when any current collection item is unavailable. An +operator can explicitly request `allow_partial=true` for diagnostics. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3979e55 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,17 @@ +services: + progression-web: + build: . + platform: linux/amd64 + restart: unless-stopped + ports: + - "8000:8000" + environment: + PROGRESSION_ADMIN_TOKEN: ${PROGRESSION_ADMIN_TOKEN:-} + MODLIST_GAME_VERSION: ${MODLIST_GAME_VERSION:-1.6} + volumes: + - progression-data:/srv/progression/.data + - progression-workshop:/opt/steamcmd/steamapps + +volumes: + progression-data: + progression-workshop: diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..4c553e1 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,23 @@ +# Published image. Use a version such as 1.0.0 instead of latest to pin a release. +PROGRESSION_IMAGE=git.hudsonriggs.systems/hriggs/progressionmods +PROGRESSION_IMAGE_TAG=latest + +# Public listener. Change the external port without changing the container. +PROGRESSION_BIND_ADDRESS=0.0.0.0 +PROGRESSION_EXTERNAL_PORT=8000 + +# Protects POST /api/refresh. Set a long random value before public deployment. +PROGRESSION_ADMIN_TOKEN=change-me +MODLIST_GAME_VERSION=1.6 + +# Progression Steam Workshop collections. +CORE_COLLECTION_URL=https://steamcommunity.com/workshop/filedetails/?id=3521297585 +CONTENT_COLLECTION_URL=https://steamcommunity.com/sharedfiles/filedetails/?id=3521319712 +COSMETICS_COLLECTION_URL=https://steamcommunity.com/sharedfiles/filedetails/?id=3637541646 + +# SteamCMD download and hourly refresh behavior. +STEAMCMD_BATCH_SIZE=25 +STEAMCMD_TIMEOUT_SECONDS=1800 +REFRESH_INTERVAL_SECONDS=3600 +REFRESH_ON_STARTUP=true + diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..a07cdfa --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,17 @@ +# Progression deployment + +This deployment pulls the published container from the Gitea registry. No +repository checkout or local image build is required on the server. + +1. Copy `.env.example` to `.env` and change `PROGRESSION_ADMIN_TOKEN`. +2. Set `PROGRESSION_EXTERNAL_PORT` if port `8000` is already in use. +3. Start the service: + +```sh +docker compose pull +docker compose up -d +``` + +The SQLite database and Steam Workshop files are stored in named Docker volumes +and survive image upgrades. To update later, run the same two commands again. + diff --git a/deploy/compose.yaml b/deploy/compose.yaml new file mode 100644 index 0000000..0cfdc2e --- /dev/null +++ b/deploy/compose.yaml @@ -0,0 +1,33 @@ +name: progression-mods + +services: + progression-web: + image: ${PROGRESSION_IMAGE:-git.hudsonriggs.systems/hriggs/progressionmods}:${PROGRESSION_IMAGE_TAG:-latest} + platform: linux/amd64 + pull_policy: always + restart: unless-stopped + ports: + - "${PROGRESSION_BIND_ADDRESS:-0.0.0.0}:${PROGRESSION_EXTERNAL_PORT:-8000}:8000" + environment: + PROGRESSION_HOST: 0.0.0.0 + PROGRESSION_PORT: 8000 + PROGRESSION_DB_PATH: /srv/progression/.data/progression.db + PROGRESSION_ADMIN_TOKEN: ${PROGRESSION_ADMIN_TOKEN:-} + MODLIST_GAME_VERSION: ${MODLIST_GAME_VERSION:-1.6} + CORE_COLLECTION_URL: ${CORE_COLLECTION_URL:-https://steamcommunity.com/workshop/filedetails/?id=3521297585} + CONTENT_COLLECTION_URL: ${CONTENT_COLLECTION_URL:-https://steamcommunity.com/sharedfiles/filedetails/?id=3521319712} + COSMETICS_COLLECTION_URL: ${COSMETICS_COLLECTION_URL:-https://steamcommunity.com/sharedfiles/filedetails/?id=3637541646} + STEAMCMD_PATH: /opt/steamcmd/steamcmd.sh + STEAMCMD_WORKSHOP_DIR: /opt/steamcmd/steamapps/workshop/content/294100 + STEAMCMD_BATCH_SIZE: ${STEAMCMD_BATCH_SIZE:-25} + STEAMCMD_TIMEOUT_SECONDS: ${STEAMCMD_TIMEOUT_SECONDS:-1800} + REFRESH_INTERVAL_SECONDS: ${REFRESH_INTERVAL_SECONDS:-3600} + REFRESH_ON_STARTUP: ${REFRESH_ON_STARTUP:-true} + volumes: + - progression-data:/srv/progression/.data + - progression-workshop:/opt/steamcmd/steamapps + +volumes: + progression-data: + progression-workshop: + diff --git a/progression_backend/__init__.py b/progression_backend/__init__.py new file mode 100644 index 0000000..27580dc --- /dev/null +++ b/progression_backend/__init__.py @@ -0,0 +1,3 @@ +"""Progression web backend.""" + +__version__ = "0.1.0" diff --git a/progression_backend/__main__.py b/progression_backend/__main__.py new file mode 100644 index 0000000..1b299cc --- /dev/null +++ b/progression_backend/__main__.py @@ -0,0 +1,18 @@ +import uvicorn + +from .config import Settings + + +def main() -> None: + settings = Settings.load() + uvicorn.run( + "progression_backend.app:app", + host=settings.host, + port=settings.port, + reload=False, + workers=1, + ) + + +if __name__ == "__main__": + main() diff --git a/progression_backend/app.py b/progression_backend/app.py new file mode 100644 index 0000000..6132ba6 --- /dev/null +++ b/progression_backend/app.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import secrets +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Annotated +from urllib.parse import urlparse + +import requests +from fastapi import ( + FastAPI, + File, + Header, + HTTPException, + Query, + UploadFile, + status, +) +from fastapi.responses import FileResponse, RedirectResponse, Response +from fastapi.staticfiles import StaticFiles + +from .config import Settings +from .database import Database +from .modlist import core_items, generate_rml, parse_active_mods, workshop_items +from .refresh import RefreshCoordinator, RefreshService +from .workshop import SteamCmdRunner + + +DESKTOP_RELEASES_API = ( + "https://git.hudsonriggs.systems/api/v1/repos/HRiggs/ProgressionMods/releases" +) +DESKTOP_RELEASE_HOST = "git.hudsonriggs.systems" + + +def latest_windows_release_url() -> str | None: + """Return the newest Windows executable, including prerelease builds.""" + response = requests.get( + DESKTOP_RELEASES_API, + headers={"Accept": "application/json"}, + timeout=10, + ) + response.raise_for_status() + releases = response.json() + if not isinstance(releases, list): + raise ValueError("Unexpected Gitea releases response") + + published = sorted( + (release for release in releases if not release.get("draft")), + key=lambda release: release.get("published_at") or release.get("created_at") or "", + reverse=True, + ) + stable = [release for release in published if not release.get("prerelease")] + candidates = stable or published + for release in candidates: + assets = release.get("assets") or [] + executables = [ + asset + for asset in assets + if str(asset.get("name") or "").lower().endswith(".exe") + ] + executables.sort( + key=lambda asset: ( + str(asset.get("name") or "").lower() != "progressionloader.exe", + str(asset.get("name") or "").lower(), + ) + ) + for asset in executables: + download_url = str(asset.get("browser_download_url") or "") + parsed = urlparse(download_url) + if parsed.scheme == "https" and parsed.netloc == DESKTOP_RELEASE_HOST: + return download_url + return None + + +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or Settings.load() + database = Database(settings.database_path) + database.initialize() + runner = SteamCmdRunner( + settings.steamcmd_path, + settings.workshop_content_dir, + settings.steamcmd_timeout_seconds, + ) + service = RefreshService(settings, database, runner=runner) + coordinator = RefreshCoordinator( + service, settings.refresh_interval_seconds, settings.refresh_on_startup + ) + + @asynccontextmanager + async def lifespan(_: FastAPI): + await coordinator.start() + yield + await coordinator.stop() + + app = FastAPI( + title="Progression Mods Backend", + version="0.1.0", + lifespan=lifespan, + docs_url="/api/docs", + redoc_url=None, + ) + app.state.settings = settings + app.state.database = database + app.state.coordinator = coordinator + app.state.runner = runner + + @app.get("/api/health") + async def health() -> dict: + return { + "status": "ok", + "steamcmd_available": runner.available(), + "workshop_content_dir": str(settings.workshop_content_dir), + "scheduler": coordinator.state(), + } + + @app.get("/api/status") + async def dashboard_status() -> dict: + return { + **database.dashboard(), + "scheduler": coordinator.state(), + "steamcmd_available": runner.available(), + "game_version": settings.game_version, + } + + @app.get("/api/analytics") + async def analytics(limit: int = Query(24, ge=1, le=168)) -> dict: + return {"runs": database.runs(limit)} + + @app.get("/api/analytics/downloads") + async def download_analytics( + run_id: str | None = None, + limit: int = Query(100, ge=1, le=500), + ) -> dict: + if not run_id: + runs = database.runs(1) + run_id = runs[0]["id"] if runs else None + return { + "run_id": run_id, + "attempts": database.download_attempts(run_id, limit) if run_id else [], + } + + @app.get("/api/mods") + async def mods( + offset: int = Query(0, ge=0), + limit: int = Query(100, ge=1, le=500), + state: str | None = Query(None, pattern="^(ready|pending|failed)$"), + ) -> dict: + clauses = [ + "workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members)" + ] + parameters: list[object] = [] + if state: + clauses.append("status = ?") + parameters.append(state) + where = " AND ".join(clauses) + total = database.fetch_one( + f"SELECT COUNT(*) AS count FROM mods WHERE {where}", parameters + ) or {"count": 0} + rows = database.fetch_all( + f""" + SELECT workshop_id, title, package_id, display_name, time_updated, + metadata_file_size, disk_size, status, error, last_checked_at, + last_downloaded_at + FROM mods WHERE {where} + ORDER BY CAST(workshop_id AS INTEGER) + LIMIT ? OFFSET ? + """, + [*parameters, limit, offset], + ) + return { + "total": total["count"], + "offset": offset, + "limit": limit, + "items": rows, + } + + @app.post("/api/refresh", status_code=status.HTTP_202_ACCEPTED) + async def refresh(authorization: str | None = Header(None)) -> dict: + if settings.admin_token: + expected = f"Bearer {settings.admin_token}" + if not authorization or not secrets.compare_digest(authorization, expected): + raise HTTPException(status_code=401, detail="Invalid admin token") + run_id, started = await coordinator.trigger("manual") + return {"run_id": run_id, "started": started} + + def ready_workshop_rows(allow_partial: bool) -> list[dict]: + rows = database.current_mods() + failed = [row for row in rows if row.get("status") != "ready"] + if failed and not allow_partial: + raise HTTPException( + status_code=409, + detail={ + "message": "The mod list is incomplete", + "not_ready": len(failed), + "hint": "Run a refresh or request allow_partial=true", + }, + ) + if not rows: + raise HTTPException( + status_code=409, detail="No collection data is available yet" + ) + return rows + + @app.get("/api/modlists/vanilla") + async def vanilla_modlist( + dlc: Annotated[list[str] | None, Query()] = None, + allow_partial: Annotated[bool, Query()] = False, + ) -> Response: + rows = ready_workshop_rows(allow_partial) + rml = generate_rml( + [*core_items(dlc or []), *workshop_items(rows)], settings.game_version + ) + return Response( + rml, + media_type="application/xml", + headers={ + "Content-Disposition": 'attachment; filename="ProgressionVanilla.rml"' + }, + ) + + @app.get("/api/downloads/windows", include_in_schema=False) + def latest_windows_download() -> RedirectResponse: + try: + download_url = latest_windows_release_url() + except (requests.RequestException, TypeError, ValueError) as error: + raise HTTPException( + status_code=502, + detail="The Windows download could not be resolved from Gitea", + ) from error + if not download_url: + raise HTTPException( + status_code=404, + detail="No Windows executable is attached to a published release", + ) + return RedirectResponse(download_url, status_code=307) + + @app.post("/api/modlists/homebrew") + async def homebrew_modlist( + mods_config: Annotated[UploadFile, File()], + allow_partial: Annotated[bool, Query()] = False, + ) -> Response: + if not mods_config.filename or not mods_config.filename.lower().endswith( + ".xml" + ): + raise HTTPException(status_code=400, detail="Upload ModsConfig.xml") + content = await mods_config.read(5 * 1024 * 1024 + 1) + try: + current = parse_active_mods(content) + except ValueError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + rows = ready_workshop_rows(allow_partial) + by_package = { + str(row.get("package_id") or "").lower(): row + for row in rows + if row.get("package_id") + } + hydrated = [] + for item in current: + match = by_package.get(item.package_id.lower()) + if match: + hydrated.extend(workshop_items([match])) + else: + hydrated.append(item) + rml = generate_rml([*hydrated, *workshop_items(rows)], settings.game_version) + return Response( + rml, + media_type="application/xml", + headers={ + "Content-Disposition": 'attachment; filename="ProgressionHomebrew.rml"' + }, + ) + + @app.get("/", include_in_schema=False) + async def index() -> FileResponse: + return FileResponse(settings.static_dir / "index.html") + + app.mount("/static", StaticFiles(directory=settings.static_dir), name="static") + art_dir = Path(settings.static_dir).parents[1] / "art" + if art_dir.is_dir(): + app.mount("/art", StaticFiles(directory=art_dir), name="art") + return app + + +app = create_app() diff --git a/progression_backend/config.py b/progression_backend/config.py new file mode 100644 index 0000000..3109e80 --- /dev/null +++ b/progression_backend/config.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +from dotenv import load_dotenv + +DEFAULT_COLLECTIONS = { + "Core": "3521297585", + "Content": "3521319712", + "Cosmetics": "3637541646", +} + + +def _as_bool(value: str | None, default: bool) -> bool: + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _collection_id(value: str | None, fallback: str) -> str: + """Accept either a raw Workshop id or one of the existing URL settings.""" + if not value: + return fallback + digits = "".join(character for character in value if character.isdigit()) + return digits or fallback + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + database_path: Path + steamcmd_path: str + workshop_content_dir: Path + steamcmd_batch_size: int + steamcmd_timeout_seconds: int + refresh_interval_seconds: int + refresh_on_startup: bool + game_version: str + admin_token: str | None + collections: dict[str, str] + static_dir: Path + + @classmethod + def load(cls, root: Path | None = None) -> Settings: + root = (root or Path.cwd()).resolve() + load_dotenv(root / ".env") + default_steamcmd = ".steamcmd/steamcmd.exe" if os.name == "nt" else "steamcmd" + + return cls( + host=os.getenv("PROGRESSION_HOST", "127.0.0.1"), + port=int(os.getenv("PROGRESSION_PORT", "8000")), + database_path=( + root / os.getenv("PROGRESSION_DB_PATH", ".data/progression.db") + ).resolve(), + steamcmd_path=os.getenv("STEAMCMD_PATH", default_steamcmd), + workshop_content_dir=( + root + / os.getenv( + "STEAMCMD_WORKSHOP_DIR", + ".steamcmd/steamapps/workshop/content/294100", + ) + ).resolve(), + steamcmd_batch_size=max(1, int(os.getenv("STEAMCMD_BATCH_SIZE", "25"))), + steamcmd_timeout_seconds=max( + 60, int(os.getenv("STEAMCMD_TIMEOUT_SECONDS", "1800")) + ), + refresh_interval_seconds=max( + 60, int(os.getenv("REFRESH_INTERVAL_SECONDS", "3600")) + ), + refresh_on_startup=_as_bool(os.getenv("REFRESH_ON_STARTUP"), True), + game_version=os.getenv("MODLIST_GAME_VERSION", "1.6").strip() or "1.6", + admin_token=os.getenv("PROGRESSION_ADMIN_TOKEN") or None, + collections={ + "Core": _collection_id( + os.getenv("CORE_COLLECTION_URL"), DEFAULT_COLLECTIONS["Core"] + ), + "Content": _collection_id( + os.getenv("CONTENT_COLLECTION_URL"), DEFAULT_COLLECTIONS["Content"] + ), + "Cosmetics": _collection_id( + os.getenv("COSMETICS_COLLECTION_URL"), + DEFAULT_COLLECTIONS["Cosmetics"], + ), + }, + static_dir=root / "progression_backend" / "static", + ) diff --git a/progression_backend/database.py b/progression_backend/database.py new file mode 100644 index 0000000..687b8dc --- /dev/null +++ b/progression_backend/database.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +import json +import sqlite3 +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS refresh_runs ( + id TEXT PRIMARY KEY, + trigger TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + status TEXT NOT NULL, + collection_items INTEGER NOT NULL DEFAULT 0, + unique_items INTEGER NOT NULL DEFAULT 0, + ready_items INTEGER NOT NULL DEFAULT 0, + downloads_requested INTEGER NOT NULL DEFAULT 0, + downloads_succeeded INTEGER NOT NULL DEFAULT 0, + downloads_failed INTEGER NOT NULL DEFAULT 0, + bytes_before INTEGER NOT NULL DEFAULT 0, + bytes_after INTEGER NOT NULL DEFAULT 0, + bytes_downloaded INTEGER NOT NULL DEFAULT 0, + cache_bytes INTEGER NOT NULL DEFAULT 0, + elapsed_seconds REAL NOT NULL DEFAULT 0, + average_mbps REAL NOT NULL DEFAULT 0, + error TEXT +); + +CREATE TABLE IF NOT EXISTS mods ( + workshop_id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + package_id TEXT, + steam_app_id TEXT, + display_name TEXT, + time_updated INTEGER NOT NULL DEFAULT 0, + metadata_file_size INTEGER NOT NULL DEFAULT 0, + disk_size INTEGER NOT NULL DEFAULT 0, + local_path TEXT, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + last_checked_at TEXT, + last_downloaded_at TEXT +); + +CREATE TABLE IF NOT EXISTS collection_members ( + collection_id TEXT NOT NULL, + collection_name TEXT NOT NULL, + workshop_id TEXT NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (collection_id, workshop_id) +); + +CREATE TABLE IF NOT EXISTS download_attempts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + batch_number INTEGER NOT NULL, + item_count INTEGER NOT NULL, + workshop_ids TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + elapsed_seconds REAL NOT NULL, + exit_code INTEGER, + status TEXT NOT NULL, + output_tail TEXT, + FOREIGN KEY (run_id) REFERENCES refresh_runs(id) +); + +CREATE INDEX IF NOT EXISTS idx_runs_started ON refresh_runs(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_mods_status ON mods(status); +CREATE INDEX IF NOT EXISTS idx_attempts_run ON download_attempts(run_id); +""" + + +class Database: + def __init__(self, path: Path): + self.path = path + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + self.path.parent.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(self.path, timeout=30) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA journal_mode = WAL") + try: + yield connection + connection.commit() + finally: + connection.close() + + def initialize(self) -> None: + with self.connect() as connection: + connection.executescript(SCHEMA) + + def execute(self, sql: str, parameters: Sequence[Any] = ()) -> None: + with self.connect() as connection: + connection.execute(sql, parameters) + + def fetch_one( + self, sql: str, parameters: Sequence[Any] = () + ) -> dict[str, Any] | None: + with self.connect() as connection: + row = connection.execute(sql, parameters).fetchone() + return dict(row) if row else None + + def fetch_all( + self, sql: str, parameters: Sequence[Any] = () + ) -> list[dict[str, Any]]: + with self.connect() as connection: + rows = connection.execute(sql, parameters).fetchall() + return [dict(row) for row in rows] + + def create_run(self, run_id: str, trigger: str, started_at: str) -> None: + self.execute( + "INSERT INTO refresh_runs (id, trigger, started_at, status) VALUES (?, ?, ?, 'running')", + (run_id, trigger, started_at), + ) + + def finish_run(self, run_id: str, values: dict[str, Any]) -> None: + allowed = { + "finished_at", + "status", + "collection_items", + "unique_items", + "ready_items", + "downloads_requested", + "downloads_succeeded", + "downloads_failed", + "bytes_before", + "bytes_after", + "bytes_downloaded", + "cache_bytes", + "elapsed_seconds", + "average_mbps", + "error", + } + selected = {key: value for key, value in values.items() if key in allowed} + assignments = ", ".join(f"{key} = ?" for key in selected) + self.execute( + f"UPDATE refresh_runs SET {assignments} WHERE id = ?", + (*selected.values(), run_id), + ) + + def replace_collection_members( + self, collections: dict[str, tuple[str, list[str]]] + ) -> None: + with self.connect() as connection: + connection.execute("DELETE FROM collection_members") + for name, (collection_id, workshop_ids) in collections.items(): + connection.executemany( + """ + INSERT INTO collection_members + (collection_id, collection_name, workshop_id, position) + VALUES (?, ?, ?, ?) + """, + [ + (collection_id, name, workshop_id, position) + for position, workshop_id in enumerate(workshop_ids) + ], + ) + + def _upsert_mod( + self, connection: sqlite3.Connection, values: dict[str, Any] + ) -> None: + columns = [ + "workshop_id", + "title", + "package_id", + "steam_app_id", + "display_name", + "time_updated", + "metadata_file_size", + "disk_size", + "local_path", + "status", + "error", + "last_checked_at", + "last_downloaded_at", + ] + workshop_id = str(values["workshop_id"]) + existing_row = connection.execute( + "SELECT * FROM mods WHERE workshop_id = ?", (workshop_id,) + ).fetchone() + existing = dict(existing_row) if existing_row else {} + defaults = { + "title": "", + "time_updated": 0, + "metadata_file_size": 0, + "disk_size": 0, + "status": "pending", + } + payload: dict[str, Any] = {"workshop_id": workshop_id} + for column in columns[1:]: + candidate = values.get(column) + if candidate is not None: + payload[column] = candidate + elif column in existing: + payload[column] = existing[column] + else: + payload[column] = defaults.get(column) + updates = ", ".join( + f"{column} = excluded.{column}" + for column in columns + if column != "workshop_id" + ) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f""" + INSERT INTO mods ({", ".join(columns)}) VALUES ({placeholders}) + ON CONFLICT(workshop_id) DO UPDATE SET {updates} + """, + tuple(payload[column] for column in columns), + ) + + def upsert_mod(self, values: dict[str, Any]) -> None: + with self.connect() as connection: + self._upsert_mod(connection, values) + + def upsert_mods(self, values: Sequence[dict[str, Any]]) -> None: + """Upsert a metadata page in one transaction instead of per-mod commits.""" + with self.connect() as connection: + for value in values: + self._upsert_mod(connection, value) + + def add_download_attempt(self, values: dict[str, Any]) -> None: + self.execute( + """ + INSERT INTO download_attempts + (run_id, batch_number, item_count, workshop_ids, started_at, + finished_at, elapsed_seconds, exit_code, status, output_tail) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + values["run_id"], + values["batch_number"], + values["item_count"], + json.dumps(values["workshop_ids"]), + values["started_at"], + values["finished_at"], + values["elapsed_seconds"], + values.get("exit_code"), + values["status"], + values.get("output_tail"), + ), + ) + + def dashboard(self) -> dict[str, Any]: + recent_runs = self.runs(1) + latest = recent_runs[0] if recent_runs else None + last_update = self.fetch_one( + """ + SELECT finished_at + FROM refresh_runs + WHERE finished_at IS NOT NULL AND unique_items > 0 + ORDER BY finished_at DESC + LIMIT 1 + """ + ) + totals = ( + self.fetch_one( + """ + SELECT COUNT(*) AS total_mods, + SUM(CASE WHEN status = 'ready' THEN 1 ELSE 0 END) AS ready_mods, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed_mods, + COALESCE(SUM(disk_size), 0) AS cache_bytes + FROM mods + WHERE workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members) + """ + ) + or {} + ) + collections = self.fetch_all( + """ + SELECT collection_name AS name, collection_id AS id, COUNT(*) AS item_count + FROM collection_members + GROUP BY collection_name, collection_id + ORDER BY CASE collection_name + WHEN 'Core' THEN 1 WHEN 'Content' THEN 2 WHEN 'Cosmetics' THEN 3 ELSE 4 END + """ + ) + return { + "latest_run": latest, + "rml_updated_at": last_update["finished_at"] if last_update else None, + "totals": totals, + "collections": collections, + } + + def runs(self, limit: int = 48) -> list[dict[str, Any]]: + return self.fetch_all( + """ + SELECT r.*, + COALESCE(SUM(a.elapsed_seconds), 0) AS download_elapsed_seconds, + COUNT(a.id) AS download_batches + FROM refresh_runs r + LEFT JOIN download_attempts a ON a.run_id = r.id + GROUP BY r.id + ORDER BY r.started_at DESC + LIMIT ? + """, + (limit,), + ) + + def download_attempts(self, run_id: str, limit: int = 100) -> list[dict[str, Any]]: + return self.fetch_all( + """ + SELECT batch_number, item_count, workshop_ids, started_at, finished_at, + elapsed_seconds, exit_code, status + FROM download_attempts + WHERE run_id = ? + ORDER BY batch_number + LIMIT ? + """, + (run_id, limit), + ) + + def current_mods(self) -> list[dict[str, Any]]: + return self.fetch_all( + """ + SELECT * FROM mods + WHERE workshop_id IN (SELECT DISTINCT workshop_id FROM collection_members) + ORDER BY CAST(workshop_id AS INTEGER) + """ + ) + + def mod_map(self, workshop_ids: Sequence[str]) -> dict[str, dict[str, Any]]: + if not workshop_ids: + return {} + placeholders = ",".join("?" for _ in workshop_ids) + rows = self.fetch_all( + f"SELECT * FROM mods WHERE workshop_id IN ({placeholders})", workshop_ids + ) + return {row["workshop_id"]: row for row in rows} diff --git a/progression_backend/modlist.py b/progression_backend/modlist.py new file mode 100644 index 0000000..954a349 --- /dev/null +++ b/progression_backend/modlist.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import io +import xml.etree.ElementTree as ET +from collections.abc import Iterable +from dataclasses import dataclass + +DLC = { + "royalty": ("ludeon.rimworld.royalty", "Royalty"), + "ideology": ("ludeon.rimworld.ideology", "Ideology"), + "biotech": ("ludeon.rimworld.biotech", "Biotech"), + "anomaly": ("ludeon.rimworld.anomaly", "Anomaly"), + "odyssey": ("ludeon.rimworld.odyssey", "Odyssey"), +} +BASE_GAME = ("ludeon.rimworld", "Core") + + +@dataclass(frozen=True) +class ModListItem: + package_id: str + workshop_id: str + name: str + + +def core_items(selected_dlc: Iterable[str]) -> list[ModListItem]: + result = [ModListItem(BASE_GAME[0], "0", BASE_GAME[1])] + normalized = {value.strip().lower() for value in selected_dlc} + for key, (package_id, name) in DLC.items(): + if key in normalized: + result.append(ModListItem(package_id, "0", name)) + return result + + +def workshop_items(rows: Iterable[dict]) -> list[ModListItem]: + result: list[ModListItem] = [] + for row in rows: + if row.get("status") != "ready" or not row.get("package_id"): + continue + # The Workshop id is the correct RimWorld modSteamIds value. The old loader + # sometimes substituted About.xml's steamAppId, which can be the game app id. + result.append( + ModListItem( + package_id=str(row["package_id"]), + workshop_id=str(row["workshop_id"]), + name=str( + row.get("display_name") or row.get("title") or row["package_id"] + ), + ) + ) + return result + + +def deduplicate(items: Iterable[ModListItem]) -> list[ModListItem]: + result: list[ModListItem] = [] + seen_packages: set[str] = set() + seen_workshop_ids: set[str] = set() + for item in items: + package_key = item.package_id.lower() + if package_key in seen_packages: + continue + if item.workshop_id != "0" and item.workshop_id in seen_workshop_ids: + continue + seen_packages.add(package_key) + if item.workshop_id != "0": + seen_workshop_ids.add(item.workshop_id) + result.append(item) + return result + + +def parse_active_mods(xml_bytes: bytes) -> list[ModListItem]: + if len(xml_bytes) > 5 * 1024 * 1024: + raise ValueError("ModsConfig.xml is larger than 5 MB") + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as error: + raise ValueError(f"ModsConfig.xml is invalid: {error}") from error + active = root.find("activeMods") + if active is None: + raise ValueError("ModsConfig.xml has no activeMods section") + items: list[ModListItem] = [] + for entry in active.findall("li"): + package_id = (entry.text or "").strip() + if package_id: + items.append(ModListItem(package_id, "0", package_id)) + return items + + +def _append_list(parent: ET.Element, name: str, values: Iterable[str]) -> None: + element = ET.SubElement(parent, name) + for value in values: + ET.SubElement(element, "li").text = value + + +def generate_rml(items: Iterable[ModListItem], game_version: str) -> bytes: + normalized = deduplicate(items) + root = ET.Element("savedModList") + meta = ET.SubElement(root, "meta") + ET.SubElement(meta, "gameVersion").text = game_version + _append_list(meta, "modIds", (item.package_id for item in normalized)) + _append_list(meta, "modSteamIds", (item.workshop_id for item in normalized)) + _append_list(meta, "modNames", (item.name for item in normalized)) + + mod_list = ET.SubElement(root, "modList") + _append_list(mod_list, "ids", (item.package_id for item in normalized)) + _append_list(mod_list, "names", (item.name for item in normalized)) + + ET.indent(root, space="\t") + output = io.BytesIO() + ET.ElementTree(root).write(output, encoding="utf-8", xml_declaration=True) + return output.getvalue() diff --git a/progression_backend/refresh.py b/progression_backend/refresh.py new file mode 100644 index 0000000..1cb0eb5 --- /dev/null +++ b/progression_backend/refresh.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +import asyncio +import time +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any + +from .config import Settings +from .database import Database +from .steam_api import SteamWorkshopClient, WorkshopDetails +from .workshop import SteamCmdRunner, directory_size, parse_about_xml, utc_now + + +def _batches(values: list[str], size: int) -> list[list[str]]: + return [values[index : index + size] for index in range(0, len(values), size)] + + +class RefreshService: + def __init__( + self, + settings: Settings, + database: Database, + steam: SteamWorkshopClient | None = None, + runner: SteamCmdRunner | None = None, + ): + self.settings = settings + self.database = database + self.steam = steam or SteamWorkshopClient() + self.runner = runner or SteamCmdRunner( + settings.steamcmd_path, + settings.workshop_content_dir, + settings.steamcmd_timeout_seconds, + ) + + def _store_metadata( + self, + detail: WorkshopDetails, + previous: dict[str, Any] | None, + stale: bool, + ) -> dict[str, Any]: + invalid_reason = None + if detail.result != 1: + invalid_reason = f"Steam returned result {detail.result}" + elif detail.consumer_app_id not in (0, 294100): + invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}" + return { + "workshop_id": detail.workshop_id, + "title": detail.title, + "package_id": None, + "steam_app_id": None, + "display_name": None, + "time_updated": detail.time_updated, + "metadata_file_size": detail.file_size, + "disk_size": None, + "local_path": None, + "status": "failed" + if invalid_reason + else ("pending" if stale else (previous or {}).get("status", "pending")), + "error": invalid_reason, + "last_checked_at": utc_now(), + "last_downloaded_at": None, + } + + def _inspection_values( + self, + workshop_id: str, + downloaded: bool, + forced_error: str | None = None, + ) -> tuple[bool, dict[str, Any]]: + mod_path = self.settings.workshop_content_dir / workshop_id + if forced_error: + return False, { + "workshop_id": workshop_id, + "status": "failed", + "error": forced_error, + "disk_size": directory_size(mod_path), + "local_path": str(mod_path) if mod_path.exists() else None, + "last_checked_at": utc_now(), + "last_downloaded_at": utc_now() if downloaded else None, + } + try: + parsed = parse_about_xml(mod_path) + return True, { + "workshop_id": workshop_id, + "package_id": parsed.package_id, + "steam_app_id": parsed.declared_steam_app_id, + "display_name": parsed.display_name, + "disk_size": directory_size(mod_path), + "local_path": str(mod_path), + "status": "ready", + "error": "", + "last_checked_at": utc_now(), + "last_downloaded_at": utc_now() if downloaded else None, + } + except (OSError, ValueError) as error: + return False, { + "workshop_id": workshop_id, + "status": "failed", + "error": str(error), + "disk_size": directory_size(mod_path), + "local_path": str(mod_path) if mod_path.exists() else None, + "last_checked_at": utc_now(), + "last_downloaded_at": utc_now() if downloaded else None, + } + + def refresh(self, run_id: str, trigger: str) -> None: + started_at = utc_now() + started = time.monotonic() + self.database.create_run(run_id, trigger, started_at) + metrics: dict[str, Any] = { + "collection_items": 0, + "unique_items": 0, + "ready_items": 0, + "downloads_requested": 0, + "downloads_succeeded": 0, + "downloads_failed": 0, + "bytes_before": directory_size(self.settings.workshop_content_dir), + "bytes_after": 0, + "bytes_downloaded": 0, + "cache_bytes": 0, + "status": "failed", + "error": None, + } + + try: + collection_ids = list(self.settings.collections.values()) + response = self.steam.get_collections(collection_ids) + named_collections: dict[str, tuple[str, list[str]]] = {} + all_ids: list[str] = [] + for name, collection_id in self.settings.collections.items(): + workshop_ids = response.get(collection_id) + if workshop_ids is None: + raise RuntimeError( + f"Steam did not return collection {collection_id}" + ) + named_collections[name] = (collection_id, workshop_ids) + all_ids.extend(workshop_ids) + self.database.replace_collection_members(named_collections) + + unique_ids = sorted(set(all_ids), key=int) + metrics["collection_items"] = len(all_ids) + metrics["unique_items"] = len(unique_ids) + details = self.steam.get_item_details(unique_ids) + previous = self.database.mod_map(unique_ids) + + targets: list[str] = [] + metadata_rows: list[dict[str, Any]] = [] + for workshop_id in unique_ids: + detail = details.get(workshop_id) + if not detail: + self.database.upsert_mod( + { + "workshop_id": workshop_id, + "status": "failed", + "error": "Steam returned no item metadata", + "last_checked_at": utc_now(), + } + ) + continue + old = previous.get(workshop_id) + local_path = self.settings.workshop_content_dir / workshop_id + stale = ( + old is None + or int(old.get("time_updated") or 0) != detail.time_updated + or old.get("status") != "ready" + or not local_path.is_dir() + ) + metadata_rows.append(self._store_metadata(detail, old, stale)) + if ( + detail.result == 1 + and detail.consumer_app_id in (0, 294100) + and stale + ): + targets.append(workshop_id) + + self.database.upsert_mods(metadata_rows) + + metrics["downloads_requested"] = len(targets) + if targets and not self.runner.available(): + reason = ( + f"SteamCMD was not found at '{self.settings.steamcmd_path}'. " + "Set STEAMCMD_PATH to enable anonymous downloads." + ) + inspection_rows: list[dict[str, Any]] = [] + for workshop_id in targets: + mod_path = self.settings.workshop_content_dir / workshop_id + success, values = self._inspection_values( + workshop_id, + downloaded=False, + forced_error=None if mod_path.is_dir() else reason, + ) + inspection_rows.append(values) + if not success: + metrics["downloads_failed"] += 1 + self.database.upsert_mods(inspection_rows) + else: + for batch_number, batch in enumerate( + _batches(targets, self.settings.steamcmd_batch_size), start=1 + ): + result = self.runner.run_batch(batch) + self.database.add_download_attempt( + { + "run_id": run_id, + "batch_number": batch_number, + "item_count": len(batch), + "workshop_ids": batch, + "started_at": result.started_at, + "finished_at": result.finished_at, + "elapsed_seconds": result.elapsed_seconds, + "exit_code": result.exit_code, + "status": result.status, + "output_tail": result.output_tail, + } + ) + inspection_rows = [] + for workshop_id in batch: + failure = ( + "SteamCMD reported a download failure" + if workshop_id in result.failed_ids + else None + ) + success, values = self._inspection_values( + workshop_id, downloaded=True, forced_error=failure + ) + inspection_rows.append(values) + if success: + metrics["downloads_succeeded"] += 1 + else: + metrics["downloads_failed"] += 1 + self.database.upsert_mods(inspection_rows) + + # Re-inspect ready cached items whose metadata has never been parsed by this backend. + current = self.database.mod_map(unique_ids) + pending_rows = [] + for workshop_id in unique_ids: + row = current.get(workshop_id) + if row and row.get("status") == "pending": + _, values = self._inspection_values(workshop_id, downloaded=False) + pending_rows.append(values) + if pending_rows: + self.database.upsert_mods(pending_rows) + + metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir) + metrics["cache_bytes"] = metrics["bytes_after"] + metrics["bytes_downloaded"] = max( + metrics["bytes_after"] - metrics["bytes_before"], 0 + ) + final_rows = self.database.mod_map(unique_ids) + metrics["ready_items"] = sum( + 1 for row in final_rows.values() if row.get("status") == "ready" + ) + metrics["status"] = ( + "completed" + if metrics["ready_items"] == metrics["unique_items"] + else "partial" + ) + if metrics["status"] == "partial" and not metrics["error"]: + metrics["error"] = ( + f"{metrics['unique_items'] - metrics['ready_items']} items are not ready" + ) + except Exception as error: # noqa: BLE001 - the run must always persist a terminal state + metrics["status"] = "failed" + metrics["error"] = str(error) + metrics["bytes_after"] = directory_size(self.settings.workshop_content_dir) + metrics["cache_bytes"] = metrics["bytes_after"] + finally: + elapsed = max(time.monotonic() - started, 0.001) + metrics["elapsed_seconds"] = round(elapsed, 3) + metrics["average_mbps"] = round( + (metrics["bytes_downloaded"] * 8 / 1_000_000) / elapsed, 3 + ) + metrics["finished_at"] = utc_now() + self.database.finish_run(run_id, metrics) + + +class RefreshCoordinator: + def __init__( + self, service: RefreshService, interval_seconds: int, refresh_on_startup: bool + ): + self.service = service + self.interval_seconds = interval_seconds + self.refresh_on_startup = refresh_on_startup + self.current_run_id: str | None = None + self.next_refresh_at: str | None = None + self._refresh_task: asyncio.Task[None] | None = None + self._scheduler_task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + + async def start(self) -> None: + if self._scheduler_task: + return + self._scheduler_task = asyncio.create_task(self._scheduler_loop()) + + async def stop(self) -> None: + self._stop.set() + if self._scheduler_task: + self._scheduler_task.cancel() + await asyncio.gather(self._scheduler_task, return_exceptions=True) + + async def trigger(self, trigger: str) -> tuple[str, bool]: + if self._refresh_task and not self._refresh_task.done(): + return self.current_run_id or "", False + run_id = uuid.uuid4().hex + self.current_run_id = run_id + + async def run() -> None: + try: + await asyncio.to_thread(self.service.refresh, run_id, trigger) + finally: + self.current_run_id = None + + self._refresh_task = asyncio.create_task(run()) + return run_id, True + + async def _scheduler_loop(self) -> None: + if self.refresh_on_startup: + await self.trigger("startup") + while not self._stop.is_set(): + next_time = datetime.now(timezone.utc) + timedelta( + seconds=self.interval_seconds + ) + self.next_refresh_at = next_time.isoformat() + try: + await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds) + except asyncio.TimeoutError: + await self.trigger("scheduled") + + def state(self) -> dict[str, Any]: + return { + "running": bool(self._refresh_task and not self._refresh_task.done()), + "current_run_id": self.current_run_id, + "next_refresh_at": self.next_refresh_at, + "interval_seconds": self.interval_seconds, + } diff --git a/progression_backend/static/app.js b/progression_backend/static/app.js new file mode 100644 index 0000000..e7ab46b --- /dev/null +++ b/progression_backend/static/app.js @@ -0,0 +1,360 @@ +const $ = (selector) => document.querySelector(selector); +const $$ = (selector) => [...document.querySelectorAll(selector)]; + +const INTRO_MINIMUM_MS = 900; +const REQUIRED_EXPANSIONS = { + royalty: { name: "Royalty", art: "/art/NotOwned_Royalty.png" }, + ideology: { name: "Ideology", art: "/art/NotOwned_Ideology.png" }, + biotech: { name: "Biotech", art: "/art/NotOwned_Biotech.png" }, + anomaly: { name: "Anomaly", art: "/art/NotOwned_Anomaly.png" }, + odyssey: { name: "Odyssey", art: null } +}; +const INSTALL_PATHS = { + windows: { + label: "Windows", + path: "%USERPROFILE%\\AppData\\LocalLow\\Ludeon Studios\\RimWorld by Ludeon Studios\\ModLists" + }, + macos: { + label: "macOS", + path: "~/Library/Application Support/RimWorld/ModLists" + }, + linux: { + label: "Linux", + path: "~/.config/unity3d/Ludeon Studios/RimWorld by Ludeon Studios/ModLists" + } +}; +const state = { + currentStep: 0, + listType: "standard", + sortAcknowledged: false, + polling: null, + rmlReady: false +}; + +function detectedOperatingSystem() { + const platform = String(navigator.userAgentData?.platform || navigator.platform || navigator.userAgent || "").toLowerCase(); + if (platform.includes("mac")) return "macos"; + if (platform.includes("linux") || platform.includes("x11")) return "linux"; + return "windows"; +} + +function renderInstallLocation() { + const install = INSTALL_PATHS[detectedOperatingSystem()]; + $("#detected-os-name").textContent = install.label; + $("#detected-os-path").textContent = install.path; +} + +function resetListType() { + const standard = $('input[name="list-type"][value="standard"]'); + const acknowledgement = $("#acknowledge-autosort"); + standard.checked = true; + state.listType = "standard"; + state.sortAcknowledged = false; + $("#homebrew-settings").hidden = true; + $("#autosort-confirmation").classList.remove("acknowledged"); + acknowledgement.setAttribute("aria-pressed", "false"); + acknowledgement.textContent = "Yes, I understand"; + setMessage("", "", "#wizard-message"); + updateDownloadAvailability(); +} + +function formatDate(value, fallback = "Not available") { + if (!value) return fallback; + return new Intl.DateTimeFormat(undefined, { + month: "long", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "2-digit" + }).format(new Date(value)); +} + +function formatRelativeDate(value) { + if (!value) return "Waiting for the first refresh"; + const elapsedSeconds = Math.max(0, Math.round((Date.now() - new Date(value).getTime()) / 1000)); + if (elapsedSeconds < 60) return "Updated just now"; + const minutes = Math.floor(elapsedSeconds / 60); + if (minutes < 60) return `Updated ${minutes} minute${minutes === 1 ? "" : "s"} ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `Updated ${hours} hour${hours === 1 ? "" : "s"} ago`; + const days = Math.floor(hours / 24); + return `Updated ${days} day${days === 1 ? "" : "s"} ago`; +} + +function setMessage(text, kind = "", selector = "#action-message") { + const element = $(selector); + element.textContent = text; + element.className = `${element.dataset.baseClass || element.className.split(" ")[0]} ${kind}`.trim(); +} + +async function api(path, options = {}) { + const response = await fetch(path, options); + const contentType = response.headers.get("content-type") || ""; + if (!response.ok) { + let detail = `${response.status} ${response.statusText}`; + if (contentType.includes("json")) { + const body = await response.json(); + detail = typeof body.detail === "string" ? body.detail : body.detail?.message || detail; + } + throw new Error(detail); + } + return contentType.includes("json") ? response.json() : response; +} + +function renderServiceState(data) { + const element = $("#service-state"); + const label = element.querySelector("span:last-child"); + const total = Number(data.totals?.total_mods || 0); + const ready = Number(data.totals?.ready_mods || 0); + element.className = "service-state"; + + if (data.scheduler?.running) { + element.classList.add("working"); + label.textContent = "Updating now"; + } else if (data.latest_run?.status === "failed") { + element.classList.add("failed"); + label.textContent = "Update needs attention"; + } else if (!data.steamcmd_available) { + element.classList.add("failed"); + label.textContent = "Downloads unavailable"; + } else if (ready < total) { + element.classList.add("working"); + label.textContent = "Preparing RML"; + } else { + element.classList.add("ready"); + label.textContent = "Ready"; + } +} + +function updateDownloadAvailability() { + const downloadButton = $("#download-list"); + const availability = $("#rml-availability"); + const needsFile = state.listType === "homebrew" && !$("#mods-config").files[0]; + + downloadButton.disabled = !state.rmlReady || needsFile || !state.sortAcknowledged; + if (!state.rmlReady) { + availability.textContent = "The current RML is still being prepared. You can review every step now and download when it is ready."; + availability.className = "availability waiting"; + } else if (needsFile) { + availability.textContent = "Choose ModsConfig.xml to finish your homebrew list."; + availability.className = "availability waiting"; + } else if (!state.sortAcknowledged) { + availability.textContent = "Confirm the auto-sort step above to unlock your download."; + availability.className = "availability waiting"; + } else { + availability.textContent = state.listType === "homebrew" + ? "Your homebrew list is ready to build." + : "The current Progression list is ready to download."; + availability.className = "availability ready"; + } +} + +function renderDashboard(data) { + renderServiceState(data); + const totals = data.totals || {}; + const total = Number(totals.total_mods || 0); + const ready = Number(totals.ready_mods || 0); + const counts = Object.fromEntries( + (data.collections || []).map(collection => [String(collection.name).toLowerCase(), Number(collection.item_count || 0)]) + ); + + $("#count-total").textContent = total ? total.toLocaleString() : "—"; + for (const category of ["core", "content", "cosmetics"]) { + const formatted = counts[category]?.toLocaleString() || "—"; + $(`#count-${category}`).textContent = formatted; + $(`[data-collection-count="${category}"]`).textContent = formatted; + } + + const updatedAt = data.rml_updated_at; + $("#last-updated").textContent = data.scheduler?.running ? "Updating now" : formatRelativeDate(updatedAt); + $("#last-updated").title = formatDate(updatedAt); + + state.rmlReady = total > 0 && ready === total && !data.scheduler?.running; + const nextRefresh = data.scheduler?.next_refresh_at; + if (data.scheduler?.running) { + $("#update-detail").textContent = "The hourly Workshop refresh is currently running."; + } else if (!state.rmlReady) { + $("#update-detail").textContent = "The current RML is being prepared from the latest Workshop data."; + } else { + $("#update-detail").textContent = nextRefresh + ? `Next automatic check ${formatDate(nextRefresh)}.` + : "Workshop data refreshes every hour."; + } + updateDownloadAvailability(); +} + +async function loadDashboard() { + try { + renderDashboard(await api("/api/status")); + } catch (error) { + const element = $("#service-state"); + element.className = "service-state failed"; + element.querySelector("span:last-child").textContent = "Backend unavailable"; + $("#last-updated").textContent = "Unavailable"; + $("#update-detail").textContent = "The service could not be reached."; + state.rmlReady = false; + updateDownloadAvailability(); + setMessage(error.message, "error"); + } +} + +function showStep(step) { + const panels = $$(".wizard-step"); + const progressItems = $$(".wizard-progress li"); + state.currentStep = Math.max(0, Math.min(step, panels.length - 1)); + + panels.forEach((panel, index) => { + const active = index === state.currentStep; + panel.hidden = !active; + panel.classList.toggle("active", active); + }); + + progressItems.forEach((item, index) => { + item.classList.toggle("active", index === state.currentStep); + item.classList.toggle("complete", index < state.currentStep); + if (index === state.currentStep) item.setAttribute("aria-current", "step"); + else item.removeAttribute("aria-current"); + }); + + if (state.currentStep === 2) updateExpansionRequirements(); +} + +function updateExpansionRequirements() { + const inputs = $$("#dlc-options input"); + const missing = inputs + .filter(input => !input.checked) + .map(input => REQUIRED_EXPANSIONS[input.value]); + const warning = $("#expansion-warning"); + const iconList = $("#missing-expansion-icons"); + + inputs.forEach(input => { + const isMissing = !input.checked; + input.closest("label").classList.toggle("missing", isMissing); + input.setAttribute("aria-invalid", String(isMissing)); + }); + + warning.hidden = missing.length === 0; + $("#expansions-next").disabled = missing.length > 0; + if (!missing.length) { + iconList.replaceChildren(); + $("#expansion-warning-text").textContent = ""; + return; + } + + const names = missing.map(expansion => expansion.name); + $("#expansion-warning-text").textContent = `${names.join(", ")} ${missing.length === 1 ? "is" : "are"} required before you can continue.`; + iconList.innerHTML = missing.map(expansion => expansion.art + ? `` + : `!ODYSSEYREQUIRED` + ).join(""); +} + +function selectedDlcQuery() { + return $$("#dlc-options input:checked") + .map(input => `dlc=${encodeURIComponent(input.value)}`) + .join("&"); +} + +async function downloadResponse(responsePromise, filename) { + const response = await responsePromise; + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} + +$("#start-wizard").addEventListener("click", () => { + resetListType(); + showStep(0); + $("#download-wizard").showModal(); +}); + +$("#close-wizard").addEventListener("click", () => $("#download-wizard").close()); + +$("#download-wizard").addEventListener("click", event => { + if (event.target === event.currentTarget) event.currentTarget.close(); +}); + +$$('[data-next]').forEach(button => button.addEventListener("click", () => showStep(state.currentStep + 1))); +$$('[data-back]').forEach(button => button.addEventListener("click", () => showStep(state.currentStep - 1))); + +$("#setup-confirm").addEventListener("change", event => { + $("#setup-next").disabled = !event.target.checked; +}); + +$$("#dlc-options input").forEach(input => input.addEventListener("change", updateExpansionRequirements)); + +$("#show-all-paths").addEventListener("click", event => { + const paths = $("#all-install-paths"); + const show = paths.hidden; + paths.hidden = !show; + event.currentTarget.setAttribute("aria-expanded", String(show)); + event.currentTarget.textContent = show ? "Hide other OS" : "Show other OS"; +}); + +$("#acknowledge-autosort").addEventListener("click", event => { + state.sortAcknowledged = true; + $("#autosort-confirmation").classList.add("acknowledged"); + event.currentTarget.setAttribute("aria-pressed", "true"); + event.currentTarget.textContent = "Understood ✓"; + updateDownloadAvailability(); +}); + +$$('input[name="list-type"]').forEach(input => input.addEventListener("change", event => { + state.listType = event.target.value; + $("#homebrew-settings").hidden = state.listType !== "homebrew"; + setMessage("", "", "#wizard-message"); + updateDownloadAvailability(); +})); + +$("#mods-config").addEventListener("change", event => { + const file = event.target.files[0]; + $("#file-label").textContent = file?.name || "Choose ModsConfig.xml"; + $(".file-picker").classList.toggle("has-file", Boolean(file)); + updateDownloadAvailability(); +}); + +$("#download-list").addEventListener("click", async () => { + const button = $("#download-list"); + button.disabled = true; + setMessage("Building your mod list…", "", "#wizard-message"); + + try { + if (state.listType === "homebrew") { + const file = $("#mods-config").files[0]; + if (!file) throw new Error("Choose ModsConfig.xml before building a homebrew list."); + const form = new FormData(); + form.append("mods_config", file); + await downloadResponse(api("/api/modlists/homebrew", { method: "POST", body: form }), "ProgressionHomebrew.rml"); + setMessage("ProgressionHomebrew.rml is ready. Auto-sort it after loading.", "success", "#wizard-message"); + } else { + const query = selectedDlcQuery(); + await downloadResponse(api(`/api/modlists/vanilla${query ? `?${query}` : ""}`), "ProgressionVanilla.rml"); + setMessage("ProgressionVanilla.rml is ready.", "success", "#wizard-message"); + } + } catch (error) { + setMessage(error.message, "error", "#wizard-message"); + } finally { + updateDownloadAvailability(); + } +}); + +async function start() { + $("#action-message").dataset.baseClass = "action-message"; + $("#wizard-message").dataset.baseClass = "wizard-message"; + renderInstallLocation(); + const minimumIntro = new Promise(resolve => window.setTimeout(resolve, INTRO_MINIMUM_MS)); + await Promise.all([loadDashboard(), minimumIntro]); + document.body.classList.add("is-ready"); + window.setTimeout(() => { + $("#intro").hidden = true; + }, 850); +} + +start(); +state.polling = window.setInterval(loadDashboard, 60_000); diff --git a/progression_backend/static/index.html b/progression_backend/static/index.html new file mode 100644 index 0000000..3f7c14d --- /dev/null +++ b/progression_backend/static/index.html @@ -0,0 +1,254 @@ + + + + + + + + Progression — RimWorld Modpack + + + + + + +
+
+ + RimWorld Progression + +
+ + Connecting +
+
+ +
+
+
+

Built for beginners.
A current mod list in one click.

+

Choose the expansions you own and download a ready-to-load RimWorld mod list.

+ + + + + + + Download the Windows app + Latest ProgressionLoader.exe + + + + +
+ RML last updated + Checking… + Workshop data refreshes every hour. +
+
+ +
+ +
+
+ Total + — + unique mods +
+
+ Core + — + foundation +
+
+ Content + — + gameplay +
+
+ Cosmetics + — + visuals +
+
+ +

+
+ + +
+ + +
+
+
+ Progression Loader + Build your mod list +
+ +
+ +
    +
  1. 1Collections
    2. +
  2. 2Setup
    3. +
  3. 3Expansions
    4. +
  4. 4Download
    5. +
+ +
+
+

Step one

+

Steam Workshop collections

+

The service keeps Progression’s three source collections current for you.

+
+
✓Core— mods
+
✓Content— mods
+
✓Cosmetics— mods
+
+
+ +
+
+ + + + + + +
+
+ + + + + diff --git a/progression_backend/static/styles.css b/progression_backend/static/styles.css new file mode 100644 index 0000000..41fabf4 --- /dev/null +++ b/progression_backend/static/styles.css @@ -0,0 +1,623 @@ +@font-face { + font-family: "Progression Georgia"; + src: url("/art/georgia.ttf") format("truetype"); + font-display: swap; + font-style: normal; + font-weight: 400; +} + +@font-face { + font-family: "Progression Georgia"; + src: url("/art/georgiab.ttf") format("truetype"); + font-display: swap; + font-style: normal; + font-weight: 700; +} + +@font-face { + font-family: "Progression Georgia"; + src: url("/art/georgiaz.ttf") format("truetype"); + font-display: swap; + font-style: italic; + font-weight: 700; +} + +:root { + color-scheme: dark; + --bg: #151515; + --surface: #212121; + --surface-raised: #2b2b2b; + --surface-deep: #171a1c; + --line: #4d4d4d; + --line-soft: #373737; + --text: #f5f2ec; + --body: #c8c8c8; + --muted: #979797; + --yellow: #f0a621; + --green: #79a557; + --green-bright: #8fbd30; + --purple: #cf83ce; + --red: #ff6863; + --pink: #ef718f; + --radius: 22px; + --font-serif: "Progression Georgia", Georgia, "Times New Roman", serif; + --font-ui: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +* { box-sizing: border-box; } +html { min-width: 320px; background: var(--bg); } + +body { + min-width: 320px; + min-height: 100vh; + margin: 0; + background: + radial-gradient(circle at 85% 8%, rgba(84, 114, 79, .12), transparent 27rem), + linear-gradient(180deg, #171717 0, #101214 100%); + color: var(--text); + font-family: var(--font-ui); + font-size: 16px; + line-height: 1.5; +} + +button, input { font: inherit; } +button { color: inherit; } +img { display: block; max-width: 100%; } +h1, h2, p { margin: 0; } + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.intro { + position: fixed; + z-index: 100; + inset: 0; + display: grid; + place-items: center; + overflow: hidden; + background: #0b0b0c url("/art/Desolation.png") center / cover no-repeat; + transition: opacity .7s ease, visibility .7s ease; +} + +.intro-shade { + position: absolute; + inset: 0; + background: + radial-gradient(circle at center, rgba(21, 21, 21, .08), rgba(21, 21, 21, .76) 72%), + linear-gradient(rgba(21, 21, 21, .36), rgba(21, 21, 21, .66)); +} + +.intro-content { + position: relative; + width: min(72vw, 760px); + text-align: center; + animation: intro-rise .85s cubic-bezier(.2, .72, .24, 1) both; +} + +.intro-logo { width: 100%; filter: drop-shadow(0 10px 32px rgba(0, 0, 0, .7)); } +.intro-content p { + margin-top: 20px; + color: rgba(245, 242, 236, .65); + font-family: var(--font-serif); + font-size: clamp(.85rem, 1.3vw, 1.05rem); + font-style: italic; + letter-spacing: .12em; + text-transform: uppercase; +} + +.is-ready .intro { opacity: 0; visibility: hidden; } + +.page-shell { + width: min(1220px, calc(100% - 48px)); + margin: 0 auto; + padding-bottom: 26px; + opacity: 0; + transform: translateY(10px); + transition: opacity .55s ease .18s, transform .55s ease .18s; +} + +.is-ready .page-shell { opacity: 1; transform: none; } + +.site-header { + min-height: 98px; + display: flex; + align-items: center; + gap: 24px; +} + +.brand { width: clamp(260px, 35vw, 420px); } +.brand img { width: 100%; } + +.service-state { + margin-left: auto; + display: flex; + align-items: center; + gap: 9px; + color: var(--muted); + font-size: .78rem; + font-weight: 800; + letter-spacing: .08em; + text-transform: uppercase; +} + +.state-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--yellow); + box-shadow: 0 0 14px rgba(240, 166, 33, .5); +} + +.service-state.ready .state-dot { background: #75bf77; box-shadow: 0 0 14px rgba(117, 191, 119, .5); } +.service-state.failed .state-dot { background: var(--red); box-shadow: none; } +.service-state.working .state-dot { animation: pulse 1.2s ease-in-out infinite; } + +main { display: grid; gap: 18px; padding-top: 2px; } + +.panel { + border: 1px solid var(--line); + border-radius: var(--radius); + background: var(--surface); + box-shadow: 0 18px 42px rgba(0, 0, 0, .18); +} + +.hero { + position: relative; + min-height: 430px; + overflow: hidden; + display: grid; + grid-template-columns: minmax(0, 1.55fr) minmax(280px, .72fr); + align-items: stretch; +} + +.hero::before { + content: ""; + position: absolute; + inset: 0; + pointer-events: none; + background: linear-gradient(110deg, rgba(20, 25, 29, .97) 0 60%, rgba(28, 43, 38, .82) 100%); +} + +.hero-copy { + position: relative; + z-index: 1; + align-self: center; + padding: clamp(34px, 5vw, 60px); +} + +h1, h2 { font-family: var(--font-serif); } + +h1 { + max-width: 760px; + font-size: clamp(2.35rem, 5.4vw, 4.7rem); + font-weight: 700; + letter-spacing: -.035em; + line-height: .97; +} + +h1 em { color: #e7ffe0; font-size: .68em; } + +.hero-summary { + max-width: 58ch; + margin-top: 22px; + color: var(--body); + font-family: var(--font-serif); + font-size: clamp(1rem, 1.5vw, 1.16rem); + line-height: 1.55; +} + +.start-button { + width: min(100%, 620px); + min-height: 86px; + margin-top: 28px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 24px; + padding: 15px 18px 15px 22px; + border: 1px solid #8bad3f; + border-radius: 12px; + background: linear-gradient(120deg, #6f981f, #84ad28); + box-shadow: inset 0 1px rgba(255,255,255,.15), 0 12px 26px rgba(50, 80, 15, .19); + color: #f8ffe9; + cursor: pointer; + text-align: left; + transition: transform .18s ease, filter .18s ease, box-shadow .18s ease; +} + +.start-button:hover { transform: translateY(-2px); filter: brightness(1.06); box-shadow: inset 0 1px rgba(255,255,255,.15), 0 16px 32px rgba(50, 80, 15, .25); } +.start-button:focus-visible { outline: 3px solid rgba(183, 226, 83, .35); outline-offset: 3px; } +.start-button > span:first-child { display: grid; } +.start-button strong { font-family: var(--font-serif); font-size: clamp(1.15rem, 2.3vw, 1.55rem); line-height: 1.15; } +.start-button small { margin-top: 3px; color: rgba(248,255,233,.76); font-size: .76rem; font-weight: 800; letter-spacing: .1em; text-transform: uppercase; } +.start-arrow { width: 44px; height: 44px; display: grid; place-items: center; flex: 0 0 auto; border-radius: 50%; background: rgba(18, 38, 8, .23); font-size: 1.35rem; transition: transform .18s ease; } +.start-button:hover .start-arrow { transform: translateX(3px); } + +.windows-download { + width: min(100%, 620px); + min-height: 62px; + margin-top: 10px; + display: grid; + grid-template-columns: 36px minmax(0, 1fr) 28px; + align-items: center; + gap: 12px; + padding: 10px 15px; + border: 1px solid #4c565e; + border-radius: 10px; + background: rgba(19, 24, 27, .82); + color: var(--text); + text-decoration: none; + transition: border-color .18s ease, background .18s ease, transform .18s ease; +} +.windows-download:hover { transform: translateY(-1px); border-color: #7a8994; background: rgba(25, 31, 35, .94); } +.windows-download:focus-visible { outline: 3px solid rgba(151, 179, 198, .25); outline-offset: 3px; } +.windows-mark { width: 34px; height: 34px; display: grid; place-items: center; border-radius: 7px; background: #27343c; color: #b9d9eb; font-size: 1.35rem; line-height: 1; } +.windows-download > span:nth-child(2) { min-width: 0; display: grid; } +.windows-download strong { font-family: var(--font-serif); font-size: .98rem; } +.windows-download small { color: var(--muted); font-size: .72rem; } +.windows-download-arrow { color: #afc6d4; font-size: 1.1rem; text-align: center; } + +.freshness { + width: fit-content; + margin-top: 24px; + display: grid; + grid-template-columns: auto auto; + column-gap: 13px; + align-items: baseline; + font-variant-numeric: tabular-nums; +} + +.freshness > span { + color: var(--muted); + font-size: .72rem; + font-weight: 800; + letter-spacing: .07em; + text-transform: uppercase; +} + +.freshness strong { color: var(--text); font-family: var(--font-serif); font-size: 1.04rem; } +.freshness small { grid-column: 1 / -1; margin-top: 2px; color: var(--muted); } + +.hero-art { + position: relative; + z-index: 1; + min-height: 290px; + overflow: hidden; + display: grid; + place-items: center; + padding: 30px; + background: + repeating-linear-gradient(135deg, rgba(255,255,255,.026) 0 18px, transparent 18px 36px), + linear-gradient(145deg, #2b3033, #1e2d28); + border-left: 1px solid rgba(255,255,255,.09); +} + +.hero-art img { + position: relative; + z-index: 1; + width: min(62%, 220px); + filter: drop-shadow(0 18px 16px rgba(0, 0, 0, .45)); +} + +.count-grid { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; } + +.count-card { + position: relative; + overflow: hidden; + min-height: 132px; + padding: 20px 22px; + border: 1px solid var(--line-soft); + border-radius: 16px; + background: var(--surface-raised); +} + +.count-card::after { + content: ""; + position: absolute; + top: 0; + right: 0; + width: 72px; + height: 5px; + background: var(--card-accent, var(--yellow)); +} + +.count-total { --card-accent: var(--yellow); } +.count-core { --card-accent: #d7d7d7; } +.count-content { --card-accent: var(--green); } +.count-cosmetics { --card-accent: var(--purple); } +.count-label { color: var(--body); font-family: var(--font-serif); font-size: 1rem; font-weight: 700; } +.count-card strong { display: block; margin-top: 4px; font-family: var(--font-serif); font-size: clamp(2rem, 3vw, 2.75rem); font-weight: 700; line-height: 1; } +.count-card small { color: var(--muted); } + +.action-message { min-height: 18px; color: var(--muted); font-size: .86rem; text-align: center; } +.action-message.error, .wizard-message.error { color: #ff918d; } +.action-message.success, .wizard-message.success { color: #a2d596; } + +footer { + min-height: 88px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; + color: #747474; + font-size: .76rem; +} + +.footer-meta { display: flex; align-items: center; gap: 10px; } +.art-credit { padding: 4px 8px; border: 1px solid #484848; border-radius: 999px; color: #999; font-family: var(--font-serif); font-style: italic; } + +.hr-link { + display: inline-flex; + align-items: center; + gap: 10px; + color: #f16365; + text-decoration: none; + transition: color .16s ease, transform .16s ease; +} + +.hr-link:hover { color: #ff7b7d; transform: translateY(-1px); } +.hr-link img { width: 42px; height: 42px; object-fit: contain; } + +.wizard { + width: min(820px, calc(100% - 30px)); + max-width: none; + max-height: min(94vh, 820px); + margin: auto; + padding: 0; + overflow: auto; + border: 0; + border-radius: 22px; + background: transparent; + color: var(--text); + opacity: 0; + transform: translateY(12px) scale(.988); + transition: opacity .22s ease, transform .22s ease, overlay .22s allow-discrete, display .22s allow-discrete; +} + +.wizard[open] { opacity: 1; transform: none; } + +@starting-style { + .wizard[open] { opacity: 0; transform: translateY(12px) scale(.988); } +} + +.wizard::backdrop { + background: rgba(6, 8, 9, .82); + backdrop-filter: blur(7px); + animation: backdrop-in .22s ease both; +} + +.wizard-shell { + overflow: hidden; + border: 1px solid #555; + border-radius: 22px; + background: + repeating-linear-gradient(135deg, rgba(255,255,255,.017) 0 18px, transparent 18px 36px), + #202224; + box-shadow: 0 28px 90px rgba(0,0,0,.58); +} + +.wizard-header { + min-height: 76px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 24px; + padding: 15px 20px 14px 24px; + border-bottom: 1px solid #393b3c; + background: rgba(17, 20, 22, .78); +} + +.wizard-header > div { display: grid; } +.wizard-eyebrow, .step-kicker { color: var(--yellow); font-size: .67rem; font-weight: 850; letter-spacing: .13em; text-transform: uppercase; } +.wizard-header strong { font-family: var(--font-serif); font-size: 1.18rem; } +.icon-button { width: 38px; height: 38px; display: grid; place-items: center; border: 1px solid #454849; border-radius: 50%; background: #292c2e; color: var(--muted); cursor: pointer; font-size: 1.4rem; line-height: 1; } +.icon-button:hover { border-color: #727576; color: var(--text); } + +.wizard-progress { + position: relative; + display: grid; + grid-template-columns: repeat(4, 1fr); + margin: 0; + padding: 20px 46px 18px; + list-style: none; + border-bottom: 1px solid #37393a; + background: rgba(29, 31, 32, .9); +} + +.wizard-progress::before { content: ""; position: absolute; top: 35px; left: 14%; right: 14%; height: 1px; background: #4a4c4d; } +.wizard-progress li { position: relative; z-index: 1; display: grid; justify-items: center; gap: 5px; color: #777; } +.wizard-progress li span { width: 30px; height: 30px; display: grid; place-items: center; border: 1px solid #505253; border-radius: 50%; background: #242627; font-family: var(--font-serif); font-size: .84rem; transition: background .22s ease, border-color .22s ease, color .22s ease; } +.wizard-progress li small { font-size: .68rem; font-weight: 750; letter-spacing: .04em; } +.wizard-progress li.active { color: var(--text); } +.wizard-progress li.active span { border-color: #a78340; background: #524326; color: #ffe6a4; } +.wizard-progress li.complete { color: #91b989; } +.wizard-progress li.complete span { border-color: #5d7d57; background: #33452f; color: #c9efc0; } + +.wizard-viewport { min-height: 405px; } +.wizard-step { padding: 28px 42px 26px; } +.wizard-step.active { animation: step-enter .3s cubic-bezier(.2,.72,.24,1) both; } +.wizard-step h2 { margin-top: 4px; font-size: clamp(1.8rem, 4vw, 2.65rem); line-height: 1.08; } +.step-lede { max-width: 60ch; margin-top: 10px; color: var(--muted); } + +.collection-checks { display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px; margin-top: 28px; } +.collection-checks > div { min-width: 0; display: flex; align-items: center; gap: 11px; padding: 15px; border: 1px solid #414443; border-radius: 10px; background: rgba(17,20,19,.72); } +.collection-checks > div > span:last-child { min-width: 0; display: grid; } +.collection-checks strong { font-family: var(--font-serif); } +.collection-checks small { color: var(--muted); } +.collection-checks b { color: #c8e9bd; font-weight: 800; } +.checkmark { width: 30px; height: 30px; display: grid; place-items: center; flex: 0 0 auto; border-radius: 50%; background: #33452f; color: #bfe6b5; font-weight: 900; } + +.setup-list { display: grid; gap: 9px; margin: 24px 0 20px; padding: 0; list-style: none; } +.setup-list li { display: flex; align-items: center; gap: 11px; padding: 10px 13px; border-radius: 8px; background: rgba(15,18,17,.58); color: var(--body); } +.setup-list li span { color: #95ca89; font-weight: 900; } +.confirm-row { width: fit-content; display: flex; align-items: center; gap: 10px; color: var(--text); cursor: pointer; font-weight: 750; } +.confirm-row input { width: 18px; height: 18px; accent-color: #759f23; } + +.dlc-grid { display: grid; grid-template-columns: repeat(5, 1fr); gap: 8px; min-width: 0; margin: 27px 0 16px; padding: 0; border: 0; } +.dlc-grid label { min-width: 0; cursor: pointer; } +.dlc-grid input { position: absolute; opacity: 0; pointer-events: none; } +.dlc-grid span { min-height: 58px; display: grid; place-items: center; padding: 8px 5px; border: 1px solid var(--line); border-radius: 9px; background: #1b1b1b; color: var(--muted); font-size: .78rem; font-weight: 800; transition: border-color .16s ease, background .16s ease, color .16s ease, transform .16s ease; } +.dlc-grid input:checked + span { border-color: #807551; background: #3d392c; color: #fff7c8; } +.dlc-grid label.missing span { border-color: #9b4446; background: #392426; color: #ffaaa7; } +.dlc-grid label:hover span { transform: translateY(-1px); } +.dlc-grid input:focus-visible + span { outline: 3px solid rgba(240,166,33,.32); outline-offset: 2px; } +.expansion-warning { + display: flex; + align-items: center; + justify-content: space-between; + gap: 20px; + margin: 13px 0; + padding: 11px 14px; + border: 1px solid #814347; + border-radius: 11px; + background: + repeating-linear-gradient(135deg, rgba(255,95,91,.045) 0 12px, transparent 12px 24px), + #2d2022; + animation: warning-enter .24s cubic-bezier(.2,.72,.24,1) both; +} + +.expansion-warning[hidden] { display: none; } +.expansion-warning-copy { min-width: 0; display: grid; } +.expansion-warning-copy > span { color: #ff7773; font-size: .64rem; font-weight: 850; letter-spacing: .13em; text-transform: uppercase; } +.expansion-warning-copy strong { margin-top: 2px; font-family: var(--font-serif); font-size: 1.05rem; } +.expansion-warning-copy p { margin-top: 2px; color: #cba9a8; font-size: .78rem; } +.missing-expansion-icons { flex: 0 0 auto; display: flex; align-items: center; gap: 7px; } +.missing-expansion-icons img { width: 58px; height: 58px; object-fit: contain; } +.missing-expansion-fallback { width: 58px; height: 58px; display: grid; place-items: center; align-content: center; border: 2px solid #a8b2bd; clip-path: polygon(50% 0, 92% 24%, 92% 76%, 50% 100%, 8% 76%, 8% 24%); background: #31363a; text-align: center; } +.missing-expansion-fallback b { color: #ff3f3a; font-family: var(--font-serif); font-size: 1.7rem; line-height: .75; } +.missing-expansion-fallback small { margin-top: 3px; color: #d9dde0; font-size: .44rem; font-style: normal; font-weight: 900; letter-spacing: .03em; } +.missing-expansion-fallback em { color: #fff; font-size: .46rem; font-style: normal; font-weight: 900; } + +.list-type-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; margin-top: 24px; } +.list-type-grid label { cursor: pointer; } +.list-type-grid input { position: absolute; opacity: 0; pointer-events: none; } +.list-type-grid label > span { min-height: 84px; display: grid; align-content: center; padding: 15px 17px; border: 1px solid #484a4b; border-radius: 10px; background: #191b1c; transition: border-color .16s ease, background .16s ease, transform .16s ease; } +.list-type-grid label:hover > span { transform: translateY(-1px); } +.list-type-grid strong { font-family: var(--font-serif); font-size: 1.05rem; } +.list-type-grid small { color: var(--muted); } +.list-type-grid input:checked + span { border-color: #987c47; background: #332f25; } +.list-type-grid label:last-child input:checked + span { border-color: #8a608d; background: #332735; } +.list-type-grid input:focus-visible + span { outline: 3px solid rgba(240,166,33,.25); outline-offset: 2px; } + +.homebrew-upload { margin-top: 12px; animation: step-enter .25s ease both; } +.homebrew-upload[hidden] { display: none; } +.file-picker { min-height: 61px; min-width: 0; display: flex; align-items: center; gap: 14px; padding: 9px 10px 9px 14px; border: 1px dashed #665669; border-radius: 9px; background: #1b191c; cursor: pointer; } +.file-picker:hover, .file-picker.has-file { border-color: var(--purple); } +.file-picker input { position: absolute; opacity: 0; pointer-events: none; } +.file-picker input:focus-visible + .file-picker-copy { outline: 3px solid rgba(207,131,206,.28); outline-offset: 3px; } +.file-picker-copy { min-width: 0; display: grid; } +.file-picker-copy strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.file-picker-copy small { color: var(--muted); } +.file-action { margin-left: auto; padding: 7px 10px; border-radius: 6px; background: #302a31; color: #edc9ed; font-size: .78rem; font-weight: 800; } +.homebrew-upload p { margin-top: 8px; color: var(--muted); font-size: .78rem; } +.homebrew-upload p strong { color: #d6b7d6; } + +.install-location { margin-top: 14px; padding: 12px 13px; border: 1px solid #454849; border-radius: 10px; background: rgba(17, 19, 20, .72); } +.install-location-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 16px; } +.install-location-heading > div { min-width: 0; display: grid; } +.install-location-heading > div > span { color: var(--yellow); font-size: .62rem; font-weight: 850; letter-spacing: .11em; text-transform: uppercase; } +.install-location-heading strong { margin-top: 2px; font-family: var(--font-serif); font-size: .91rem; } +.path-toggle { flex: 0 0 auto; border: 0; padding: 3px 0; background: transparent; color: #c6b77f; cursor: pointer; font-size: .72rem; font-weight: 800; text-decoration: underline; text-underline-offset: 3px; } +.path-toggle:hover { color: #ffe5a0; } +.detected-path, .all-install-paths code { display: block; overflow-wrap: anywhere; color: #bad1b4; font-family: Consolas, "Cascadia Mono", monospace; font-size: .7rem; line-height: 1.45; } +.detected-path { margin-top: 8px; padding: 7px 9px; border-radius: 6px; background: #111413; } +.all-install-paths { display: grid; gap: 8px; margin-top: 10px; padding-top: 10px; border-top: 1px solid #393c3d; } +.all-install-paths[hidden] { display: none; } +.all-install-paths > div { display: grid; grid-template-columns: 62px minmax(0, 1fr); gap: 10px; } +.all-install-paths strong { color: var(--body); font-size: .7rem; } + +.autosort-confirmation { margin-top: 14px; } +.autosort-card { min-height: 118px; display: grid; grid-template-columns: 72px minmax(0, 1fr); align-items: center; gap: 18px; padding: 17px 19px; border: 1px solid #8b6728; border-radius: 12px 12px 0 0; background: linear-gradient(125deg, rgba(73, 51, 17, .84), rgba(38, 32, 22, .9)); box-shadow: inset 0 1px rgba(255, 227, 157, .08); } +.autosort-mark { width: 64px; height: 64px; display: grid; place-items: center; border: 1px solid #aa7c2d; border-radius: 10px; background: #322615; color: #ffc75d; font-family: var(--font-serif); font-size: 2.35rem; font-weight: 900; line-height: 1; } +.autosort-card > div { min-width: 0; } +.autosort-eyebrow { color: #f4b947; font-size: .63rem; font-weight: 900; letter-spacing: .12em; text-transform: uppercase; } +.autosort-card h3 { margin-top: 2px; font-family: var(--font-serif); font-size: 1.35rem; line-height: 1.1; } +.autosort-card p { margin-top: 6px; color: #c8bda7; font-size: .79rem; line-height: 1.45; } +.autosort-card p strong { color: #ffe0a0; } +.autosort-acknowledge { width: 100%; min-height: 44px; border: 1px solid #8b6728; border-top: 0; border-radius: 0 0 12px 12px; background: #211d16; color: #f4d28e; cursor: pointer; font-weight: 850; transition: background .16s ease, color .16s ease, border-color .16s ease; } +.autosort-acknowledge:hover { background: #2c2519; color: #ffe4a8; } +.autosort-acknowledge:focus-visible { outline: 3px solid rgba(240, 166, 33, .3); outline-offset: 2px; } +.autosort-confirmation.acknowledged .autosort-card { border-color: #56734f; background: linear-gradient(125deg, rgba(42, 64, 36, .75), rgba(29, 38, 27, .9)); } +.autosort-confirmation.acknowledged .autosort-mark { border-color: #688a60; background: #293a25; color: #bde5b4; } +.autosort-confirmation.acknowledged .autosort-acknowledge { border-color: #56734f; background: #31472d; color: #ccefc5; } + +.availability { margin-top: 14px; padding: 9px 11px; border-radius: 7px; font-size: .79rem; } +.availability.waiting { background: #302b20; color: #cdbd91; } +.availability.ready { background: #273424; color: #a9d49e; } +.wizard-message { min-height: 22px; margin-top: 6px; color: var(--muted); font-size: .8rem; } + +.wizard-actions { display: flex; justify-content: space-between; gap: 10px; margin-top: 22px; } +.wizard-actions-end { justify-content: flex-end; margin-top: 72px; } +.button { min-height: 46px; display: inline-flex; align-items: center; justify-content: center; gap: 28px; border: 1px solid transparent; border-radius: 8px; padding: 10px 20px; cursor: pointer; font-weight: 850; transition: transform .15s ease, filter .15s ease, border-color .15s ease; } +.button:hover { transform: translateY(-1px); filter: brightness(1.08); } +.button:disabled { cursor: not-allowed; opacity: .42; transform: none; filter: none; } +.button-primary { min-width: 150px; background: #729f20; color: #f7ffe9; } +.button-back { border-color: #4b4d4e; background: transparent; color: var(--muted); } +.button-download { min-width: 230px; justify-content: space-between; background: #729f20; color: #f7ffe9; } + +@keyframes intro-rise { from { opacity: 0; transform: translateY(16px) scale(.985); } to { opacity: 1; transform: none; } } +@keyframes pulse { 50% { opacity: .45; transform: scale(.78); } } +@keyframes backdrop-in { from { background: rgba(6,8,9,0); backdrop-filter: blur(0); } } +@keyframes step-enter { from { opacity: 0; transform: translateX(9px); } to { opacity: 1; transform: none; } } +@keyframes warning-enter { from { opacity: 0; transform: translateY(-5px); } to { opacity: 1; transform: none; } } + +@media (max-width: 980px) { + .hero { grid-template-columns: 1fr minmax(230px, .52fr); } + .hero-copy { padding: 38px; } + .count-grid { grid-template-columns: repeat(2, 1fr); } +} + +@media (max-width: 700px) { + .page-shell { width: min(calc(100% - 24px), 1220px); } + .site-header { min-height: 78px; } + .brand { width: min(67vw, 330px); } + .service-state { font-size: 0; } + .hero { grid-template-columns: 1fr; } + .hero-copy { padding: 30px 24px; } + .hero-art { min-height: 180px; border-top: 1px solid rgba(255,255,255,.09); border-left: 0; } + .hero-art img { width: 122px; } + .freshness { grid-template-columns: 1fr; } + .freshness small { grid-column: auto; } + footer { align-items: flex-start; gap: 10px; } + .hr-link span { display: none; } + .wizard { width: min(100% - 18px, 820px); max-height: 94vh; } + .wizard-header { min-height: 66px; padding-left: 18px; } + .wizard-progress { padding: 16px 16px 14px; } + .wizard-progress::before { top: 31px; } + .wizard-progress li small { display: none; } + .wizard-step { padding: 27px 20px 24px; } + .wizard-viewport { min-height: 470px; } + .collection-checks { grid-template-columns: 1fr; } + .wizard-actions-end { margin-top: 30px; } + .dlc-grid { grid-template-columns: repeat(2, 1fr); } + .expansion-warning { align-items: flex-start; flex-direction: column; } + .missing-expansion-icons { max-width: 100%; flex-wrap: wrap; } + .list-type-grid { grid-template-columns: 1fr; } + .install-location-heading { align-items: flex-start; flex-direction: column; gap: 6px; } + .autosort-card { grid-template-columns: 54px minmax(0, 1fr); gap: 13px; padding: 15px; } + .autosort-mark { width: 50px; height: 50px; font-size: 1.9rem; } +} + +@media (max-width: 430px) { + .count-grid { grid-template-columns: 1fr; } + .count-card { min-height: 100px; } + .start-button { min-height: 78px; padding-left: 16px; } + .footer-meta { align-items: flex-start; flex-direction: column; gap: 5px; } + .wizard-actions { display: grid; grid-template-columns: 1fr 1.5fr; } + .button, .button-download { min-width: 0; } +} + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { animation: none !important; scroll-behavior: auto !important; transition: none !important; } + .intro { display: none; } + .page-shell { opacity: 1; transform: none; } +} diff --git a/progression_backend/steam_api.py b/progression_backend/steam_api.py new file mode 100644 index 0000000..8916efc --- /dev/null +++ b/progression_backend/steam_api.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +COLLECTION_DETAILS_URL = ( + "https://api.steampowered.com/ISteamRemoteStorage/GetCollectionDetails/v1/" +) +PUBLISHED_FILE_DETAILS_URL = ( + "https://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1/" +) + + +@dataclass(frozen=True) +class WorkshopDetails: + workshop_id: str + title: str + time_updated: int + file_size: int + consumer_app_id: int + result: int + + +def _chunks(values: list[str], size: int) -> Iterable[list[str]]: + for index in range(0, len(values), size): + yield values[index : index + size] + + +class SteamWorkshopClient: + def __init__(self, timeout: int = 30): + self.timeout = timeout + self.session = requests.Session() + retries = Retry( + total=4, + backoff_factor=0.75, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"POST"}), + ) + self.session.mount("https://", HTTPAdapter(max_retries=retries)) + self.session.headers.update({"User-Agent": "ProgressionMods/1.0"}) + + def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]: + payload: dict[str, str | int] = {"collectioncount": len(collection_ids)} + for index, collection_id in enumerate(collection_ids): + payload[f"publishedfileids[{index}]"] = collection_id + + response = self.session.post( + COLLECTION_DETAILS_URL, data=payload, timeout=self.timeout + ) + response.raise_for_status() + details = response.json().get("response", {}).get("collectiondetails", []) + + result: dict[str, list[str]] = {} + for detail in details: + collection_id = str(detail.get("publishedfileid", "")) + if int(detail.get("result", 0)) != 1: + raise RuntimeError(f"Steam rejected collection {collection_id}") + children = sorted( + detail.get("children", []), + key=lambda child: int(child.get("sortorder", 0)), + ) + result[collection_id] = [ + str(child["publishedfileid"]) for child in children + ] + return result + + def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: + result: dict[str, WorkshopDetails] = {} + for batch in _chunks(workshop_ids, 100): + payload: dict[str, str | int] = {"itemcount": len(batch)} + for index, workshop_id in enumerate(batch): + payload[f"publishedfileids[{index}]"] = workshop_id + response = self.session.post( + PUBLISHED_FILE_DETAILS_URL, data=payload, timeout=self.timeout + ) + response.raise_for_status() + details = ( + response.json().get("response", {}).get("publishedfiledetails", []) + ) + for detail in details: + workshop_id = str(detail.get("publishedfileid", "")) + result[workshop_id] = WorkshopDetails( + workshop_id=workshop_id, + title=str(detail.get("title", "")), + time_updated=int(detail.get("time_updated", 0) or 0), + file_size=int(detail.get("file_size", 0) or 0), + consumer_app_id=int(detail.get("consumer_app_id", 0) or 0), + result=int(detail.get("result", 0) or 0), + ) + return result diff --git a/progression_backend/workshop.py b/progression_backend/workshop.py new file mode 100644 index 0000000..3e382cb --- /dev/null +++ b/progression_backend/workshop.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import re +import shutil +import subprocess +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +RIMWORLD_APP_ID = "294100" +FAILED_ITEM_PATTERN = re.compile( + r"(?:ERROR!\s*)?Downloading item\s+(\d+)\s+failed", re.IGNORECASE +) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def directory_size(path: Path) -> int: + if not path.exists(): + return 0 + total = 0 + for entry in path.rglob("*"): + try: + if entry.is_file(): + total += entry.stat().st_size + except OSError: + continue + return total + + +def find_about_xml(mod_path: Path) -> Path | None: + direct = mod_path / "About" / "About.xml" + if direct.is_file(): + return direct + try: + for candidate in mod_path.rglob("*.xml"): + if ( + candidate.name.lower() == "about.xml" + and candidate.parent.name.lower() == "about" + ): + return candidate + except OSError: + return None + return None + + +def _child_text(root: ET.Element, name: str) -> str | None: + wanted = name.lower() + for child in root: + if child.tag.lower() == wanted and child.text: + return child.text.strip() + return None + + +@dataclass(frozen=True) +class ParsedMod: + package_id: str + display_name: str + declared_steam_app_id: str | None + + +def parse_about_xml(mod_path: Path) -> ParsedMod: + about_xml = find_about_xml(mod_path) + if not about_xml: + raise ValueError("About/About.xml not found") + try: + root = ET.parse(about_xml).getroot() + except ET.ParseError as error: + raise ValueError(f"About.xml is invalid XML: {error}") from error + + package_id = _child_text(root, "packageId") + if not package_id: + raise ValueError("packageId not found in About.xml") + display_name = _child_text(root, "name") or package_id + return ParsedMod( + package_id=package_id.lower(), + display_name=display_name, + declared_steam_app_id=_child_text(root, "steamAppId"), + ) + + +@dataclass(frozen=True) +class SteamCmdBatchResult: + workshop_ids: list[str] + started_at: str + finished_at: str + elapsed_seconds: float + exit_code: int | None + status: str + failed_ids: set[str] + output_tail: str + + +class SteamCmdRunner: + def __init__( + self, + executable: str, + content_dir: Path, + timeout_seconds: int, + ): + self.executable = executable + self.content_dir = content_dir + self.timeout_seconds = timeout_seconds + + def executable_path(self) -> str | None: + configured = Path(self.executable) + if configured.is_file(): + return str(configured.resolve()) + return shutil.which(self.executable) + + def available(self) -> bool: + return self.executable_path() is not None + + def run_batch(self, workshop_ids: list[str]) -> SteamCmdBatchResult: + executable = self.executable_path() + if not executable: + raise FileNotFoundError( + f"SteamCMD was not found at '{self.executable}'. Set STEAMCMD_PATH." + ) + invalid = [item for item in workshop_ids if not item.isdigit()] + if invalid: + raise ValueError(f"Invalid Workshop ids: {', '.join(invalid)}") + + command = [executable, "+login", "anonymous"] + for workshop_id in workshop_ids: + command.extend( + [ + "+workshop_download_item", + RIMWORLD_APP_ID, + workshop_id, + "validate", + ] + ) + command.append("+quit") + + started_at = utc_now() + started = time.monotonic() + exit_code: int | None = None + status = "completed" + output = "" + try: + completed = subprocess.run( + command, + cwd=str(Path(executable).resolve().parent), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=self.timeout_seconds, + check=False, + shell=False, + ) + exit_code = completed.returncode + output = f"{completed.stdout}\n{completed.stderr}".strip() + if completed.returncode != 0: + status = "failed" + except subprocess.TimeoutExpired as error: + status = "timeout" + stdout = ( + error.stdout.decode(errors="replace") + if isinstance(error.stdout, bytes) + else error.stdout + ) + stderr = ( + error.stderr.decode(errors="replace") + if isinstance(error.stderr, bytes) + else error.stderr + ) + output = f"{stdout or ''}\n{stderr or ''}".strip() + + failed_ids = set(FAILED_ITEM_PATTERN.findall(output)) + for workshop_id in workshop_ids: + if not (self.content_dir / workshop_id).is_dir(): + failed_ids.add(workshop_id) + if not failed_ids: + # SteamCMD's bootstrapper can return 1 after updating and relaunching even + # though every requested item was installed successfully. + status = "completed" + elif len(failed_ids) < len(workshop_ids): + status = "partial" + else: + status = "failed" + + return SteamCmdBatchResult( + workshop_ids=workshop_ids, + started_at=started_at, + finished_at=utc_now(), + elapsed_seconds=round(time.monotonic() - started, 3), + exit_code=exit_code, + status=status, + failed_ids=failed_ids, + output_tail=output[-8000:], + ) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..89bd54b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements-web.txt +httpx2>=0.1.0 diff --git a/requirements-web.txt b/requirements-web.txt new file mode 100644 index 0000000..271ad51 --- /dev/null +++ b/requirements-web.txt @@ -0,0 +1,5 @@ +requests>=2.25.1 +python-dotenv>=0.19.0 +fastapi>=0.115.0,<1.0 +uvicorn[standard]>=0.30.0,<1.0 +python-multipart>=0.0.12,<1.0 diff --git a/requirements.txt b/requirements.txt index 12ae88b..6001a32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,7 @@ pyglet>=1.5.0 Pillow>=8.0.0 packaging>=21.0 customtkinter>=5.0.0 -rich>=13.0.0 \ No newline at end of file +rich>=13.0.0 +fastapi>=0.115.0,<1.0 +uvicorn[standard]>=0.30.0,<1.0 +python-multipart>=0.0.12,<1.0 diff --git a/tests/test_backend.py b/tests/test_backend.py new file mode 100644 index 0000000..7886606 --- /dev/null +++ b/tests/test_backend.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import tempfile +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path +from unittest.mock import Mock, patch + +from fastapi.testclient import TestClient + +from progression_backend.app import create_app +from progression_backend.app import latest_windows_release_url +from progression_backend.config import Settings +from progression_backend.database import Database +from progression_backend.modlist import ( + ModListItem, + core_items, + generate_rml, + parse_active_mods, + workshop_items, +) +from progression_backend.refresh import RefreshService +from progression_backend.steam_api import WorkshopDetails +from progression_backend.workshop import SteamCmdBatchResult, parse_about_xml, utc_now + + +class FakeSteam: + def get_collections(self, collection_ids: list[str]) -> dict[str, list[str]]: + return {collection_id: ["123456"] for collection_id in collection_ids} + + def get_item_details(self, workshop_ids: list[str]) -> dict[str, WorkshopDetails]: + return { + workshop_id: WorkshopDetails( + workshop_id=workshop_id, + title="Test & Mod", + time_updated=123, + file_size=456, + consumer_app_id=294100, + result=1, + ) + for workshop_id in workshop_ids + } + + +class FakeRunner: + def __init__(self, content_dir: Path): + self.content_dir = content_dir + + def available(self) -> bool: + return True + + def run_batch(self, workshop_ids: list[str]) -> SteamCmdBatchResult: + for workshop_id in workshop_ids: + about = self.content_dir / workshop_id / "About" + about.mkdir(parents=True) + (about / "About.xml").write_text( + "Test & Mod" + "Author.Test" + "294100", + encoding="utf-8", + ) + timestamp = utc_now() + return SteamCmdBatchResult( + workshop_ids=workshop_ids, + started_at=timestamp, + finished_at=timestamp, + elapsed_seconds=0.1, + exit_code=0, + status="completed", + failed_ids=set(), + output_tail="Success", + ) + + +class BackendTests(unittest.TestCase): + @patch("progression_backend.app.requests.get") + def test_latest_windows_release_includes_prereleases(self, get: Mock) -> None: + response = Mock() + response.json.return_value = [ + { + "tag_name": "0.2.2", + "published_at": "2026-06-13T02:37:15Z", + "draft": False, + "prerelease": True, + "assets": [ + { + "name": "ProgressionLoader.exe", + "browser_download_url": "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/releases/download/0.2.2/ProgressionLoader.exe", + } + ], + } + ] + get.return_value = response + + self.assertEqual( + latest_windows_release_url(), + "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/releases/download/0.2.2/ProgressionLoader.exe", + ) + response.raise_for_status.assert_called_once_with() + + @patch("progression_backend.app.requests.get") + def test_latest_windows_release_prefers_stable_build(self, get: Mock) -> None: + response = Mock() + response.json.return_value = [ + { + "published_at": "2026-07-01T00:00:00Z", + "draft": False, + "prerelease": True, + "assets": [ + { + "name": "ProgressionLoader.exe", + "browser_download_url": "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/releases/download/0.3.0-rc1/ProgressionLoader.exe", + } + ], + }, + { + "published_at": "2026-06-30T00:00:00Z", + "draft": False, + "prerelease": False, + "assets": [ + { + "name": "ProgressionLoader.exe", + "browser_download_url": "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/releases/download/0.2.3/ProgressionLoader.exe", + } + ], + }, + ] + get.return_value = response + + self.assertEqual( + latest_windows_release_url(), + "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/releases/download/0.2.3/ProgressionLoader.exe", + ) + + def test_about_parser_normalizes_package_id(self) -> None: + with tempfile.TemporaryDirectory() as directory: + about = Path(directory) / "About" + about.mkdir() + (about / "About.xml").write_text( + "A & BAuthor.Mod", + encoding="utf-8", + ) + parsed = parse_about_xml(Path(directory)) + self.assertEqual(parsed.package_id, "author.mod") + self.assertEqual(parsed.display_name, "A & B") + + def test_rml_uses_workshop_id_and_escapes_names(self) -> None: + payload = generate_rml( + [ + *core_items(["royalty"]), + ModListItem("author.mod", "123456", "A & B"), + ], + "1.6", + ) + root = ET.fromstring(payload) + self.assertEqual(root.findtext("meta/gameVersion"), "1.6") + self.assertEqual( + [node.text for node in root.findall("meta/modSteamIds/li")], + ["0", "0", "123456"], + ) + self.assertIn(b"A & B", payload) + + def test_homebrew_parser_reads_active_mods(self) -> None: + items = parse_active_mods( + b"
  • ludeon.rimworld
    • " + b"
  • author.mod
    • " + ) + self.assertEqual( + [item.package_id for item in items], ["ludeon.rimworld", "author.mod"] + ) + + def test_refresh_persists_download_analytics(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + settings = Settings( + host="127.0.0.1", + port=8000, + database_path=root / "progression.db", + steamcmd_path="steamcmd", + workshop_content_dir=root / "workshop" / "294100", + steamcmd_batch_size=25, + steamcmd_timeout_seconds=60, + refresh_interval_seconds=3600, + refresh_on_startup=False, + game_version="1.6", + admin_token=None, + collections={"Core": "1", "Content": "2", "Cosmetics": "3"}, + static_dir=root, + ) + database = Database(settings.database_path) + database.initialize() + service = RefreshService( + settings, + database, + steam=FakeSteam(), + runner=FakeRunner(settings.workshop_content_dir), + ) + service.refresh("test-run", "test") + + run = database.fetch_one( + "SELECT * FROM refresh_runs WHERE id = ?", ("test-run",) + ) + self.assertIsNotNone(run) + self.assertEqual(run["status"], "completed", run) + self.assertEqual(run["unique_items"], 1) + self.assertEqual(run["downloads_succeeded"], 1) + mod = database.fetch_one("SELECT * FROM mods WHERE workshop_id = '123456'") + self.assertEqual(mod["package_id"], "author.test") + self.assertEqual(mod["steam_app_id"], "294100") + generated = workshop_items([mod]) + self.assertEqual(generated[0].workshop_id, "123456") + + def test_api_serves_dashboard_and_reports_empty_modlist(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + project_root = Path(__file__).resolve().parents[1] + settings = Settings( + host="127.0.0.1", + port=8000, + database_path=root / "progression.db", + steamcmd_path="missing-steamcmd", + workshop_content_dir=root / "workshop" / "294100", + steamcmd_batch_size=25, + steamcmd_timeout_seconds=60, + refresh_interval_seconds=3600, + refresh_on_startup=False, + game_version="1.6", + admin_token=None, + collections={"Core": "1", "Content": "2", "Cosmetics": "3"}, + static_dir=project_root / "progression_backend" / "static", + ) + with TestClient(create_app(settings)) as client: + self.assertEqual(client.get("/").status_code, 200) + self.assertEqual(client.get("/art/GameTitle.png").status_code, 200) + self.assertEqual(client.get("/api/health").json()["status"], "ok") + self.assertIn("rml_updated_at", client.get("/api/status").json()) + response = client.get("/api/modlists/vanilla") + self.assertEqual(response.status_code, 409) + + release_url = ( + "https://git.hudsonriggs.systems/HRiggs/ProgressionMods/" + "releases/download/0.2.2/ProgressionLoader.exe" + ) + with patch( + "progression_backend.app.latest_windows_release_url", + return_value=release_url, + ): + response = client.get( + "/api/downloads/windows", follow_redirects=False + ) + self.assertEqual(response.status_code, 307) + self.assertEqual(response.headers["location"], release_url) + + +if __name__ == "__main__": + unittest.main()