This commit is contained in:
@@ -22,6 +22,12 @@ result, and completion percentage in Dockhand or with `docker compose logs -f`.
|
|||||||
Failed batches also include SteamCMD's output tail so network, subscription, and
|
Failed batches also include SteamCMD's output tail so network, subscription, and
|
||||||
filesystem errors are visible directly in the container log.
|
filesystem errors are visible directly in the container log.
|
||||||
|
|
||||||
|
SteamCMD is forced to use `/opt/steamcmd` so Workshop content always lands in
|
||||||
|
the mounted volume. On the first `I/O Operation Failed` result, transient `.acf`,
|
||||||
|
download, and temp state is moved into `.progression-recovery`; completed mod
|
||||||
|
directories are left untouched. Startup also checks that the volume has enough
|
||||||
|
free space for the remaining Steam metadata estimate.
|
||||||
|
|
||||||
The image does not switch Unix users at startup because some rootless and
|
The image does not switch Unix users at startup because some rootless and
|
||||||
capability-restricted container hosts block `setuid`. The process remains inside
|
capability-restricted container hosts block `setuid`. The process remains inside
|
||||||
the container's user namespace and writes directly to its named volumes.
|
the container's user namespace and writes directly to its named volumes.
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import shutil
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
@@ -246,6 +247,27 @@ class RefreshService:
|
|||||||
total_targets,
|
total_targets,
|
||||||
(overall_timeout_seconds or 0) / 3600,
|
(overall_timeout_seconds or 0) / 3600,
|
||||||
)
|
)
|
||||||
|
if targets:
|
||||||
|
remaining_bytes = sum(
|
||||||
|
max(int(details[workshop_id].file_size or 0), 0)
|
||||||
|
for workshop_id in targets
|
||||||
|
if workshop_id in details
|
||||||
|
)
|
||||||
|
workshop_root = self.settings.workshop_content_dir.parent.parent
|
||||||
|
workshop_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
free_bytes = shutil.disk_usage(workshop_root).free
|
||||||
|
required_bytes = int(remaining_bytes * 1.1)
|
||||||
|
logger.info(
|
||||||
|
"Workshop capacity check: %.2f GiB expected, %.2f GiB free",
|
||||||
|
remaining_bytes / (1024**3),
|
||||||
|
free_bytes / (1024**3),
|
||||||
|
)
|
||||||
|
if required_bytes and free_bytes < required_bytes:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Insufficient Workshop volume space: "
|
||||||
|
f"{free_bytes / (1024**3):.2f} GiB free, "
|
||||||
|
f"approximately {required_bytes / (1024**3):.2f} GiB required"
|
||||||
|
)
|
||||||
if targets and not self.runner.available():
|
if targets and not self.runner.available():
|
||||||
reason = (
|
reason = (
|
||||||
f"SteamCMD was not found at '{self.settings.steamcmd_path}'. "
|
f"SteamCMD was not found at '{self.settings.steamcmd_path}'. "
|
||||||
@@ -305,6 +327,15 @@ class RefreshService:
|
|||||||
result.elapsed_seconds,
|
result.elapsed_seconds,
|
||||||
result.output_tail or "(no SteamCMD output)",
|
result.output_tail or "(no SteamCMD output)",
|
||||||
)
|
)
|
||||||
|
if result.has_io_failure:
|
||||||
|
recover = getattr(self.runner, "recover_io_failure", None)
|
||||||
|
moved = recover() if callable(recover) else []
|
||||||
|
if moved:
|
||||||
|
logger.warning(
|
||||||
|
"SteamCMD I/O recovery quarantined transient "
|
||||||
|
"Workshop state: %s",
|
||||||
|
", ".join(str(path) for path in moved),
|
||||||
|
)
|
||||||
self.database.add_download_attempt(
|
self.database.add_download_attempt(
|
||||||
{
|
{
|
||||||
"run_id": run_id,
|
"run_id": run_id,
|
||||||
|
|||||||
@@ -95,6 +95,10 @@ class SteamCmdBatchResult:
|
|||||||
failed_ids: set[str]
|
failed_ids: set[str]
|
||||||
output_tail: str
|
output_tail: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_io_failure(self) -> bool:
|
||||||
|
return "i/o operation failed" in self.output_tail.lower()
|
||||||
|
|
||||||
|
|
||||||
class SteamCmdRunner:
|
class SteamCmdRunner:
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -106,6 +110,16 @@ class SteamCmdRunner:
|
|||||||
self.executable = executable
|
self.executable = executable
|
||||||
self.content_dir = content_dir
|
self.content_dir = content_dir
|
||||||
self.timeout_seconds = timeout_seconds
|
self.timeout_seconds = timeout_seconds
|
||||||
|
self._io_recovery_attempted = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def install_root(self) -> Path:
|
||||||
|
try:
|
||||||
|
return self.content_dir.parents[3]
|
||||||
|
except IndexError as error:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid SteamCMD Workshop content path: {self.content_dir}"
|
||||||
|
) from error
|
||||||
|
|
||||||
def executable_path(self) -> str | None:
|
def executable_path(self) -> str | None:
|
||||||
configured = Path(self.executable)
|
configured = Path(self.executable)
|
||||||
@@ -130,6 +144,39 @@ class SteamCmdRunner:
|
|||||||
"Check the container volume ownership."
|
"Check the container volume ownership."
|
||||||
) from error
|
) from error
|
||||||
|
|
||||||
|
def recover_io_failure(self) -> list[Path]:
|
||||||
|
"""Quarantine Steam's transient Workshop state without touching mods."""
|
||||||
|
if self._io_recovery_attempted:
|
||||||
|
return []
|
||||||
|
self._io_recovery_attempted = True
|
||||||
|
workshop_dir = self.content_dir.parent.parent
|
||||||
|
candidates = [
|
||||||
|
workshop_dir / f"appworkshop_{RIMWORLD_APP_ID}.acf",
|
||||||
|
workshop_dir / "downloads" / RIMWORLD_APP_ID,
|
||||||
|
workshop_dir / "temp" / RIMWORLD_APP_ID,
|
||||||
|
]
|
||||||
|
existing = [candidate for candidate in candidates if candidate.exists()]
|
||||||
|
if not existing:
|
||||||
|
return []
|
||||||
|
|
||||||
|
timestamp = (
|
||||||
|
datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
+ f"-{time.time_ns()}"
|
||||||
|
)
|
||||||
|
recovery_dir = workshop_dir / ".progression-recovery" / timestamp
|
||||||
|
recovery_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
moved: list[Path] = []
|
||||||
|
for candidate in existing:
|
||||||
|
label = (
|
||||||
|
candidate.name
|
||||||
|
if candidate.is_file()
|
||||||
|
else f"{candidate.parent.name}-{candidate.name}"
|
||||||
|
)
|
||||||
|
destination = recovery_dir / label
|
||||||
|
shutil.move(str(candidate), str(destination))
|
||||||
|
moved.append(destination)
|
||||||
|
return moved
|
||||||
|
|
||||||
def run_batch(
|
def run_batch(
|
||||||
self,
|
self,
|
||||||
workshop_ids: list[str],
|
workshop_ids: list[str],
|
||||||
@@ -145,7 +192,13 @@ class SteamCmdRunner:
|
|||||||
raise ValueError(f"Invalid Workshop ids: {', '.join(invalid)}")
|
raise ValueError(f"Invalid Workshop ids: {', '.join(invalid)}")
|
||||||
self.ensure_workshop_writable()
|
self.ensure_workshop_writable()
|
||||||
|
|
||||||
command = [executable, "+login", "anonymous"]
|
command = [
|
||||||
|
executable,
|
||||||
|
"+force_install_dir",
|
||||||
|
str(self.install_root),
|
||||||
|
"+login",
|
||||||
|
"anonymous",
|
||||||
|
]
|
||||||
for workshop_id in workshop_ids:
|
for workshop_id in workshop_ids:
|
||||||
command.extend(
|
command.extend(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -197,6 +197,60 @@ class BackendTests(unittest.TestCase):
|
|||||||
(content_dir.parent.parent / ".progression-write-test").exists()
|
(content_dir.parent.parent / ".progression-write-test").exists()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch("progression_backend.workshop.subprocess.run")
|
||||||
|
def test_steamcmd_runner_forces_mounted_install_root(self, run: Mock) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
root = Path(directory)
|
||||||
|
executable = root / "steamcmd"
|
||||||
|
executable.write_text("", encoding="utf-8")
|
||||||
|
content_dir = (
|
||||||
|
root / "steamapps" / "workshop" / "content" / "294100"
|
||||||
|
)
|
||||||
|
run.return_value = Mock(returncode=0, stdout="", stderr="")
|
||||||
|
runner = SteamCmdRunner(str(executable), content_dir, 60)
|
||||||
|
|
||||||
|
runner.run_batch(["123456"])
|
||||||
|
|
||||||
|
command = run.call_args.args[0]
|
||||||
|
self.assertEqual(
|
||||||
|
command[:5],
|
||||||
|
[
|
||||||
|
str(executable.resolve()),
|
||||||
|
"+force_install_dir",
|
||||||
|
str(root),
|
||||||
|
"+login",
|
||||||
|
"anonymous",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_steamcmd_io_recovery_preserves_completed_mods(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
content_dir = (
|
||||||
|
Path(directory)
|
||||||
|
/ "steamapps"
|
||||||
|
/ "workshop"
|
||||||
|
/ "content"
|
||||||
|
/ "294100"
|
||||||
|
)
|
||||||
|
completed_mod = content_dir / "123456" / "About"
|
||||||
|
completed_mod.mkdir(parents=True)
|
||||||
|
(completed_mod / "About.xml").write_text("<ModMetaData />")
|
||||||
|
workshop_dir = content_dir.parent.parent
|
||||||
|
manifest = workshop_dir / "appworkshop_294100.acf"
|
||||||
|
manifest.write_text("stale", encoding="utf-8")
|
||||||
|
staging = workshop_dir / "downloads" / "294100"
|
||||||
|
staging.mkdir(parents=True)
|
||||||
|
(staging / "partial.bin").write_bytes(b"partial")
|
||||||
|
runner = SteamCmdRunner("steamcmd", content_dir, 60)
|
||||||
|
|
||||||
|
moved = runner.recover_io_failure()
|
||||||
|
|
||||||
|
self.assertEqual(len(moved), 2)
|
||||||
|
self.assertFalse(manifest.exists())
|
||||||
|
self.assertFalse(staging.exists())
|
||||||
|
self.assertTrue((completed_mod / "About.xml").is_file())
|
||||||
|
self.assertEqual(runner.recover_io_failure(), [])
|
||||||
|
|
||||||
def test_rml_uses_workshop_id_and_escapes_names(self) -> None:
|
def test_rml_uses_workshop_id_and_escapes_names(self) -> None:
|
||||||
payload = generate_rml(
|
payload = generate_rml(
|
||||||
[
|
[
|
||||||
|
|||||||
Reference in New Issue
Block a user