fix: download cases
Build and Publish Container / Publish Linux container (push) Successful in 3m53s
Build and Publish Container / Publish Linux container (push) Successful in 3m53s
This commit is contained in:
@@ -48,22 +48,26 @@ class RefreshService:
|
||||
rows = self.database.current_mods()
|
||||
if not rows:
|
||||
return True
|
||||
return any(
|
||||
row.get("status") != "ready"
|
||||
or not (self.settings.workshop_content_dir / row["workshop_id"]).is_dir()
|
||||
for row in rows
|
||||
)
|
||||
for row in rows:
|
||||
if row.get("status") == "skipped":
|
||||
continue
|
||||
if row.get("status") != "ready" or not (
|
||||
self.settings.workshop_content_dir / row["workshop_id"]
|
||||
).is_dir():
|
||||
return True
|
||||
return False
|
||||
|
||||
def _store_metadata(
|
||||
self,
|
||||
detail: WorkshopDetails,
|
||||
previous: dict[str, Any] | None,
|
||||
stale: bool,
|
||||
skip_reason: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
invalid_reason = None
|
||||
if detail.result != 1:
|
||||
invalid_reason = skip_reason
|
||||
if invalid_reason is None and detail.result != 1:
|
||||
invalid_reason = f"Steam returned result {detail.result}"
|
||||
elif detail.consumer_app_id not in (0, 294100):
|
||||
elif invalid_reason is None and detail.consumer_app_id not in (0, 294100):
|
||||
invalid_reason = f"Workshop item belongs to app {detail.consumer_app_id}"
|
||||
return {
|
||||
"workshop_id": detail.workshop_id,
|
||||
@@ -75,7 +79,7 @@ class RefreshService:
|
||||
"metadata_file_size": detail.file_size,
|
||||
"disk_size": None,
|
||||
"local_path": None,
|
||||
"status": "failed"
|
||||
"status": "skipped"
|
||||
if invalid_reason
|
||||
else ("pending" if stale else (previous or {}).get("status", "pending")),
|
||||
"error": invalid_reason,
|
||||
@@ -176,7 +180,7 @@ class RefreshService:
|
||||
report(stage="collections", completed_mods=0, total_mods=0, percent=0.0)
|
||||
remaining_time()
|
||||
collection_ids = list(self.settings.collections.values())
|
||||
response = self.steam.get_collections(collection_ids)
|
||||
response, collection_reference_ids = self.steam.get_collections(collection_ids)
|
||||
named_collections: dict[str, tuple[str, list[str]]] = {}
|
||||
all_ids: list[str] = []
|
||||
for name, collection_id in self.settings.collections.items():
|
||||
@@ -203,7 +207,7 @@ class RefreshService:
|
||||
self.database.upsert_mod(
|
||||
{
|
||||
"workshop_id": workshop_id,
|
||||
"status": "failed",
|
||||
"status": "skipped",
|
||||
"error": "Steam returned no item metadata",
|
||||
"last_checked_at": utc_now(),
|
||||
}
|
||||
@@ -217,10 +221,18 @@ class RefreshService:
|
||||
or old.get("status") != "ready"
|
||||
or not local_path.is_dir()
|
||||
)
|
||||
metadata_rows.append(self._store_metadata(detail, old, stale))
|
||||
skip_reason = (
|
||||
"Workshop entry is a collection reference"
|
||||
if workshop_id in collection_reference_ids
|
||||
else None
|
||||
)
|
||||
metadata_rows.append(
|
||||
self._store_metadata(detail, old, stale, skip_reason=skip_reason)
|
||||
)
|
||||
if (
|
||||
detail.result == 1
|
||||
and detail.consumer_app_id in (0, 294100)
|
||||
and workshop_id not in collection_reference_ids
|
||||
and stale
|
||||
):
|
||||
targets.append(workshop_id)
|
||||
@@ -230,6 +242,14 @@ class RefreshService:
|
||||
metrics["downloads_requested"] = len(targets)
|
||||
total_targets = len(targets)
|
||||
completed_targets = 0
|
||||
current_after_metadata = self.database.mod_map(unique_ids)
|
||||
resolved_before = sum(
|
||||
1
|
||||
for row in current_after_metadata.values()
|
||||
if row.get("status") in {"ready", "skipped"}
|
||||
)
|
||||
progress_total = len(unique_ids) if initial_download else total_targets
|
||||
progress_completed = resolved_before if initial_download else 0
|
||||
title_by_id = {
|
||||
workshop_id: details[workshop_id].title
|
||||
for workshop_id in targets
|
||||
@@ -237,14 +257,21 @@ class RefreshService:
|
||||
}
|
||||
report(
|
||||
stage="downloading" if targets else "inspecting",
|
||||
completed_mods=0,
|
||||
total_mods=total_targets,
|
||||
percent=100.0 if not targets else 0.0,
|
||||
completed_mods=progress_completed,
|
||||
total_mods=progress_total,
|
||||
percent=round(
|
||||
(progress_completed / max(progress_total, 1)) * 100, 1
|
||||
)
|
||||
if progress_total
|
||||
else 100.0,
|
||||
)
|
||||
if initial_download:
|
||||
logger.info(
|
||||
"Initial Workshop download started: %d mods, %.1f-hour hard limit",
|
||||
"Initial Workshop download started: %d mods need download, "
|
||||
"%d/%d collection entries already resolved, %.1f-hour hard limit",
|
||||
total_targets,
|
||||
resolved_before,
|
||||
len(unique_ids),
|
||||
(overall_timeout_seconds or 0) / 3600,
|
||||
)
|
||||
if targets:
|
||||
@@ -291,9 +318,21 @@ class RefreshService:
|
||||
batch_number = 0
|
||||
pass_number = 1
|
||||
while download_queue:
|
||||
batches = _batches(
|
||||
download_queue, self.settings.steamcmd_batch_size
|
||||
# SteamCMD stops processing the rest of a command when one
|
||||
# Workshop item times out. Keep the first pass efficient,
|
||||
# then isolate every remaining item so one bad mod cannot
|
||||
# permanently block the items that followed it in a batch.
|
||||
pass_batch_size = (
|
||||
self.settings.steamcmd_batch_size if pass_number == 1 else 1
|
||||
)
|
||||
batches = _batches(download_queue, pass_batch_size)
|
||||
if pass_number > 1:
|
||||
logger.info(
|
||||
"Initial Workshop download retry pass %d is "
|
||||
"isolating %d remaining mods individually",
|
||||
pass_number,
|
||||
len(download_queue),
|
||||
)
|
||||
for pass_batch_number, batch in enumerate(batches, start=1):
|
||||
remaining = remaining_time()
|
||||
batch_number += 1
|
||||
@@ -353,7 +392,12 @@ class RefreshService:
|
||||
inspection_rows = []
|
||||
for workshop_id in batch:
|
||||
failure = (
|
||||
"SteamCMD reported a download failure"
|
||||
(
|
||||
"SteamCMD did not complete this item; "
|
||||
"queued for retry"
|
||||
if initial_download
|
||||
else "SteamCMD reported a download failure"
|
||||
)
|
||||
if workshop_id in result.failed_ids
|
||||
else None
|
||||
)
|
||||
@@ -367,16 +411,25 @@ class RefreshService:
|
||||
else:
|
||||
metrics["downloads_failed"] += 1
|
||||
completed_targets = len(ready_target_ids)
|
||||
progress_completed = (
|
||||
resolved_before + completed_targets
|
||||
if initial_download
|
||||
else completed_targets
|
||||
)
|
||||
percent = round(
|
||||
(completed_targets / max(total_targets, 1)) * 100,
|
||||
(progress_completed / max(progress_total, 1)) * 100,
|
||||
1,
|
||||
)
|
||||
title = title_by_id.get(workshop_id, "Unknown mod")
|
||||
outcome = "ready" if success else "failed"
|
||||
outcome = (
|
||||
"ready"
|
||||
if success
|
||||
else ("retrying" if initial_download else "failed")
|
||||
)
|
||||
report(
|
||||
stage="downloading",
|
||||
completed_mods=completed_targets,
|
||||
total_mods=total_targets,
|
||||
completed_mods=progress_completed,
|
||||
total_mods=progress_total,
|
||||
percent=percent,
|
||||
current_workshop_id=workshop_id,
|
||||
current_title=title,
|
||||
@@ -387,8 +440,8 @@ class RefreshService:
|
||||
"Initial mod download"
|
||||
if initial_download
|
||||
else "Mod refresh",
|
||||
completed_targets,
|
||||
total_targets,
|
||||
progress_completed,
|
||||
progress_total,
|
||||
percent,
|
||||
outcome,
|
||||
title,
|
||||
@@ -438,23 +491,28 @@ class RefreshService:
|
||||
metrics["ready_items"] = sum(
|
||||
1 for row in final_rows.values() if row.get("status") == "ready"
|
||||
)
|
||||
resolved_items = sum(
|
||||
1
|
||||
for row in final_rows.values()
|
||||
if row.get("status") in {"ready", "skipped"}
|
||||
)
|
||||
metrics["status"] = (
|
||||
"completed"
|
||||
if metrics["ready_items"] == metrics["unique_items"]
|
||||
if resolved_items == metrics["unique_items"]
|
||||
else "partial"
|
||||
)
|
||||
if metrics["status"] == "partial" and not metrics["error"]:
|
||||
metrics["error"] = (
|
||||
f"{metrics['unique_items'] - metrics['ready_items']} items are not ready"
|
||||
f"{metrics['unique_items'] - resolved_items} items are not resolved"
|
||||
)
|
||||
report(
|
||||
stage=metrics["status"],
|
||||
completed_mods=completed_targets,
|
||||
total_mods=total_targets,
|
||||
completed_mods=resolved_items,
|
||||
total_mods=metrics["unique_items"],
|
||||
percent=round(
|
||||
(completed_targets / max(total_targets, 1)) * 100, 1
|
||||
(resolved_items / max(metrics["unique_items"], 1)) * 100, 1
|
||||
)
|
||||
if total_targets
|
||||
if metrics["unique_items"]
|
||||
else 100.0,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - the run must always persist a terminal state
|
||||
|
||||
Reference in New Issue
Block a user