versioning: 0.0.4, feat: create listing, source image

2026-05-08 00:02:59 -04:00
parent e2f87481d6
commit 97c751c585
7 changed files with 297 additions and 12 deletions
@@ -1,6 +1,6 @@
 [project]
 name = "traderai"
-version = "0.0.3"
+version = "0.0.4"
 description = "Local Ollama-powered assistant for UEX marketplace workflows."
 requires-python = ">=3.11"
 dependencies = [
@@ -37,3 +37,4 @@ include = ["traderai*"]



+
@@ -230,7 +230,10 @@ class FakeCornerstone:
            "url": f"{self.base_url}/ShipSalvageMods1/{item_id}",
            "html": """
                <html>
-                  <head><title>Star Citizen - Salvage modifier - Abrade Scraper Module</title></head>
+                  <head>
+                    <title>Star Citizen - Salvage modifier - Abrade Scraper Module</title>
+                    <meta property="og:image" content="/images/abrade.png">
+                  </head>
                  <body>
                    <table>
                      <tr><td>NAME</td><td>Abrade Scraper Module</td></tr>
@@ -246,6 +249,15 @@ class FakeCornerstone:
            """,
        }

+    async def get_image_data(self, url, max_bytes=10_000_000):
+        assert url == f"{self.base_url}/images/abrade.png"
+        return {
+            "url": url,
+            "content_type": "image/png",
+            "size_bytes": 12,
+            "image_data": "ZmFrZS1pbWFnZQ==",
+        }
+

@pytest.mark.asyncio
 async def test_search_marketplace_listings_filters_locally():
@@ -379,6 +391,8 @@ def test_schemas_expose_cornerstone_item_tools():

    assert "search_cornerstone_items" in names
    assert "get_cornerstone_item_locations" in names
+    assert "get_cornerstone_item_media" in names
+    assert "draft_marketplace_listing_with_cornerstone_image" in names


@pytest.mark.asyncio
@@ -441,18 +455,64 @@ async def test_get_cornerstone_item_locations_parses_store_prices():
    ]


+@pytest.mark.asyncio
+async def test_get_cornerstone_item_media_returns_absolute_image_urls():
+    registry = ToolRegistry(FakeUEX(), cornerstone=FakeCornerstone())
+
+    result = await registry.get_cornerstone_item_media(query="abrade")
+
+    assert result["media"] == [
+        {
+            "url": "https://finder.cstone.test/images/abrade.png",
+            "source": "og:image",
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_draft_marketplace_listing_with_cornerstone_image_adds_image_data_and_redacts_display():
+    registry = ToolRegistry(FakeUEX(), cornerstone=FakeCornerstone())
+
+    result = await registry.draft_marketplace_listing_with_cornerstone_image(
+        item_query="abrade",
+        id_category=3,
+        operation="sell",
+        type="item",
+        unit="unit",
+        title="Abrade Scraper Module",
+        description="Clean module, ready for pickup.",
+        price=21250,
+        currency="UEC",
+        language="en_US",
+        source="purchased_in_game",
+        in_stock=1,
+    )
+
+    pending = result["pending_action"]
+    stored = registry.pending_actions[pending["id"]]
+
+    assert pending["endpoint"] == "marketplace_advertise"
+    assert pending["payload"]["image_data"].startswith("<base64 image data redacted")
+    assert stored.payload["image_data"] == "ZmFrZS1pbWFnZQ=="
+    assert pending["metadata"]["cornerstone_image_status"] == "included"
+
+
 def test_parse_cornerstone_item_page_extracts_locations():
    parsed = parse_cornerstone_item_page(
        """
-        <html><head><title>Star Citizen - Food - Whamburger</title></head>
+        <html><head><title>Star Citizen - Food - Whamburger</title><meta property="og:image" content="/img/wham.png"></head>
        <body><table><tr><td>NAME</td><td>Whamburger</td></tr></table>
+        <img src="https://example.test/extra.png" alt="Whamburger">
        <table><tr><th>LOCATION</th><th>BASE PRICE</th><th>VERIFIED</th></tr>
        <tr><td>Stanton - Area18 - Cubby Blast</td><td>9</td><td>2956-01-01</td></tr></table></body></html>
-        """
+        """,
+        "https://finder.cstone.test/Search/item-wham",
    )

    assert parsed["name"] == "Whamburger"
    assert parsed["locations"][0]["base_price"] == 9
+    assert parsed["media"][0]["url"] == "https://finder.cstone.test/img/wham.png"
+    assert parsed["media"][1]["url"] == "https://example.test/extra.png"


@pytest.mark.asyncio
@@ -20,6 +20,7 @@ Use the specific UEX tool for the needed endpoint, such as get_uex_commodities_p
 When the user asks for history, trends, changes over time, or past prices, prefer the summarize_uex_*_history tools when available; use search_uex_api_index(history_only=true) if you need to discover history endpoints.
 Use SCMDB tools when the user asks about Star Citizen missions/contracts, mission rewards, payouts, reputation gains, item rewards, blueprint rewards, or hauling mission cargo. Prefer SCMDB live data unless the user asks for PTU or a specific game version.
 Use Cornerstone tools when the user asks where an item is sold, which shops carry an item, item store locations, in-game item base prices, or Universal Item Finder data.
+When drafting UEX marketplace item posts that need images, use Cornerstone media tools or draft_marketplace_listing_with_cornerstone_image so the pending listing can include UEX image_data sourced from Cornerstone.
 Prefer open and current UEX marketplace information. Do not use historical sale data, completed sale records, or sale/average-history information unless the user explicitly asks for historical sales.
 Treat UEX marketplace prices as in-game aUEC/UEC credits, never real-world dollars, unless the user explicitly says otherwise.
 For marketplace writes, draft the exact pending action and tell the user what will be sent; never claim it was sent until approval succeeds.
@@ -474,7 +475,7 @@ class OllamaAgent:
                "label": action.label,
                "method": action.method,
                "endpoint": action.endpoint,
-                "payload": action.payload,
+                "payload": self.tools._display_payload(action.payload) if hasattr(self.tools, "_display_payload") else action.payload,
                "metadata": action.metadata or {},
            }
            for action in self.tools.pending_actions.values()
@@ -524,6 +525,7 @@ class OllamaAgent:
            "get_scmdb_mission_rewards": "Fetching SCMDB mission rewards",
            "search_cornerstone_items": "Searching Cornerstone items",
            "get_cornerstone_item_locations": "Fetching Cornerstone item locations",
+            "get_cornerstone_item_media": "Fetching Cornerstone item media",
            "uex_api_catalog": "Checking UEX API catalog",
            "uex_get": "Fetching UEX data",
            "uex_draft_post": "Drafting UEX write for approval",
@@ -534,6 +536,7 @@ class OllamaAgent:
            "get_negotiation_messages": "Reading negotiation messages",
            "draft_negotiation_message": "Drafting message for approval",
            "draft_marketplace_listing": "Drafting listing for approval",
+            "draft_marketplace_listing_with_cornerstone_image": "Drafting listing with Cornerstone image",
            "check_uex_notifications": "Checking UEX notifications",
        }
        return labels.get(name, f"Running {name}")
@@ -1,8 +1,10 @@
 from __future__ import annotations

 from html.parser import HTMLParser
+import base64
 import json
 from typing import Any
+from urllib.parse import urljoin

 import httpx

@@ -41,6 +43,23 @@ class CornerstoneClient:
            raise CornerstoneError(f"Cornerstone HTTP {response.status_code}: {response.text[:240]}")
        return {"url": str(response.url), "html": response.text}

+    async def get_image_data(self, url: str, max_bytes: int = 10_000_000) -> dict[str, Any]:
+        async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
+            response = await client.get(url, headers={"Accept": "image/png,image/jpeg,image/*"})
+        if response.status_code >= 400:
+            raise CornerstoneError(f"Cornerstone image HTTP {response.status_code}: {response.text[:240]}")
+        content_type = response.headers.get("content-type", "").split(";")[0].strip().casefold()
+        if content_type not in {"image/jpeg", "image/jpg", "image/png"}:
+            raise CornerstoneError(f"Cornerstone image was not JPG or PNG: {content_type or 'unknown content type'}")
+        if len(response.content) > max_bytes:
+            raise CornerstoneError(f"Cornerstone image is larger than {max_bytes} bytes.")
+        return {
+            "url": str(response.url),
+            "content_type": content_type,
+            "size_bytes": len(response.content),
+            "image_data": base64.b64encode(response.content).decode("ascii"),
+        }
+
    async def _get_json(self, path: str) -> Any:
        async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
            response = await client.get(f"{self.base_url}/{path.lstrip('/')}", headers={"Accept": "application/json"})
@@ -58,6 +77,7 @@ class CornerstonePageParser(HTMLParser):
        super().__init__(convert_charrefs=True)
        self.title = ""
        self.tables: list[list[list[str]]] = []
+        self.images: list[dict[str, str]] = []
        self._skip_depth = 0
        self._in_title = False
        self._current_table: list[list[str]] | None = None
@@ -73,6 +93,29 @@ class CornerstonePageParser(HTMLParser):
            return
        if tag == "title":
            self._in_title = True
+        elif tag == "meta":
+            attr_map = self._attrs(attrs)
+            name = (attr_map.get("property") or attr_map.get("name") or "").casefold()
+            content = attr_map.get("content") or ""
+            if content and name in {"og:image", "twitter:image", "twitter:image:src"}:
+                self.images.append({"url": content, "source": name})
+        elif tag == "link":
+            attr_map = self._attrs(attrs)
+            rel = (attr_map.get("rel") or "").casefold()
+            href = attr_map.get("href") or ""
+            if href and "image_src" in rel:
+                self.images.append({"url": href, "source": "link:image_src"})
+        elif tag == "img":
+            attr_map = self._attrs(attrs)
+            url = attr_map.get("src") or attr_map.get("data-src") or attr_map.get("data-original") or ""
+            if url:
+                self.images.append(
+                    {
+                        "url": url,
+                        "alt": attr_map.get("alt") or "",
+                        "source": "img",
+                    }
+                )
        elif tag == "table":
            self._current_table = []
        elif tag == "tr" and self._current_table is not None:
@@ -110,8 +153,12 @@ class CornerstonePageParser(HTMLParser):
        if self._current_cell is not None:
            self._current_cell.append(data)

+    @staticmethod
+    def _attrs(attrs: list[tuple[str, str | None]]) -> dict[str, str]:
+        return {key.casefold(): value or "" for key, value in attrs}

-def parse_cornerstone_item_page(html: str) -> dict[str, Any]:
+
+def parse_cornerstone_item_page(html: str, page_url: str | None = None) -> dict[str, Any]:
    parser = CornerstonePageParser()
    parser.feed(html)
    info: dict[str, Any] = {"page_title": " ".join(parser.title.split())}
@@ -142,6 +189,9 @@ def parse_cornerstone_item_page(html: str) -> dict[str, Any]:
                    general[key] = value

    info["name"] = general.get("name") or _name_from_title(info["page_title"])
+    media = _dedupe_media(parser.images, page_url)
+    if media:
+        info["media"] = media
    if general:
        info["general"] = general
    info["locations"] = locations
@@ -157,3 +207,20 @@ def _name_from_title(title: str) -> str | None:
    if " - " not in title:
        return title or None
    return title.rsplit(" - ", 1)[-1].strip() or None
+
+
+def _dedupe_media(images: list[dict[str, str]], page_url: str | None = None) -> list[dict[str, str]]:
+    media = []
+    seen = set()
+    for image in images:
+        raw_url = (image.get("url") or "").strip()
+        if not raw_url or raw_url.startswith("data:"):
+            continue
+        url = urljoin(page_url or "", raw_url)
+        if url in seen:
+            continue
+        seen.add(url)
+        item = dict(image)
+        item["url"] = url
+        media.append(item)
+    return media
@@ -196,6 +196,8 @@ class ToolRegistry:
            "get_scmdb_mission_rewards": self.get_scmdb_mission_rewards,
            "search_cornerstone_items": self.search_cornerstone_items,
            "get_cornerstone_item_locations": self.get_cornerstone_item_locations,
+            "get_cornerstone_item_media": self.get_cornerstone_item_media,
+            "draft_marketplace_listing_with_cornerstone_image": self.draft_marketplace_listing_with_cornerstone_image,
        }
        self.handlers["uex_api_catalog"] = self.uex_api_catalog
        self.handlers["uex_get"] = self.uex_get
@@ -311,7 +313,7 @@ class ToolRegistry:
                "type": "function",
                "function": {
                    "name": "draft_marketplace_listing",
-                    "description": "Draft a new UEX marketplace listing. Listing prices are in-game aUEC/UEC credits, not real-world dollars. This creates a pending action that must be approved before posting.",
+                    "description": "Draft a new UEX marketplace listing. Listing prices are in-game aUEC/UEC credits, not real-world dollars. This creates a pending action that must be approved before posting. Prefer draft_marketplace_listing_with_cornerstone_image for item posts when a Cornerstone image is useful.",
                    "parameters": {
                        "type": "object",
                        "required": ["id_category", "operation", "type", "unit", "title", "description", "price", "currency", "language"],
@@ -332,8 +334,12 @@ class ToolRegistry:
                            "source": {"type": "string"},
                            "availability": {"type": "string"},
                            "in_stock": {"type": "integer"},
+                            "durability": {"type": "integer", "minimum": 0, "maximum": 100},
+                            "video_url": {"type": "string"},
+                            "image_data": {"type": "string", "description": "Base64 JPG or PNG image data for UEX upload."},
                            "hours_expiration": {"type": "integer"},
                            "is_hidden": {"type": "integer", "enum": [0, 1]},
+                            "is_tv_allowed": {"type": "integer", "enum": [0, 1]},
                            "is_production": {"type": "integer", "enum": [0, 1], "default": 1},
                        },
                    },
@@ -512,7 +518,7 @@ class ToolRegistry:
                "label": action.label,
                "method": action.method,
                "endpoint": action.endpoint,
-                "payload": action.payload,
+                "payload": self._display_payload(action.payload),
                "metadata": action.metadata or {},
            },
        }
@@ -973,6 +979,60 @@ class ToolRegistry:
                    },
                },
            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_cornerstone_item_media",
+                    "description": "Fetch Cornerstone item page media, especially image URLs that can be used when drafting UEX marketplace listings.",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "id": {"type": "string", "description": "Cornerstone item id from search_cornerstone_items."},
+                            "query": {"type": "string", "description": "Item name if id is not known."},
+                            "limit": {"type": "integer", "minimum": 1, "maximum": 10, "default": 5},
+                        },
+                    },
+                },
+            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "draft_marketplace_listing_with_cornerstone_image",
+                    "description": "Draft a UEX marketplace listing and source the listing image from Cornerstone. The image is downloaded as base64 image_data and included in the pending action. Nothing is posted until user approval.",
+                    "parameters": {
+                        "type": "object",
+                        "required": ["item_query", "id_category", "operation", "type", "unit", "title", "description", "price", "currency", "language"],
+                        "properties": {
+                            "item_query": {"type": "string", "description": "Cornerstone item name to source an image from."},
+                            "cornerstone_id": {"type": "string", "description": "Cornerstone item id, if already known."},
+                            "id_item": {"type": "integer"},
+                            "id_star_system": {"type": "integer"},
+                            "id_terminal": {"type": "integer"},
+                            "id_organization": {"type": "integer"},
+                            "id_category": {"type": "integer"},
+                            "operation": {"type": "string", "enum": ["buy", "sell", "rent", "trade"]},
+                            "type": {"type": "string", "enum": ["item", "service", "contract"]},
+                            "unit": {"type": "string"},
+                            "title": {"type": "string"},
+                            "description": {"type": "string"},
+                            "price": {"type": "number"},
+                            "currency": {"type": "string", "enum": ["UEC"]},
+                            "language": {"type": "string", "default": "en_US"},
+                            "location": {"type": "string"},
+                            "source": {"type": "string", "enum": ["looted", "pledged", "purchased_in_game", "pirated", "gifted"]},
+                            "availability": {"type": "string"},
+                            "in_stock": {"type": "integer"},
+                            "durability": {"type": "integer", "minimum": 0, "maximum": 100},
+                            "video_url": {"type": "string"},
+                            "hours_expiration": {"type": "integer"},
+                            "is_hidden": {"type": "integer", "enum": [0, 1]},
+                            "is_tv_allowed": {"type": "integer", "enum": [0, 1]},
+                            "is_production": {"type": "integer", "enum": [0, 1], "default": 1},
+                            "require_image": {"type": "boolean", "default": False, "description": "Return an error instead of drafting if no Cornerstone JPG/PNG image can be sourced."},
+                        },
+                    },
+                },
+            },
        ]

    @classmethod
@@ -1167,6 +1227,55 @@ class ToolRegistry:
    async def draft_marketplace_listing(self, **payload: Any) -> dict[str, Any]:
        return self._pending("Post marketplace listing", "marketplace_advertise", payload)

+    async def draft_marketplace_listing_with_cornerstone_image(
+        self,
+        item_query: str,
+        cornerstone_id: str | None = None,
+        **payload: Any,
+    ) -> dict[str, Any]:
+        require_image = bool(payload.pop("require_image", False))
+        item = await self._resolve_cornerstone_item(id=cornerstone_id, query=item_query)
+        if not item:
+            return {"error": "No Cornerstone item matched. Provide cornerstone_id or a more specific item_query."}
+
+        page = await self.cornerstone.get_item_page(str(item["id"]))
+        parsed = parse_cornerstone_item_page(page["html"], page["url"])
+        media = parsed.get("media") or []
+        image_result: dict[str, Any] | None = None
+        image_error = ""
+        for media_item in media:
+            try:
+                image_result = await self.cornerstone.get_image_data(media_item["url"])
+                break
+            except Exception as exc:
+                image_error = str(exc)
+
+        if image_result:
+            payload["image_data"] = image_result["image_data"]
+        elif require_image:
+            return {
+                "error": "Cornerstone item matched, but no usable JPG/PNG image could be sourced.",
+                "cornerstone": {
+                    "item": {"id": item.get("id"), "name": parsed.get("name") or item.get("name")},
+                    "url": page["url"],
+                    "media": media,
+                    "image_error": image_error,
+                },
+            }
+
+        payload.setdefault("id_item", self._int_or_none(item.get("id")))
+        metadata = {
+            "cornerstone_item_id": item.get("id"),
+            "cornerstone_item_name": parsed.get("name") or item.get("name"),
+            "cornerstone_url": page["url"],
+            "cornerstone_image_url": image_result.get("url") if image_result else None,
+            "cornerstone_image_content_type": image_result.get("content_type") if image_result else None,
+            "cornerstone_image_size_bytes": image_result.get("size_bytes") if image_result else None,
+            "cornerstone_image_status": "included" if image_result else "not_found",
+            "cornerstone_image_error": image_error or None,
+        }
+        return self._pending("Post marketplace listing with Cornerstone image", "marketplace_advertise", payload, metadata=metadata)
+
    async def remember_user_fact(self, content: str, kind: str = "note", importance: int = 3) -> dict[str, Any]:
        if self.memory is None:
            return {"error": "Memory store is not configured."}
@@ -1430,7 +1539,7 @@ class ToolRegistry:
            return {"error": "No Cornerstone item matched. Provide an id or a more specific query."}

        page = await self.cornerstone.get_item_page(str(item["id"]))
-        parsed = parse_cornerstone_item_page(page["html"])
+        parsed = parse_cornerstone_item_page(page["html"], page["url"])
        locations = parsed.get("locations") or []
        location_filter = (location or "").casefold().strip()
        if location_filter:
@@ -1455,6 +1564,34 @@ class ToolRegistry:
            "locations": locations[:limit],
        }

+    async def get_cornerstone_item_media(
+        self,
+        id: str | None = None,
+        query: str | None = None,
+        limit: int = 5,
+    ) -> dict[str, Any]:
+        item = await self._resolve_cornerstone_item(id=id, query=query)
+        if not item:
+            return {"error": "No Cornerstone item matched. Provide an id or a more specific query."}
+
+        page = await self.cornerstone.get_item_page(str(item["id"]))
+        parsed = parse_cornerstone_item_page(page["html"], page["url"])
+        media = parsed.get("media") or []
+        limit = max(1, min(limit, 10))
+        return {
+            "source": self.cornerstone.base_url,
+            "url": page["url"],
+            "item": {
+                "id": item.get("id"),
+                "name": parsed.get("name") or item.get("name"),
+                "sold": bool(item.get("sold")),
+                "general": parsed.get("general") or {},
+            },
+            "returned": min(len(media), limit),
+            "truncated": len(media) > limit,
+            "media": media[:limit],
+        }
+
    def _pending(
        self,
        label: str,
@@ -1474,12 +1611,27 @@ class ToolRegistry:
                "label": label,
                "method": method,
                "endpoint": endpoint,
-                "payload": payload,
+                "payload": self._display_payload(payload),
                "metadata": metadata,
                "approval_required": self.require_write_approval,
            }
        }

+    @staticmethod
+    def _display_payload(payload: dict[str, Any]) -> dict[str, Any]:
+        display = dict(payload)
+        image_data = display.get("image_data")
+        if isinstance(image_data, str) and image_data:
+            display["image_data"] = f"<base64 image data redacted; {len(image_data)} characters>"
+        return display
+
+    @staticmethod
+    def _int_or_none(value: Any) -> int | None:
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return None
+
    def _record_pending_action_result(self, action: PendingAction, result_kind: str, result: dict[str, Any]) -> None:
        metadata = action.metadata or {}
        plan_id = metadata.get("plan_id")
@@ -1,6 +1,6 @@
 from __future__ import annotations

-__version__ = "0.0.3"
+__version__ = "0.0.4"

 RELEASES_URL = "https://git.hudsonriggs.systems/LambdaBankingConglomerate/TraderAI/releases"
 RELEASES_API_URL = "https://git.hudsonriggs.systems/api/v1/repos/LambdaBankingConglomerate/TraderAI/releases"
@@ -9,3 +9,4 @@ RELEASES_API_URL = "https://git.hudsonriggs.systems/api/v1/repos/LambdaBankingCo



+
@@ -755,7 +755,7 @@ wheels = [

 [[package]]
 name = "traderai"
-version = "0.0.3"
+version = "0.0.4"
 source = { virtual = "." }
 dependencies = [
    { name = "apscheduler" },
@@ -1049,3 +1049,4 @@ wheels = [



+