From 6bd1e81a5191ba1055ae7fd0c3dc9c21934838e9 Mon Sep 17 00:00:00 2001 From: HRiggs Date: Fri, 8 May 2026 14:48:51 -0400 Subject: [PATCH] versioning: 0.0.6, ux: move buttons, feat: add cloud providers, feat: increese tool call limit --- .env.example | 4 + README.md | 5 +- pyproject.toml | 3 +- tests/test_agent.py | 30 +++ tests/test_tools.py | 26 ++ traderai/agent.py | 567 +++++++++++++++++++++++++++++++++++--------- traderai/config.py | 18 +- traderai/server.py | 109 ++++++++- traderai/tools.py | 111 ++++++++- traderai/version.py | 3 +- uv.lock | 3 +- web/app.js | 258 +++++++++++++++++--- web/index.html | 62 +++-- web/styles.css | 102 +++++++- 14 files changed, 1103 insertions(+), 198 deletions(-) diff --git a/.env.example b/.env.example index f380ad7..ce3ff30 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,10 @@ +MODEL_PROVIDER=ollama OLLAMA_BASE_URL=http://localhost:11434 OLLAMA_MODEL=qwen3.5:9b OLLAMA_NUM_CTX=64512 +OPENAI_BASE_URL=https://api.openai.com/v1 +OPENAI_MODEL=gpt-5.3-codex +OPENAI_API_KEY= UEX_BASE_URL=https://api.uexcorp.space/2.0 SCMDB_BASE_URL=https://scmdb.net CORNERSTONE_BASE_URL=https://finder.cstone.space diff --git a/README.md b/README.md index 50f5a00..2d3a93d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TraderAI -Local Ollama-powered chat for UEX marketplace workflows. +Local Ollama- or OpenAI-powered chat for UEX marketplace workflows. ## What It Does @@ -25,6 +25,7 @@ Local Ollama-powered chat for UEX marketplace workflows. ``` 3. Create `.env` from `.env.example` and set `UEX_SECRET_KEY` and/or `UEX_BEARER_TOKEN` if you want authenticated actions. + If you want to use OpenAI instead of Ollama, set `MODEL_PROVIDER=openai`, set `OPENAI_API_KEY`, and optionally change `OPENAI_MODEL` from the default `gpt-5.3-codex`. `SCMDB_BASE_URL` defaults to `https://scmdb.net`. `CORNERSTONE_BASE_URL` defaults to `https://finder.cstone.space`. 4. Install and run: @@ -38,7 +39,7 @@ Local Ollama-powered chat for UEX marketplace workflows. ## Notes -Ollama runs locally at `http://localhost:11434` by default. This app talks to Ollama's native chat API with tool schemas, then executes approved UEX calls in the FastAPI backend. `OLLAMA_NUM_CTX` controls the per-request Ollama context window; `64512` is the default because Ollama recommends at least 64k tokens for agent-style workflows when hardware allows it. +Ollama runs locally at `http://localhost:11434` by default. This app can talk to either Ollama's native chat API or OpenAI's Chat Completions API with tool schemas, then executes approved UEX calls in the FastAPI backend. `OLLAMA_NUM_CTX` controls the per-request Ollama context window; `64512` is the default because Ollama recommends at least 64k tokens for agent-style workflows when hardware allows it. ## Releases And Updates diff --git a/pyproject.toml b/pyproject.toml index 73c8a34..316b97d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "traderai" -version = "0.0.5" +version = "0.0.6" description = "Local Ollama-powered assistant for UEX marketplace workflows." requires-python = ">=3.11" dependencies = [ @@ -39,3 +39,4 @@ include = ["traderai*"] + diff --git a/tests/test_agent.py b/tests/test_agent.py index 0039bfa..7f551c6 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -64,6 +64,19 @@ class TitleAgent(OllamaAgent): return {"message": {"role": "assistant", "content": "Done"}} +class ImageCaptureAgent(OllamaAgent): + def __init__(self, memory): + super().__init__("http://127.0.0.1:1", "missing-model", EmptyTools(), memory=memory) + self.last_messages = None + + async def ensure_available(self): + return None + + async def _chat_once(self, query="", messages=None, **kwargs): + self.last_messages = messages + return {"message": {"role": "assistant", "content": "Seen"}} + + class SlowToolTools(EmptyTools): schemas = [ { @@ -229,6 +242,23 @@ async def test_first_chat_message_generates_thread_title(tmp_path): assert memory.get_thread(thread["id"])["title"] == "UEX Market Check" +@pytest.mark.asyncio +async def test_chat_includes_pasted_images_and_memory_note(tmp_path): + memory = MemoryStore(str(tmp_path / "memory.sqlite3")) + agent = ImageCaptureAgent(memory) + + result = await agent.chat( + "", + images=[{"name": "listing.png", "content_type": "image/png", "image_data": "ZmFrZS1pbWFnZQ=="}], + ) + + assert result["message"] == "Seen" + user_message = next(message for message in reversed(agent.last_messages) if message.get("role") == "user") + assert user_message["images"] == ["ZmFrZS1pbWFnZQ=="] + assert user_message["content"] == "Please analyze the attached image." + assert "[Attached 1 pasted image]" in memory.recent_conversation()[-2]["content"] + + @pytest.mark.asyncio async def test_chat_events_returns_fallback_after_slow_tool_and_empty_final_response(tmp_path): memory = MemoryStore(str(tmp_path / "memory.sqlite3")) diff --git a/tests/test_tools.py b/tests/test_tools.py index 2a0aa26..a0dd1ce 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -497,6 +497,32 @@ async def test_draft_marketplace_listing_with_cornerstone_image_adds_image_data_ assert pending["metadata"]["cornerstone_image_status"] == "included" +@pytest.mark.asyncio +async def test_draft_marketplace_listing_can_reuse_pasted_chat_image(): + registry = ToolRegistry(FakeUEX()) + + with registry.chat_image_scope([{"name": "listing.png", "content_type": "image/png", "image_data": "ZmFrZS1pbWFnZQ=="}]): + result = await registry.draft_marketplace_listing( + id_category=3, + operation="sell", + type="item", + unit="unit", + title="Abrade Scraper Module", + description="Clean module, ready for pickup.", + price=21250, + currency="UEC", + language="en_US", + use_attached_image=True, + ) + + pending = result["pending_action"] + stored = registry.pending_actions[pending["id"]] + assert pending["payload"]["image_data"].startswith(" None: self.base_url = base_url.rstrip("/") self.model = model @@ -45,9 +48,13 @@ class OllamaAgent: self.memory = memory self.user_name = user_name self.num_ctx = num_ctx + self.provider = provider.strip().casefold() or "ollama" + self.api_key = api_key self.thread_messages: dict[str, list[dict[str, Any]]] = {} async def health(self) -> dict[str, Any]: + if self.provider == "openai": + return await self._openai_health() try: async with httpx.AsyncClient(timeout=3) as client: response = await client.get(f"{self.base_url}/api/tags") @@ -77,60 +84,74 @@ class OllamaAgent: if not health["online"]: raise OllamaUnavailable(health["message"]) - async def chat(self, content: str, thread_id: str | None = DEFAULT_THREAD_ID) -> dict[str, Any]: + async def chat( + self, + content: str, + thread_id: str | None = DEFAULT_THREAD_ID, + images: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: await self.ensure_available() resolved_thread_id = self._thread_id(thread_id) messages = self._messages_for_thread(resolved_thread_id) previous_interaction = self.memory.last_interaction(resolved_thread_id) if self.memory else None + normalized_images = self._normalize_images(images) + prompt_text = self._prompt_text(content, len(normalized_images)) + memory_content = self._conversation_content(content, len(normalized_images)) if self.memory: - self.memory.add_conversation("user", content, resolved_thread_id) - await self._title_first_message(resolved_thread_id, content, previous_interaction) - messages.append({"role": "user", "content": content}) + self.memory.add_conversation("user", memory_content, resolved_thread_id) + await self._title_first_message(resolved_thread_id, prompt_text, previous_interaction) + messages.append(self._user_message(prompt_text, normalized_images)) last_tool_results: list[dict[str, Any]] = [] - for _ in range(5): - try: - response = await self._ollama_chat( - content, - messages, - previous_interaction=previous_interaction, - thread_id=resolved_thread_id, - ) - except Exception as exc: - if not last_tool_results: - raise - answer = self._tool_result_fallback( - last_tool_results, - f"The local model stopped after the tool call: {exc}", - ) - messages.append({"role": "assistant", "content": answer}) - if self.memory: - self.memory.add_conversation("assistant", answer, resolved_thread_id) - return {"message": answer, "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} - message = response.get("message") or {} - tool_calls = message.get("tool_calls") or [] - if not tool_calls: - answer = message.get("content", "") - if not answer.strip(): - answer = self._empty_response_fallback(last_tool_results) - messages.append({"role": "assistant", "content": answer}) - if self.memory: - self.memory.add_conversation("assistant", answer, resolved_thread_id) - return {"message": answer, "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} - - messages.append(message) - for call in tool_calls: - name, arguments = self._extract_call(call) - result = await self.tools.execute(name, arguments) - last_tool_results.append({"tool": name, "result": result}) - messages.append({"role": "tool", "tool_name": name, "content": json.dumps(result)}) + image_scope = self.tools.chat_image_scope(normalized_images) if hasattr(self.tools, "chat_image_scope") else nullcontext() + with image_scope: + for _ in range(10): + try: + response = await self._chat_once( + prompt_text, + messages, + previous_interaction=previous_interaction, + thread_id=resolved_thread_id, + ) + except Exception as exc: + if not last_tool_results: + raise + answer = self._tool_result_fallback( + last_tool_results, + f"The {self._provider_label()} stopped after the tool call: {exc}", + ) + messages.append({"role": "assistant", "content": answer}) + if self.memory: + self.memory.add_conversation("assistant", answer, resolved_thread_id) + return {"message": answer, "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} + message = response.get("message") or {} + tool_calls = message.get("tool_calls") or [] + if not tool_calls: + answer = message.get("content", "") + if not answer.strip(): + answer = self._empty_response_fallback(last_tool_results) + messages.append({"role": "assistant", "content": answer}) + if self.memory: + self.memory.add_conversation("assistant", answer, resolved_thread_id) + return {"message": answer, "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} + messages.append(message) + for call in tool_calls: + name, arguments = self._extract_call(call) + result = await self.tools.execute(name, arguments) + last_tool_results.append({"tool": name, "result": result}) + messages.append({"role": "tool", "tool_name": name, "tool_call_id": call.get("id"), "content": json.dumps(result)}) fallback = "I hit the tool-call limit while working on that. Try narrowing the request or approve any pending action first." messages.append({"role": "assistant", "content": fallback}) if self.memory: self.memory.add_conversation("assistant", fallback, resolved_thread_id) return {"message": fallback, "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} - async def chat_events(self, content: str, thread_id: str | None = DEFAULT_THREAD_ID) -> AsyncIterator[dict[str, Any]]: + async def chat_events( + self, + content: str, + thread_id: str | None = DEFAULT_THREAD_ID, + images: list[dict[str, Any]] | None = None, + ) -> AsyncIterator[dict[str, Any]]: health = await self.health() if not health["online"]: yield {"type": "warning", "message": health["message"]} @@ -140,74 +161,77 @@ class OllamaAgent: resolved_thread_id = self._thread_id(thread_id) messages = self._messages_for_thread(resolved_thread_id) previous_interaction = self.memory.last_interaction(resolved_thread_id) if self.memory else None + normalized_images = self._normalize_images(images) + prompt_text = self._prompt_text(content, len(normalized_images)) + memory_content = self._conversation_content(content, len(normalized_images)) if self.memory: - self.memory.add_conversation("user", content, resolved_thread_id) - await self._title_first_message(resolved_thread_id, content, previous_interaction) - messages.append({"role": "user", "content": content}) + self.memory.add_conversation("user", memory_content, resolved_thread_id) + await self._title_first_message(resolved_thread_id, prompt_text, previous_interaction) + messages.append(self._user_message(prompt_text, normalized_images)) yield {"type": "status", "message": "Thinking"} last_tool_results: list[dict[str, Any]] = [] + image_scope = self.tools.chat_image_scope(normalized_images) if hasattr(self.tools, "chat_image_scope") else nullcontext() + with image_scope: + for _ in range(10): + assistant_message: dict[str, Any] = {"role": "assistant", "content": ""} + tool_calls: list[dict[str, Any]] = [] - for _ in range(5): - assistant_message: dict[str, Any] = {"role": "assistant", "content": ""} - tool_calls: list[dict[str, Any]] = [] - - try: - async for event in self._ollama_chat_stream( - content, - messages, - previous_interaction=previous_interaction, - thread_id=resolved_thread_id, - ): - message = event.get("message") or {} - chunk = message.get("content") or "" - if chunk: - assistant_message["content"] += chunk - yield {"type": "token", "content": chunk} - if message.get("tool_calls"): - tool_calls.extend(message["tool_calls"]) - if event.get("done"): - metrics = self._stream_metrics(event) - if metrics: - yield {"type": "metrics", **metrics} - except Exception as exc: - if not last_tool_results: - yield {"type": "warning", "message": f"Chat failed before any tool result was available: {exc}"} + try: + async for event in self._chat_stream_once( + prompt_text, + messages, + previous_interaction=previous_interaction, + thread_id=resolved_thread_id, + ): + message = event.get("message") or {} + chunk = message.get("content") or "" + if chunk: + assistant_message["content"] += chunk + yield {"type": "token", "content": chunk} + if message.get("tool_calls"): + tool_calls.extend(message["tool_calls"]) + if event.get("done"): + metrics = self._stream_metrics(event) + if metrics: + yield {"type": "metrics", **metrics} + except Exception as exc: + if not last_tool_results: + yield {"type": "warning", "message": f"Chat failed before any tool result was available: {exc}"} + yield {"type": "done", "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} + return + fallback = self._tool_result_fallback( + last_tool_results, + f"The {self._provider_label()} stopped after the tool call: {exc}", + ) + assistant_message["content"] = fallback + messages.append(assistant_message) + if self.memory: + self.memory.add_conversation("assistant", fallback, resolved_thread_id) + yield {"type": "token", "content": fallback} yield {"type": "done", "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} return - fallback = self._tool_result_fallback( - last_tool_results, - f"The local model stopped after the tool call: {exc}", - ) - assistant_message["content"] = fallback + + if not tool_calls: + if not assistant_message.get("content", "").strip(): + fallback = self._empty_response_fallback(last_tool_results) + assistant_message["content"] = fallback + yield {"type": "token", "content": fallback} + messages.append(assistant_message) + if self.memory: + self.memory.add_conversation("assistant", assistant_message.get("content", ""), resolved_thread_id) + yield {"type": "done", "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} + return + + assistant_message["tool_calls"] = tool_calls messages.append(assistant_message) - if self.memory: - self.memory.add_conversation("assistant", fallback, resolved_thread_id) - yield {"type": "token", "content": fallback} - yield {"type": "done", "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} - return - - if not tool_calls: - if not assistant_message.get("content", "").strip(): - fallback = self._empty_response_fallback(last_tool_results) - assistant_message["content"] = fallback - yield {"type": "token", "content": fallback} - messages.append(assistant_message) - if self.memory: - self.memory.add_conversation("assistant", assistant_message.get("content", ""), resolved_thread_id) - yield {"type": "done", "pending_actions": self._pending_payloads(), "thread_id": resolved_thread_id} - return - - assistant_message["tool_calls"] = tool_calls - messages.append(assistant_message) - for call in tool_calls: - name, arguments = self._extract_call(call) - yield {"type": "status", "message": self._tool_status(name)} - result = await self.tools.execute(name, arguments) - last_tool_results.append({"tool": name, "result": result}) - messages.append({"role": "tool", "tool_name": name, "content": json.dumps(result)}) - - yield {"type": "status", "message": "Writing response"} + for call in tool_calls: + name, arguments = self._extract_call(call) + yield {"type": "status", "message": self._tool_status(name)} + result = await self.tools.execute(name, arguments) + last_tool_results.append({"tool": name, "result": result}) + messages.append({"role": "tool", "tool_name": name, "tool_call_id": call.get("id"), "content": json.dumps(result)}) + yield {"type": "status", "message": "Writing response"} fallback = "I hit the tool-call limit while working on that. Try narrowing the request or approve any pending action first." messages.append({"role": "assistant", "content": fallback}) if self.memory: @@ -221,9 +245,9 @@ class OllamaAgent: previous_interaction = self.memory.last_interaction("wake") if self.memory else None messages.append({"role": "user", "content": wake_message}) last_tool_results: list[dict[str, Any]] = [] - for _ in range(5): + for _ in range(10): try: - response = await self._ollama_chat( + response = await self._chat_once( wake_message, messages, previous_interaction=previous_interaction, @@ -234,7 +258,7 @@ class OllamaAgent: raise content = self._tool_result_fallback( last_tool_results, - f"The local model stopped after the wake-job tool call: {exc}", + f"The {self._provider_label()} stopped after the wake-job tool call: {exc}", ) messages.append({"role": "assistant", "content": content}) if self.memory: @@ -258,8 +282,7 @@ class OllamaAgent: name, arguments = self._extract_call(call) result = await self.tools.execute(name, arguments) last_tool_results.append({"tool": name, "result": result}) - messages.append({"role": "tool", "tool_name": name, "content": json.dumps(result)}) - + messages.append({"role": "tool", "tool_name": name, "tool_call_id": call.get("id"), "content": json.dumps(result)}) content = "I hit the tool-call limit while running this scheduled wake job. Check the job prompt or pending approvals." messages.append({"role": "assistant", "content": content}) if self.memory: @@ -267,6 +290,51 @@ class OllamaAgent: self.memory.add_conversation("assistant", content, "wake") return content + async def _chat_once( + self, + query: str = "", + messages: list[dict[str, Any]] | None = None, + previous_interaction: dict[str, Any] | None = None, + thread_id: str | None = DEFAULT_THREAD_ID, + ) -> dict[str, Any]: + if self.provider == "openai": + return await self._openai_chat( + query, + messages, + previous_interaction=previous_interaction, + thread_id=thread_id, + ) + return await self._ollama_chat( + query, + messages, + previous_interaction=previous_interaction, + thread_id=thread_id, + ) + + async def _chat_stream_once( + self, + query: str = "", + messages: list[dict[str, Any]] | None = None, + previous_interaction: dict[str, Any] | None = None, + thread_id: str | None = DEFAULT_THREAD_ID, + ) -> AsyncIterator[dict[str, Any]]: + if self.provider == "openai": + async for event in self._openai_chat_stream( + query, + messages, + previous_interaction=previous_interaction, + thread_id=thread_id, + ): + yield event + return + async for event in self._ollama_chat_stream( + query, + messages, + previous_interaction=previous_interaction, + thread_id=thread_id, + ): + yield event + async def _ollama_chat( self, query: str = "", @@ -322,6 +390,103 @@ class OllamaAgent: if line: yield json.loads(line) + async def _openai_chat( + self, + query: str = "", + messages: list[dict[str, Any]] | None = None, + previous_interaction: dict[str, Any] | None = None, + thread_id: str | None = DEFAULT_THREAD_ID, + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=120) as client: + response = await client.post( + f"{self.base_url}/chat/completions", + headers=self._openai_headers(), + json={ + "model": self.model, + "messages": self._openai_messages( + query, + messages or self._messages_for_thread(thread_id), + previous_interaction=previous_interaction, + thread_id=thread_id, + ), + "tools": self.tools.schemas, + "stream": False, + }, + ) + response.raise_for_status() + body = response.json() + choice = (body.get("choices") or [{}])[0] + message = choice.get("message") or {} + return { + "message": { + "role": message.get("role", "assistant"), + "content": message.get("content") or "", + "tool_calls": message.get("tool_calls") or [], + } + } + + async def _openai_chat_stream( + self, + query: str = "", + messages: list[dict[str, Any]] | None = None, + previous_interaction: dict[str, Any] | None = None, + thread_id: str | None = DEFAULT_THREAD_ID, + ) -> AsyncIterator[dict[str, Any]]: + tool_calls: dict[int, dict[str, Any]] = {} + async with httpx.AsyncClient(timeout=120) as client: + async with client.stream( + "POST", + f"{self.base_url}/chat/completions", + headers=self._openai_headers(), + json={ + "model": self.model, + "messages": self._openai_messages( + query, + messages or self._messages_for_thread(thread_id), + previous_interaction=previous_interaction, + thread_id=thread_id, + ), + "tools": self.tools.schemas, + "stream": True, + }, + ) as response: + response.raise_for_status() + async for line in response.aiter_lines(): + if not line or not line.startswith("data:"): + continue + payload = line.removeprefix("data:").strip() + if not payload: + continue + if payload == "[DONE]": + break + event = json.loads(payload) + choice = (event.get("choices") or [{}])[0] + delta = choice.get("delta") or {} + content = delta.get("content") or "" + if content: + yield {"message": {"role": "assistant", "content": content}} + for tool_call in delta.get("tool_calls") or []: + self._merge_openai_tool_call(tool_calls, tool_call) + finish_reason = choice.get("finish_reason") + if finish_reason: + yield { + "message": { + "role": "assistant", + "content": "", + "tool_calls": self._ordered_tool_calls(tool_calls), + }, + "done": True, + } + return + yield { + "message": { + "role": "assistant", + "content": "", + "tool_calls": self._ordered_tool_calls(tool_calls), + }, + "done": True, + } + def _messages_with_context( self, query: str, @@ -329,21 +494,146 @@ class OllamaAgent: previous_interaction: dict[str, Any] | None = None, thread_id: str | None = DEFAULT_THREAD_ID, ) -> list[dict[str, Any]]: - context = self._runtime_context(query, previous_interaction=previous_interaction, thread_id=thread_id) + attached_image_count = 0 + for message in reversed(messages): + if message.get("role") != "user": + continue + attached_image_count = len(message.get("images") or []) + break + context = self._runtime_context( + query, + previous_interaction=previous_interaction, + thread_id=thread_id, + attached_image_count=attached_image_count, + ) if not context: return messages return [messages[0], {"role": "system", "content": context}, *messages[1:]] + async def _openai_health(self) -> dict[str, Any]: + if not self.api_key: + return { + "online": False, + "model": self.model, + "base_url": self.base_url, + "provider": "openai", + "model_available": False, + "models": [], + "message": "OpenAI is selected, but no OpenAI API key is configured.", + "detail": "", + } + try: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(f"{self.base_url}/models", headers=self._openai_headers()) + response.raise_for_status() + body = response.json() + except (httpx.HTTPError, ValueError) as exc: + return { + "online": False, + "model": self.model, + "base_url": self.base_url, + "provider": "openai", + "model_available": False, + "models": [], + "message": f"OpenAI is unreachable at {self.base_url} or rejected the API key.", + "detail": str(exc), + } + models = sorted(item.get("id") for item in body.get("data", []) if item.get("id")) + return { + "online": True, + "model": self.model, + "base_url": self.base_url, + "provider": "openai", + "model_available": self.model in models, + "models": models, + "message": "OpenAI is online.", + } + + def _openai_headers(self) -> dict[str, str]: + return { + "Authorization": f"Bearer {self.api_key or ''}", + "Content-Type": "application/json", + } + + def _openai_messages( + self, + query: str, + messages: list[dict[str, Any]], + previous_interaction: dict[str, Any] | None = None, + thread_id: str | None = DEFAULT_THREAD_ID, + ) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for message in self._messages_with_context( + query, + messages, + previous_interaction=previous_interaction, + thread_id=thread_id, + ): + role = message.get("role") + if role not in {"system", "user", "assistant", "tool"}: + continue + entry: dict[str, Any] = {"role": role, "content": message.get("content", "")} + if role == "user" and message.get("images"): + text_content = message.get("content", "") + content_parts: list[dict[str, Any]] = [] + content_types = list(message.get("image_content_types") or []) + if text_content: + content_parts.append({"type": "text", "text": text_content}) + for index, image_data in enumerate(message.get("images") or []): + content_type = content_types[index] if index < len(content_types) else "image/png" + content_parts.append( + { + "type": "image_url", + "image_url": {"url": f"data:{content_type};base64,{image_data}"}, + } + ) + entry["content"] = content_parts + if role == "assistant" and message.get("tool_calls"): + entry["tool_calls"] = message["tool_calls"] + if role == "tool": + entry["tool_call_id"] = message.get("tool_call_id") or message.get("tool_name") or "tool" + normalized.append(entry) + return normalized + + def _provider_label(self) -> str: + return "OpenAI model" if self.provider == "openai" else "local model" + + @staticmethod + def _merge_openai_tool_call(target: dict[int, dict[str, Any]], delta: dict[str, Any]) -> None: + index = int(delta.get("index") or 0) + current = target.setdefault(index, {"id": delta.get("id"), "type": "function", "function": {"name": "", "arguments": ""}}) + if delta.get("id"): + current["id"] = delta["id"] + function = delta.get("function") or {} + current_function = current.setdefault("function", {"name": "", "arguments": ""}) + if function.get("name"): + current_function["name"] += function["name"] + if function.get("arguments"): + current_function["arguments"] += function["arguments"] + + @staticmethod + def _ordered_tool_calls(tool_calls: dict[int, dict[str, Any]]) -> list[dict[str, Any]]: + return [tool_calls[index] for index in sorted(tool_calls)] + def _runtime_context( self, query: str, previous_interaction: dict[str, Any] | None = None, thread_id: str | None = DEFAULT_THREAD_ID, + attached_image_count: int = 0, ) -> str: local_zone = get_localzone() parts = [ f"Current local date/time: {iso_now()} UTC; {iso_now_in_zone(local_zone)} {local_zone}.", ] + if attached_image_count: + label = "image" if attached_image_count == 1 else "images" + parts.append( + f"Current user message includes {attached_image_count} pasted {label}. " + "You can inspect them visually. If the user wants one reused in a marketplace listing draft, " + "call draft_marketplace_listing or draft_marketplace_listing_with_cornerstone_image with " + "use_attached_image=true and attached_image_index when needed." + ) uex = getattr(self.tools, "uex", None) if uex: auth_methods = [] @@ -433,6 +723,24 @@ class OllamaAgent: f"Message: {first_message[:800]}" ) try: + if self.provider == "openai": + async with httpx.AsyncClient(timeout=20) as client: + response = await client.post( + f"{self.base_url}/chat/completions", + headers=self._openai_headers(), + json={ + "model": self.model, + "messages": [ + {"role": "system", "content": "You write short chat titles."}, + {"role": "user", "content": prompt}, + ], + "stream": False, + }, + ) + response.raise_for_status() + choice = (response.json().get("choices") or [{}])[0] + message = choice.get("message") or {} + return self._clean_generated_title(message.get("content", "")) async with httpx.AsyncClient(timeout=20) as client: response = await client.post( f"{self.base_url}/api/chat", @@ -489,10 +797,10 @@ class OllamaAgent: @staticmethod def _empty_response_fallback(tool_results: list[dict[str, Any]]) -> str: if not tool_results: - return "I did not get a usable response from the local model. Please try again, or narrow the request a bit." + return "I did not get a usable response from the model. Please try again, or narrow the request a bit." return OllamaAgent._tool_result_fallback( tool_results, - "I completed the tool call, but the local model did not write a final answer.", + "I completed the tool call, but the model did not write a final answer.", ) @staticmethod @@ -633,6 +941,47 @@ class OllamaAgent: arguments = json.loads(arguments or "{}") return name, arguments + @staticmethod + def _normalize_images(images: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for image in images or []: + if not isinstance(image, dict): + continue + image_data = str(image.get("image_data") or "").strip() + if not image_data: + continue + normalized.append( + { + "name": str(image.get("name") or "").strip() or "pasted-image.png", + "content_type": str(image.get("content_type") or "image/png").strip() or "image/png", + "image_data": image_data, + } + ) + return normalized + + @staticmethod + def _prompt_text(content: str, image_count: int) -> str: + text = content.strip() + if text: + return text + return "Please analyze the attached image." if image_count == 1 else "Please analyze the attached images." + + @staticmethod + def _conversation_content(content: str, image_count: int) -> str: + text = content.strip() + if not image_count: + return text + note = f"[Attached {image_count} pasted image{'s' if image_count != 1 else ''}]" + return f"{text}\n\n{note}" if text else note + + @staticmethod + def _user_message(content: str, images: list[dict[str, Any]]) -> dict[str, Any]: + message: dict[str, Any] = {"role": "user", "content": content} + if images: + message["images"] = [image["image_data"] for image in images] + message["image_content_types"] = [image["content_type"] for image in images] + return message + class OllamaUnavailable(RuntimeError): pass diff --git a/traderai/config.py b/traderai/config.py index beebca1..2bc1fa7 100644 --- a/traderai/config.py +++ b/traderai/config.py @@ -11,12 +11,16 @@ from pydantic_settings import BaseSettings, SettingsConfigDict CONFIG_FIELDS: dict[str, dict[str, Any]] = { + "model_provider": {"env": "MODEL_PROVIDER", "type": "string", "secret": False}, "ollama_base_url": {"env": "OLLAMA_BASE_URL", "type": "string", "secret": False}, "ollama_model": {"env": "OLLAMA_MODEL", "type": "string", "secret": False}, "ollama_num_ctx": {"env": "OLLAMA_NUM_CTX", "type": "integer", "secret": False}, + "openai_base_url": {"env": "OPENAI_BASE_URL", "type": "string", "secret": False}, + "openai_model": {"env": "OPENAI_MODEL", "type": "string", "secret": False}, "uex_base_url": {"env": "UEX_BASE_URL", "type": "string", "secret": False}, "scmdb_base_url": {"env": "SCMDB_BASE_URL", "type": "string", "secret": False}, "cornerstone_base_url": {"env": "CORNERSTONE_BASE_URL", "type": "string", "secret": False}, + "openai_api_key": {"env": "OPENAI_API_KEY", "type": "string", "secret": True}, "uex_secret_key": {"env": "UEX_SECRET_KEY", "type": "string", "secret": True}, "uex_bearer_token": {"env": "UEX_BEARER_TOKEN", "type": "string", "secret": True}, "traderai_user_name": {"env": "TRADERAI_USER_NAME", "type": "string", "secret": False}, @@ -62,12 +66,16 @@ class Settings(BaseSettings): env_file_encoding="utf-8", ) + model_provider: str = "ollama" ollama_base_url: str = "http://localhost:11434" ollama_model: str = "qwen3.5:9b" ollama_num_ctx: int = 64512 + openai_base_url: str = "https://api.openai.com/v1" + openai_model: str = "gpt-5.3-codex" uex_base_url: str = "https://api.uexcorp.space/2.0" scmdb_base_url: str = "https://scmdb.net" cornerstone_base_url: str = "https://finder.cstone.space" + openai_api_key: str | None = Field(default=None) uex_secret_key: str | None = Field(default=None) uex_bearer_token: str | None = Field(default=None) traderai_user_name: str | None = Field(default=None) @@ -75,11 +83,17 @@ class Settings(BaseSettings): uex_notification_poll_seconds: int = 60 require_write_approval: bool = True - @field_validator("uex_secret_key", "uex_bearer_token", "traderai_user_name", mode="before") + @field_validator("openai_api_key", "uex_secret_key", "uex_bearer_token", "traderai_user_name", mode="before") @classmethod def _blank_optional(cls, value: Any) -> Any: return None if value == "" else value + @field_validator("model_provider", mode="before") + @classmethod + def _normalize_model_provider(cls, value: Any) -> str: + text = str(value or "ollama").strip().casefold() + return text if text in {"ollama", "openai"} else "ollama" + @field_validator("traderai_memory_path", mode="before") @classmethod def _blank_memory_path(cls, value: Any) -> Any: @@ -137,7 +151,7 @@ def save_settings(values: dict[str, Any]) -> dict[str, Any]: def _coerce_value(key: str, value: Any) -> Any: field_type = CONFIG_FIELDS[key]["type"] if value == "": - return None if key in {"uex_secret_key", "uex_bearer_token", "traderai_user_name"} else "" + return None if key in {"openai_api_key", "uex_secret_key", "uex_bearer_token", "traderai_user_name"} else "" if field_type == "integer": return int(value) if field_type == "boolean": diff --git a/traderai/server.py b/traderai/server.py index 383c7be..422a86d 100644 --- a/traderai/server.py +++ b/traderai/server.py @@ -39,6 +39,13 @@ def resource_path(*parts: str) -> Path: class ChatRequest(BaseModel): message: str thread_id: str | None = DEFAULT_THREAD_ID + images: list["ChatImageRequest"] = [] + + +class ChatImageRequest(BaseModel): + name: str = "pasted-image.png" + content_type: str = "image/png" + image_data: str class ChatThreadRequest(BaseModel): @@ -114,12 +121,14 @@ def create_app() -> FastAPI: plan_runner = ContinualPlanRunner(plan_store, tools, memory) tools.plan_runner = plan_runner agent = OllamaAgent( - settings.ollama_base_url, - settings.ollama_model, + settings.openai_base_url if settings.model_provider == "openai" else settings.ollama_base_url, + settings.openai_model if settings.model_provider == "openai" else settings.ollama_model, tools, memory=memory, user_name=settings.traderai_user_name, num_ctx=settings.ollama_num_ctx, + provider=settings.model_provider, + api_key=settings.openai_api_key, ) plan_runner.bind_agent(agent) scheduler.bind_agent(agent) @@ -171,6 +180,7 @@ def create_app() -> FastAPI: async def health() -> dict: return { "ollama": await agent.health(), + "model_provider": settings.model_provider, "user": memory.get_profile(), "jobs": scheduler.list_jobs(), "app_data_dir": settings_payload()["app_data_dir"], @@ -190,7 +200,19 @@ def create_app() -> FastAPI: @app.get("/api/ollama/status") async def ollama_status() -> dict: - return await inspect_ollama() + return await inspect_model_provider() + + @app.get("/api/openai/models") + async def openai_models() -> dict: + status = await inspect_openai() + return { + "provider": "openai", + "configured_model": status.get("configured_model"), + "models": status.get("models", []), + "message": status.get("message", ""), + "detail": status.get("detail", ""), + "online": status.get("online", False), + } @app.post("/api/ollama/launch") async def launch_ollama() -> dict: @@ -201,7 +223,7 @@ def create_app() -> FastAPI: popen_hidden(command) except OSError as exc: raise HTTPException(status_code=500, detail=f"Could not launch Ollama: {exc}") from exc - status = await inspect_ollama() + status = await inspect_model_provider() status["message"] = "Ollama launch requested." return status @@ -218,7 +240,7 @@ def create_app() -> FastAPI: popen_hidden([str(cli), "pull", model]) except OSError as exc: raise HTTPException(status_code=500, detail=f"Could not start model install: {exc}") from exc - status = await inspect_ollama() + status = await inspect_model_provider() status["message"] = f"Started installing model {model}." return status @@ -298,14 +320,22 @@ def create_app() -> FastAPI: @app.post("/api/chat") async def chat(request: ChatRequest) -> dict: try: - return await agent.chat(request.message, thread_id=request.thread_id) + return await agent.chat( + request.message, + thread_id=request.thread_id, + images=[image.model_dump() for image in request.images], + ) except OllamaUnavailable as exc: raise HTTPException(status_code=503, detail=str(exc)) from exc @app.post("/api/chat/stream") async def chat_stream(request: ChatRequest) -> StreamingResponse: async def events(): - async for event in agent.chat_events(request.message, thread_id=request.thread_id): + async for event in agent.chat_events( + request.message, + thread_id=request.thread_id, + images=[image.model_dump() for image in request.images], + ): yield f"data: {json.dumps(event)}\n\n" return StreamingResponse(events(), media_type="text/event-stream") @@ -475,6 +505,60 @@ def negotiation_identifier_params(identifier: str) -> dict[str, Any]: return {"hash": value} +async def inspect_model_provider() -> dict[str, Any]: + settings = get_settings() + if settings.model_provider == "openai": + return await inspect_openai() + return await inspect_ollama() + + +async def inspect_openai() -> dict[str, Any]: + settings = get_settings() + models: list[str] = [] + online = False + detail = "" + if not settings.openai_api_key: + return { + "installed": True, + "running": False, + "online": False, + "provider": "openai", + "model_available": False, + "configured_model": settings.openai_model, + "base_url": settings.openai_base_url, + "models": [], + "message": "OpenAI is selected, but no API key is configured.", + "detail": "", + } + + try: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get( + f"{settings.openai_base_url.rstrip('/')}/models", + headers={"Authorization": f"Bearer {settings.openai_api_key}"}, + ) + response.raise_for_status() + body = response.json() + online = True + models = sorted(item.get("id") for item in body.get("data", []) if item.get("id")) + except (httpx.HTTPError, ValueError) as exc: + detail = str(exc) + + model_available = settings.openai_model in models + return { + "installed": True, + "running": online, + "online": online, + "provider": "openai", + "model_available": model_available, + "configured_model": settings.openai_model, + "base_url": settings.openai_base_url, + "models": models, + "message": openai_status_message(online, bool(settings.openai_api_key), model_available, settings.openai_model), + "detail": detail, + } + + async def inspect_ollama() -> dict[str, Any]: settings = get_settings() executable = find_ollama_executable() @@ -500,6 +584,7 @@ async def inspect_ollama() -> dict[str, Any]: "installed": installed, "running": online, "online": online, + "provider": "ollama", "model_available": model_available, "configured_model": settings.ollama_model, "base_url": settings.ollama_base_url, @@ -514,6 +599,16 @@ async def inspect_ollama() -> dict[str, Any]: } +def openai_status_message(running: bool, configured: bool, model_available: bool, model: str) -> str: + if not configured: + return "OpenAI API key is not configured." + if not running: + return "OpenAI is not reachable with the configured key." + if not model_available: + return f'OpenAI is reachable, but model "{model}" was not returned by the API.' + return "OpenAI is ready." + + def ollama_status_message(installed: bool, running: bool, model_available: bool, model: str) -> str: if not installed: return "Ollama is not installed." diff --git a/traderai/tools.py b/traderai/tools.py index 6d6cb17..936506a 100644 --- a/traderai/tools.py +++ b/traderai/tools.py @@ -1,6 +1,8 @@ from __future__ import annotations import uuid +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass from typing import Any, Awaitable, Callable @@ -172,6 +174,7 @@ class ToolRegistry: self.plan_store = plan_store self.plan_runner = plan_runner self.pending_actions: dict[str, PendingAction] = {} + self._chat_images_var: ContextVar[list[dict[str, Any]]] = ContextVar("chat_images", default=[]) self.handlers: dict[str, ToolHandler] = { "search_marketplace_listings": self.search_marketplace_listings, "get_marketplace_listing": self.get_marketplace_listing, @@ -334,10 +337,19 @@ class ToolRegistry: "source": {"type": "string"}, "availability": {"type": "string"}, "in_stock": {"type": "integer"}, - "durability": {"type": "integer", "minimum": 0, "maximum": 100}, - "video_url": {"type": "string"}, - "image_data": {"type": "string", "description": "Base64 JPG or PNG image data for UEX upload."}, - "hours_expiration": {"type": "integer"}, + "durability": {"type": "integer", "minimum": 0, "maximum": 100}, + "video_url": {"type": "string"}, + "image_data": {"type": "string", "description": "Base64 JPG or PNG image data for UEX upload."}, + "use_attached_image": { + "type": "boolean", + "description": "When true, reuse an image pasted into the current chat as the listing image_data.", + }, + "attached_image_index": { + "type": "integer", + "minimum": 0, + "description": "Zero-based pasted image index to reuse when use_attached_image is true.", + }, + "hours_expiration": {"type": "integer"}, "is_hidden": {"type": "integer", "enum": [0, 1]}, "is_tv_allowed": {"type": "integer", "enum": [0, 1]}, "is_production": {"type": "integer", "enum": [0, 1], "default": 1}, @@ -495,6 +507,14 @@ class ToolRegistry: except Exception as exc: return {"error": str(exc)} + @contextmanager + def chat_image_scope(self, images: list[dict[str, Any]] | None): + token = self._chat_images_var.set(self._normalize_chat_images(images)) + try: + yield + finally: + self._chat_images_var.reset(token) + async def approve(self, action_id: str) -> dict[str, Any]: action = self.pending_actions.pop(action_id, None) if not action: @@ -1020,11 +1040,21 @@ class ToolRegistry: "language": {"type": "string", "default": "en_US"}, "location": {"type": "string"}, "source": {"type": "string", "enum": ["looted", "pledged", "purchased_in_game", "pirated", "gifted"]}, - "availability": {"type": "string"}, - "in_stock": {"type": "integer"}, - "durability": {"type": "integer", "minimum": 0, "maximum": 100}, - "video_url": {"type": "string"}, - "hours_expiration": {"type": "integer"}, + "availability": {"type": "string"}, + "in_stock": {"type": "integer"}, + "durability": {"type": "integer", "minimum": 0, "maximum": 100}, + "video_url": {"type": "string"}, + "image_data": {"type": "string", "description": "Base64 JPG or PNG image data for UEX upload."}, + "use_attached_image": { + "type": "boolean", + "description": "When true, reuse an image pasted into the current chat as the listing image_data instead of sourcing from Cornerstone.", + }, + "attached_image_index": { + "type": "integer", + "minimum": 0, + "description": "Zero-based pasted image index to reuse when use_attached_image is true.", + }, + "hours_expiration": {"type": "integer"}, "is_hidden": {"type": "integer", "enum": [0, 1]}, "is_tv_allowed": {"type": "integer", "enum": [0, 1]}, "is_production": {"type": "integer", "enum": [0, 1], "default": 1}, @@ -1225,7 +1255,15 @@ class ToolRegistry: return self._pending("Send negotiation message", "marketplace_negotiations_messages", payload, metadata=metadata) async def draft_marketplace_listing(self, **payload: Any) -> dict[str, Any]: - return self._pending("Post marketplace listing", "marketplace_advertise", payload) + attached_image = self._attach_chat_image(payload) + if attached_image.get("error"): + return {"error": attached_image["error"]} + return self._pending( + "Post marketplace listing", + "marketplace_advertise", + payload, + metadata=attached_image.get("metadata"), + ) async def draft_marketplace_listing_with_cornerstone_image( self, @@ -1234,6 +1272,9 @@ class ToolRegistry: **payload: Any, ) -> dict[str, Any]: require_image = bool(payload.pop("require_image", False)) + attached_image = self._attach_chat_image(payload) + if attached_image.get("error"): + return {"error": attached_image["error"]} item = await self._resolve_cornerstone_item(id=cornerstone_id, query=item_query) if not item: return {"error": "No Cornerstone item matched. Provide cornerstone_id or a more specific item_query."} @@ -1250,9 +1291,9 @@ class ToolRegistry: except Exception as exc: image_error = str(exc) - if image_result: + if image_result and not payload.get("image_data"): payload["image_data"] = image_result["image_data"] - elif require_image: + elif require_image and not payload.get("image_data"): return { "error": "Cornerstone item matched, but no usable JPG/PNG image could be sourced.", "cornerstone": { @@ -1271,9 +1312,11 @@ class ToolRegistry: "cornerstone_image_url": image_result.get("url") if image_result else None, "cornerstone_image_content_type": image_result.get("content_type") if image_result else None, "cornerstone_image_size_bytes": image_result.get("size_bytes") if image_result else None, - "cornerstone_image_status": "included" if image_result else "not_found", + "cornerstone_image_status": "user_attached" if attached_image.get("metadata") else ("included" if image_result else "not_found"), "cornerstone_image_error": image_error or None, } + if attached_image.get("metadata"): + metadata.update(attached_image["metadata"]) return self._pending("Post marketplace listing with Cornerstone image", "marketplace_advertise", payload, metadata=metadata) async def remember_user_fact(self, content: str, kind: str = "note", importance: int = 3) -> dict[str, Any]: @@ -1625,6 +1668,48 @@ class ToolRegistry: display["image_data"] = f"" return display + def _attach_chat_image(self, payload: dict[str, Any]) -> dict[str, Any]: + attached_index = payload.pop("attached_image_index", None) + use_attached_image = bool(payload.pop("use_attached_image", False) or attached_index is not None) + if payload.get("image_data") or not use_attached_image: + return {} + image = self._chat_image(attached_index or 0) + if not image: + return {"error": "No pasted chat image is available at the requested attached_image_index."} + payload["image_data"] = image["image_data"] + return { + "metadata": { + "attached_chat_image_name": image.get("name"), + "attached_chat_image_content_type": image.get("content_type"), + "attached_chat_image_index": attached_index or 0, + "attached_chat_image_status": "included", + } + } + + def _chat_image(self, index: int) -> dict[str, Any] | None: + images = self._chat_images_var.get() + if 0 <= index < len(images): + return images[index] + return None + + @staticmethod + def _normalize_chat_images(images: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + for image in images or []: + if not isinstance(image, dict): + continue + image_data = str(image.get("image_data") or "").strip() + if not image_data: + continue + normalized.append( + { + "name": str(image.get("name") or "").strip() or "pasted-image.png", + "content_type": str(image.get("content_type") or "image/png").strip() or "image/png", + "image_data": image_data, + } + ) + return normalized + @staticmethod def _int_or_none(value: Any) -> int | None: try: diff --git a/traderai/version.py b/traderai/version.py index c1d94eb..26e3669 100644 --- a/traderai/version.py +++ b/traderai/version.py @@ -1,6 +1,6 @@ from __future__ import annotations -__version__ = "0.0.5" +__version__ = "0.0.6" RELEASES_URL = "https://git.hudsonriggs.systems/LambdaBankingConglomerate/TraderAI/releases" RELEASES_API_URL = "https://git.hudsonriggs.systems/api/v1/repos/LambdaBankingConglomerate/TraderAI/releases" @@ -11,3 +11,4 @@ RELEASES_API_URL = "https://git.hudsonriggs.systems/api/v1/repos/LambdaBankingCo + diff --git a/uv.lock b/uv.lock index fc85d92..5503199 100644 --- a/uv.lock +++ b/uv.lock @@ -755,7 +755,7 @@ wheels = [ [[package]] name = "traderai" -version = "0.0.5" +version = "0.0.6" source = { virtual = "." } dependencies = [ { name = "apscheduler" }, @@ -1051,3 +1051,4 @@ wheels = [ + diff --git a/web/app.js b/web/app.js index 72e04ef..b593c17 100644 --- a/web/app.js +++ b/web/app.js @@ -1,5 +1,6 @@ const form = document.getElementById("chat-form"); const input = document.getElementById("message-input"); +const composerImagesEl = document.getElementById("composer-images"); const messages = document.getElementById("messages"); const statusEl = document.getElementById("status"); const pendingEl = document.getElementById("pending-actions"); @@ -25,6 +26,7 @@ const ollamaDownloadButton = document.getElementById("ollama-download"); const ollamaInstallButton = document.getElementById("ollama-install"); const ollamaLaunchButton = document.getElementById("ollama-launch"); const ollamaPullButton = document.getElementById("ollama-pull"); +const openaiModelsRefreshButton = document.getElementById("openai-models-refresh"); const ollamaStatusEl = document.getElementById("ollama-status"); const ollamaMessageEl = document.getElementById("ollama-message"); const updateCheckButton = document.getElementById("update-check"); @@ -61,25 +63,53 @@ let latestUpdate = null; let currentThreadId = "default"; let currentNegotiationId = null; let latestOllamaStatus = null; +let composerImages = []; const clickedOllamaActions = new Set(); if (window.lucide) { window.lucide.createIcons(); } -function addMessage(role, text) { +function addMessage(role, text, options = {}) { const node = document.createElement("div"); node.className = `message ${role}`; - setMessageMarkdown(node, text); + setMessageMarkdown(node, text, options); messages.appendChild(node); messages.scrollTop = messages.scrollHeight; return node; } -function setMessageMarkdown(node, text) { +function setMessageMarkdown(node, text, options = {}) { const body = node.querySelector(".message-body") || node; - body.innerHTML = renderMarkdown(text); - enhanceNegotiationLinks(body); + body.innerHTML = ""; + const attachedImages = options.images || []; + if (attachedImages.length) { + body.appendChild(renderImageGallery(attachedImages)); + } + if (text) { + const markdown = document.createElement("div"); + markdown.innerHTML = renderMarkdown(text); + body.appendChild(markdown); + enhanceNegotiationLinks(markdown); + } +} + +function renderImageGallery(images) { + const gallery = document.createElement("div"); + gallery.className = "message-images"; + for (const image of images) { + const card = document.createElement("div"); + card.className = "message-image"; + const preview = document.createElement("img"); + preview.src = image.preview_url || `data:${image.content_type || "image/png"};base64,${image.image_data}`; + preview.alt = image.name || "Attached image"; + const label = document.createElement("span"); + label.className = "message-image-label"; + label.textContent = image.name || "Attached image"; + card.append(preview, label); + gallery.appendChild(card); + } + return gallery; } function setMessageActivity(node, text, active = false) { @@ -459,6 +489,74 @@ function escapeHtml(text) { .replace(/'/g, "'"); } +function composerImageId() { + if (window.crypto?.randomUUID) return window.crypto.randomUUID(); + return `image-${Date.now()}-${Math.random().toString(16).slice(2)}`; +} + +function readFileAsDataUrl(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result || "")); + reader.onerror = () => reject(reader.error || new Error(`Could not read ${file.name || "image"}`)); + reader.readAsDataURL(file); + }); +} + +async function addComposerImages(files) { + const additions = []; + for (const file of files) { + if (!file || !String(file.type || "").startsWith("image/")) continue; + const previewUrl = await readFileAsDataUrl(file); + const [, imageData = ""] = previewUrl.split(",", 2); + if (!imageData) continue; + additions.push({ + id: composerImageId(), + name: file.name || `pasted-image-${composerImages.length + additions.length + 1}.png`, + content_type: file.type || "image/png", + image_data: imageData, + preview_url: previewUrl, + }); + } + if (!additions.length) return; + composerImages = [...composerImages, ...additions]; + renderComposerImages(); +} + +function removeComposerImage(imageId) { + composerImages = composerImages.filter((image) => image.id !== imageId); + renderComposerImages(); +} + +function clearComposerImages() { + composerImages = []; + renderComposerImages(); +} + +function renderComposerImages() { + if (!composerImagesEl) return; + composerImagesEl.innerHTML = ""; + composerImagesEl.hidden = !composerImages.length; + for (const image of composerImages) { + const card = document.createElement("div"); + card.className = "composer-image"; + const preview = document.createElement("img"); + preview.src = image.preview_url; + preview.alt = image.name || "Pasted image"; + const remove = document.createElement("button"); + remove.type = "button"; + remove.className = "composer-image-remove"; + remove.textContent = "×"; + remove.title = "Remove image"; + remove.addEventListener("click", () => removeComposerImage(image.id)); + const label = document.createElement("span"); + label.className = "composer-image-name"; + label.textContent = image.name || "Pasted image"; + card.append(preview, remove, label); + composerImagesEl.appendChild(card); + } +} + function formatMetrics(event) { const read = formatTokenMetric(event.reading_tokens, event.reading_tokens_per_second); const wrote = formatTokenMetric(event.writing_tokens, event.writing_tokens_per_second); @@ -494,9 +592,13 @@ const configFieldIds = { }; const ollamaFieldIds = { + model_provider: "model-provider", ollama_base_url: "ollama-base-url", ollama_model: "ollama-model", ollama_num_ctx: "ollama-num-ctx", + openai_base_url: "openai-base-url", + openai_api_key: "openai-api-key", + openai_model: "openai-model", }; async function refreshConfig() { @@ -527,7 +629,12 @@ function renderConfig(config) { for (const [key, id] of Object.entries(ollamaFieldIds)) { const field = document.getElementById(id); if (!field) continue; - field.value = values[key] ?? ""; + if (field.type === "password") { + field.value = ""; + field.placeholder = secretsConfigured[key] ? "Configured" : ""; + } else { + field.value = values[key] ?? ""; + } } configPathsEl.textContent = `App data: ${config.app_data_dir}\nConfig: ${config.config_path}\nLog: ${config.log_path}\nEdge profile: ${config.edge_profile_dir}`; configStatusEl.textContent = ""; @@ -565,7 +672,7 @@ async function saveOllamaConfig(event) { if (!field) continue; values[key] = field.value; } - setOllamaMessage("Saving Ollama config"); + setOllamaMessage("Saving provider config"); try { const response = await fetch("/api/config", { method: "POST", @@ -577,46 +684,71 @@ async function saveOllamaConfig(event) { setOllamaMessage(result.message || "Saved"); await refreshOllamaStatus(); } catch (error) { - setOllamaMessage(`Ollama config save failed: ${fetchErrorMessage(error)}`); + setOllamaMessage(`Provider config save failed: ${fetchErrorMessage(error)}`); } } async function refreshOllamaStatus() { if (!ollamaStatusEl) return; - ollamaStatusEl.textContent = "Checking Ollama"; + ollamaStatusEl.textContent = "Checking provider"; try { const response = await fetch("/api/ollama/status"); const status = await response.json(); renderOllamaStatus(status); } catch (error) { - ollamaStatusEl.textContent = `Ollama status failed: ${error.message}`; + ollamaStatusEl.textContent = `Provider status failed: ${error.message}`; } } function renderOllamaStatus(status) { if (!ollamaStatusEl) return; latestOllamaStatus = status; + const provider = status.provider === "openai" ? "OpenAI" : "Ollama"; const models = status.models?.length ? status.models.join(", ") : "None detected"; - const pillClass = status.installed && status.running && status.model_available ? "status-pill" : "status-pill warning"; + const ready = status.provider === "openai" + ? Boolean(status.online && status.model_available) + : Boolean(status.installed && status.running && status.model_available); + const pillClass = ready ? "status-pill" : "status-pill warning"; + const detailItems = [ + ollamaStatusItem("Provider", provider), + ollamaStatusItem("Model", status.configured_model || ""), + ollamaStatusItem("URL", status.base_url || ""), + ]; + if (status.provider !== "openai") { + detailItems.splice(1, 0, ollamaStatusItem("Installed", status.installed ? "Yes" : "No")); + detailItems.splice(2, 0, ollamaStatusItem("Running", status.running ? "Yes" : "No")); + detailItems.push(ollamaStatusItem("Pulled", status.model_available ? "Yes" : "No")); + if (status.can_auto_install) detailItems.push(ollamaStatusItem("Auto Install", "Available")); + if (status.num_ctx) detailItems.push(ollamaStatusItem("Context", status.num_ctx)); + } else { + detailItems.splice(1, 0, ollamaStatusItem("Connected", status.online ? "Yes" : "No")); + } ollamaStatusEl.innerHTML = `
${escapeHtml(status.message || "Unknown")}
- ${ollamaStatusItem("Installed", status.installed ? "Yes" : "No")} - ${ollamaStatusItem("Running", status.running ? "Yes" : "No")} - ${ollamaStatusItem("Model", status.configured_model || "")} - ${ollamaStatusItem("Pulled", status.model_available ? "Yes" : "No")} - ${ollamaStatusItem("URL", status.base_url || "")} - ${status.can_auto_install ? ollamaStatusItem("Auto Install", "Available") : ""} + ${detailItems.join("")}
- ${ollamaStatusItem("Installed Models", models)} + ${ollamaStatusItem(status.provider === "openai" ? "Available Models" : "Installed Models", models)} ${status.detail ? ollamaStatusItem("Detail", status.detail) : ""} `; + if (ollamaDownloadButton) ollamaDownloadButton.hidden = status.provider === "openai"; if (ollamaInstallButton) { - ollamaInstallButton.hidden = !status.can_auto_install; + ollamaInstallButton.hidden = status.provider === "openai" || !status.can_auto_install; ollamaInstallButton.disabled = Boolean(status.installed) || !status.can_auto_install; } - if (ollamaLaunchButton) ollamaLaunchButton.disabled = !status.installed || Boolean(status.running); - if (ollamaPullButton) ollamaPullButton.disabled = !status.running || Boolean(status.model_available); + if (ollamaLaunchButton) { + ollamaLaunchButton.hidden = status.provider === "openai"; + ollamaLaunchButton.disabled = !status.installed || Boolean(status.running); + } + if (ollamaPullButton) { + ollamaPullButton.hidden = status.provider === "openai"; + ollamaPullButton.disabled = !status.running || Boolean(status.model_available); + } + if (openaiModelsRefreshButton) { + openaiModelsRefreshButton.hidden = status.provider !== "openai"; + openaiModelsRefreshButton.disabled = false; + } + renderProviderModelOptions(status.models || []); updateOllamaAttention(status); } @@ -659,12 +791,15 @@ function setOllamaButtonAttention(button, action, active) { function updateOllamaAttention(status = null) { const currentStatus = status || latestOllamaStatus; if (!currentStatus) return; - const ready = Boolean(currentStatus.installed && currentStatus.running && currentStatus.model_available); + const ready = currentStatus.provider === "openai" + ? Boolean(currentStatus.online && currentStatus.model_available) + : Boolean(currentStatus.installed && currentStatus.running && currentStatus.model_available); ollamaToggle?.classList.toggle("attention-pulse", !ready); - setOllamaButtonAttention(ollamaDownloadButton, "download", !currentStatus.installed); - setOllamaButtonAttention(ollamaInstallButton, "install", !currentStatus.installed && currentStatus.can_auto_install); - setOllamaButtonAttention(ollamaLaunchButton, "launch", currentStatus.installed && !currentStatus.running); - setOllamaButtonAttention(ollamaPullButton, "pull", currentStatus.running && !currentStatus.model_available); + setOllamaButtonAttention(ollamaDownloadButton, "download", currentStatus.provider !== "openai" && !currentStatus.installed); + setOllamaButtonAttention(ollamaInstallButton, "install", currentStatus.provider !== "openai" && !currentStatus.installed && currentStatus.can_auto_install); + setOllamaButtonAttention(ollamaLaunchButton, "launch", currentStatus.provider !== "openai" && currentStatus.installed && !currentStatus.running); + setOllamaButtonAttention(ollamaPullButton, "pull", currentStatus.provider !== "openai" && currentStatus.running && !currentStatus.model_available); + setOllamaButtonAttention(openaiModelsRefreshButton, "openai-models", currentStatus.provider === "openai" && !currentStatus.model_available); if (ready) clickedOllamaActions.clear(); } @@ -672,6 +807,31 @@ function configuredOllamaModel() { return document.getElementById("ollama-model")?.value || ""; } +function renderProviderModelOptions(models) { + const datalist = document.getElementById("provider-models"); + if (!datalist) return; + datalist.innerHTML = ""; + for (const model of models) { + const option = document.createElement("option"); + option.value = model; + datalist.appendChild(option); + } +} + +async function refreshOpenAIModels() { + setOllamaMessage("Loading OpenAI models"); + try { + const response = await fetch("/api/openai/models"); + const result = await response.json(); + if (!response.ok) throw new Error(result.detail || `HTTP ${response.status}`); + renderProviderModelOptions(result.models || []); + setOllamaMessage(result.message || "Loaded OpenAI models"); + await refreshOllamaStatus(); + } catch (error) { + setOllamaMessage(`OpenAI models failed: ${fetchErrorMessage(error)}`); + } +} + async function checkForUpdate(promptUser = false) { if (!updateStatusEl) return; updateStatusEl.textContent = "Checking releases"; @@ -1234,15 +1394,17 @@ async function checkHealth() { const response = await fetch("/api/health"); const result = await response.json(); const health = result.ollama || {}; + const provider = health.provider === "openai" ? "OpenAI" : "Ollama"; ollamaOnline = Boolean(health.online); if (!ollamaOnline) { statusEl.textContent = "Offline"; - setWarning("Ollama needs attention. Open the Ollama tab and use the pulsing action button."); + setWarning(`${provider} needs attention. Open the model provider tab and use the pulsing action button.`); ollamaToggle?.classList.add("attention-pulse"); return false; } if (health.model_available === false) { - setWarning(`Ollama needs the configured model "${health.model}". Open the Ollama tab and use Install Model.`); + const action = health.provider === "openai" ? "Load OpenAI Models." : "Install Model."; + setWarning(`${provider} needs the configured model "${health.model}". Open the model provider tab and use ${action}`); ollamaToggle?.classList.add("attention-pulse"); } else { setWarning(""); @@ -1253,7 +1415,7 @@ async function checkHealth() { } catch (error) { ollamaOnline = false; statusEl.textContent = "Offline"; - setWarning("Could not check Ollama health. Open the Ollama tab and use the pulsing action button."); + setWarning("Could not check the active model provider. Open the model provider tab and use the pulsing action button."); ollamaToggle?.classList.add("attention-pulse"); return false; } @@ -1426,6 +1588,23 @@ input.addEventListener("keydown", async (event) => { } }); +input.addEventListener("paste", async (event) => { + const clipboardItems = [...(event.clipboardData?.items || [])]; + const imageFiles = clipboardItems + .filter((item) => item.kind === "file" && String(item.type || "").startsWith("image/")) + .map((item) => item.getAsFile()) + .filter(Boolean); + if (!imageFiles.length) return; + if (!event.clipboardData?.getData("text/plain")) { + event.preventDefault(); + } + try { + await addComposerImages(imageFiles); + } catch (error) { + setWarning(`Image paste failed: ${fetchErrorMessage(error)}`); + } +}); + memoryRefreshButton?.addEventListener("click", refreshMemory); memoryClearButton?.addEventListener("click", clearMemory); configRefreshButton?.addEventListener("click", refreshConfig); @@ -1458,6 +1637,10 @@ ollamaPullButton?.addEventListener("click", () => { markOllamaActionClicked("pull"); postOllamaAction("/api/ollama/pull", { body: { model: configuredOllamaModel() } }); }); +openaiModelsRefreshButton?.addEventListener("click", () => { + markOllamaActionClicked("openai-models"); + refreshOpenAIModels(); +}); updateCheckButton?.addEventListener("click", checkForUpdate); updateInstallButton?.addEventListener("click", installUpdate); updateOpenReleasesButton?.addEventListener("click", openReleasesPage); @@ -1471,15 +1654,22 @@ updateModalInstall?.addEventListener("click", installUpdate); async function sendMessage() { const message = input.value.trim(); - if (!message || input.disabled) return; + const attachedImages = composerImages.map(({ name, content_type, image_data, preview_url }) => ({ + name, + content_type, + image_data, + preview_url, + })); + if ((!message && !attachedImages.length) || input.disabled) return; const healthy = await checkHealth(); if (!healthy) { - addMessage("assistant warning-message", "Ollama needs attention before chat can continue. Open the Ollama tab and press the pulsing action button, then try again."); + addMessage("assistant warning-message", "The active model provider needs attention before chat can continue. Open the model provider tab and press the pulsing action button, then try again."); return; } input.value = ""; + clearComposerImages(); input.disabled = true; - addMessage("user", message); + addMessage("user", message, { images: attachedImages }); const assistantNode = addMessage("assistant streaming", ""); ensureStreamingChrome(assistantNode); let assistantText = ""; @@ -1491,7 +1681,7 @@ async function sendMessage() { const response = await fetch("/api/chat/stream", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ message, thread_id: currentThreadId }), + body: JSON.stringify({ message, thread_id: currentThreadId, images: attachedImages }), }); if (!response.ok || !response.body) { throw new Error(`HTTP ${response.status}`); @@ -1537,7 +1727,7 @@ async function sendMessage() { } } catch (error) { const message = error.message.includes("503") - ? "Ollama needs attention before chat can continue. Open the Ollama tab and press the pulsing action button, then try again." + ? "The active model provider needs attention before chat can continue. Open the model provider tab and press the pulsing action button, then try again." : `Chat failed: ${error.message}`; setWarning(message); setMessageMarkdown(assistantNode, message); diff --git a/web/index.html b/web/index.html index 4c6e48a..b52f702 100644 --- a/web/index.html +++ b/web/index.html @@ -57,12 +57,15 @@
-
-
- - -
-
+
+
+
+ + +
+ +
+
diff --git a/web/styles.css b/web/styles.css index e6c9cdb..868a5a5 100644 --- a/web/styles.css +++ b/web/styles.css @@ -269,6 +269,8 @@ body::before { } .actions { + display: flex; + flex-direction: column; padding: 28px; overflow: auto; min-height: 0; @@ -555,6 +557,38 @@ h2 { background: rgba(255, 250, 240, 0.96); } +.message-images { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(132px, 1fr)); + gap: 10px; + margin-bottom: 12px; +} + +.message-image { + overflow: hidden; + border: 1px solid rgba(88, 66, 47, 0.18); + border-radius: 14px; + background: rgba(255, 255, 255, 0.78); +} + +.message-image img { + display: block; + width: 100%; + aspect-ratio: 1 / 1; + object-fit: cover; +} + +.message-image-label { + display: block; + padding: 8px 10px; + color: #6d5b4e; + font-size: 12px; + font-weight: 700; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + .message.warning-message { border-color: rgba(212, 175, 55, 0.6); background: #f5eac4; @@ -720,6 +754,60 @@ h2 { padding: 20px; } +.composer-main { + display: grid; + gap: 12px; +} + +.composer-images { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 10px; +} + +.composer-image { + position: relative; + overflow: hidden; + border: 1px solid rgba(88, 66, 47, 0.16); + border-radius: 14px; + background: rgba(255, 255, 255, 0.88); + box-shadow: 0 12px 26px rgba(38, 58, 27, 0.08); +} + +.composer-image img { + display: block; + width: 100%; + aspect-ratio: 1 / 1; + object-fit: cover; +} + +.composer-image-name { + display: block; + padding: 8px 10px 10px; + color: #6d5b4e; + font-size: 12px; + font-weight: 700; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.composer-image-remove { + position: absolute; + top: 8px; + right: 8px; + width: 28px; + height: 28px; + min-height: 28px; + padding: 0; + border-radius: 999px; + border: 1px solid rgba(88, 66, 47, 0.18); + background: rgba(255, 250, 240, 0.92); + color: var(--brown); + font-size: 16px; + line-height: 1; +} + textarea { width: 100%; min-height: 58px; @@ -1005,7 +1093,7 @@ button.secondary { } .side-section { - margin-bottom: 28px; + margin-bottom: 0; } .side-section + .side-section { @@ -1014,8 +1102,14 @@ button.secondary { } .sidebar-tools { - display: grid; + display: flex; + flex-direction: column; gap: 14px; + margin-top: auto; + position: sticky; + bottom: -28px; + padding-bottom: 28px; + background: linear-gradient(180deg, rgba(247, 241, 220, 0) 0%, var(--cream) 22%, var(--cream) 100%); } .sidebar-tool-buttons { @@ -1110,8 +1204,8 @@ button.secondary { } .sidebar-panel { - padding-top: 12px; - border-top: 1px solid var(--line); + padding-bottom: 12px; + border-bottom: 1px solid var(--line); } .config-form {