feat(instance-chat): full multimodal attachment support via OpenClaw bridge

After verifying that the OpenClaw gateway's chat.send WebSocket RPC
accepts an 'attachments' array (confirmed from openclaw/openclaw source
and documentation), implement end-to-end image/file attachment support
for instance chat:

Bridge (openclaw-client.ts):
- chatSendAndWait() now accepts optional `attachments[]` parameter
- Passes attachments to chat.send RPC only when non-empty

Bridge (index.ts):
- /task-async accepts `attachments[]` from request body
- Forwards to chatSendAndWait unchanged

Backend (agent.controller.ts):
- executeInstanceTask() accepts IT0 attachment format
  { base64Data, mediaType, fileName? }
- Converts to OpenClaw format { name, mimeType, media: "data:..." }
- Saves attachments to conversation history via contextService
- Forwards to bridge via bridgeAttachments spread

Flutter (agent_instance_chat_remote_datasource.dart):
- createTask() now includes attachments in POST body when present

Flutter (chat_page.dart):
- Reverted Fix 5 (disabled button) — attachment button fully enabled
  in instance mode since the bridge now supports it

Attachment format (OpenClaw wire):
  { name: string, mimeType: string, media: "data:<mime>;base64,<data>" }

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-03-09 21:18:14 -07:00
parent ea3cbf64a5
commit c9ee93fffd
5 changed files with 39 additions and 18 deletions

View File

@ -29,7 +29,7 @@ class AgentInstanceChatDatasource implements ChatRemoteDatasource {
data: { data: {
'prompt': message, 'prompt': message,
if (sessionId != 'new') 'sessionId': sessionId, if (sessionId != 'new') 'sessionId': sessionId,
// Note: attachments are not yet supported for instance chat if (attachments != null && attachments.isNotEmpty) 'attachments': attachments,
}, },
); );
return response.data as Map<String, dynamic>; return response.data as Map<String, dynamic>;

View File

@ -720,16 +720,10 @@ class _ChatPageState extends ConsumerState<ChatPage> {
if (!isStreaming) if (!isStreaming)
Padding( Padding(
padding: const EdgeInsets.only(left: 4), padding: const EdgeInsets.only(left: 4),
child: Tooltip( child: IconButton(
message: widget.agentName != null icon: const Icon(Icons.add_circle_outline, size: 22),
? '附件功能暂不支持智能体对话' tooltip: AppLocalizations.of(context).chatAddImageTooltip,
: AppLocalizations.of(context).chatAddImageTooltip, onPressed: isAwaitingApproval ? null : _showAttachmentOptions,
child: IconButton(
icon: const Icon(Icons.add_circle_outline, size: 22),
onPressed: (isAwaitingApproval || widget.agentName != null)
? null
: _showAttachmentOptions,
),
), ),
), ),
Expanded( Expanded(

View File

@ -145,6 +145,12 @@ app.post('/task-async', async (req, res) => {
const timeoutSeconds: number = req.body.timeoutSeconds ?? 120; // 2 min default for async tasks const timeoutSeconds: number = req.body.timeoutSeconds ?? 120; // 2 min default for async tasks
const idempotencyKey: string = req.body.idempotencyKey ?? crypto.randomUUID(); const idempotencyKey: string = req.body.idempotencyKey ?? crypto.randomUUID();
const callbackData = req.body.callbackData ?? {}; const callbackData = req.body.callbackData ?? {};
// Optional attachments — passed through to OpenClaw chat.send unchanged.
// Expected format: [{ name, mimeType, media }] where media is a data-URI.
const attachments: Array<{ name: string; mimeType: string; media: string }> | undefined =
Array.isArray(req.body.attachments) && req.body.attachments.length > 0
? req.body.attachments
: undefined;
// Return immediately — LLM runs in background // Return immediately — LLM runs in background
res.json({ ok: true, pending: true }); res.json({ ok: true, pending: true });
@ -162,6 +168,7 @@ app.post('/task-async', async (req, res) => {
message: req.body.prompt, message: req.body.prompt,
idempotencyKey, idempotencyKey,
timeoutMs: timeoutSeconds * 1000, timeoutMs: timeoutSeconds * 1000,
attachments,
}).then((reply: string) => { }).then((reply: string) => {
postCallback({ ok: true, result: reply, callbackData }); postCallback({ ok: true, result: reply, callbackData });
}).catch((err: Error) => { }).catch((err: Error) => {

View File

@ -298,17 +298,23 @@ export class OpenClawClient {
message: string; message: string;
idempotencyKey: string; idempotencyKey: string;
timeoutMs?: number; timeoutMs?: number;
/** Optional media attachments (images, PDFs, etc.) in OpenClaw format. */
attachments?: Array<{ name: string; mimeType: string; media: string }>;
}): Promise<string> { }): Promise<string> {
const timeoutMs = params.timeoutMs ?? 30_000; const timeoutMs = params.timeoutMs ?? 30_000;
// Send chat.send — resolves immediately with { runId, status: "started" } // Send chat.send — resolves immediately with { runId, status: "started" }
const chatSendParams: Record<string, unknown> = {
sessionKey: params.sessionKey,
message: params.message,
idempotencyKey: params.idempotencyKey,
};
if (params.attachments && params.attachments.length > 0) {
chatSendParams['attachments'] = params.attachments;
}
const ack = await this.rpc( const ack = await this.rpc(
'chat.send', 'chat.send',
{ chatSendParams,
sessionKey: params.sessionKey,
message: params.message,
idempotencyKey: params.idempotencyKey,
},
10_000, // 10s for the initial ack 10_000, // 10s for the initial ack
) as { runId: string; status: string }; ) as { runId: string; status: string };

View File

@ -827,7 +827,11 @@ export class AgentController {
@TenantId() tenantId: string, @TenantId() tenantId: string,
@Req() req: any, @Req() req: any,
@Param('instanceId') instanceId: string, @Param('instanceId') instanceId: string,
@Body() body: { prompt: string; sessionId?: string }, @Body() body: {
prompt: string;
sessionId?: string;
attachments?: Array<{ base64Data: string; mediaType: string; fileName?: string }>;
},
) { ) {
const instance = await this.instanceRepository.findById(instanceId); const instance = await this.instanceRepository.findById(instanceId);
if (!instance) throw new NotFoundException(`Instance ${instanceId} not found`); if (!instance) throw new NotFoundException(`Instance ${instanceId} not found`);
@ -883,7 +887,7 @@ export class AgentController {
await this.taskRepository.save(task); await this.taskRepository.save(task);
// Persist user message for display in conversation history // Persist user message for display in conversation history
await this.contextService.saveUserMessage(session.id, body.prompt); await this.contextService.saveUserMessage(session.id, body.prompt, body.attachments);
// The OpenClaw bridge tracks conversation context internally via sessionKey. // The OpenClaw bridge tracks conversation context internally via sessionKey.
// We use our DB session ID as the key so each session has isolated context. // We use our DB session ID as the key so each session has isolated context.
@ -891,6 +895,15 @@ export class AgentController {
const callbackUrl = `${process.env.AGENT_SERVICE_PUBLIC_URL}/api/v1/agent/instances/openclaw-app-callback`; const callbackUrl = `${process.env.AGENT_SERVICE_PUBLIC_URL}/api/v1/agent/instances/openclaw-app-callback`;
const bridgeUrl = `http://${instance.serverHost}:${instance.hostPort}/task-async`; const bridgeUrl = `http://${instance.serverHost}:${instance.hostPort}/task-async`;
// Convert IT0 attachment format → OpenClaw format for the bridge.
// IT0: { base64Data, mediaType, fileName? }
// OpenClaw: { name, mimeType, media: "data:<mimeType>;base64,<data>" }
const bridgeAttachments = body.attachments?.map((att) => ({
name: att.fileName ?? 'attachment',
mimeType: att.mediaType,
media: `data:${att.mediaType};base64,${att.base64Data}`,
}));
this.logger.log( this.logger.log(
`[Task ${task.id}] Routing to OpenClaw instance ${instanceId} @ ${bridgeUrl}, session=${session.id}`, `[Task ${task.id}] Routing to OpenClaw instance ${instanceId} @ ${bridgeUrl}, session=${session.id}`,
); );
@ -946,6 +959,7 @@ export class AgentController {
idempotencyKey: task.id, idempotencyKey: task.id,
callbackUrl, callbackUrl,
callbackData: { sessionId: session.id, taskId: task.id }, callbackData: { sessionId: session.id, taskId: task.id },
...(bridgeAttachments && bridgeAttachments.length > 0 && { attachments: bridgeAttachments }),
}), }),
}) })
.then(async (res) => { .then(async (res) => {