feat(instance-chat): full multimodal attachment support via OpenClaw bridge
After verifying that the OpenClaw gateway's chat.send WebSocket RPC
accepts an 'attachments' array (confirmed from openclaw/openclaw source
and documentation), implement end-to-end image/file attachment support
for instance chat:
Bridge (openclaw-client.ts):
- chatSendAndWait() now accepts optional `attachments[]` parameter
- Passes attachments to chat.send RPC only when non-empty
Bridge (index.ts):
- /task-async accepts `attachments[]` from request body
- Forwards to chatSendAndWait unchanged
Backend (agent.controller.ts):
- executeInstanceTask() accepts IT0 attachment format
{ base64Data, mediaType, fileName? }
- Converts to OpenClaw format { name, mimeType, media: "data:..." }
- Saves attachments to conversation history via contextService
- Forwards to bridge via bridgeAttachments spread
Flutter (agent_instance_chat_remote_datasource.dart):
- createTask() now includes attachments in POST body when present
Flutter (chat_page.dart):
- Reverted Fix 5 (disabled button) — attachment button fully enabled
in instance mode since the bridge now supports it
Attachment format (OpenClaw wire):
{ name: string, mimeType: string, media: "data:<mime>;base64,<data>" }
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
ea3cbf64a5
commit
c9ee93fffd
|
|
@ -29,7 +29,7 @@ class AgentInstanceChatDatasource implements ChatRemoteDatasource {
|
||||||
data: {
|
data: {
|
||||||
'prompt': message,
|
'prompt': message,
|
||||||
if (sessionId != 'new') 'sessionId': sessionId,
|
if (sessionId != 'new') 'sessionId': sessionId,
|
||||||
// Note: attachments are not yet supported for instance chat
|
if (attachments != null && attachments.isNotEmpty) 'attachments': attachments,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
return response.data as Map<String, dynamic>;
|
return response.data as Map<String, dynamic>;
|
||||||
|
|
|
||||||
|
|
@ -720,16 +720,10 @@ class _ChatPageState extends ConsumerState<ChatPage> {
|
||||||
if (!isStreaming)
|
if (!isStreaming)
|
||||||
Padding(
|
Padding(
|
||||||
padding: const EdgeInsets.only(left: 4),
|
padding: const EdgeInsets.only(left: 4),
|
||||||
child: Tooltip(
|
|
||||||
message: widget.agentName != null
|
|
||||||
? '附件功能暂不支持智能体对话'
|
|
||||||
: AppLocalizations.of(context).chatAddImageTooltip,
|
|
||||||
child: IconButton(
|
child: IconButton(
|
||||||
icon: const Icon(Icons.add_circle_outline, size: 22),
|
icon: const Icon(Icons.add_circle_outline, size: 22),
|
||||||
onPressed: (isAwaitingApproval || widget.agentName != null)
|
tooltip: AppLocalizations.of(context).chatAddImageTooltip,
|
||||||
? null
|
onPressed: isAwaitingApproval ? null : _showAttachmentOptions,
|
||||||
: _showAttachmentOptions,
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
Expanded(
|
Expanded(
|
||||||
|
|
|
||||||
|
|
@ -145,6 +145,12 @@ app.post('/task-async', async (req, res) => {
|
||||||
const timeoutSeconds: number = req.body.timeoutSeconds ?? 120; // 2 min default for async tasks
|
const timeoutSeconds: number = req.body.timeoutSeconds ?? 120; // 2 min default for async tasks
|
||||||
const idempotencyKey: string = req.body.idempotencyKey ?? crypto.randomUUID();
|
const idempotencyKey: string = req.body.idempotencyKey ?? crypto.randomUUID();
|
||||||
const callbackData = req.body.callbackData ?? {};
|
const callbackData = req.body.callbackData ?? {};
|
||||||
|
// Optional attachments — passed through to OpenClaw chat.send unchanged.
|
||||||
|
// Expected format: [{ name, mimeType, media }] where media is a data-URI.
|
||||||
|
const attachments: Array<{ name: string; mimeType: string; media: string }> | undefined =
|
||||||
|
Array.isArray(req.body.attachments) && req.body.attachments.length > 0
|
||||||
|
? req.body.attachments
|
||||||
|
: undefined;
|
||||||
|
|
||||||
// Return immediately — LLM runs in background
|
// Return immediately — LLM runs in background
|
||||||
res.json({ ok: true, pending: true });
|
res.json({ ok: true, pending: true });
|
||||||
|
|
@ -162,6 +168,7 @@ app.post('/task-async', async (req, res) => {
|
||||||
message: req.body.prompt,
|
message: req.body.prompt,
|
||||||
idempotencyKey,
|
idempotencyKey,
|
||||||
timeoutMs: timeoutSeconds * 1000,
|
timeoutMs: timeoutSeconds * 1000,
|
||||||
|
attachments,
|
||||||
}).then((reply: string) => {
|
}).then((reply: string) => {
|
||||||
postCallback({ ok: true, result: reply, callbackData });
|
postCallback({ ok: true, result: reply, callbackData });
|
||||||
}).catch((err: Error) => {
|
}).catch((err: Error) => {
|
||||||
|
|
|
||||||
|
|
@ -298,17 +298,23 @@ export class OpenClawClient {
|
||||||
message: string;
|
message: string;
|
||||||
idempotencyKey: string;
|
idempotencyKey: string;
|
||||||
timeoutMs?: number;
|
timeoutMs?: number;
|
||||||
|
/** Optional media attachments (images, PDFs, etc.) in OpenClaw format. */
|
||||||
|
attachments?: Array<{ name: string; mimeType: string; media: string }>;
|
||||||
}): Promise<string> {
|
}): Promise<string> {
|
||||||
const timeoutMs = params.timeoutMs ?? 30_000;
|
const timeoutMs = params.timeoutMs ?? 30_000;
|
||||||
|
|
||||||
// Send chat.send — resolves immediately with { runId, status: "started" }
|
// Send chat.send — resolves immediately with { runId, status: "started" }
|
||||||
const ack = await this.rpc(
|
const chatSendParams: Record<string, unknown> = {
|
||||||
'chat.send',
|
|
||||||
{
|
|
||||||
sessionKey: params.sessionKey,
|
sessionKey: params.sessionKey,
|
||||||
message: params.message,
|
message: params.message,
|
||||||
idempotencyKey: params.idempotencyKey,
|
idempotencyKey: params.idempotencyKey,
|
||||||
},
|
};
|
||||||
|
if (params.attachments && params.attachments.length > 0) {
|
||||||
|
chatSendParams['attachments'] = params.attachments;
|
||||||
|
}
|
||||||
|
const ack = await this.rpc(
|
||||||
|
'chat.send',
|
||||||
|
chatSendParams,
|
||||||
10_000, // 10s for the initial ack
|
10_000, // 10s for the initial ack
|
||||||
) as { runId: string; status: string };
|
) as { runId: string; status: string };
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -827,7 +827,11 @@ export class AgentController {
|
||||||
@TenantId() tenantId: string,
|
@TenantId() tenantId: string,
|
||||||
@Req() req: any,
|
@Req() req: any,
|
||||||
@Param('instanceId') instanceId: string,
|
@Param('instanceId') instanceId: string,
|
||||||
@Body() body: { prompt: string; sessionId?: string },
|
@Body() body: {
|
||||||
|
prompt: string;
|
||||||
|
sessionId?: string;
|
||||||
|
attachments?: Array<{ base64Data: string; mediaType: string; fileName?: string }>;
|
||||||
|
},
|
||||||
) {
|
) {
|
||||||
const instance = await this.instanceRepository.findById(instanceId);
|
const instance = await this.instanceRepository.findById(instanceId);
|
||||||
if (!instance) throw new NotFoundException(`Instance ${instanceId} not found`);
|
if (!instance) throw new NotFoundException(`Instance ${instanceId} not found`);
|
||||||
|
|
@ -883,7 +887,7 @@ export class AgentController {
|
||||||
await this.taskRepository.save(task);
|
await this.taskRepository.save(task);
|
||||||
|
|
||||||
// Persist user message for display in conversation history
|
// Persist user message for display in conversation history
|
||||||
await this.contextService.saveUserMessage(session.id, body.prompt);
|
await this.contextService.saveUserMessage(session.id, body.prompt, body.attachments);
|
||||||
|
|
||||||
// The OpenClaw bridge tracks conversation context internally via sessionKey.
|
// The OpenClaw bridge tracks conversation context internally via sessionKey.
|
||||||
// We use our DB session ID as the key so each session has isolated context.
|
// We use our DB session ID as the key so each session has isolated context.
|
||||||
|
|
@ -891,6 +895,15 @@ export class AgentController {
|
||||||
const callbackUrl = `${process.env.AGENT_SERVICE_PUBLIC_URL}/api/v1/agent/instances/openclaw-app-callback`;
|
const callbackUrl = `${process.env.AGENT_SERVICE_PUBLIC_URL}/api/v1/agent/instances/openclaw-app-callback`;
|
||||||
const bridgeUrl = `http://${instance.serverHost}:${instance.hostPort}/task-async`;
|
const bridgeUrl = `http://${instance.serverHost}:${instance.hostPort}/task-async`;
|
||||||
|
|
||||||
|
// Convert IT0 attachment format → OpenClaw format for the bridge.
|
||||||
|
// IT0: { base64Data, mediaType, fileName? }
|
||||||
|
// OpenClaw: { name, mimeType, media: "data:<mimeType>;base64,<data>" }
|
||||||
|
const bridgeAttachments = body.attachments?.map((att) => ({
|
||||||
|
name: att.fileName ?? 'attachment',
|
||||||
|
mimeType: att.mediaType,
|
||||||
|
media: `data:${att.mediaType};base64,${att.base64Data}`,
|
||||||
|
}));
|
||||||
|
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`[Task ${task.id}] Routing to OpenClaw instance ${instanceId} @ ${bridgeUrl}, session=${session.id}`,
|
`[Task ${task.id}] Routing to OpenClaw instance ${instanceId} @ ${bridgeUrl}, session=${session.id}`,
|
||||||
);
|
);
|
||||||
|
|
@ -946,6 +959,7 @@ export class AgentController {
|
||||||
idempotencyKey: task.id,
|
idempotencyKey: task.id,
|
||||||
callbackUrl,
|
callbackUrl,
|
||||||
callbackData: { sessionId: session.id, taskId: task.id },
|
callbackData: { sessionId: session.id, taskId: task.id },
|
||||||
|
...(bridgeAttachments && bridgeAttachments.length > 0 && { attachments: bridgeAttachments }),
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.then(async (res) => {
|
.then(async (res) => {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue