feat(admin): add multimodal image paste support to all admin chat interfaces

支持管理员在3个管理聊天界面（系统总监、评估指令、收集指令）中通过粘贴板粘贴图片，实现与管理Agent的多模态对话。 **新增文件:** - `shared/hooks/useImagePaste.ts`: 共享 hook，处理剪贴板图片粘贴、 base64 转换、待发送图片管理、多模态内容块构建 **后端改动 (conversation-service):** - 3个管理聊天服务 (system-supervisor-chat, directive-chat, collection-directive-chat): chat() 方法参数类型从 `content: string` 改为 `content: Anthropic.MessageParam['content']`，支持接收图片块 - 3个管理控制器 (admin-supervisor, admin-assessment-directive, admin-collection-directive): DTO content 类型改为 `any` 以透传前端发送的多模态内容 **前端改动 (admin-client):** - 3个 API 类型文件: ChatMessage.content 类型扩展为 `string | ContentBlock[]` - SupervisorPage: 集成 useImagePaste hook，添加 onPaste 处理、待发送图片预览（64x64 缩略图+删除按钮）、消息中图片渲染 - DirectiveChatDrawer: 同上，48x48 缩略图适配 Drawer 宽度 - CollectionChatDrawer: 同上 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-08 21:18:57 -08:00 · 2026-02-08 21:18:57 -08:00 · 1f6d473649
parent 3b6e1586b7
commit 1f6d473649
13 changed files with 213 additions and 31 deletions
--- a/packages/admin-client/src/features/assessment-config/infrastructure/assessment-config.api.ts
+++ b/packages/admin-client/src/features/assessment-config/infrastructure/assessment-config.api.ts
@ -46,7 +46,7 @@ export interface UpdateDirectiveDto {

 export interface ChatMessage {
  role: 'user' | 'assistant';
-  content: string;
+  content: string | import('../../../shared/hooks/useImagePaste').ContentBlock[];
 }

 export interface ChatResponse {
--- a/packages/admin-client/src/features/assessment-config/presentation/components/DirectiveChatDrawer.tsx
+++ b/packages/admin-client/src/features/assessment-config/presentation/components/DirectiveChatDrawer.tsx
@ -1,9 +1,10 @@
 import { useState, useEffect, useRef } from 'react';
 import { Drawer, Button, Input, Tag, Spin } from 'antd';
-import { RobotOutlined, SendOutlined } from '@ant-design/icons';
+import { RobotOutlined, SendOutlined, CloseCircleOutlined, PictureOutlined } from '@ant-design/icons';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import { useDirectiveChat } from '../../application/useAssessmentConfig';
+import { useImagePaste } from '../../../../shared/hooks/useImagePaste';
 import type { ChatMessage } from '../../infrastructure/assessment-config.api';

 interface DirectiveChatDrawerProps {
@ -16,6 +17,7 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)
  const [inputValue, setInputValue] = useState('');
  const chatMutation = useDirectiveChat();
  const messagesEndRef = useRef<HTMLDivElement>(null);
+  const { pendingImages, handlePaste, removePendingImage, clearPendingImages, buildContent } = useImagePaste();

  useEffect(() => {
    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
@ -25,17 +27,20 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)
    if (!open) {
      setMessages([]);
      setInputValue('');
+      clearPendingImages();
    }
-  }, [open]);
+  }, [open, clearPendingImages]);

  const handleSend = async () => {
    const text = inputValue.trim();
-    if (!text || chatMutation.isPending) return;
+    if ((!text && pendingImages.length === 0) || chatMutation.isPending) return;

-    const userMsg: ChatMessage = { role: 'user', content: text };
+    const content = buildContent(text);
+    const userMsg: ChatMessage = { role: 'user', content };
    const newMessages = [...messages, userMsg];
    setMessages(newMessages);
    setInputValue('');
+    clearPendingImages();

    try {
      const result = await chatMutation.mutateAsync(newMessages);
@ -99,11 +104,23 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)
            {msg.role === 'assistant' ? (
              <div className="supervisor-markdown">
                <ReactMarkdown remarkPlugins={[remarkGfm]}>
-                  {msg.content}
+                  {typeof msg.content === 'string' ? msg.content : ''}
                </ReactMarkdown>
              </div>
            ) : (
-              <div className="whitespace-pre-wrap">{msg.content}</div>
+              <div className="whitespace-pre-wrap">
+                {typeof msg.content === 'string' ? msg.content : (
+                  <>
+                    {msg.content.map((block, bi) =>
+                      block.type === 'image' ? (
+                        <img key={bi} src={`data:${block.source.media_type};base64,${block.source.data}`} alt="uploaded" style={{ maxWidth: 160, maxHeight: 160, borderRadius: 4, display: 'block', marginBottom: 4 }} />
+                      ) : (
+                        <span key={bi}>{block.text}</span>
+                      )
+                    )}
+                  </>
+                )}
+              </div>
            )}
          </div>
        ))}
@ -125,6 +142,17 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)

      {/* Input area */}
      <div style={{ borderTop: '1px solid #f0f0f0', padding: 12 }}>
+        {pendingImages.length > 0 && (
+          <div style={{ display: 'flex', gap: 8, marginBottom: 8, flexWrap: 'wrap' }}>
+            {pendingImages.map((img, idx) => (
+              <div key={idx} style={{ position: 'relative', display: 'inline-block' }}>
+                <img src={img.preview} alt="pending" style={{ width: 48, height: 48, objectFit: 'cover', borderRadius: 4, border: '1px solid #d9d9d9' }} />
+                <CloseCircleOutlined onClick={() => removePendingImage(idx)} style={{ position: 'absolute', top: -6, right: -6, fontSize: 14, color: '#ff4d4f', cursor: 'pointer', background: '#fff', borderRadius: '50%' }} />
+              </div>
+            ))}
+            <Tag icon={<PictureOutlined />} color="blue" style={{ height: 22, marginTop: 14 }}>{pendingImages.length} 张图片</Tag>
+          </div>
+        )}
        <Input.TextArea
          value={inputValue}
          onChange={(e) => setInputValue(e.target.value)}
@ -134,7 +162,8 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)
              handleSend();
            }
          }}
-          placeholder="输入指令..."
+          onPaste={handlePaste}
+          placeholder="输入指令...（支持粘贴图片）"
          autoSize={{ minRows: 1, maxRows: 4 }}
          disabled={chatMutation.isPending}
        />
@ -143,7 +172,7 @@ export function DirectiveChatDrawer({ open, onClose }: DirectiveChatDrawerProps)
          icon={<SendOutlined />}
          onClick={handleSend}
          loading={chatMutation.isPending}
-          disabled={!inputValue.trim()}
+          disabled={!inputValue.trim() && pendingImages.length === 0}
          className="mt-2"
          block
        >
--- a/packages/admin-client/src/features/collection-config/infrastructure/collection-config.api.ts
+++ b/packages/admin-client/src/features/collection-config/infrastructure/collection-config.api.ts
@ -46,7 +46,7 @@ export interface UpdateDirectiveDto {

 export interface ChatMessage {
  role: 'user' | 'assistant';
-  content: string;
+  content: string | import('../../../shared/hooks/useImagePaste').ContentBlock[];
 }

 export interface ChatResponse {
--- a/packages/admin-client/src/features/collection-config/presentation/components/CollectionChatDrawer.tsx
+++ b/packages/admin-client/src/features/collection-config/presentation/components/CollectionChatDrawer.tsx
@ -1,9 +1,10 @@
 import { useState, useEffect, useRef } from 'react';
 import { Drawer, Button, Input, Tag, Spin } from 'antd';
-import { RobotOutlined, SendOutlined } from '@ant-design/icons';
+import { RobotOutlined, SendOutlined, CloseCircleOutlined, PictureOutlined } from '@ant-design/icons';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import { useDirectiveChat } from '../../application/useCollectionConfig';
+import { useImagePaste } from '../../../../shared/hooks/useImagePaste';
 import type { ChatMessage } from '../../infrastructure/collection-config.api';

 interface CollectionChatDrawerProps {
@ -16,6 +17,7 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp
  const [inputValue, setInputValue] = useState('');
  const chatMutation = useDirectiveChat();
  const messagesEndRef = useRef<HTMLDivElement>(null);
+  const { pendingImages, handlePaste, removePendingImage, clearPendingImages, buildContent } = useImagePaste();

  useEffect(() => {
    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
@ -25,17 +27,20 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp
    if (!open) {
      setMessages([]);
      setInputValue('');
+      clearPendingImages();
    }
-  }, [open]);
+  }, [open, clearPendingImages]);

  const handleSend = async () => {
    const text = inputValue.trim();
-    if (!text || chatMutation.isPending) return;
+    if ((!text && pendingImages.length === 0) || chatMutation.isPending) return;

-    const userMsg: ChatMessage = { role: 'user', content: text };
+    const content = buildContent(text);
+    const userMsg: ChatMessage = { role: 'user', content };
    const newMessages = [...messages, userMsg];
    setMessages(newMessages);
    setInputValue('');
+    clearPendingImages();

    try {
      const result = await chatMutation.mutateAsync(newMessages);
@ -99,11 +104,23 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp
            {msg.role === 'assistant' ? (
              <div className="supervisor-markdown">
                <ReactMarkdown remarkPlugins={[remarkGfm]}>
-                  {msg.content}
+                  {typeof msg.content === 'string' ? msg.content : ''}
                </ReactMarkdown>
              </div>
            ) : (
-              <div className="whitespace-pre-wrap">{msg.content}</div>
+              <div className="whitespace-pre-wrap">
+                {typeof msg.content === 'string' ? msg.content : (
+                  <>
+                    {msg.content.map((block, bi) =>
+                      block.type === 'image' ? (
+                        <img key={bi} src={`data:${block.source.media_type};base64,${block.source.data}`} alt="uploaded" style={{ maxWidth: 160, maxHeight: 160, borderRadius: 4, display: 'block', marginBottom: 4 }} />
+                      ) : (
+                        <span key={bi}>{block.text}</span>
+                      )
+                    )}
+                  </>
+                )}
+              </div>
            )}
          </div>
        ))}
@ -125,6 +142,17 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp

      {/* Input area */}
      <div style={{ borderTop: '1px solid #f0f0f0', padding: 12 }}>
+        {pendingImages.length > 0 && (
+          <div style={{ display: 'flex', gap: 8, marginBottom: 8, flexWrap: 'wrap' }}>
+            {pendingImages.map((img, idx) => (
+              <div key={idx} style={{ position: 'relative', display: 'inline-block' }}>
+                <img src={img.preview} alt="pending" style={{ width: 48, height: 48, objectFit: 'cover', borderRadius: 4, border: '1px solid #d9d9d9' }} />
+                <CloseCircleOutlined onClick={() => removePendingImage(idx)} style={{ position: 'absolute', top: -6, right: -6, fontSize: 14, color: '#ff4d4f', cursor: 'pointer', background: '#fff', borderRadius: '50%' }} />
+              </div>
+            ))}
+            <Tag icon={<PictureOutlined />} color="blue" style={{ height: 22, marginTop: 14 }}>{pendingImages.length} 张图片</Tag>
+          </div>
+        )}
        <Input.TextArea
          value={inputValue}
          onChange={(e) => setInputValue(e.target.value)}
@ -134,7 +162,8 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp
              handleSend();
            }
          }}
-          placeholder="输入收集指令..."
+          onPaste={handlePaste}
+          placeholder="输入收集指令...（支持粘贴图片）"
          autoSize={{ minRows: 1, maxRows: 4 }}
          disabled={chatMutation.isPending}
        />
@ -143,7 +172,7 @@ export function CollectionChatDrawer({ open, onClose }: CollectionChatDrawerProp
          icon={<SendOutlined />}
          onClick={handleSend}
          loading={chatMutation.isPending}
-          disabled={!inputValue.trim()}
+          disabled={!inputValue.trim() && pendingImages.length === 0}
          className="mt-2"
          block
        >
--- a/packages/admin-client/src/features/supervisor/infrastructure/supervisor.api.ts
+++ b/packages/admin-client/src/features/supervisor/infrastructure/supervisor.api.ts
@ -1,8 +1,9 @@
 import api from '../../../shared/utils/api';
+import type { ContentBlock } from '../../../shared/hooks/useImagePaste';

 export interface ChatMessage {
  role: 'user' | 'assistant';
-  content: string;
+  content: string | ContentBlock[];
 }

 export interface SupervisorChatResponse {
--- a/packages/admin-client/src/features/supervisor/presentation/pages/SupervisorPage.tsx
+++ b/packages/admin-client/src/features/supervisor/presentation/pages/SupervisorPage.tsx
@ -8,10 +8,13 @@ import {
  HeartOutlined,
  DollarOutlined,
  ClearOutlined,
+  CloseCircleOutlined,
+  PictureOutlined,
 } from '@ant-design/icons';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
 import { useSupervisorChat } from '../../application/useSupervisor';
+import { useImagePaste } from '../../../../shared/hooks/useImagePaste';
 import type { ChatMessage } from '../../infrastructure/supervisor.api';

 const { Title, Text } = Typography;
@ -28,19 +31,22 @@ export function SupervisorPage() {
  const [inputValue, setInputValue] = useState('');
  const chatMutation = useSupervisorChat();
  const messagesEndRef = useRef<HTMLDivElement>(null);
+  const { pendingImages, handlePaste, removePendingImage, clearPendingImages, buildContent } = useImagePaste();

  useEffect(() => {
    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
  }, [messages]);

  const handleSend = async (text?: string) => {
-    const content = (text || inputValue).trim();
-    if (!content || chatMutation.isPending) return;
+    const rawText = (text || inputValue).trim();
+    if ((!rawText && pendingImages.length === 0) || chatMutation.isPending) return;

+    const content = text ? text : buildContent(rawText);
    const userMsg: ChatMessage = { role: 'user', content };
    const newMessages = [...messages, userMsg];
    setMessages(newMessages);
    setInputValue('');
+    clearPendingImages();

    try {
      const result = await chatMutation.mutateAsync(newMessages);
@ -142,11 +148,28 @@ export function SupervisorPage() {
              {msg.role === 'assistant' ? (
                <div className="supervisor-markdown">
                  <ReactMarkdown remarkPlugins={[remarkGfm]}>
-                    {msg.content}
+                    {typeof msg.content === 'string' ? msg.content : ''}
                  </ReactMarkdown>
                </div>
              ) : (
-                <div style={{ whiteSpace: 'pre-wrap', lineHeight: 1.7 }}>{msg.content}</div>
+                <div style={{ whiteSpace: 'pre-wrap', lineHeight: 1.7 }}>
+                  {typeof msg.content === 'string' ? msg.content : (
+                    <>
+                      {msg.content.map((block, bi) =>
+                        block.type === 'image' ? (
+                          <img
+                            key={bi}
+                            src={`data:${block.source.media_type};base64,${block.source.data}`}
+                            alt="uploaded"
+                            style={{ maxWidth: 200, maxHeight: 200, borderRadius: 4, display: 'block', marginBottom: 4 }}
+                          />
+                        ) : (
+                          <span key={bi}>{block.text}</span>
+                        )
+                      )}
+                    </>
+                  )}
+                </div>
              )}
            </div>
          ))}
@ -168,6 +191,27 @@ export function SupervisorPage() {

        {/* Input */}
        <div style={{ borderTop: '1px solid #f0f0f0', padding: 12 }}>
+          {/* Pending image previews */}
+          {pendingImages.length > 0 && (
+            <div style={{ display: 'flex', gap: 8, marginBottom: 8, flexWrap: 'wrap' }}>
+              {pendingImages.map((img, idx) => (
+                <div key={idx} style={{ position: 'relative', display: 'inline-block' }}>
+                  <img
+                    src={img.preview}
+                    alt="pending"
+                    style={{ width: 64, height: 64, objectFit: 'cover', borderRadius: 4, border: '1px solid #d9d9d9' }}
+                  />
+                  <CloseCircleOutlined
+                    onClick={() => removePendingImage(idx)}
+                    style={{ position: 'absolute', top: -6, right: -6, fontSize: 16, color: '#ff4d4f', cursor: 'pointer', background: '#fff', borderRadius: '50%' }}
+                  />
+                </div>
+              ))}
+              <Tag icon={<PictureOutlined />} color="blue" style={{ height: 24, marginTop: 20 }}>
+                {pendingImages.length} 张图片
+              </Tag>
+            </div>
+          )}
          <Input.TextArea
            value={inputValue}
            onChange={(e) => setInputValue(e.target.value)}
@ -177,7 +221,8 @@ export function SupervisorPage() {
                handleSend();
              }
            }}
-            placeholder="输入您想了解的系统信息..."
+            onPaste={handlePaste}
+            placeholder="输入您想了解的系统信息...（支持粘贴图片）"
            autoSize={{ minRows: 1, maxRows: 4 }}
            disabled={chatMutation.isPending}
          />
@ -204,7 +249,7 @@ export function SupervisorPage() {
                icon={<SendOutlined />}
                onClick={() => handleSend()}
                loading={chatMutation.isPending}
-                disabled={!inputValue.trim()}
+                disabled={!inputValue.trim() && pendingImages.length === 0}
              >
                发送
              </Button>
--- a/packages/admin-client/src/shared/hooks/useImagePaste.ts
+++ b/packages/admin-client/src/shared/hooks/useImagePaste.ts
@ -0,0 +1,78 @@
+import { useState, useCallback } from 'react';
+
+export interface PendingImage {
+  base64: string;
+  mediaType: string;
+  preview: string; // data URL for <img> display
+}
+
+export type ContentBlock =
+  | { type: 'text'; text: string }
+  | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
+
+/**
+ * Hook for handling clipboard image paste in admin chat interfaces.
+ * Manages pending images and builds multimodal content blocks for Claude API.
+ */
+export function useImagePaste() {
+  const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);
+
+  const handlePaste = useCallback((e: React.ClipboardEvent) => {
+    const items = e.clipboardData?.items;
+    if (!items) return;
+
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i];
+      if (item.type.startsWith('image/')) {
+        e.preventDefault();
+        const file = item.getAsFile();
+        if (!file) continue;
+
+        const reader = new FileReader();
+        reader.onload = () => {
+          const dataUrl = reader.result as string;
+          const base64 = dataUrl.split(',')[1];
+          const mediaType = file.type;
+          setPendingImages(prev => [...prev, { base64, mediaType, preview: dataUrl }]);
+        };
+        reader.readAsDataURL(file);
+        break; // one image per paste
+      }
+    }
+  }, []);
+
+  const removePendingImage = useCallback((index: number) => {
+    setPendingImages(prev => prev.filter((_, i) => i !== index));
+  }, []);
+
+  const clearPendingImages = useCallback(() => {
+    setPendingImages([]);
+  }, []);
+
+  /**
+   * Build message content: plain string if text-only, array of blocks if images attached.
+   */
+  const buildContent = useCallback((text: string): string | ContentBlock[] => {
+    if (pendingImages.length === 0) return text;
+
+    const blocks: ContentBlock[] = [];
+    for (const img of pendingImages) {
+      blocks.push({
+        type: 'image',
+        source: { type: 'base64', media_type: img.mediaType, data: img.base64 },
+      });
+    }
+    if (text) {
+      blocks.push({ type: 'text', text });
+    }
+    return blocks;
+  }, [pendingImages]);
+
+  return {
+    pendingImages,
+    handlePaste,
+    removePendingImage,
+    clearPendingImages,
+    buildContent,
+  };
+}
--- a/packages/services/conversation-service/src/adapters/inbound/admin-assessment-directive.controller.ts
+++ b/packages/services/conversation-service/src/adapters/inbound/admin-assessment-directive.controller.ts
@ -90,7 +90,7 @@ export class AdminAssessmentDirectiveController {
  @HttpCode(HttpStatus.OK)
  async chat(
    @Headers('authorization') auth: string,
-    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: string }> },
+    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: any }> },
  ) {
    const admin = this.verifyAdmin(auth);
    if (!this.chatService) {
--- a/packages/services/conversation-service/src/adapters/inbound/admin-collection-directive.controller.ts
+++ b/packages/services/conversation-service/src/adapters/inbound/admin-collection-directive.controller.ts
@ -82,7 +82,7 @@ export class AdminCollectionDirectiveController {
  @HttpCode(HttpStatus.OK)
  async chat(
    @Headers('authorization') auth: string,
-    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: string }> },
+    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: any }> },
  ) {
    const admin = this.verifyAdmin(auth);
    if (!this.chatService) {
--- a/packages/services/conversation-service/src/adapters/inbound/admin-supervisor.controller.ts
+++ b/packages/services/conversation-service/src/adapters/inbound/admin-supervisor.controller.ts
@ -46,7 +46,7 @@ export class AdminSupervisorController {
  @HttpCode(HttpStatus.OK)
  async chat(
    @Headers('authorization') auth: string,
-    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: string }> },
+    @Body() dto: { messages: Array<{ role: 'user' | 'assistant'; content: any }> },
  ) {
    this.verifyAdmin(auth);

--- a/packages/services/conversation-service/src/infrastructure/agents/admin/collection-directive-chat.service.ts
+++ b/packages/services/conversation-service/src/infrastructure/agents/admin/collection-directive-chat.service.ts
@ -134,7 +134,7 @@ export class CollectionDirectiveChatService {
  ) {}

  async chat(
-    messages: Array<{ role: 'user' | 'assistant'; content: string }>,
+    messages: Array<{ role: 'user' | 'assistant'; content: Anthropic.MessageParam['content'] }>,
    adminId: string,
    tenantId: string | null,
  ): Promise<CollectionChatResult> {
--- a/packages/services/conversation-service/src/infrastructure/agents/admin/directive-chat.service.ts
+++ b/packages/services/conversation-service/src/infrastructure/agents/admin/directive-chat.service.ts
@ -134,7 +134,7 @@ export class DirectiveChatService {
  ) {}

  async chat(
-    messages: Array<{ role: 'user' | 'assistant'; content: string }>,
+    messages: Array<{ role: 'user' | 'assistant'; content: Anthropic.MessageParam['content'] }>,
    adminId: string,
    tenantId: string | null,
  ): Promise<ChatResult> {
--- a/packages/services/conversation-service/src/infrastructure/agents/admin/system-supervisor-chat.service.ts
+++ b/packages/services/conversation-service/src/infrastructure/agents/admin/system-supervisor-chat.service.ts
@ -134,7 +134,7 @@ export class SystemSupervisorChatService {
  ) {}

  async chat(
-    messages: Array<{ role: 'user' | 'assistant'; content: string }>,
+    messages: Array<{ role: 'user' | 'assistant'; content: Anthropic.MessageParam['content'] }>,
  ): Promise<SupervisorChatResult> {
    if (!this.anthropic) {
      return { reply: 'AI 服务不可用，请检查 Anthropic API 配置。' };