From 5076e16cc7f6707c8985a6376ffa7ec986c3da6d Mon Sep 17 00:00:00 2001 From: hailin Date: Sat, 7 Feb 2026 04:31:53 -0800 Subject: [PATCH] feat: add document support for Claude + image thumbnail generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Coordinator now sends all attachment types to Claude: - Images → native image blocks (existing) - PDF → native document blocks (Claude PDF support) - Text files (txt, csv, json, md) → text blocks with filename Extracted common buildAttachmentBlocks() helper. 2. File-service generates thumbnails on image upload: - Uses sharp to resize to 400x400 max (inside fit, no upscale) - Output as WebP at 80% quality for smaller file size - Stored in MinIO under thumbnails/ prefix - Generated for both direct upload and presigned URL confirm - Non-blocking: thumbnail failure doesn't break upload Co-Authored-By: Claude Opus 4.6 --- .../coordinator/coordinator-agent.service.ts | 71 +++++++++++------ .../src/application/services/file.service.ts | 76 ++++++++++++++++++- 2 files changed, 122 insertions(+), 25 deletions(-) diff --git a/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts b/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts index 1ae775c..8661803 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts @@ -328,6 +328,51 @@ export class CoordinatorAgentService implements OnModuleInit { // Message Building // ============================================================ + /** + * 将附件转换为 Claude API content blocks + * - image → { type: 'image', source: { type: 'url', url } } + * - PDF → { type: 'document', source: { type: 'url', url }, title } + * - text/csv/json/md → 嵌入为带文件名提示的 text block + */ + private buildAttachmentBlocks(attachments: FileAttachment[]): any[] { + const blocks: any[] = []; + + for (const att of attachments) { + if (!att.downloadUrl) continue; + + if (att.type === 'image') { + blocks.push({ + type: 'image', + source: { type: 'url', url: att.downloadUrl }, + }); + } else if (att.mimeType === 'application/pdf') { + // Claude 原生支持 PDF 文档 + blocks.push({ + type: 'document', + source: { type: 'url', url: att.downloadUrl }, + title: att.originalName, + }); + } else if (this.isTextBasedMime(att.mimeType)) { + // 文本类文件：提示模型通过 URL 获取内容 + blocks.push({ + type: 'text', + text: `[附件: ${att.originalName} (${att.mimeType})] — 文件内容可通过URL获取: ${att.downloadUrl}`, + }); + } + } + + return blocks; + } + + /** 判断是否为可直接阅读的文本类 MIME */ + private isTextBasedMime(mimeType: string): boolean { + const textMimes = [ + 'text/plain', 'text/csv', 'text/markdown', + 'application/json', + ]; + return textMimes.includes(mimeType) || mimeType.startsWith('text/'); + } + private buildMessages( context: LegacyConversationContext, userContent: string, @@ -339,21 +384,7 @@ export class CoordinatorAgentService implements OnModuleInit { if (context.previousMessages) { for (const msg of context.previousMessages) { if (msg.attachments?.length) { - // Multimodal message with images - const contentBlocks: any[] = []; - - for (const att of msg.attachments) { - if (att.type === 'image' && att.downloadUrl) { - contentBlocks.push({ - type: 'image', - source: { - type: 'url', - url: att.downloadUrl, - }, - }); - } - } - + const contentBlocks = this.buildAttachmentBlocks(msg.attachments); contentBlocks.push({ type: 'text', text: msg.content }); messages.push({ role: msg.role, content: contentBlocks }); } else { @@ -364,15 +395,7 @@ export class CoordinatorAgentService implements OnModuleInit { // Build current user message if (attachments?.length) { - const contentBlocks: any[] = []; - for (const att of attachments) { - if (att.type === 'image' && att.downloadUrl) { - contentBlocks.push({ - type: 'image', - source: { type: 'url', url: att.downloadUrl }, - }); - } - } + const contentBlocks = this.buildAttachmentBlocks(attachments); contentBlocks.push({ type: 'text', text: userContent }); messages.push({ role: 'user', content: contentBlocks }); } else { diff --git a/packages/services/file-service/src/application/services/file.service.ts b/packages/services/file-service/src/application/services/file.service.ts index 7ae1b18..6548dc5 100644 --- a/packages/services/file-service/src/application/services/file.service.ts +++ b/packages/services/file-service/src/application/services/file.service.ts @@ -7,6 +7,7 @@ import { } from '@nestjs/common'; import { v4 as uuidv4 } from 'uuid'; import * as mimeTypes from 'mime-types'; +import sharp from 'sharp'; import { FileEntity, FileType, FileStatus } from '../../domain/entities/file.entity'; import { IFileRepository, FILE_REPOSITORY } from '../../domain/repositories/file.repository.interface'; import { MinioStorageAdapter } from '../../adapters/outbound/storage/minio-storage.adapter'; @@ -38,6 +39,16 @@ const ALLOWED_DOCUMENT_TYPES = [ const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB +// 缩略图配置 +const THUMBNAIL_MAX_WIDTH = 400; +const THUMBNAIL_MAX_HEIGHT = 400; +const THUMBNAIL_QUALITY = 80; + +/** sharp 支持生成缩略图的 MIME 类型 (svg 不支持 resize) */ +const THUMBNAIL_SUPPORTED_MIMES = [ + 'image/jpeg', 'image/png', 'image/webp', 'image/gif', +]; + @Injectable() export class FileService { private readonly logger = new Logger(FileService.name); @@ -126,6 +137,20 @@ export class FileService { // 更新文件状态 file.confirmUpload(fileSize); + + // 生成缩略图（大文件上传路径） + if (file.type === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(file.mimeType) && !file.thumbnailPath) { + try { + const buffer = await this.storageAdapter.downloadFile(file.storagePath); + const thumbnailPath = await this.generateThumbnail(buffer, file.storagePath, file.mimeType); + if (thumbnailPath) { + file.thumbnailPath = thumbnailPath; + } + } catch (err) { + this.logger.warn(`Thumbnail generation failed on confirm for ${fileId}: ${err}`); + } + } + await this.fileRepo.update(file); return this.toResponseDto(file); @@ -167,6 +192,16 @@ export class FileService { 'x-amz-meta-user-id': userId, }); + // 生成缩略图 (仅图片，fire-and-forget 不阻塞主流程) + let thumbnailPath: string | null = null; + if (fileType === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(mimetype)) { + try { + thumbnailPath = await this.generateThumbnail(buffer, objectName, mimetype); + } catch (err) { + this.logger.warn(`Thumbnail generation failed for ${fileId}: ${err}`); + } + } + // 创建文件记录 const fileEntity = FileEntity.create({ id: fileId, @@ -180,9 +215,13 @@ export class FileService { status: FileStatus.READY, }); + if (thumbnailPath) { + fileEntity.thumbnailPath = thumbnailPath; + } + await this.fileRepo.save(fileEntity); - this.logger.log(`File uploaded: ${fileId} by user ${userId}`); + this.logger.log(`File uploaded: ${fileId} by user ${userId}${thumbnailPath ? ' (with thumbnail)' : ''}`); return this.toResponseDto(fileEntity); } @@ -253,6 +292,41 @@ export class FileService { this.logger.log(`File deleted: ${fileId} by user ${userId}`); } + /** + * 生成缩略图并上传到 MinIO + * @returns thumbnailPath 或 null + */ + private async generateThumbnail( + imageBuffer: Buffer, + originalPath: string, + mimeType: string, + ): Promise { + // 将原始路径 uploads/images/.../xxx.png → thumbnails/images/.../xxx.webp + const thumbnailPath = originalPath + .replace(/^uploads\//, 'thumbnails/') + .replace(/\.[^.]+$/, '.webp'); + + const thumbnailBuffer = await sharp(imageBuffer) + .resize(THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT, { + fit: 'inside', + withoutEnlargement: true, + }) + .webp({ quality: THUMBNAIL_QUALITY }) + .toBuffer(); + + await this.storageAdapter.uploadFile( + thumbnailPath, + thumbnailBuffer, + 'image/webp', + ); + + this.logger.debug( + `Thumbnail generated: ${thumbnailPath} (${thumbnailBuffer.length} bytes)`, + ); + + return thumbnailPath; + } + /** * 确定文件类型 */