feat: add document support for Claude + image thumbnail generation

1. Coordinator now sends all attachment types to Claude:
   - Images → native image blocks (existing)
   - PDF → native document blocks (Claude PDF support)
   - Text files (txt, csv, json, md) → text blocks with filename
   Extracted common buildAttachmentBlocks() helper.

2. File-service generates thumbnails on image upload:
   - Uses sharp to resize to 400x400 max (inside fit, no upscale)
   - Output as WebP at 80% quality for smaller file size
   - Stored in MinIO under thumbnails/ prefix
   - Generated for both direct upload and presigned URL confirm
   - Non-blocking: thumbnail failure doesn't break upload

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-02-07 04:31:53 -08:00
parent cf2fd07ead
commit 5076e16cc7
2 changed files with 122 additions and 25 deletions

View File

@ -328,6 +328,51 @@ export class CoordinatorAgentService implements OnModuleInit {
// Message Building
// ============================================================
/**
* Claude API content blocks
* - image { type: 'image', source: { type: 'url', url } }
* - PDF { type: 'document', source: { type: 'url', url }, title }
* - text/csv/json/md text block
*/
private buildAttachmentBlocks(attachments: FileAttachment[]): any[] {
const blocks: any[] = [];
for (const att of attachments) {
if (!att.downloadUrl) continue;
if (att.type === 'image') {
blocks.push({
type: 'image',
source: { type: 'url', url: att.downloadUrl },
});
} else if (att.mimeType === 'application/pdf') {
// Claude 原生支持 PDF 文档
blocks.push({
type: 'document',
source: { type: 'url', url: att.downloadUrl },
title: att.originalName,
});
} else if (this.isTextBasedMime(att.mimeType)) {
// 文本类文件:提示模型通过 URL 获取内容
blocks.push({
type: 'text',
text: `[附件: ${att.originalName} (${att.mimeType})] — 文件内容可通过URL获取: ${att.downloadUrl}`,
});
}
}
return blocks;
}
/** 判断是否为可直接阅读的文本类 MIME */
private isTextBasedMime(mimeType: string): boolean {
const textMimes = [
'text/plain', 'text/csv', 'text/markdown',
'application/json',
];
return textMimes.includes(mimeType) || mimeType.startsWith('text/');
}
private buildMessages(
context: LegacyConversationContext,
userContent: string,
@ -339,21 +384,7 @@ export class CoordinatorAgentService implements OnModuleInit {
if (context.previousMessages) {
for (const msg of context.previousMessages) {
if (msg.attachments?.length) {
// Multimodal message with images
const contentBlocks: any[] = [];
for (const att of msg.attachments) {
if (att.type === 'image' && att.downloadUrl) {
contentBlocks.push({
type: 'image',
source: {
type: 'url',
url: att.downloadUrl,
},
});
}
}
const contentBlocks = this.buildAttachmentBlocks(msg.attachments);
contentBlocks.push({ type: 'text', text: msg.content });
messages.push({ role: msg.role, content: contentBlocks });
} else {
@ -364,15 +395,7 @@ export class CoordinatorAgentService implements OnModuleInit {
// Build current user message
if (attachments?.length) {
const contentBlocks: any[] = [];
for (const att of attachments) {
if (att.type === 'image' && att.downloadUrl) {
contentBlocks.push({
type: 'image',
source: { type: 'url', url: att.downloadUrl },
});
}
}
const contentBlocks = this.buildAttachmentBlocks(attachments);
contentBlocks.push({ type: 'text', text: userContent });
messages.push({ role: 'user', content: contentBlocks });
} else {

View File

@ -7,6 +7,7 @@ import {
} from '@nestjs/common';
import { v4 as uuidv4 } from 'uuid';
import * as mimeTypes from 'mime-types';
import sharp from 'sharp';
import { FileEntity, FileType, FileStatus } from '../../domain/entities/file.entity';
import { IFileRepository, FILE_REPOSITORY } from '../../domain/repositories/file.repository.interface';
import { MinioStorageAdapter } from '../../adapters/outbound/storage/minio-storage.adapter';
@ -38,6 +39,16 @@ const ALLOWED_DOCUMENT_TYPES = [
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
// 缩略图配置
const THUMBNAIL_MAX_WIDTH = 400;
const THUMBNAIL_MAX_HEIGHT = 400;
const THUMBNAIL_QUALITY = 80;
/** sharp 支持生成缩略图的 MIME 类型 (svg 不支持 resize) */
const THUMBNAIL_SUPPORTED_MIMES = [
'image/jpeg', 'image/png', 'image/webp', 'image/gif',
];
@Injectable()
export class FileService {
private readonly logger = new Logger(FileService.name);
@ -126,6 +137,20 @@ export class FileService {
// 更新文件状态
file.confirmUpload(fileSize);
// 生成缩略图(大文件上传路径)
if (file.type === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(file.mimeType) && !file.thumbnailPath) {
try {
const buffer = await this.storageAdapter.downloadFile(file.storagePath);
const thumbnailPath = await this.generateThumbnail(buffer, file.storagePath, file.mimeType);
if (thumbnailPath) {
file.thumbnailPath = thumbnailPath;
}
} catch (err) {
this.logger.warn(`Thumbnail generation failed on confirm for ${fileId}: ${err}`);
}
}
await this.fileRepo.update(file);
return this.toResponseDto(file);
@ -167,6 +192,16 @@ export class FileService {
'x-amz-meta-user-id': userId,
});
// 生成缩略图 (仅图片fire-and-forget 不阻塞主流程)
let thumbnailPath: string | null = null;
if (fileType === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(mimetype)) {
try {
thumbnailPath = await this.generateThumbnail(buffer, objectName, mimetype);
} catch (err) {
this.logger.warn(`Thumbnail generation failed for ${fileId}: ${err}`);
}
}
// 创建文件记录
const fileEntity = FileEntity.create({
id: fileId,
@ -180,9 +215,13 @@ export class FileService {
status: FileStatus.READY,
});
if (thumbnailPath) {
fileEntity.thumbnailPath = thumbnailPath;
}
await this.fileRepo.save(fileEntity);
this.logger.log(`File uploaded: ${fileId} by user ${userId}`);
this.logger.log(`File uploaded: ${fileId} by user ${userId}${thumbnailPath ? ' (with thumbnail)' : ''}`);
return this.toResponseDto(fileEntity);
}
@ -253,6 +292,41 @@ export class FileService {
this.logger.log(`File deleted: ${fileId} by user ${userId}`);
}
/**
* MinIO
* @returns thumbnailPath null
*/
private async generateThumbnail(
imageBuffer: Buffer,
originalPath: string,
mimeType: string,
): Promise<string | null> {
// 将原始路径 uploads/images/.../xxx.png → thumbnails/images/.../xxx.webp
const thumbnailPath = originalPath
.replace(/^uploads\//, 'thumbnails/')
.replace(/\.[^.]+$/, '.webp');
const thumbnailBuffer = await sharp(imageBuffer)
.resize(THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT, {
fit: 'inside',
withoutEnlargement: true,
})
.webp({ quality: THUMBNAIL_QUALITY })
.toBuffer();
await this.storageAdapter.uploadFile(
thumbnailPath,
thumbnailBuffer,
'image/webp',
);
this.logger.debug(
`Thumbnail generated: ${thumbnailPath} (${thumbnailBuffer.length} bytes)`,
);
return thumbnailPath;
}
/**
*
*/