feat: add document support for Claude + image thumbnail generation
1. Coordinator now sends all attachment types to Claude: - Images → native image blocks (existing) - PDF → native document blocks (Claude PDF support) - Text files (txt, csv, json, md) → text blocks with filename Extracted common buildAttachmentBlocks() helper. 2. File-service generates thumbnails on image upload: - Uses sharp to resize to 400x400 max (inside fit, no upscale) - Output as WebP at 80% quality for smaller file size - Stored in MinIO under thumbnails/ prefix - Generated for both direct upload and presigned URL confirm - Non-blocking: thumbnail failure doesn't break upload Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cf2fd07ead
commit
5076e16cc7
|
|
@ -328,6 +328,51 @@ export class CoordinatorAgentService implements OnModuleInit {
|
|||
// Message Building
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* 将附件转换为 Claude API content blocks
|
||||
* - image → { type: 'image', source: { type: 'url', url } }
|
||||
* - PDF → { type: 'document', source: { type: 'url', url }, title }
|
||||
* - text/csv/json/md → 嵌入为带文件名提示的 text block
|
||||
*/
|
||||
private buildAttachmentBlocks(attachments: FileAttachment[]): any[] {
|
||||
const blocks: any[] = [];
|
||||
|
||||
for (const att of attachments) {
|
||||
if (!att.downloadUrl) continue;
|
||||
|
||||
if (att.type === 'image') {
|
||||
blocks.push({
|
||||
type: 'image',
|
||||
source: { type: 'url', url: att.downloadUrl },
|
||||
});
|
||||
} else if (att.mimeType === 'application/pdf') {
|
||||
// Claude 原生支持 PDF 文档
|
||||
blocks.push({
|
||||
type: 'document',
|
||||
source: { type: 'url', url: att.downloadUrl },
|
||||
title: att.originalName,
|
||||
});
|
||||
} else if (this.isTextBasedMime(att.mimeType)) {
|
||||
// 文本类文件:提示模型通过 URL 获取内容
|
||||
blocks.push({
|
||||
type: 'text',
|
||||
text: `[附件: ${att.originalName} (${att.mimeType})] — 文件内容可通过URL获取: ${att.downloadUrl}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/** 判断是否为可直接阅读的文本类 MIME */
|
||||
private isTextBasedMime(mimeType: string): boolean {
|
||||
const textMimes = [
|
||||
'text/plain', 'text/csv', 'text/markdown',
|
||||
'application/json',
|
||||
];
|
||||
return textMimes.includes(mimeType) || mimeType.startsWith('text/');
|
||||
}
|
||||
|
||||
private buildMessages(
|
||||
context: LegacyConversationContext,
|
||||
userContent: string,
|
||||
|
|
@ -339,21 +384,7 @@ export class CoordinatorAgentService implements OnModuleInit {
|
|||
if (context.previousMessages) {
|
||||
for (const msg of context.previousMessages) {
|
||||
if (msg.attachments?.length) {
|
||||
// Multimodal message with images
|
||||
const contentBlocks: any[] = [];
|
||||
|
||||
for (const att of msg.attachments) {
|
||||
if (att.type === 'image' && att.downloadUrl) {
|
||||
contentBlocks.push({
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'url',
|
||||
url: att.downloadUrl,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const contentBlocks = this.buildAttachmentBlocks(msg.attachments);
|
||||
contentBlocks.push({ type: 'text', text: msg.content });
|
||||
messages.push({ role: msg.role, content: contentBlocks });
|
||||
} else {
|
||||
|
|
@ -364,15 +395,7 @@ export class CoordinatorAgentService implements OnModuleInit {
|
|||
|
||||
// Build current user message
|
||||
if (attachments?.length) {
|
||||
const contentBlocks: any[] = [];
|
||||
for (const att of attachments) {
|
||||
if (att.type === 'image' && att.downloadUrl) {
|
||||
contentBlocks.push({
|
||||
type: 'image',
|
||||
source: { type: 'url', url: att.downloadUrl },
|
||||
});
|
||||
}
|
||||
}
|
||||
const contentBlocks = this.buildAttachmentBlocks(attachments);
|
||||
contentBlocks.push({ type: 'text', text: userContent });
|
||||
messages.push({ role: 'user', content: contentBlocks });
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
} from '@nestjs/common';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import * as mimeTypes from 'mime-types';
|
||||
import sharp from 'sharp';
|
||||
import { FileEntity, FileType, FileStatus } from '../../domain/entities/file.entity';
|
||||
import { IFileRepository, FILE_REPOSITORY } from '../../domain/repositories/file.repository.interface';
|
||||
import { MinioStorageAdapter } from '../../adapters/outbound/storage/minio-storage.adapter';
|
||||
|
|
@ -38,6 +39,16 @@ const ALLOWED_DOCUMENT_TYPES = [
|
|||
|
||||
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
|
||||
|
||||
// 缩略图配置
|
||||
const THUMBNAIL_MAX_WIDTH = 400;
|
||||
const THUMBNAIL_MAX_HEIGHT = 400;
|
||||
const THUMBNAIL_QUALITY = 80;
|
||||
|
||||
/** sharp 支持生成缩略图的 MIME 类型 (svg 不支持 resize) */
|
||||
const THUMBNAIL_SUPPORTED_MIMES = [
|
||||
'image/jpeg', 'image/png', 'image/webp', 'image/gif',
|
||||
];
|
||||
|
||||
@Injectable()
|
||||
export class FileService {
|
||||
private readonly logger = new Logger(FileService.name);
|
||||
|
|
@ -126,6 +137,20 @@ export class FileService {
|
|||
|
||||
// 更新文件状态
|
||||
file.confirmUpload(fileSize);
|
||||
|
||||
// 生成缩略图(大文件上传路径)
|
||||
if (file.type === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(file.mimeType) && !file.thumbnailPath) {
|
||||
try {
|
||||
const buffer = await this.storageAdapter.downloadFile(file.storagePath);
|
||||
const thumbnailPath = await this.generateThumbnail(buffer, file.storagePath, file.mimeType);
|
||||
if (thumbnailPath) {
|
||||
file.thumbnailPath = thumbnailPath;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(`Thumbnail generation failed on confirm for ${fileId}: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
await this.fileRepo.update(file);
|
||||
|
||||
return this.toResponseDto(file);
|
||||
|
|
@ -167,6 +192,16 @@ export class FileService {
|
|||
'x-amz-meta-user-id': userId,
|
||||
});
|
||||
|
||||
// 生成缩略图 (仅图片,fire-and-forget 不阻塞主流程)
|
||||
let thumbnailPath: string | null = null;
|
||||
if (fileType === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(mimetype)) {
|
||||
try {
|
||||
thumbnailPath = await this.generateThumbnail(buffer, objectName, mimetype);
|
||||
} catch (err) {
|
||||
this.logger.warn(`Thumbnail generation failed for ${fileId}: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 创建文件记录
|
||||
const fileEntity = FileEntity.create({
|
||||
id: fileId,
|
||||
|
|
@ -180,9 +215,13 @@ export class FileService {
|
|||
status: FileStatus.READY,
|
||||
});
|
||||
|
||||
if (thumbnailPath) {
|
||||
fileEntity.thumbnailPath = thumbnailPath;
|
||||
}
|
||||
|
||||
await this.fileRepo.save(fileEntity);
|
||||
|
||||
this.logger.log(`File uploaded: ${fileId} by user ${userId}`);
|
||||
this.logger.log(`File uploaded: ${fileId} by user ${userId}${thumbnailPath ? ' (with thumbnail)' : ''}`);
|
||||
|
||||
return this.toResponseDto(fileEntity);
|
||||
}
|
||||
|
|
@ -253,6 +292,41 @@ export class FileService {
|
|||
this.logger.log(`File deleted: ${fileId} by user ${userId}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成缩略图并上传到 MinIO
|
||||
* @returns thumbnailPath 或 null
|
||||
*/
|
||||
private async generateThumbnail(
|
||||
imageBuffer: Buffer,
|
||||
originalPath: string,
|
||||
mimeType: string,
|
||||
): Promise<string | null> {
|
||||
// 将原始路径 uploads/images/.../xxx.png → thumbnails/images/.../xxx.webp
|
||||
const thumbnailPath = originalPath
|
||||
.replace(/^uploads\//, 'thumbnails/')
|
||||
.replace(/\.[^.]+$/, '.webp');
|
||||
|
||||
const thumbnailBuffer = await sharp(imageBuffer)
|
||||
.resize(THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT, {
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.webp({ quality: THUMBNAIL_QUALITY })
|
||||
.toBuffer();
|
||||
|
||||
await this.storageAdapter.uploadFile(
|
||||
thumbnailPath,
|
||||
thumbnailBuffer,
|
||||
'image/webp',
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
`Thumbnail generated: ${thumbnailPath} (${thumbnailBuffer.length} bytes)`,
|
||||
);
|
||||
|
||||
return thumbnailPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 确定文件类型
|
||||
*/
|
||||
|
|
|
|||
Loading…
Reference in New Issue