feat: add document support for Claude + image thumbnail generation
1. Coordinator now sends all attachment types to Claude: - Images → native image blocks (existing) - PDF → native document blocks (Claude PDF support) - Text files (txt, csv, json, md) → text blocks with filename Extracted common buildAttachmentBlocks() helper. 2. File-service generates thumbnails on image upload: - Uses sharp to resize to 400x400 max (inside fit, no upscale) - Output as WebP at 80% quality for smaller file size - Stored in MinIO under thumbnails/ prefix - Generated for both direct upload and presigned URL confirm - Non-blocking: thumbnail failure doesn't break upload Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cf2fd07ead
commit
5076e16cc7
|
|
@ -328,6 +328,51 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
// Message Building
|
// Message Building
|
||||||
// ============================================================
|
// ============================================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将附件转换为 Claude API content blocks
|
||||||
|
* - image → { type: 'image', source: { type: 'url', url } }
|
||||||
|
* - PDF → { type: 'document', source: { type: 'url', url }, title }
|
||||||
|
* - text/csv/json/md → 嵌入为带文件名提示的 text block
|
||||||
|
*/
|
||||||
|
private buildAttachmentBlocks(attachments: FileAttachment[]): any[] {
|
||||||
|
const blocks: any[] = [];
|
||||||
|
|
||||||
|
for (const att of attachments) {
|
||||||
|
if (!att.downloadUrl) continue;
|
||||||
|
|
||||||
|
if (att.type === 'image') {
|
||||||
|
blocks.push({
|
||||||
|
type: 'image',
|
||||||
|
source: { type: 'url', url: att.downloadUrl },
|
||||||
|
});
|
||||||
|
} else if (att.mimeType === 'application/pdf') {
|
||||||
|
// Claude 原生支持 PDF 文档
|
||||||
|
blocks.push({
|
||||||
|
type: 'document',
|
||||||
|
source: { type: 'url', url: att.downloadUrl },
|
||||||
|
title: att.originalName,
|
||||||
|
});
|
||||||
|
} else if (this.isTextBasedMime(att.mimeType)) {
|
||||||
|
// 文本类文件:提示模型通过 URL 获取内容
|
||||||
|
blocks.push({
|
||||||
|
type: 'text',
|
||||||
|
text: `[附件: ${att.originalName} (${att.mimeType})] — 文件内容可通过URL获取: ${att.downloadUrl}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return blocks;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 判断是否为可直接阅读的文本类 MIME */
|
||||||
|
private isTextBasedMime(mimeType: string): boolean {
|
||||||
|
const textMimes = [
|
||||||
|
'text/plain', 'text/csv', 'text/markdown',
|
||||||
|
'application/json',
|
||||||
|
];
|
||||||
|
return textMimes.includes(mimeType) || mimeType.startsWith('text/');
|
||||||
|
}
|
||||||
|
|
||||||
private buildMessages(
|
private buildMessages(
|
||||||
context: LegacyConversationContext,
|
context: LegacyConversationContext,
|
||||||
userContent: string,
|
userContent: string,
|
||||||
|
|
@ -339,21 +384,7 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
if (context.previousMessages) {
|
if (context.previousMessages) {
|
||||||
for (const msg of context.previousMessages) {
|
for (const msg of context.previousMessages) {
|
||||||
if (msg.attachments?.length) {
|
if (msg.attachments?.length) {
|
||||||
// Multimodal message with images
|
const contentBlocks = this.buildAttachmentBlocks(msg.attachments);
|
||||||
const contentBlocks: any[] = [];
|
|
||||||
|
|
||||||
for (const att of msg.attachments) {
|
|
||||||
if (att.type === 'image' && att.downloadUrl) {
|
|
||||||
contentBlocks.push({
|
|
||||||
type: 'image',
|
|
||||||
source: {
|
|
||||||
type: 'url',
|
|
||||||
url: att.downloadUrl,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
contentBlocks.push({ type: 'text', text: msg.content });
|
contentBlocks.push({ type: 'text', text: msg.content });
|
||||||
messages.push({ role: msg.role, content: contentBlocks });
|
messages.push({ role: msg.role, content: contentBlocks });
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -364,15 +395,7 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
|
|
||||||
// Build current user message
|
// Build current user message
|
||||||
if (attachments?.length) {
|
if (attachments?.length) {
|
||||||
const contentBlocks: any[] = [];
|
const contentBlocks = this.buildAttachmentBlocks(attachments);
|
||||||
for (const att of attachments) {
|
|
||||||
if (att.type === 'image' && att.downloadUrl) {
|
|
||||||
contentBlocks.push({
|
|
||||||
type: 'image',
|
|
||||||
source: { type: 'url', url: att.downloadUrl },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
contentBlocks.push({ type: 'text', text: userContent });
|
contentBlocks.push({ type: 'text', text: userContent });
|
||||||
messages.push({ role: 'user', content: contentBlocks });
|
messages.push({ role: 'user', content: contentBlocks });
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { v4 as uuidv4 } from 'uuid';
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
import * as mimeTypes from 'mime-types';
|
import * as mimeTypes from 'mime-types';
|
||||||
|
import sharp from 'sharp';
|
||||||
import { FileEntity, FileType, FileStatus } from '../../domain/entities/file.entity';
|
import { FileEntity, FileType, FileStatus } from '../../domain/entities/file.entity';
|
||||||
import { IFileRepository, FILE_REPOSITORY } from '../../domain/repositories/file.repository.interface';
|
import { IFileRepository, FILE_REPOSITORY } from '../../domain/repositories/file.repository.interface';
|
||||||
import { MinioStorageAdapter } from '../../adapters/outbound/storage/minio-storage.adapter';
|
import { MinioStorageAdapter } from '../../adapters/outbound/storage/minio-storage.adapter';
|
||||||
|
|
@ -38,6 +39,16 @@ const ALLOWED_DOCUMENT_TYPES = [
|
||||||
|
|
||||||
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
|
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
|
||||||
|
|
||||||
|
// 缩略图配置
|
||||||
|
const THUMBNAIL_MAX_WIDTH = 400;
|
||||||
|
const THUMBNAIL_MAX_HEIGHT = 400;
|
||||||
|
const THUMBNAIL_QUALITY = 80;
|
||||||
|
|
||||||
|
/** sharp 支持生成缩略图的 MIME 类型 (svg 不支持 resize) */
|
||||||
|
const THUMBNAIL_SUPPORTED_MIMES = [
|
||||||
|
'image/jpeg', 'image/png', 'image/webp', 'image/gif',
|
||||||
|
];
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class FileService {
|
export class FileService {
|
||||||
private readonly logger = new Logger(FileService.name);
|
private readonly logger = new Logger(FileService.name);
|
||||||
|
|
@ -126,6 +137,20 @@ export class FileService {
|
||||||
|
|
||||||
// 更新文件状态
|
// 更新文件状态
|
||||||
file.confirmUpload(fileSize);
|
file.confirmUpload(fileSize);
|
||||||
|
|
||||||
|
// 生成缩略图(大文件上传路径)
|
||||||
|
if (file.type === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(file.mimeType) && !file.thumbnailPath) {
|
||||||
|
try {
|
||||||
|
const buffer = await this.storageAdapter.downloadFile(file.storagePath);
|
||||||
|
const thumbnailPath = await this.generateThumbnail(buffer, file.storagePath, file.mimeType);
|
||||||
|
if (thumbnailPath) {
|
||||||
|
file.thumbnailPath = thumbnailPath;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`Thumbnail generation failed on confirm for ${fileId}: ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
await this.fileRepo.update(file);
|
await this.fileRepo.update(file);
|
||||||
|
|
||||||
return this.toResponseDto(file);
|
return this.toResponseDto(file);
|
||||||
|
|
@ -167,6 +192,16 @@ export class FileService {
|
||||||
'x-amz-meta-user-id': userId,
|
'x-amz-meta-user-id': userId,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// 生成缩略图 (仅图片,fire-and-forget 不阻塞主流程)
|
||||||
|
let thumbnailPath: string | null = null;
|
||||||
|
if (fileType === FileType.IMAGE && THUMBNAIL_SUPPORTED_MIMES.includes(mimetype)) {
|
||||||
|
try {
|
||||||
|
thumbnailPath = await this.generateThumbnail(buffer, objectName, mimetype);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`Thumbnail generation failed for ${fileId}: ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 创建文件记录
|
// 创建文件记录
|
||||||
const fileEntity = FileEntity.create({
|
const fileEntity = FileEntity.create({
|
||||||
id: fileId,
|
id: fileId,
|
||||||
|
|
@ -180,9 +215,13 @@ export class FileService {
|
||||||
status: FileStatus.READY,
|
status: FileStatus.READY,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (thumbnailPath) {
|
||||||
|
fileEntity.thumbnailPath = thumbnailPath;
|
||||||
|
}
|
||||||
|
|
||||||
await this.fileRepo.save(fileEntity);
|
await this.fileRepo.save(fileEntity);
|
||||||
|
|
||||||
this.logger.log(`File uploaded: ${fileId} by user ${userId}`);
|
this.logger.log(`File uploaded: ${fileId} by user ${userId}${thumbnailPath ? ' (with thumbnail)' : ''}`);
|
||||||
|
|
||||||
return this.toResponseDto(fileEntity);
|
return this.toResponseDto(fileEntity);
|
||||||
}
|
}
|
||||||
|
|
@ -253,6 +292,41 @@ export class FileService {
|
||||||
this.logger.log(`File deleted: ${fileId} by user ${userId}`);
|
this.logger.log(`File deleted: ${fileId} by user ${userId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生成缩略图并上传到 MinIO
|
||||||
|
* @returns thumbnailPath 或 null
|
||||||
|
*/
|
||||||
|
private async generateThumbnail(
|
||||||
|
imageBuffer: Buffer,
|
||||||
|
originalPath: string,
|
||||||
|
mimeType: string,
|
||||||
|
): Promise<string | null> {
|
||||||
|
// 将原始路径 uploads/images/.../xxx.png → thumbnails/images/.../xxx.webp
|
||||||
|
const thumbnailPath = originalPath
|
||||||
|
.replace(/^uploads\//, 'thumbnails/')
|
||||||
|
.replace(/\.[^.]+$/, '.webp');
|
||||||
|
|
||||||
|
const thumbnailBuffer = await sharp(imageBuffer)
|
||||||
|
.resize(THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT, {
|
||||||
|
fit: 'inside',
|
||||||
|
withoutEnlargement: true,
|
||||||
|
})
|
||||||
|
.webp({ quality: THUMBNAIL_QUALITY })
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
await this.storageAdapter.uploadFile(
|
||||||
|
thumbnailPath,
|
||||||
|
thumbnailBuffer,
|
||||||
|
'image/webp',
|
||||||
|
);
|
||||||
|
|
||||||
|
this.logger.debug(
|
||||||
|
`Thumbnail generated: ${thumbnailPath} (${thumbnailBuffer.length} bytes)`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return thumbnailPath;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 确定文件类型
|
* 确定文件类型
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue