diff --git a/database/migrations/20260206_add_evaluation_rules.sql b/database/migrations/20260206_add_evaluation_rules.sql new file mode 100644 index 0000000..b4c6b79 --- /dev/null +++ b/database/migrations/20260206_add_evaluation_rules.sql @@ -0,0 +1,26 @@ +-- Evaluation Rules: Admin-configurable quality gate rules per consulting stage +-- Zero rows = zero checks = current behavior preserved + +BEGIN; + +CREATE TABLE IF NOT EXISTS evaluation_rules ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id uuid, + stage varchar(50) NOT NULL, -- consulting stage (greeting, info_collection, etc.) or '*' for all stages + rule_type varchar(50) NOT NULL, -- FIELD_COMPLETENESS, ASSESSMENT_QUALITY, RESPONSE_LENGTH, MUST_CONTAIN, STAGE_MIN_TURNS, CONVERSION_SIGNAL + name varchar(255) NOT NULL, + description text, + config jsonb NOT NULL DEFAULT '{}', -- rule-specific parameters + enabled boolean DEFAULT true, + priority integer DEFAULT 0, -- lower = run first + failure_action varchar(20) NOT NULL DEFAULT 'WARN_AND_PASS', -- RETRY, SUPPLEMENT, WARN_AND_PASS, ESCALATE + created_by uuid, + updated_by uuid, + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now() +); + +CREATE INDEX idx_evaluation_rules_tenant ON evaluation_rules(tenant_id); +CREATE INDEX idx_evaluation_rules_tenant_stage ON evaluation_rules(tenant_id, stage, enabled); + +COMMIT; diff --git a/packages/services/conversation-service/src/adapters/inbound/admin-evaluation-rule.controller.ts b/packages/services/conversation-service/src/adapters/inbound/admin-evaluation-rule.controller.ts new file mode 100644 index 0000000..4c5ec61 --- /dev/null +++ b/packages/services/conversation-service/src/adapters/inbound/admin-evaluation-rule.controller.ts @@ -0,0 +1,289 @@ +/** + * Admin Evaluation Rule Controller + * 评估规则管理 API — 供 admin-client 使用 + * + * 管理员可以: + * 1. 为不同咨询阶段配置评估规则 + * 2. 开关规则 + * 3. 测试规则(dry-run) + * 4. 清除规则缓存 + */ + +import { + Controller, + Get, + Post, + Put, + Delete, + Param, + Body, + Query, + Headers, + HttpCode, + HttpStatus, + UnauthorizedException, + BadRequestException, +} from '@nestjs/common'; +import * as jwt from 'jsonwebtoken'; +import { v4 as uuidv4 } from 'uuid'; +import { EvaluationGateService } from '../../infrastructure/agents/coordinator/evaluation-gate.service'; +import { + IEvaluationRuleRepository, + EVALUATION_RULE_REPOSITORY, +} from '../../domain/repositories/evaluation-rule.repository.interface'; +import { + EvaluationRuleEntity, + EvaluationRuleType, + EvaluationFailureAction, +} from '../../domain/entities/evaluation-rule.entity'; +import { + CreateEvaluationRuleDto, + UpdateEvaluationRuleDto, + TestEvaluationDto, +} from '../../application/dtos/evaluation-rule.dto'; +import { Inject } from '@nestjs/common'; + +interface AdminPayload { + id: string; + username: string; + role: string; + tenantId?: string; +} + +@Controller('conversations/admin/evaluation-rules') +export class AdminEvaluationRuleController { + constructor( + @Inject(EVALUATION_RULE_REPOSITORY) + private readonly repo: IEvaluationRuleRepository, + private readonly evaluationGate: EvaluationGateService, + ) {} + + private verifyAdmin(authorization: string): AdminPayload { + const token = authorization?.replace('Bearer ', ''); + if (!token) { + throw new UnauthorizedException('Missing token'); + } + try { + const secret = process.env.JWT_SECRET || 'your-jwt-secret-key'; + return jwt.verify(token, secret) as AdminPayload; + } catch { + throw new UnauthorizedException('Invalid token'); + } + } + + // GET /conversations/admin/evaluation-rules + @Get() + async listRules( + @Headers('authorization') auth: string, + @Query('stage') stage?: string, + ) { + const admin = this.verifyAdmin(auth); + const tenantId = admin.tenantId || null; + + const allRules = await this.repo.findAllByTenant(tenantId); + const filtered = stage + ? allRules.filter(r => r.stage === stage || r.stage === '*') + : allRules; + + return { + success: true, + data: { + items: filtered.map(r => this.toResponse(r)), + total: filtered.length, + ruleTypes: Object.values(EvaluationRuleType), + failureActions: Object.values(EvaluationFailureAction), + stages: ['*', 'greeting', 'needs_discovery', 'info_collection', 'assessment', 'recommendation', 'objection_handling', 'conversion', 'handoff'], + }, + }; + } + + // GET /conversations/admin/evaluation-rules/:id + @Get(':id') + async getRule( + @Headers('authorization') auth: string, + @Param('id') id: string, + ) { + this.verifyAdmin(auth); + + const rule = await this.repo.findById(id); + if (!rule) { + return { success: false, error: 'Rule not found' }; + } + + return { success: true, data: this.toResponse(rule) }; + } + + // POST /conversations/admin/evaluation-rules + @Post() + @HttpCode(HttpStatus.CREATED) + async createRule( + @Headers('authorization') auth: string, + @Body() dto: CreateEvaluationRuleDto, + ) { + const admin = this.verifyAdmin(auth); + const tenantId = admin.tenantId || null; + + if (!dto.name || !dto.stage || !dto.ruleType || !dto.config) { + throw new BadRequestException('name, stage, ruleType, and config are required'); + } + + if (!Object.values(EvaluationRuleType).includes(dto.ruleType)) { + throw new BadRequestException(`Invalid ruleType. Valid types: ${Object.values(EvaluationRuleType).join(', ')}`); + } + + if (dto.failureAction && !Object.values(EvaluationFailureAction).includes(dto.failureAction)) { + throw new BadRequestException(`Invalid failureAction. Valid actions: ${Object.values(EvaluationFailureAction).join(', ')}`); + } + + const rule = EvaluationRuleEntity.create({ + id: uuidv4(), + tenantId, + stage: dto.stage, + ruleType: dto.ruleType, + name: dto.name, + description: dto.description, + config: dto.config, + priority: dto.priority, + failureAction: dto.failureAction, + createdBy: admin.id, + }); + + const saved = await this.repo.save(rule); + + // Invalidate cache + this.evaluationGate.clearCache(tenantId || undefined); + + return { success: true, data: this.toResponse(saved) }; + } + + // PUT /conversations/admin/evaluation-rules/:id + @Put(':id') + async updateRule( + @Headers('authorization') auth: string, + @Param('id') id: string, + @Body() dto: UpdateEvaluationRuleDto, + ) { + const admin = this.verifyAdmin(auth); + + const rule = await this.repo.findById(id); + if (!rule) { + return { success: false, error: 'Rule not found' }; + } + + if (dto.stage !== undefined) rule.stage = dto.stage; + if (dto.ruleType !== undefined) rule.ruleType = dto.ruleType; + if (dto.name !== undefined) rule.name = dto.name; + if (dto.description !== undefined) rule.description = dto.description; + if (dto.config !== undefined) rule.config = dto.config; + if (dto.priority !== undefined) rule.priority = dto.priority; + if (dto.failureAction !== undefined) rule.failureAction = dto.failureAction; + if (dto.enabled !== undefined) rule.enabled = dto.enabled; + rule.updatedBy = admin.id; + rule.updatedAt = new Date(); + + const updated = await this.repo.update(rule); + + // Invalidate cache + this.evaluationGate.clearCache(rule.tenantId || undefined); + + return { success: true, data: this.toResponse(updated) }; + } + + // DELETE /conversations/admin/evaluation-rules/:id + @Delete(':id') + async deleteRule( + @Headers('authorization') auth: string, + @Param('id') id: string, + ) { + this.verifyAdmin(auth); + + const rule = await this.repo.findById(id); + if (!rule) { + return { success: false, error: 'Rule not found' }; + } + + await this.repo.delete(id); + + // Invalidate cache + this.evaluationGate.clearCache(rule.tenantId || undefined); + + return { success: true }; + } + + // POST /conversations/admin/evaluation-rules/:id/toggle + @Post(':id/toggle') + async toggleRule( + @Headers('authorization') auth: string, + @Param('id') id: string, + ) { + const admin = this.verifyAdmin(auth); + + const rule = await this.repo.findById(id); + if (!rule) { + return { success: false, error: 'Rule not found' }; + } + + rule.toggle(); + rule.updatedBy = admin.id; + const updated = await this.repo.update(rule); + + // Invalidate cache + this.evaluationGate.clearCache(rule.tenantId || undefined); + + return { success: true, data: this.toResponse(updated) }; + } + + // POST /conversations/admin/evaluation-rules/test + @Post('test') + async testEvaluation( + @Headers('authorization') auth: string, + @Body() dto: TestEvaluationDto, + ) { + const admin = this.verifyAdmin(auth); + const tenantId = admin.tenantId || null; + + if (!dto.stage || !dto.responseText) { + throw new BadRequestException('stage and responseText are required'); + } + + const result = await this.evaluationGate.evaluate(tenantId, { + stage: dto.stage, + collectedInfo: dto.collectedInfo || null, + assessmentResult: dto.assessmentResult || null, + responseText: dto.responseText, + turnCount: dto.turnCount || 1, + messageCount: dto.messageCount || 1, + hasConverted: false, + agentsUsed: [], + }); + + return { success: true, data: result }; + } + + // DELETE /conversations/admin/evaluation-rules/cache + @Delete('cache') + async clearCache(@Headers('authorization') auth: string) { + this.verifyAdmin(auth); + this.evaluationGate.clearCache(); + return { success: true, message: 'Cache cleared' }; + } + + private toResponse(rule: EvaluationRuleEntity) { + return { + id: rule.id, + tenantId: rule.tenantId, + stage: rule.stage, + ruleType: rule.ruleType, + name: rule.name, + description: rule.description, + config: rule.config, + enabled: rule.enabled, + priority: rule.priority, + failureAction: rule.failureAction, + createdBy: rule.createdBy, + updatedBy: rule.updatedBy, + createdAt: rule.createdAt, + updatedAt: rule.updatedAt, + }; + } +} diff --git a/packages/services/conversation-service/src/application/dtos/evaluation-rule.dto.ts b/packages/services/conversation-service/src/application/dtos/evaluation-rule.dto.ts new file mode 100644 index 0000000..11c1177 --- /dev/null +++ b/packages/services/conversation-service/src/application/dtos/evaluation-rule.dto.ts @@ -0,0 +1,37 @@ +import { + EvaluationRuleTypeValue, + EvaluationFailureActionValue, +} from '../../domain/entities/evaluation-rule.entity'; + +export interface CreateEvaluationRuleDto { + stage: string; + ruleType: EvaluationRuleTypeValue; + name: string; + description?: string; + config: Record; + priority?: number; + failureAction?: EvaluationFailureActionValue; +} + +export interface UpdateEvaluationRuleDto { + stage?: string; + ruleType?: EvaluationRuleTypeValue; + name?: string; + description?: string; + config?: Record; + priority?: number; + failureAction?: EvaluationFailureActionValue; + enabled?: boolean; +} + +export interface TestEvaluationDto { + stage: string; + responseText: string; + collectedInfo?: Record; + assessmentResult?: { + topRecommended: string[]; + suitabilityScore: number; + }; + turnCount?: number; + messageCount?: number; +} diff --git a/packages/services/conversation-service/src/conversation/conversation.module.ts b/packages/services/conversation-service/src/conversation/conversation.module.ts index a7fae40..d677f47 100644 --- a/packages/services/conversation-service/src/conversation/conversation.module.ts +++ b/packages/services/conversation-service/src/conversation/conversation.module.ts @@ -15,11 +15,12 @@ import { ConversationController } from '../adapters/inbound/conversation.control import { InternalConversationController } from '../adapters/inbound/internal.controller'; import { AdminConversationController } from '../adapters/inbound/admin-conversation.controller'; import { AdminMcpController } from '../adapters/inbound/admin-mcp.controller'; +import { AdminEvaluationRuleController } from '../adapters/inbound/admin-evaluation-rule.controller'; import { ConversationGateway } from '../adapters/inbound/conversation.gateway'; @Module({ imports: [TypeOrmModule.forFeature([ConversationORM, MessageORM, TokenUsageORM, AgentExecutionORM])], - controllers: [ConversationController, InternalConversationController, AdminConversationController, AdminMcpController], + controllers: [ConversationController, InternalConversationController, AdminConversationController, AdminMcpController, AdminEvaluationRuleController], providers: [ ConversationService, ConversationGateway, diff --git a/packages/services/conversation-service/src/domain/entities/evaluation-rule.entity.ts b/packages/services/conversation-service/src/domain/entities/evaluation-rule.entity.ts new file mode 100644 index 0000000..c4342f8 --- /dev/null +++ b/packages/services/conversation-service/src/domain/entities/evaluation-rule.entity.ts @@ -0,0 +1,138 @@ +/** + * Evaluation Rule Domain Entity + * 评估规则 — 管理员可配置的质量门控规则 + */ + +export const EvaluationRuleType = { + FIELD_COMPLETENESS: 'FIELD_COMPLETENESS', + ASSESSMENT_QUALITY: 'ASSESSMENT_QUALITY', + RESPONSE_LENGTH: 'RESPONSE_LENGTH', + MUST_CONTAIN: 'MUST_CONTAIN', + STAGE_MIN_TURNS: 'STAGE_MIN_TURNS', + CONVERSION_SIGNAL: 'CONVERSION_SIGNAL', +} as const; + +export type EvaluationRuleTypeValue = + (typeof EvaluationRuleType)[keyof typeof EvaluationRuleType]; + +export const EvaluationFailureAction = { + RETRY: 'RETRY', + SUPPLEMENT: 'SUPPLEMENT', + WARN_AND_PASS: 'WARN_AND_PASS', + ESCALATE: 'ESCALATE', +} as const; + +export type EvaluationFailureActionValue = + (typeof EvaluationFailureAction)[keyof typeof EvaluationFailureAction]; + +/** Severity ordering for resolving multiple failures */ +const ACTION_SEVERITY: Record = { + [EvaluationFailureAction.WARN_AND_PASS]: 0, + [EvaluationFailureAction.SUPPLEMENT]: 1, + [EvaluationFailureAction.RETRY]: 2, + [EvaluationFailureAction.ESCALATE]: 3, +}; + +export function getHighestSeverityAction( + actions: EvaluationFailureActionValue[], +): EvaluationFailureActionValue { + if (actions.length === 0) return EvaluationFailureAction.WARN_AND_PASS; + return actions.reduce((highest, current) => + ACTION_SEVERITY[current] > ACTION_SEVERITY[highest] ? current : highest, + ); +} + +export class EvaluationRuleEntity { + readonly id: string; + tenantId: string | null; + stage: string; + ruleType: EvaluationRuleTypeValue; + name: string; + description: string | null; + config: Record; + enabled: boolean; + priority: number; + failureAction: EvaluationFailureActionValue; + createdBy: string | null; + updatedBy: string | null; + readonly createdAt: Date; + updatedAt: Date; + + private constructor(props: { + id: string; + tenantId: string | null; + stage: string; + ruleType: EvaluationRuleTypeValue; + name: string; + description: string | null; + config: Record; + enabled: boolean; + priority: number; + failureAction: EvaluationFailureActionValue; + createdBy: string | null; + updatedBy: string | null; + createdAt: Date; + updatedAt: Date; + }) { + Object.assign(this, props); + } + + static create(props: { + id: string; + tenantId: string | null; + stage: string; + ruleType: EvaluationRuleTypeValue; + name: string; + description?: string; + config: Record; + priority?: number; + failureAction?: EvaluationFailureActionValue; + createdBy?: string; + }): EvaluationRuleEntity { + const now = new Date(); + return new EvaluationRuleEntity({ + id: props.id, + tenantId: props.tenantId, + stage: props.stage, + ruleType: props.ruleType, + name: props.name, + description: props.description || null, + config: props.config, + enabled: true, + priority: props.priority ?? 0, + failureAction: props.failureAction ?? EvaluationFailureAction.WARN_AND_PASS, + createdBy: props.createdBy || null, + updatedBy: null, + createdAt: now, + updatedAt: now, + }); + } + + static fromPersistence(props: { + id: string; + tenantId: string | null; + stage: string; + ruleType: EvaluationRuleTypeValue; + name: string; + description: string | null; + config: Record; + enabled: boolean; + priority: number; + failureAction: EvaluationFailureActionValue; + createdBy: string | null; + updatedBy: string | null; + createdAt: Date; + updatedAt: Date; + }): EvaluationRuleEntity { + return new EvaluationRuleEntity(props); + } + + toggle(): void { + this.enabled = !this.enabled; + this.updatedAt = new Date(); + } + + isActive(): boolean { + return this.enabled; + } +} diff --git a/packages/services/conversation-service/src/domain/repositories/evaluation-rule.repository.interface.ts b/packages/services/conversation-service/src/domain/repositories/evaluation-rule.repository.interface.ts new file mode 100644 index 0000000..38a83d1 --- /dev/null +++ b/packages/services/conversation-service/src/domain/repositories/evaluation-rule.repository.interface.ts @@ -0,0 +1,12 @@ +import { EvaluationRuleEntity } from '../entities/evaluation-rule.entity'; + +export interface IEvaluationRuleRepository { + findByTenantAndStage(tenantId: string | null, stage: string): Promise; + findAllByTenant(tenantId: string | null): Promise; + findById(id: string): Promise; + save(rule: EvaluationRuleEntity): Promise; + update(rule: EvaluationRuleEntity): Promise; + delete(id: string): Promise; +} + +export const EVALUATION_RULE_REPOSITORY = Symbol('IEvaluationRuleRepository'); diff --git a/packages/services/conversation-service/src/infrastructure/agents/agents.module.ts b/packages/services/conversation-service/src/infrastructure/agents/agents.module.ts index 793f01e..179c154 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/agents.module.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/agents.module.ts @@ -35,6 +35,12 @@ import { TokenUsageORM } from '../database/postgres/entities/token-usage.orm'; // MCP Integration import { McpModule } from './mcp/mcp.module'; +// Evaluation Gate +import { EvaluationGateService } from './coordinator/evaluation-gate.service'; +import { EvaluationRuleORM } from '../database/postgres/entities/evaluation-rule.orm'; +import { EvaluationRulePostgresRepository } from '../database/postgres/repositories/evaluation-rule.repository'; +import { EVALUATION_RULE_REPOSITORY } from '../../domain/repositories/evaluation-rule.repository.interface'; + /** * Anthropic Client Provider * 共享的 Anthropic SDK 实例,所有 Agent 共用 @@ -65,7 +71,7 @@ const AnthropicClientProvider = { imports: [ ConfigModule, KnowledgeModule, - TypeOrmModule.forFeature([TokenUsageORM]), + TypeOrmModule.forFeature([TokenUsageORM, EvaluationRuleORM]), McpModule, ], providers: [ @@ -89,11 +95,19 @@ const AnthropicClientProvider = { CaseAnalystService, MemoryManagerService, + // Evaluation gate + { + provide: EVALUATION_RULE_REPOSITORY, + useClass: EvaluationRulePostgresRepository, + }, + EvaluationGateService, + // Main coordinator CoordinatorAgentService, ], exports: [ CoordinatorAgentService, + EvaluationGateService, // Export specialists for potential direct use PolicyExpertService, AssessmentExpertService, diff --git a/packages/services/conversation-service/src/infrastructure/agents/coordinator/agent-loop.ts b/packages/services/conversation-service/src/infrastructure/agents/coordinator/agent-loop.ts index 372bd75..f34e639 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/coordinator/agent-loop.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/coordinator/agent-loop.ts @@ -311,8 +311,79 @@ export async function* agentLoop( (b): b is Anthropic.ToolUseBlock => b.type === 'tool_use', ); - // If no tool_use → conversation is done + // If no tool_use → conversation is done (with optional evaluation gate) if (toolUseBlocks.length === 0 || finalMessage.stop_reason === 'end_turn') { + // --- Evaluation Gate (optional, zero-config safe) --- + if (params.evaluationGate) { + try { + const gateResult = await params.evaluationGate( + currentTextContent, + currentTurn + 1, + agentsUsed, + ); + + if ( + !gateResult.passed && + (gateResult.action === 'RETRY' || gateResult.action === 'SUPPLEMENT') + ) { + // Safety: don't retry if near limits + if (currentTurn + 1 < maxTurns && currentCost + turnCost < maxBudgetUsd) { + logger.debug( + `[Turn ${currentTurn + 1}] Evaluation gate: ${gateResult.action}, retrying...`, + ); + + yield { + type: 'coordinator_thinking', + phase: 'evaluating', + message: '质量检查未通过,正在重新生成...', + timestamp: Date.now(), + }; + + const feedbackMessage: ClaudeMessage = { + role: 'user', + content: gateResult.feedbackForModel || '[系统] 请重新生成回复。', + }; + + const retryMessages: ClaudeMessage[] = [ + ...messages, + { role: 'assistant', content: finalMessage.content as ContentBlock[] }, + feedbackMessage, + ]; + + // Recurse WITHOUT evaluationGate to prevent infinite loops (max 1 retry) + yield* agentLoop( + anthropicClient, + { + ...params, + messages: retryMessages, + currentTurnCount: currentTurn + 1, + currentCostUsd: currentCost + turnCost, + evaluationGate: undefined, + }, + toolExecutor, + additionalTools, + additionalConcurrencyMap, + ); + return; + } + } + + // WARN_AND_PASS or ESCALATE: emit warning event and continue to 'end' + if (!gateResult.passed) { + yield { + type: 'evaluation_warning', + results: gateResult.results.filter(r => !r.passed), + action: gateResult.action, + timestamp: Date.now(), + }; + } + } catch (gateError) { + // Gate error is non-fatal — log and continue + logger.error(`Evaluation gate error: ${gateError}`); + } + } + // --- End Evaluation Gate --- + yield { type: 'end', totalTokens: { diff --git a/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts b/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts index 7a436fe..8aa291d 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/coordinator/coordinator-agent.service.ts @@ -48,6 +48,9 @@ import { ConversationContext as OldConversationContext } from '../../claude/clau // MCP Integration import { McpClientService } from '../mcp/mcp-client.service'; +// Evaluation Gate +import { EvaluationGateService } from './evaluation-gate.service'; + // ============================================================ // Compatibility Types (与 ClaudeAgentServiceV2 的 StreamChunk 兼容) // ============================================================ @@ -65,7 +68,7 @@ export interface FileAttachment { /** 兼容旧版 StreamChunk 格式 */ export interface StreamChunk { type: 'text' | 'tool_use' | 'tool_result' | 'end' | 'usage' | 'stage_change' | 'state_update' | 'error' - | 'agent_start' | 'agent_progress' | 'agent_complete' | 'coordinator_thinking'; + | 'agent_start' | 'agent_progress' | 'agent_complete' | 'coordinator_thinking' | 'evaluation_warning'; content?: string; toolName?: string; toolInput?: Record; @@ -83,6 +86,9 @@ export interface StreamChunk { phase?: string; durationMs?: number; success?: boolean; + // Evaluation gate fields + results?: Array<{ ruleId: string; ruleName: string; ruleType: string; passed: boolean; failureAction: string; message?: string }>; + action?: string; } /** 兼容旧版 ConversationContext */ @@ -133,6 +139,8 @@ export class CoordinatorAgentService implements OnModuleInit { private readonly memoryManager: MemoryManagerService, // MCP client private readonly mcpClient: McpClientService, + // Evaluation gate + private readonly evaluationGate: EvaluationGateService, ) {} onModuleInit() { @@ -205,7 +213,25 @@ export class CoordinatorAgentService implements OnModuleInit { // 4. Create abort controller const abortController = new AbortController(); - // 5. Build agent loop params + // 5. Build evaluation gate callback + const evaluationGateCallback = async ( + responseText: string, + turnCount: number, + agentsUsedInLoop: string[], + ) => { + return this.evaluationGate.evaluate(null, { + stage: context.consultingState?.currentStageId || null, + collectedInfo: context.consultingState?.collectedInfo || null, + assessmentResult: context.consultingState?.assessmentResult || null, + responseText, + turnCount, + messageCount: context.previousMessages?.length || 0, + hasConverted: false, + agentsUsed: agentsUsedInLoop, + }); + }; + + // 6. Build agent loop params const loopParams: AgentLoopParams = { messages: enrichedMessages, systemPrompt, @@ -217,6 +243,7 @@ export class CoordinatorAgentService implements OnModuleInit { abortSignal: abortController.signal, currentTurnCount: 0, currentCostUsd: 0, + evaluationGate: evaluationGateCallback, }; // 6. Create tool executor @@ -558,6 +585,13 @@ export class CoordinatorAgentService implements OnModuleInit { newState: event.state, }; + case 'evaluation_warning': + return { + type: 'evaluation_warning', + results: event.results, + action: event.action, + }; + default: return { type: 'text', content: '' }; } diff --git a/packages/services/conversation-service/src/infrastructure/agents/coordinator/evaluation-gate.service.ts b/packages/services/conversation-service/src/infrastructure/agents/coordinator/evaluation-gate.service.ts new file mode 100644 index 0000000..bfc8496 --- /dev/null +++ b/packages/services/conversation-service/src/infrastructure/agents/coordinator/evaluation-gate.service.ts @@ -0,0 +1,400 @@ +/** + * Evaluation Gate Service + * 评估门控引擎 — 加载管理员配置的规则,在 agent loop 终止前执行检查 + * + * 设计原则: + * - 零规则 = 零检查 = 当前行为完全保留 + * - 规则从数据库加载,内存缓存 5 分钟 + * - 每条规则的评估器是纯函数,不访问数据库 + * - 多规则失败时取最高严重性动作 + */ + +import { Injectable, Inject, Logger } from '@nestjs/common'; +import { + IEvaluationRuleRepository, + EVALUATION_RULE_REPOSITORY, +} from '../../../domain/repositories/evaluation-rule.repository.interface'; +import { + EvaluationRuleEntity, + EvaluationRuleType, + EvaluationFailureAction, + EvaluationFailureActionValue, + getHighestSeverityAction, +} from '../../../domain/entities/evaluation-rule.entity'; + +// ============================================================ +// Types +// ============================================================ + +/** Context available for rule evaluation */ +export interface EvaluationContext { + stage: string | null; + collectedInfo: Record | null; + assessmentResult: { + topRecommended: string[]; + suitabilityScore: number; + } | null; + responseText: string; + turnCount: number; + messageCount: number; + hasConverted: boolean; + agentsUsed: string[]; +} + +/** Result of a single rule check */ +export interface RuleCheckResult { + ruleId: string; + ruleName: string; + ruleType: string; + passed: boolean; + failureAction: string; + message?: string; +} + +/** Aggregate gate result */ +export interface GateResult { + passed: boolean; + results: RuleCheckResult[]; + action: 'PASS' | 'RETRY' | 'SUPPLEMENT' | 'WARN_AND_PASS' | 'ESCALATE'; + feedbackForModel?: string; +} + +// ============================================================ +// Service +// ============================================================ + +@Injectable() +export class EvaluationGateService { + private readonly logger = new Logger(EvaluationGateService.name); + private cache = new Map(); + private readonly CACHE_TTL = 5 * 60 * 1000; // 5 minutes + + constructor( + @Inject(EVALUATION_RULE_REPOSITORY) + private readonly repo: IEvaluationRuleRepository, + ) {} + + /** + * Main entry: evaluate all applicable rules + * Returns passed=true if no rules exist or all pass + */ + async evaluate(tenantId: string | null, context: EvaluationContext): Promise { + const stage = context.stage || '*'; + const rules = await this.loadRules(tenantId, stage); + + // Zero rules = zero checks + if (rules.length === 0) { + return { passed: true, results: [], action: 'PASS' }; + } + + const results: RuleCheckResult[] = []; + + for (const rule of rules) { + const check = this.evaluateRule(rule, context); + results.push({ + ruleId: rule.id, + ruleName: rule.name, + ruleType: rule.ruleType, + passed: check.passed, + failureAction: rule.failureAction, + message: check.message, + }); + } + + const failedResults = results.filter(r => !r.passed); + + if (failedResults.length === 0) { + return { passed: true, results, action: 'PASS' }; + } + + const failedActions = failedResults.map( + r => r.failureAction as EvaluationFailureActionValue, + ); + const highestAction = getHighestSeverityAction(failedActions); + + const feedbackForModel = this.buildFeedback(failedResults, highestAction); + + this.logger.warn( + `Evaluation gate: ${failedResults.length}/${results.length} rules failed, action=${highestAction}, stage=${stage}`, + ); + + return { + passed: false, + results, + action: highestAction, + feedbackForModel, + }; + } + + /** Clear cached rules (e.g., after admin updates) */ + clearCache(tenantId?: string): void { + if (tenantId) { + for (const key of this.cache.keys()) { + if (key.startsWith(tenantId)) { + this.cache.delete(key); + } + } + } else { + this.cache.clear(); + } + } + + // ============================================================ + // Rule Loading (with cache) + // ============================================================ + + private async loadRules( + tenantId: string | null, + stage: string, + ): Promise { + const cacheKey = `${tenantId || 'global'}:${stage}`; + const cached = this.cache.get(cacheKey); + + if (cached && cached.expiresAt > Date.now()) { + return cached.rules; + } + + try { + const rules = await this.repo.findByTenantAndStage(tenantId, stage); + this.cache.set(cacheKey, { + rules, + expiresAt: Date.now() + this.CACHE_TTL, + }); + return rules; + } catch (error) { + this.logger.error(`Failed to load evaluation rules: ${error}`); + // On error, return empty = no checks = safe fallback + return []; + } + } + + // ============================================================ + // Rule Evaluation (pure functions) + // ============================================================ + + private evaluateRule( + rule: EvaluationRuleEntity, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + switch (rule.ruleType) { + case EvaluationRuleType.FIELD_COMPLETENESS: + return this.checkFieldCompleteness(rule.config, context); + case EvaluationRuleType.ASSESSMENT_QUALITY: + return this.checkAssessmentQuality(rule.config, context); + case EvaluationRuleType.RESPONSE_LENGTH: + return this.checkResponseLength(rule.config, context); + case EvaluationRuleType.MUST_CONTAIN: + return this.checkMustContain(rule.config, context); + case EvaluationRuleType.STAGE_MIN_TURNS: + return this.checkStageMinTurns(rule.config, context); + case EvaluationRuleType.CONVERSION_SIGNAL: + return this.checkConversionSignal(rule.config, context); + default: + this.logger.warn(`Unknown rule type: ${rule.ruleType}`); + return { passed: true }; + } + } + + /** + * FIELD_COMPLETENESS: Check if required fields exist in collectedInfo + * config: { fields: string[], minRatio: number } + */ + private checkFieldCompleteness( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const fields = (config.fields as string[]) || []; + const minRatio = (config.minRatio as number) ?? 1.0; + + if (fields.length === 0) return { passed: true }; + + const info = context.collectedInfo || {}; + const presentFields: string[] = []; + const missingFields: string[] = []; + + for (const field of fields) { + const value = info[field]; + if (value !== undefined && value !== null && value !== '') { + presentFields.push(field); + } else { + missingFields.push(field); + } + } + + const ratio = presentFields.length / fields.length; + const passed = ratio >= minRatio; + + return { + passed, + message: passed + ? undefined + : `信息完整度不足(${presentFields.length}/${fields.length}=${(ratio * 100).toFixed(0)}%,要求≥${(minRatio * 100).toFixed(0)}%),缺少: ${missingFields.join(', ')}`, + }; + } + + /** + * ASSESSMENT_QUALITY: Check assessment result quality + * config: { minCategories?: number, minConfidence?: number, minScore?: number } + */ + private checkAssessmentQuality( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const minCategories = (config.minCategories as number) ?? 1; + const minScore = (config.minScore as number) ?? 0; + + if (!context.assessmentResult) { + return { + passed: false, + message: `尚未完成评估(要求至少评估 ${minCategories} 个类别)`, + }; + } + + const categoryCount = context.assessmentResult.topRecommended?.length || 0; + if (categoryCount < minCategories) { + return { + passed: false, + message: `推荐类别数不足(${categoryCount}/${minCategories})`, + }; + } + + if (context.assessmentResult.suitabilityScore < minScore) { + return { + passed: false, + message: `适配分数过低(${context.assessmentResult.suitabilityScore}/${minScore})`, + }; + } + + return { passed: true }; + } + + /** + * RESPONSE_LENGTH: Check response text length + * config: { min?: number, max?: number } + */ + private checkResponseLength( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const min = (config.min as number) ?? 0; + const max = (config.max as number) ?? Infinity; + const length = context.responseText.length; + + if (length < min) { + return { + passed: false, + message: `回复过短(${length}字,最少${min}字)`, + }; + } + + if (length > max) { + return { + passed: false, + message: `回复过长(${length}字,最多${max}字)`, + }; + } + + return { passed: true }; + } + + /** + * MUST_CONTAIN: Check if response contains required keywords + * config: { keywords: string[], mode: 'any' | 'all' } + */ + private checkMustContain( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const keywords = (config.keywords as string[]) || []; + const mode = (config.mode as string) || 'any'; + + if (keywords.length === 0) return { passed: true }; + + const text = context.responseText; + const foundKeywords = keywords.filter(kw => text.includes(kw)); + const missingKeywords = keywords.filter(kw => !text.includes(kw)); + + if (mode === 'all') { + const passed = missingKeywords.length === 0; + return { + passed, + message: passed + ? undefined + : `回复缺少必需关键词: ${missingKeywords.join(', ')}`, + }; + } + + // mode === 'any' + const passed = foundKeywords.length > 0; + return { + passed, + message: passed + ? undefined + : `回复未包含任何期望关键词: ${keywords.join(', ')}`, + }; + } + + /** + * STAGE_MIN_TURNS: Check minimum conversation turns + * config: { minTurns: number } + */ + private checkStageMinTurns( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const minTurns = (config.minTurns as number) ?? 1; + const passed = context.turnCount >= minTurns; + + return { + passed, + message: passed + ? undefined + : `当前轮次不足(${context.turnCount}/${minTurns}轮)`, + }; + } + + /** + * CONVERSION_SIGNAL: Check for conversion-related content + * config: { keywords?: string[] } + */ + private checkConversionSignal( + config: Record, + context: EvaluationContext, + ): { passed: boolean; message?: string } { + const defaultKeywords = ['下一步', '预约', '联系', '咨询', '费用', '付款', '签约']; + const keywords = (config.keywords as string[]) || defaultKeywords; + + const text = context.responseText; + const found = keywords.some(kw => text.includes(kw)); + + return { + passed: found, + message: found + ? undefined + : '回复中缺少转化引导内容(如下一步行动、预约、联系方式等)', + }; + } + + // ============================================================ + // Feedback Builder + // ============================================================ + + private buildFeedback( + failedResults: RuleCheckResult[], + action: EvaluationFailureActionValue, + ): string { + const lines = ['[系统评估] 回复未通过质量检查:']; + + for (const result of failedResults) { + lines.push(`- [${result.ruleName}] ${result.message || '未通过'}`); + } + + if (action === EvaluationFailureAction.RETRY) { + lines.push('请根据以上反馈重新生成回复。'); + } else if (action === EvaluationFailureAction.SUPPLEMENT) { + lines.push('请在回复中补充缺失的信息后再回复用户。'); + } + + return lines.join('\n'); + } +} diff --git a/packages/services/conversation-service/src/infrastructure/agents/types/agent.types.ts b/packages/services/conversation-service/src/infrastructure/agents/types/agent.types.ts index cd0b953..f83b782 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/types/agent.types.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/types/agent.types.ts @@ -274,6 +274,12 @@ export interface AgentLoopParams { currentTurnCount?: number; /** 已经消耗的成本(递归时累加)*/ currentCostUsd?: number; + /** 可选评估门控回调。提供时,在 yield 'end' 前调用检查回复质量。 */ + evaluationGate?: ( + responseText: string, + turnCount: number, + agentsUsed: string[], + ) => Promise; } /** Claude API 消息格式 */ diff --git a/packages/services/conversation-service/src/infrastructure/agents/types/stream.types.ts b/packages/services/conversation-service/src/infrastructure/agents/types/stream.types.ts index f49d4d0..04a85ba 100644 --- a/packages/services/conversation-service/src/infrastructure/agents/types/stream.types.ts +++ b/packages/services/conversation-service/src/infrastructure/agents/types/stream.types.ts @@ -74,7 +74,7 @@ export interface AgentCompleteEvent extends BaseStreamEvent { /** Coordinator 正在思考/编排 */ export interface CoordinatorThinkingEvent extends BaseStreamEvent { type: 'coordinator_thinking'; - phase: 'analyzing' | 'orchestrating' | 'synthesizing'; + phase: 'analyzing' | 'orchestrating' | 'synthesizing' | 'evaluating'; message?: string; } @@ -113,6 +113,20 @@ export interface EndStreamEvent extends BaseStreamEvent { agentsUsed: SpecialistAgentType[]; } +/** 评估门控警告事件 */ +export interface EvaluationWarningEvent extends BaseStreamEvent { + type: 'evaluation_warning'; + results: Array<{ + ruleId: string; + ruleName: string; + ruleType: string; + passed: boolean; + failureAction: string; + message?: string; + }>; + action: string; +} + /** 错误事件 */ export interface ErrorStreamEvent extends BaseStreamEvent { type: 'error'; @@ -137,6 +151,7 @@ export type StreamEvent = | StateUpdateEvent | UsageEvent | EndStreamEvent + | EvaluationWarningEvent | ErrorStreamEvent; // ============================================================ diff --git a/packages/services/conversation-service/src/infrastructure/database/postgres/entities/evaluation-rule.orm.ts b/packages/services/conversation-service/src/infrastructure/database/postgres/entities/evaluation-rule.orm.ts new file mode 100644 index 0000000..989c327 --- /dev/null +++ b/packages/services/conversation-service/src/infrastructure/database/postgres/entities/evaluation-rule.orm.ts @@ -0,0 +1,55 @@ +import { + Entity, + PrimaryGeneratedColumn, + Column, + CreateDateColumn, + UpdateDateColumn, + Index, +} from 'typeorm'; + +@Entity('evaluation_rules') +@Index('idx_evaluation_rules_tenant', ['tenantId']) +@Index('idx_evaluation_rules_tenant_stage', ['tenantId', 'stage', 'enabled']) +export class EvaluationRuleORM { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ name: 'tenant_id', type: 'uuid', nullable: true }) + tenantId: string | null; + + @Column({ type: 'varchar', length: 50 }) + stage: string; + + @Column({ name: 'rule_type', type: 'varchar', length: 50 }) + ruleType: string; + + @Column({ type: 'varchar', length: 255 }) + name: string; + + @Column({ type: 'text', nullable: true }) + description: string | null; + + @Column({ type: 'jsonb', default: '{}' }) + config: Record; + + @Column({ type: 'boolean', default: true }) + enabled: boolean; + + @Column({ type: 'int', default: 0 }) + priority: number; + + @Column({ name: 'failure_action', type: 'varchar', length: 20, default: 'WARN_AND_PASS' }) + failureAction: string; + + @Column({ name: 'created_by', type: 'uuid', nullable: true }) + createdBy: string | null; + + @Column({ name: 'updated_by', type: 'uuid', nullable: true }) + updatedBy: string | null; + + @CreateDateColumn({ name: 'created_at' }) + createdAt: Date; + + @UpdateDateColumn({ name: 'updated_at' }) + updatedAt: Date; +} diff --git a/packages/services/conversation-service/src/infrastructure/database/postgres/repositories/evaluation-rule.repository.ts b/packages/services/conversation-service/src/infrastructure/database/postgres/repositories/evaluation-rule.repository.ts new file mode 100644 index 0000000..9aef64d --- /dev/null +++ b/packages/services/conversation-service/src/infrastructure/database/postgres/repositories/evaluation-rule.repository.ts @@ -0,0 +1,100 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository, IsNull } from 'typeorm'; +import { EvaluationRuleORM } from '../entities/evaluation-rule.orm'; +import { + IEvaluationRuleRepository, +} from '../../../../domain/repositories/evaluation-rule.repository.interface'; +import { + EvaluationRuleEntity, + EvaluationRuleTypeValue, + EvaluationFailureActionValue, +} from '../../../../domain/entities/evaluation-rule.entity'; + +@Injectable() +export class EvaluationRulePostgresRepository implements IEvaluationRuleRepository { + constructor( + @InjectRepository(EvaluationRuleORM) + private readonly repo: Repository, + ) {} + + async findByTenantAndStage( + tenantId: string | null, + stage: string, + ): Promise { + const orms = await this.repo + .createQueryBuilder('r') + .where('(r.tenant_id = :tenantId OR r.tenant_id IS NULL)', { tenantId }) + .andWhere('(r.stage = :stage OR r.stage = :wildcard)', { stage, wildcard: '*' }) + .andWhere('r.enabled = true') + .orderBy('r.priority', 'ASC') + .getMany(); + + return orms.map(orm => this.toEntity(orm)); + } + + async findAllByTenant(tenantId: string | null): Promise { + const orms = await this.repo.find({ + where: { tenantId: tenantId ?? IsNull() }, + order: { stage: 'ASC', priority: 'ASC' }, + }); + return orms.map(orm => this.toEntity(orm)); + } + + async findById(id: string): Promise { + const orm = await this.repo.findOne({ where: { id } }); + return orm ? this.toEntity(orm) : null; + } + + async save(rule: EvaluationRuleEntity): Promise { + const orm = this.toORM(rule); + const saved = await this.repo.save(orm); + return this.toEntity(saved); + } + + async update(rule: EvaluationRuleEntity): Promise { + const orm = this.toORM(rule); + const saved = await this.repo.save(orm); + return this.toEntity(saved); + } + + async delete(id: string): Promise { + await this.repo.delete(id); + } + + private toEntity(orm: EvaluationRuleORM): EvaluationRuleEntity { + return EvaluationRuleEntity.fromPersistence({ + id: orm.id, + tenantId: orm.tenantId, + stage: orm.stage, + ruleType: orm.ruleType as EvaluationRuleTypeValue, + name: orm.name, + description: orm.description, + config: orm.config, + enabled: orm.enabled, + priority: orm.priority, + failureAction: orm.failureAction as EvaluationFailureActionValue, + createdBy: orm.createdBy, + updatedBy: orm.updatedBy, + createdAt: orm.createdAt, + updatedAt: orm.updatedAt, + }); + } + + private toORM(entity: EvaluationRuleEntity): EvaluationRuleORM { + const orm = new EvaluationRuleORM(); + orm.id = entity.id; + orm.tenantId = entity.tenantId; + orm.stage = entity.stage; + orm.ruleType = entity.ruleType; + orm.name = entity.name; + orm.description = entity.description; + orm.config = entity.config; + orm.enabled = entity.enabled; + orm.priority = entity.priority; + orm.failureAction = entity.failureAction; + orm.createdBy = entity.createdBy; + orm.updatedBy = entity.updatedBy; + return orm; + } +}