feat(agents): add admin-configurable evaluation gate for agent loop quality control
Add a configurable evaluation gate system that allows administrators to define quality rules per consulting stage. The gate checks are executed programmatically before the agent loop returns a response to the user. ## Architecture - **Zero-config safe**: Empty rules table = no checks = current behavior preserved - **Callback-based decoupling**: agent-loop.ts receives an optional callback, stays decoupled from database layer - **Max 1 retry**: On RETRY/SUPPLEMENT failure, recurse once without gate to prevent infinite loops - **Error-tolerant**: Gate exceptions are caught and logged, never block responses ## New files - `database/migrations/20260206_add_evaluation_rules.sql` — DB migration - `domain/entities/evaluation-rule.entity.ts` — Domain entity with 6 rule types (FIELD_COMPLETENESS, ASSESSMENT_QUALITY, RESPONSE_LENGTH, MUST_CONTAIN, STAGE_MIN_TURNS, CONVERSION_SIGNAL) and 4 failure actions (RETRY, SUPPLEMENT, WARN_AND_PASS, ESCALATE) - `domain/repositories/evaluation-rule.repository.interface.ts` — Repository contract - `infrastructure/database/postgres/entities/evaluation-rule.orm.ts` — TypeORM ORM entity - `infrastructure/database/postgres/repositories/evaluation-rule.repository.ts` — Repository impl - `infrastructure/agents/coordinator/evaluation-gate.service.ts` — Core evaluation engine with 5-minute rule cache, per-rule-type evaluators, severity-based action resolution, and feedback message builder for model retry - `application/dtos/evaluation-rule.dto.ts` — Create/Update/Test DTOs - `adapters/inbound/admin-evaluation-rule.controller.ts` — Admin CRUD API with 8 endpoints: list, get, create, update, delete, toggle, test (dry-run), clear cache ## Modified files - `agent.types.ts` — Add optional `evaluationGate` callback to `AgentLoopParams` - `stream.types.ts` — Add `EvaluationWarningEvent`, `'evaluating'` phase - `agent-loop.ts` — Insert gate check at termination point (line 315) - `coordinator-agent.service.ts` — Inject EvaluationGateService, build callback, handle `evaluation_warning` event in StreamChunk mapping - `agents.module.ts` — Register EvaluationRuleORM, repository, EvaluationGateService - `conversation.module.ts` — Register AdminEvaluationRuleController Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
714a674818
commit
00a0ac3820
|
|
@ -0,0 +1,26 @@
|
||||||
|
-- Evaluation Rules: Admin-configurable quality gate rules per consulting stage
|
||||||
|
-- Zero rows = zero checks = current behavior preserved
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS evaluation_rules (
|
||||||
|
id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||||
|
tenant_id uuid,
|
||||||
|
stage varchar(50) NOT NULL, -- consulting stage (greeting, info_collection, etc.) or '*' for all stages
|
||||||
|
rule_type varchar(50) NOT NULL, -- FIELD_COMPLETENESS, ASSESSMENT_QUALITY, RESPONSE_LENGTH, MUST_CONTAIN, STAGE_MIN_TURNS, CONVERSION_SIGNAL
|
||||||
|
name varchar(255) NOT NULL,
|
||||||
|
description text,
|
||||||
|
config jsonb NOT NULL DEFAULT '{}', -- rule-specific parameters
|
||||||
|
enabled boolean DEFAULT true,
|
||||||
|
priority integer DEFAULT 0, -- lower = run first
|
||||||
|
failure_action varchar(20) NOT NULL DEFAULT 'WARN_AND_PASS', -- RETRY, SUPPLEMENT, WARN_AND_PASS, ESCALATE
|
||||||
|
created_by uuid,
|
||||||
|
updated_by uuid,
|
||||||
|
created_at timestamptz DEFAULT now(),
|
||||||
|
updated_at timestamptz DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_evaluation_rules_tenant ON evaluation_rules(tenant_id);
|
||||||
|
CREATE INDEX idx_evaluation_rules_tenant_stage ON evaluation_rules(tenant_id, stage, enabled);
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -0,0 +1,289 @@
|
||||||
|
/**
|
||||||
|
* Admin Evaluation Rule Controller
|
||||||
|
* 评估规则管理 API — 供 admin-client 使用
|
||||||
|
*
|
||||||
|
* 管理员可以：
|
||||||
|
* 1. 为不同咨询阶段配置评估规则
|
||||||
|
* 2. 开关规则
|
||||||
|
* 3. 测试规则（dry-run）
|
||||||
|
* 4. 清除规则缓存
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
Controller,
|
||||||
|
Get,
|
||||||
|
Post,
|
||||||
|
Put,
|
||||||
|
Delete,
|
||||||
|
Param,
|
||||||
|
Body,
|
||||||
|
Query,
|
||||||
|
Headers,
|
||||||
|
HttpCode,
|
||||||
|
HttpStatus,
|
||||||
|
UnauthorizedException,
|
||||||
|
BadRequestException,
|
||||||
|
} from '@nestjs/common';
|
||||||
|
import * as jwt from 'jsonwebtoken';
|
||||||
|
import { v4 as uuidv4 } from 'uuid';
|
||||||
|
import { EvaluationGateService } from '../../infrastructure/agents/coordinator/evaluation-gate.service';
|
||||||
|
import {
|
||||||
|
IEvaluationRuleRepository,
|
||||||
|
EVALUATION_RULE_REPOSITORY,
|
||||||
|
} from '../../domain/repositories/evaluation-rule.repository.interface';
|
||||||
|
import {
|
||||||
|
EvaluationRuleEntity,
|
||||||
|
EvaluationRuleType,
|
||||||
|
EvaluationFailureAction,
|
||||||
|
} from '../../domain/entities/evaluation-rule.entity';
|
||||||
|
import {
|
||||||
|
CreateEvaluationRuleDto,
|
||||||
|
UpdateEvaluationRuleDto,
|
||||||
|
TestEvaluationDto,
|
||||||
|
} from '../../application/dtos/evaluation-rule.dto';
|
||||||
|
import { Inject } from '@nestjs/common';
|
||||||
|
|
||||||
|
interface AdminPayload {
|
||||||
|
id: string;
|
||||||
|
username: string;
|
||||||
|
role: string;
|
||||||
|
tenantId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Controller('conversations/admin/evaluation-rules')
|
||||||
|
export class AdminEvaluationRuleController {
|
||||||
|
constructor(
|
||||||
|
@Inject(EVALUATION_RULE_REPOSITORY)
|
||||||
|
private readonly repo: IEvaluationRuleRepository,
|
||||||
|
private readonly evaluationGate: EvaluationGateService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
private verifyAdmin(authorization: string): AdminPayload {
|
||||||
|
const token = authorization?.replace('Bearer ', '');
|
||||||
|
if (!token) {
|
||||||
|
throw new UnauthorizedException('Missing token');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const secret = process.env.JWT_SECRET || 'your-jwt-secret-key';
|
||||||
|
return jwt.verify(token, secret) as AdminPayload;
|
||||||
|
} catch {
|
||||||
|
throw new UnauthorizedException('Invalid token');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /conversations/admin/evaluation-rules
|
||||||
|
@Get()
|
||||||
|
async listRules(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Query('stage') stage?: string,
|
||||||
|
) {
|
||||||
|
const admin = this.verifyAdmin(auth);
|
||||||
|
const tenantId = admin.tenantId || null;
|
||||||
|
|
||||||
|
const allRules = await this.repo.findAllByTenant(tenantId);
|
||||||
|
const filtered = stage
|
||||||
|
? allRules.filter(r => r.stage === stage || r.stage === '*')
|
||||||
|
: allRules;
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
items: filtered.map(r => this.toResponse(r)),
|
||||||
|
total: filtered.length,
|
||||||
|
ruleTypes: Object.values(EvaluationRuleType),
|
||||||
|
failureActions: Object.values(EvaluationFailureAction),
|
||||||
|
stages: ['*', 'greeting', 'needs_discovery', 'info_collection', 'assessment', 'recommendation', 'objection_handling', 'conversion', 'handoff'],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /conversations/admin/evaluation-rules/:id
|
||||||
|
@Get(':id')
|
||||||
|
async getRule(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Param('id') id: string,
|
||||||
|
) {
|
||||||
|
this.verifyAdmin(auth);
|
||||||
|
|
||||||
|
const rule = await this.repo.findById(id);
|
||||||
|
if (!rule) {
|
||||||
|
return { success: false, error: 'Rule not found' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, data: this.toResponse(rule) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /conversations/admin/evaluation-rules
|
||||||
|
@Post()
|
||||||
|
@HttpCode(HttpStatus.CREATED)
|
||||||
|
async createRule(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Body() dto: CreateEvaluationRuleDto,
|
||||||
|
) {
|
||||||
|
const admin = this.verifyAdmin(auth);
|
||||||
|
const tenantId = admin.tenantId || null;
|
||||||
|
|
||||||
|
if (!dto.name || !dto.stage || !dto.ruleType || !dto.config) {
|
||||||
|
throw new BadRequestException('name, stage, ruleType, and config are required');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Object.values(EvaluationRuleType).includes(dto.ruleType)) {
|
||||||
|
throw new BadRequestException(`Invalid ruleType. Valid types: ${Object.values(EvaluationRuleType).join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dto.failureAction && !Object.values(EvaluationFailureAction).includes(dto.failureAction)) {
|
||||||
|
throw new BadRequestException(`Invalid failureAction. Valid actions: ${Object.values(EvaluationFailureAction).join(', ')}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const rule = EvaluationRuleEntity.create({
|
||||||
|
id: uuidv4(),
|
||||||
|
tenantId,
|
||||||
|
stage: dto.stage,
|
||||||
|
ruleType: dto.ruleType,
|
||||||
|
name: dto.name,
|
||||||
|
description: dto.description,
|
||||||
|
config: dto.config,
|
||||||
|
priority: dto.priority,
|
||||||
|
failureAction: dto.failureAction,
|
||||||
|
createdBy: admin.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
const saved = await this.repo.save(rule);
|
||||||
|
|
||||||
|
// Invalidate cache
|
||||||
|
this.evaluationGate.clearCache(tenantId || undefined);
|
||||||
|
|
||||||
|
return { success: true, data: this.toResponse(saved) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// PUT /conversations/admin/evaluation-rules/:id
|
||||||
|
@Put(':id')
|
||||||
|
async updateRule(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Param('id') id: string,
|
||||||
|
@Body() dto: UpdateEvaluationRuleDto,
|
||||||
|
) {
|
||||||
|
const admin = this.verifyAdmin(auth);
|
||||||
|
|
||||||
|
const rule = await this.repo.findById(id);
|
||||||
|
if (!rule) {
|
||||||
|
return { success: false, error: 'Rule not found' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dto.stage !== undefined) rule.stage = dto.stage;
|
||||||
|
if (dto.ruleType !== undefined) rule.ruleType = dto.ruleType;
|
||||||
|
if (dto.name !== undefined) rule.name = dto.name;
|
||||||
|
if (dto.description !== undefined) rule.description = dto.description;
|
||||||
|
if (dto.config !== undefined) rule.config = dto.config;
|
||||||
|
if (dto.priority !== undefined) rule.priority = dto.priority;
|
||||||
|
if (dto.failureAction !== undefined) rule.failureAction = dto.failureAction;
|
||||||
|
if (dto.enabled !== undefined) rule.enabled = dto.enabled;
|
||||||
|
rule.updatedBy = admin.id;
|
||||||
|
rule.updatedAt = new Date();
|
||||||
|
|
||||||
|
const updated = await this.repo.update(rule);
|
||||||
|
|
||||||
|
// Invalidate cache
|
||||||
|
this.evaluationGate.clearCache(rule.tenantId || undefined);
|
||||||
|
|
||||||
|
return { success: true, data: this.toResponse(updated) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// DELETE /conversations/admin/evaluation-rules/:id
|
||||||
|
@Delete(':id')
|
||||||
|
async deleteRule(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Param('id') id: string,
|
||||||
|
) {
|
||||||
|
this.verifyAdmin(auth);
|
||||||
|
|
||||||
|
const rule = await this.repo.findById(id);
|
||||||
|
if (!rule) {
|
||||||
|
return { success: false, error: 'Rule not found' };
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.repo.delete(id);
|
||||||
|
|
||||||
|
// Invalidate cache
|
||||||
|
this.evaluationGate.clearCache(rule.tenantId || undefined);
|
||||||
|
|
||||||
|
return { success: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /conversations/admin/evaluation-rules/:id/toggle
|
||||||
|
@Post(':id/toggle')
|
||||||
|
async toggleRule(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Param('id') id: string,
|
||||||
|
) {
|
||||||
|
const admin = this.verifyAdmin(auth);
|
||||||
|
|
||||||
|
const rule = await this.repo.findById(id);
|
||||||
|
if (!rule) {
|
||||||
|
return { success: false, error: 'Rule not found' };
|
||||||
|
}
|
||||||
|
|
||||||
|
rule.toggle();
|
||||||
|
rule.updatedBy = admin.id;
|
||||||
|
const updated = await this.repo.update(rule);
|
||||||
|
|
||||||
|
// Invalidate cache
|
||||||
|
this.evaluationGate.clearCache(rule.tenantId || undefined);
|
||||||
|
|
||||||
|
return { success: true, data: this.toResponse(updated) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /conversations/admin/evaluation-rules/test
|
||||||
|
@Post('test')
|
||||||
|
async testEvaluation(
|
||||||
|
@Headers('authorization') auth: string,
|
||||||
|
@Body() dto: TestEvaluationDto,
|
||||||
|
) {
|
||||||
|
const admin = this.verifyAdmin(auth);
|
||||||
|
const tenantId = admin.tenantId || null;
|
||||||
|
|
||||||
|
if (!dto.stage || !dto.responseText) {
|
||||||
|
throw new BadRequestException('stage and responseText are required');
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.evaluationGate.evaluate(tenantId, {
|
||||||
|
stage: dto.stage,
|
||||||
|
collectedInfo: dto.collectedInfo || null,
|
||||||
|
assessmentResult: dto.assessmentResult || null,
|
||||||
|
responseText: dto.responseText,
|
||||||
|
turnCount: dto.turnCount || 1,
|
||||||
|
messageCount: dto.messageCount || 1,
|
||||||
|
hasConverted: false,
|
||||||
|
agentsUsed: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
return { success: true, data: result };
|
||||||
|
}
|
||||||
|
|
||||||
|
// DELETE /conversations/admin/evaluation-rules/cache
|
||||||
|
@Delete('cache')
|
||||||
|
async clearCache(@Headers('authorization') auth: string) {
|
||||||
|
this.verifyAdmin(auth);
|
||||||
|
this.evaluationGate.clearCache();
|
||||||
|
return { success: true, message: 'Cache cleared' };
|
||||||
|
}
|
||||||
|
|
||||||
|
private toResponse(rule: EvaluationRuleEntity) {
|
||||||
|
return {
|
||||||
|
id: rule.id,
|
||||||
|
tenantId: rule.tenantId,
|
||||||
|
stage: rule.stage,
|
||||||
|
ruleType: rule.ruleType,
|
||||||
|
name: rule.name,
|
||||||
|
description: rule.description,
|
||||||
|
config: rule.config,
|
||||||
|
enabled: rule.enabled,
|
||||||
|
priority: rule.priority,
|
||||||
|
failureAction: rule.failureAction,
|
||||||
|
createdBy: rule.createdBy,
|
||||||
|
updatedBy: rule.updatedBy,
|
||||||
|
createdAt: rule.createdAt,
|
||||||
|
updatedAt: rule.updatedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
import {
|
||||||
|
EvaluationRuleTypeValue,
|
||||||
|
EvaluationFailureActionValue,
|
||||||
|
} from '../../domain/entities/evaluation-rule.entity';
|
||||||
|
|
||||||
|
export interface CreateEvaluationRuleDto {
|
||||||
|
stage: string;
|
||||||
|
ruleType: EvaluationRuleTypeValue;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
priority?: number;
|
||||||
|
failureAction?: EvaluationFailureActionValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UpdateEvaluationRuleDto {
|
||||||
|
stage?: string;
|
||||||
|
ruleType?: EvaluationRuleTypeValue;
|
||||||
|
name?: string;
|
||||||
|
description?: string;
|
||||||
|
config?: Record<string, unknown>;
|
||||||
|
priority?: number;
|
||||||
|
failureAction?: EvaluationFailureActionValue;
|
||||||
|
enabled?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TestEvaluationDto {
|
||||||
|
stage: string;
|
||||||
|
responseText: string;
|
||||||
|
collectedInfo?: Record<string, unknown>;
|
||||||
|
assessmentResult?: {
|
||||||
|
topRecommended: string[];
|
||||||
|
suitabilityScore: number;
|
||||||
|
};
|
||||||
|
turnCount?: number;
|
||||||
|
messageCount?: number;
|
||||||
|
}
|
||||||
|
|
@ -15,11 +15,12 @@ import { ConversationController } from '../adapters/inbound/conversation.control
|
||||||
import { InternalConversationController } from '../adapters/inbound/internal.controller';
|
import { InternalConversationController } from '../adapters/inbound/internal.controller';
|
||||||
import { AdminConversationController } from '../adapters/inbound/admin-conversation.controller';
|
import { AdminConversationController } from '../adapters/inbound/admin-conversation.controller';
|
||||||
import { AdminMcpController } from '../adapters/inbound/admin-mcp.controller';
|
import { AdminMcpController } from '../adapters/inbound/admin-mcp.controller';
|
||||||
|
import { AdminEvaluationRuleController } from '../adapters/inbound/admin-evaluation-rule.controller';
|
||||||
import { ConversationGateway } from '../adapters/inbound/conversation.gateway';
|
import { ConversationGateway } from '../adapters/inbound/conversation.gateway';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [TypeOrmModule.forFeature([ConversationORM, MessageORM, TokenUsageORM, AgentExecutionORM])],
|
imports: [TypeOrmModule.forFeature([ConversationORM, MessageORM, TokenUsageORM, AgentExecutionORM])],
|
||||||
controllers: [ConversationController, InternalConversationController, AdminConversationController, AdminMcpController],
|
controllers: [ConversationController, InternalConversationController, AdminConversationController, AdminMcpController, AdminEvaluationRuleController],
|
||||||
providers: [
|
providers: [
|
||||||
ConversationService,
|
ConversationService,
|
||||||
ConversationGateway,
|
ConversationGateway,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,138 @@
|
||||||
|
/**
|
||||||
|
* Evaluation Rule Domain Entity
|
||||||
|
* 评估规则 — 管理员可配置的质量门控规则
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const EvaluationRuleType = {
|
||||||
|
FIELD_COMPLETENESS: 'FIELD_COMPLETENESS',
|
||||||
|
ASSESSMENT_QUALITY: 'ASSESSMENT_QUALITY',
|
||||||
|
RESPONSE_LENGTH: 'RESPONSE_LENGTH',
|
||||||
|
MUST_CONTAIN: 'MUST_CONTAIN',
|
||||||
|
STAGE_MIN_TURNS: 'STAGE_MIN_TURNS',
|
||||||
|
CONVERSION_SIGNAL: 'CONVERSION_SIGNAL',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type EvaluationRuleTypeValue =
|
||||||
|
(typeof EvaluationRuleType)[keyof typeof EvaluationRuleType];
|
||||||
|
|
||||||
|
export const EvaluationFailureAction = {
|
||||||
|
RETRY: 'RETRY',
|
||||||
|
SUPPLEMENT: 'SUPPLEMENT',
|
||||||
|
WARN_AND_PASS: 'WARN_AND_PASS',
|
||||||
|
ESCALATE: 'ESCALATE',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type EvaluationFailureActionValue =
|
||||||
|
(typeof EvaluationFailureAction)[keyof typeof EvaluationFailureAction];
|
||||||
|
|
||||||
|
/** Severity ordering for resolving multiple failures */
|
||||||
|
const ACTION_SEVERITY: Record<EvaluationFailureActionValue, number> = {
|
||||||
|
[EvaluationFailureAction.WARN_AND_PASS]: 0,
|
||||||
|
[EvaluationFailureAction.SUPPLEMENT]: 1,
|
||||||
|
[EvaluationFailureAction.RETRY]: 2,
|
||||||
|
[EvaluationFailureAction.ESCALATE]: 3,
|
||||||
|
};
|
||||||
|
|
||||||
|
export function getHighestSeverityAction(
|
||||||
|
actions: EvaluationFailureActionValue[],
|
||||||
|
): EvaluationFailureActionValue {
|
||||||
|
if (actions.length === 0) return EvaluationFailureAction.WARN_AND_PASS;
|
||||||
|
return actions.reduce((highest, current) =>
|
||||||
|
ACTION_SEVERITY[current] > ACTION_SEVERITY[highest] ? current : highest,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export class EvaluationRuleEntity {
|
||||||
|
readonly id: string;
|
||||||
|
tenantId: string | null;
|
||||||
|
stage: string;
|
||||||
|
ruleType: EvaluationRuleTypeValue;
|
||||||
|
name: string;
|
||||||
|
description: string | null;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
enabled: boolean;
|
||||||
|
priority: number;
|
||||||
|
failureAction: EvaluationFailureActionValue;
|
||||||
|
createdBy: string | null;
|
||||||
|
updatedBy: string | null;
|
||||||
|
readonly createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
|
||||||
|
private constructor(props: {
|
||||||
|
id: string;
|
||||||
|
tenantId: string | null;
|
||||||
|
stage: string;
|
||||||
|
ruleType: EvaluationRuleTypeValue;
|
||||||
|
name: string;
|
||||||
|
description: string | null;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
enabled: boolean;
|
||||||
|
priority: number;
|
||||||
|
failureAction: EvaluationFailureActionValue;
|
||||||
|
createdBy: string | null;
|
||||||
|
updatedBy: string | null;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}) {
|
||||||
|
Object.assign(this, props);
|
||||||
|
}
|
||||||
|
|
||||||
|
static create(props: {
|
||||||
|
id: string;
|
||||||
|
tenantId: string | null;
|
||||||
|
stage: string;
|
||||||
|
ruleType: EvaluationRuleTypeValue;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
priority?: number;
|
||||||
|
failureAction?: EvaluationFailureActionValue;
|
||||||
|
createdBy?: string;
|
||||||
|
}): EvaluationRuleEntity {
|
||||||
|
const now = new Date();
|
||||||
|
return new EvaluationRuleEntity({
|
||||||
|
id: props.id,
|
||||||
|
tenantId: props.tenantId,
|
||||||
|
stage: props.stage,
|
||||||
|
ruleType: props.ruleType,
|
||||||
|
name: props.name,
|
||||||
|
description: props.description || null,
|
||||||
|
config: props.config,
|
||||||
|
enabled: true,
|
||||||
|
priority: props.priority ?? 0,
|
||||||
|
failureAction: props.failureAction ?? EvaluationFailureAction.WARN_AND_PASS,
|
||||||
|
createdBy: props.createdBy || null,
|
||||||
|
updatedBy: null,
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
static fromPersistence(props: {
|
||||||
|
id: string;
|
||||||
|
tenantId: string | null;
|
||||||
|
stage: string;
|
||||||
|
ruleType: EvaluationRuleTypeValue;
|
||||||
|
name: string;
|
||||||
|
description: string | null;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
enabled: boolean;
|
||||||
|
priority: number;
|
||||||
|
failureAction: EvaluationFailureActionValue;
|
||||||
|
createdBy: string | null;
|
||||||
|
updatedBy: string | null;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}): EvaluationRuleEntity {
|
||||||
|
return new EvaluationRuleEntity(props);
|
||||||
|
}
|
||||||
|
|
||||||
|
toggle(): void {
|
||||||
|
this.enabled = !this.enabled;
|
||||||
|
this.updatedAt = new Date();
|
||||||
|
}
|
||||||
|
|
||||||
|
isActive(): boolean {
|
||||||
|
return this.enabled;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
import { EvaluationRuleEntity } from '../entities/evaluation-rule.entity';
|
||||||
|
|
||||||
|
export interface IEvaluationRuleRepository {
|
||||||
|
findByTenantAndStage(tenantId: string | null, stage: string): Promise<EvaluationRuleEntity[]>;
|
||||||
|
findAllByTenant(tenantId: string | null): Promise<EvaluationRuleEntity[]>;
|
||||||
|
findById(id: string): Promise<EvaluationRuleEntity | null>;
|
||||||
|
save(rule: EvaluationRuleEntity): Promise<EvaluationRuleEntity>;
|
||||||
|
update(rule: EvaluationRuleEntity): Promise<EvaluationRuleEntity>;
|
||||||
|
delete(id: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const EVALUATION_RULE_REPOSITORY = Symbol('IEvaluationRuleRepository');
|
||||||
|
|
@ -35,6 +35,12 @@ import { TokenUsageORM } from '../database/postgres/entities/token-usage.orm';
|
||||||
// MCP Integration
|
// MCP Integration
|
||||||
import { McpModule } from './mcp/mcp.module';
|
import { McpModule } from './mcp/mcp.module';
|
||||||
|
|
||||||
|
// Evaluation Gate
|
||||||
|
import { EvaluationGateService } from './coordinator/evaluation-gate.service';
|
||||||
|
import { EvaluationRuleORM } from '../database/postgres/entities/evaluation-rule.orm';
|
||||||
|
import { EvaluationRulePostgresRepository } from '../database/postgres/repositories/evaluation-rule.repository';
|
||||||
|
import { EVALUATION_RULE_REPOSITORY } from '../../domain/repositories/evaluation-rule.repository.interface';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Anthropic Client Provider
|
* Anthropic Client Provider
|
||||||
* 共享的 Anthropic SDK 实例，所有 Agent 共用
|
* 共享的 Anthropic SDK 实例，所有 Agent 共用
|
||||||
|
|
@ -65,7 +71,7 @@ const AnthropicClientProvider = {
|
||||||
imports: [
|
imports: [
|
||||||
ConfigModule,
|
ConfigModule,
|
||||||
KnowledgeModule,
|
KnowledgeModule,
|
||||||
TypeOrmModule.forFeature([TokenUsageORM]),
|
TypeOrmModule.forFeature([TokenUsageORM, EvaluationRuleORM]),
|
||||||
McpModule,
|
McpModule,
|
||||||
],
|
],
|
||||||
providers: [
|
providers: [
|
||||||
|
|
@ -89,11 +95,19 @@ const AnthropicClientProvider = {
|
||||||
CaseAnalystService,
|
CaseAnalystService,
|
||||||
MemoryManagerService,
|
MemoryManagerService,
|
||||||
|
|
||||||
|
// Evaluation gate
|
||||||
|
{
|
||||||
|
provide: EVALUATION_RULE_REPOSITORY,
|
||||||
|
useClass: EvaluationRulePostgresRepository,
|
||||||
|
},
|
||||||
|
EvaluationGateService,
|
||||||
|
|
||||||
// Main coordinator
|
// Main coordinator
|
||||||
CoordinatorAgentService,
|
CoordinatorAgentService,
|
||||||
],
|
],
|
||||||
exports: [
|
exports: [
|
||||||
CoordinatorAgentService,
|
CoordinatorAgentService,
|
||||||
|
EvaluationGateService,
|
||||||
// Export specialists for potential direct use
|
// Export specialists for potential direct use
|
||||||
PolicyExpertService,
|
PolicyExpertService,
|
||||||
AssessmentExpertService,
|
AssessmentExpertService,
|
||||||
|
|
|
||||||
|
|
@ -311,8 +311,79 @@ export async function* agentLoop(
|
||||||
(b): b is Anthropic.ToolUseBlock => b.type === 'tool_use',
|
(b): b is Anthropic.ToolUseBlock => b.type === 'tool_use',
|
||||||
);
|
);
|
||||||
|
|
||||||
// If no tool_use → conversation is done
|
// If no tool_use → conversation is done (with optional evaluation gate)
|
||||||
if (toolUseBlocks.length === 0 || finalMessage.stop_reason === 'end_turn') {
|
if (toolUseBlocks.length === 0 || finalMessage.stop_reason === 'end_turn') {
|
||||||
|
// --- Evaluation Gate (optional, zero-config safe) ---
|
||||||
|
if (params.evaluationGate) {
|
||||||
|
try {
|
||||||
|
const gateResult = await params.evaluationGate(
|
||||||
|
currentTextContent,
|
||||||
|
currentTurn + 1,
|
||||||
|
agentsUsed,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (
|
||||||
|
!gateResult.passed &&
|
||||||
|
(gateResult.action === 'RETRY' || gateResult.action === 'SUPPLEMENT')
|
||||||
|
) {
|
||||||
|
// Safety: don't retry if near limits
|
||||||
|
if (currentTurn + 1 < maxTurns && currentCost + turnCost < maxBudgetUsd) {
|
||||||
|
logger.debug(
|
||||||
|
`[Turn ${currentTurn + 1}] Evaluation gate: ${gateResult.action}, retrying...`,
|
||||||
|
);
|
||||||
|
|
||||||
|
yield {
|
||||||
|
type: 'coordinator_thinking',
|
||||||
|
phase: 'evaluating',
|
||||||
|
message: '质量检查未通过，正在重新生成...',
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
|
const feedbackMessage: ClaudeMessage = {
|
||||||
|
role: 'user',
|
||||||
|
content: gateResult.feedbackForModel || '[系统] 请重新生成回复。',
|
||||||
|
};
|
||||||
|
|
||||||
|
const retryMessages: ClaudeMessage[] = [
|
||||||
|
...messages,
|
||||||
|
{ role: 'assistant', content: finalMessage.content as ContentBlock[] },
|
||||||
|
feedbackMessage,
|
||||||
|
];
|
||||||
|
|
||||||
|
// Recurse WITHOUT evaluationGate to prevent infinite loops (max 1 retry)
|
||||||
|
yield* agentLoop(
|
||||||
|
anthropicClient,
|
||||||
|
{
|
||||||
|
...params,
|
||||||
|
messages: retryMessages,
|
||||||
|
currentTurnCount: currentTurn + 1,
|
||||||
|
currentCostUsd: currentCost + turnCost,
|
||||||
|
evaluationGate: undefined,
|
||||||
|
},
|
||||||
|
toolExecutor,
|
||||||
|
additionalTools,
|
||||||
|
additionalConcurrencyMap,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WARN_AND_PASS or ESCALATE: emit warning event and continue to 'end'
|
||||||
|
if (!gateResult.passed) {
|
||||||
|
yield {
|
||||||
|
type: 'evaluation_warning',
|
||||||
|
results: gateResult.results.filter(r => !r.passed),
|
||||||
|
action: gateResult.action,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (gateError) {
|
||||||
|
// Gate error is non-fatal — log and continue
|
||||||
|
logger.error(`Evaluation gate error: ${gateError}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// --- End Evaluation Gate ---
|
||||||
|
|
||||||
yield {
|
yield {
|
||||||
type: 'end',
|
type: 'end',
|
||||||
totalTokens: {
|
totalTokens: {
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,9 @@ import { ConversationContext as OldConversationContext } from '../../claude/clau
|
||||||
// MCP Integration
|
// MCP Integration
|
||||||
import { McpClientService } from '../mcp/mcp-client.service';
|
import { McpClientService } from '../mcp/mcp-client.service';
|
||||||
|
|
||||||
|
// Evaluation Gate
|
||||||
|
import { EvaluationGateService } from './evaluation-gate.service';
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
// Compatibility Types (与 ClaudeAgentServiceV2 的 StreamChunk 兼容)
|
// Compatibility Types (与 ClaudeAgentServiceV2 的 StreamChunk 兼容)
|
||||||
// ============================================================
|
// ============================================================
|
||||||
|
|
@ -65,7 +68,7 @@ export interface FileAttachment {
|
||||||
/** 兼容旧版 StreamChunk 格式 */
|
/** 兼容旧版 StreamChunk 格式 */
|
||||||
export interface StreamChunk {
|
export interface StreamChunk {
|
||||||
type: 'text' | 'tool_use' | 'tool_result' | 'end' | 'usage' | 'stage_change' | 'state_update' | 'error'
|
type: 'text' | 'tool_use' | 'tool_result' | 'end' | 'usage' | 'stage_change' | 'state_update' | 'error'
|
||||||
| 'agent_start' | 'agent_progress' | 'agent_complete' | 'coordinator_thinking';
|
| 'agent_start' | 'agent_progress' | 'agent_complete' | 'coordinator_thinking' | 'evaluation_warning';
|
||||||
content?: string;
|
content?: string;
|
||||||
toolName?: string;
|
toolName?: string;
|
||||||
toolInput?: Record<string, unknown>;
|
toolInput?: Record<string, unknown>;
|
||||||
|
|
@ -83,6 +86,9 @@ export interface StreamChunk {
|
||||||
phase?: string;
|
phase?: string;
|
||||||
durationMs?: number;
|
durationMs?: number;
|
||||||
success?: boolean;
|
success?: boolean;
|
||||||
|
// Evaluation gate fields
|
||||||
|
results?: Array<{ ruleId: string; ruleName: string; ruleType: string; passed: boolean; failureAction: string; message?: string }>;
|
||||||
|
action?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 兼容旧版 ConversationContext */
|
/** 兼容旧版 ConversationContext */
|
||||||
|
|
@ -133,6 +139,8 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
private readonly memoryManager: MemoryManagerService,
|
private readonly memoryManager: MemoryManagerService,
|
||||||
// MCP client
|
// MCP client
|
||||||
private readonly mcpClient: McpClientService,
|
private readonly mcpClient: McpClientService,
|
||||||
|
// Evaluation gate
|
||||||
|
private readonly evaluationGate: EvaluationGateService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
onModuleInit() {
|
onModuleInit() {
|
||||||
|
|
@ -205,7 +213,25 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
// 4. Create abort controller
|
// 4. Create abort controller
|
||||||
const abortController = new AbortController();
|
const abortController = new AbortController();
|
||||||
|
|
||||||
// 5. Build agent loop params
|
// 5. Build evaluation gate callback
|
||||||
|
const evaluationGateCallback = async (
|
||||||
|
responseText: string,
|
||||||
|
turnCount: number,
|
||||||
|
agentsUsedInLoop: string[],
|
||||||
|
) => {
|
||||||
|
return this.evaluationGate.evaluate(null, {
|
||||||
|
stage: context.consultingState?.currentStageId || null,
|
||||||
|
collectedInfo: context.consultingState?.collectedInfo || null,
|
||||||
|
assessmentResult: context.consultingState?.assessmentResult || null,
|
||||||
|
responseText,
|
||||||
|
turnCount,
|
||||||
|
messageCount: context.previousMessages?.length || 0,
|
||||||
|
hasConverted: false,
|
||||||
|
agentsUsed: agentsUsedInLoop,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
// 6. Build agent loop params
|
||||||
const loopParams: AgentLoopParams = {
|
const loopParams: AgentLoopParams = {
|
||||||
messages: enrichedMessages,
|
messages: enrichedMessages,
|
||||||
systemPrompt,
|
systemPrompt,
|
||||||
|
|
@ -217,6 +243,7 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
abortSignal: abortController.signal,
|
abortSignal: abortController.signal,
|
||||||
currentTurnCount: 0,
|
currentTurnCount: 0,
|
||||||
currentCostUsd: 0,
|
currentCostUsd: 0,
|
||||||
|
evaluationGate: evaluationGateCallback,
|
||||||
};
|
};
|
||||||
|
|
||||||
// 6. Create tool executor
|
// 6. Create tool executor
|
||||||
|
|
@ -558,6 +585,13 @@ export class CoordinatorAgentService implements OnModuleInit {
|
||||||
newState: event.state,
|
newState: event.state,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
case 'evaluation_warning':
|
||||||
|
return {
|
||||||
|
type: 'evaluation_warning',
|
||||||
|
results: event.results,
|
||||||
|
action: event.action,
|
||||||
|
};
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return { type: 'text', content: '' };
|
return { type: 'text', content: '' };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,400 @@
|
||||||
|
/**
|
||||||
|
* Evaluation Gate Service
|
||||||
|
* 评估门控引擎 — 加载管理员配置的规则，在 agent loop 终止前执行检查
|
||||||
|
*
|
||||||
|
* 设计原则：
|
||||||
|
* - 零规则 = 零检查 = 当前行为完全保留
|
||||||
|
* - 规则从数据库加载，内存缓存 5 分钟
|
||||||
|
* - 每条规则的评估器是纯函数，不访问数据库
|
||||||
|
* - 多规则失败时取最高严重性动作
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Injectable, Inject, Logger } from '@nestjs/common';
|
||||||
|
import {
|
||||||
|
IEvaluationRuleRepository,
|
||||||
|
EVALUATION_RULE_REPOSITORY,
|
||||||
|
} from '../../../domain/repositories/evaluation-rule.repository.interface';
|
||||||
|
import {
|
||||||
|
EvaluationRuleEntity,
|
||||||
|
EvaluationRuleType,
|
||||||
|
EvaluationFailureAction,
|
||||||
|
EvaluationFailureActionValue,
|
||||||
|
getHighestSeverityAction,
|
||||||
|
} from '../../../domain/entities/evaluation-rule.entity';
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Types
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
/** Context available for rule evaluation */
|
||||||
|
export interface EvaluationContext {
|
||||||
|
stage: string | null;
|
||||||
|
collectedInfo: Record<string, unknown> | null;
|
||||||
|
assessmentResult: {
|
||||||
|
topRecommended: string[];
|
||||||
|
suitabilityScore: number;
|
||||||
|
} | null;
|
||||||
|
responseText: string;
|
||||||
|
turnCount: number;
|
||||||
|
messageCount: number;
|
||||||
|
hasConverted: boolean;
|
||||||
|
agentsUsed: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Result of a single rule check */
|
||||||
|
export interface RuleCheckResult {
|
||||||
|
ruleId: string;
|
||||||
|
ruleName: string;
|
||||||
|
ruleType: string;
|
||||||
|
passed: boolean;
|
||||||
|
failureAction: string;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Aggregate gate result */
|
||||||
|
export interface GateResult {
|
||||||
|
passed: boolean;
|
||||||
|
results: RuleCheckResult[];
|
||||||
|
action: 'PASS' | 'RETRY' | 'SUPPLEMENT' | 'WARN_AND_PASS' | 'ESCALATE';
|
||||||
|
feedbackForModel?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Service
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class EvaluationGateService {
|
||||||
|
private readonly logger = new Logger(EvaluationGateService.name);
|
||||||
|
private cache = new Map<string, { rules: EvaluationRuleEntity[]; expiresAt: number }>();
|
||||||
|
private readonly CACHE_TTL = 5 * 60 * 1000; // 5 minutes
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
@Inject(EVALUATION_RULE_REPOSITORY)
|
||||||
|
private readonly repo: IEvaluationRuleRepository,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main entry: evaluate all applicable rules
|
||||||
|
* Returns passed=true if no rules exist or all pass
|
||||||
|
*/
|
||||||
|
async evaluate(tenantId: string | null, context: EvaluationContext): Promise<GateResult> {
|
||||||
|
const stage = context.stage || '*';
|
||||||
|
const rules = await this.loadRules(tenantId, stage);
|
||||||
|
|
||||||
|
// Zero rules = zero checks
|
||||||
|
if (rules.length === 0) {
|
||||||
|
return { passed: true, results: [], action: 'PASS' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const results: RuleCheckResult[] = [];
|
||||||
|
|
||||||
|
for (const rule of rules) {
|
||||||
|
const check = this.evaluateRule(rule, context);
|
||||||
|
results.push({
|
||||||
|
ruleId: rule.id,
|
||||||
|
ruleName: rule.name,
|
||||||
|
ruleType: rule.ruleType,
|
||||||
|
passed: check.passed,
|
||||||
|
failureAction: rule.failureAction,
|
||||||
|
message: check.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedResults = results.filter(r => !r.passed);
|
||||||
|
|
||||||
|
if (failedResults.length === 0) {
|
||||||
|
return { passed: true, results, action: 'PASS' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedActions = failedResults.map(
|
||||||
|
r => r.failureAction as EvaluationFailureActionValue,
|
||||||
|
);
|
||||||
|
const highestAction = getHighestSeverityAction(failedActions);
|
||||||
|
|
||||||
|
const feedbackForModel = this.buildFeedback(failedResults, highestAction);
|
||||||
|
|
||||||
|
this.logger.warn(
|
||||||
|
`Evaluation gate: ${failedResults.length}/${results.length} rules failed, action=${highestAction}, stage=${stage}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
results,
|
||||||
|
action: highestAction,
|
||||||
|
feedbackForModel,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clear cached rules (e.g., after admin updates) */
|
||||||
|
clearCache(tenantId?: string): void {
|
||||||
|
if (tenantId) {
|
||||||
|
for (const key of this.cache.keys()) {
|
||||||
|
if (key.startsWith(tenantId)) {
|
||||||
|
this.cache.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.cache.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Rule Loading (with cache)
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
private async loadRules(
|
||||||
|
tenantId: string | null,
|
||||||
|
stage: string,
|
||||||
|
): Promise<EvaluationRuleEntity[]> {
|
||||||
|
const cacheKey = `${tenantId || 'global'}:${stage}`;
|
||||||
|
const cached = this.cache.get(cacheKey);
|
||||||
|
|
||||||
|
if (cached && cached.expiresAt > Date.now()) {
|
||||||
|
return cached.rules;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const rules = await this.repo.findByTenantAndStage(tenantId, stage);
|
||||||
|
this.cache.set(cacheKey, {
|
||||||
|
rules,
|
||||||
|
expiresAt: Date.now() + this.CACHE_TTL,
|
||||||
|
});
|
||||||
|
return rules;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to load evaluation rules: ${error}`);
|
||||||
|
// On error, return empty = no checks = safe fallback
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Rule Evaluation (pure functions)
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
private evaluateRule(
|
||||||
|
rule: EvaluationRuleEntity,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
switch (rule.ruleType) {
|
||||||
|
case EvaluationRuleType.FIELD_COMPLETENESS:
|
||||||
|
return this.checkFieldCompleteness(rule.config, context);
|
||||||
|
case EvaluationRuleType.ASSESSMENT_QUALITY:
|
||||||
|
return this.checkAssessmentQuality(rule.config, context);
|
||||||
|
case EvaluationRuleType.RESPONSE_LENGTH:
|
||||||
|
return this.checkResponseLength(rule.config, context);
|
||||||
|
case EvaluationRuleType.MUST_CONTAIN:
|
||||||
|
return this.checkMustContain(rule.config, context);
|
||||||
|
case EvaluationRuleType.STAGE_MIN_TURNS:
|
||||||
|
return this.checkStageMinTurns(rule.config, context);
|
||||||
|
case EvaluationRuleType.CONVERSION_SIGNAL:
|
||||||
|
return this.checkConversionSignal(rule.config, context);
|
||||||
|
default:
|
||||||
|
this.logger.warn(`Unknown rule type: ${rule.ruleType}`);
|
||||||
|
return { passed: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FIELD_COMPLETENESS: Check if required fields exist in collectedInfo
|
||||||
|
* config: { fields: string[], minRatio: number }
|
||||||
|
*/
|
||||||
|
private checkFieldCompleteness(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const fields = (config.fields as string[]) || [];
|
||||||
|
const minRatio = (config.minRatio as number) ?? 1.0;
|
||||||
|
|
||||||
|
if (fields.length === 0) return { passed: true };
|
||||||
|
|
||||||
|
const info = context.collectedInfo || {};
|
||||||
|
const presentFields: string[] = [];
|
||||||
|
const missingFields: string[] = [];
|
||||||
|
|
||||||
|
for (const field of fields) {
|
||||||
|
const value = info[field];
|
||||||
|
if (value !== undefined && value !== null && value !== '') {
|
||||||
|
presentFields.push(field);
|
||||||
|
} else {
|
||||||
|
missingFields.push(field);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const ratio = presentFields.length / fields.length;
|
||||||
|
const passed = ratio >= minRatio;
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed,
|
||||||
|
message: passed
|
||||||
|
? undefined
|
||||||
|
: `信息完整度不足（${presentFields.length}/${fields.length}=${(ratio * 100).toFixed(0)}%，要求≥${(minRatio * 100).toFixed(0)}%），缺少: ${missingFields.join(', ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ASSESSMENT_QUALITY: Check assessment result quality
|
||||||
|
* config: { minCategories?: number, minConfidence?: number, minScore?: number }
|
||||||
|
*/
|
||||||
|
private checkAssessmentQuality(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const minCategories = (config.minCategories as number) ?? 1;
|
||||||
|
const minScore = (config.minScore as number) ?? 0;
|
||||||
|
|
||||||
|
if (!context.assessmentResult) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `尚未完成评估（要求至少评估 ${minCategories} 个类别）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const categoryCount = context.assessmentResult.topRecommended?.length || 0;
|
||||||
|
if (categoryCount < minCategories) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `推荐类别数不足（${categoryCount}/${minCategories}）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (context.assessmentResult.suitabilityScore < minScore) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `适配分数过低（${context.assessmentResult.suitabilityScore}/${minScore}）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* RESPONSE_LENGTH: Check response text length
|
||||||
|
* config: { min?: number, max?: number }
|
||||||
|
*/
|
||||||
|
private checkResponseLength(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const min = (config.min as number) ?? 0;
|
||||||
|
const max = (config.max as number) ?? Infinity;
|
||||||
|
const length = context.responseText.length;
|
||||||
|
|
||||||
|
if (length < min) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `回复过短（${length}字，最少${min}字）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (length > max) {
|
||||||
|
return {
|
||||||
|
passed: false,
|
||||||
|
message: `回复过长（${length}字，最多${max}字）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { passed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MUST_CONTAIN: Check if response contains required keywords
|
||||||
|
* config: { keywords: string[], mode: 'any' | 'all' }
|
||||||
|
*/
|
||||||
|
private checkMustContain(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const keywords = (config.keywords as string[]) || [];
|
||||||
|
const mode = (config.mode as string) || 'any';
|
||||||
|
|
||||||
|
if (keywords.length === 0) return { passed: true };
|
||||||
|
|
||||||
|
const text = context.responseText;
|
||||||
|
const foundKeywords = keywords.filter(kw => text.includes(kw));
|
||||||
|
const missingKeywords = keywords.filter(kw => !text.includes(kw));
|
||||||
|
|
||||||
|
if (mode === 'all') {
|
||||||
|
const passed = missingKeywords.length === 0;
|
||||||
|
return {
|
||||||
|
passed,
|
||||||
|
message: passed
|
||||||
|
? undefined
|
||||||
|
: `回复缺少必需关键词: ${missingKeywords.join(', ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// mode === 'any'
|
||||||
|
const passed = foundKeywords.length > 0;
|
||||||
|
return {
|
||||||
|
passed,
|
||||||
|
message: passed
|
||||||
|
? undefined
|
||||||
|
: `回复未包含任何期望关键词: ${keywords.join(', ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* STAGE_MIN_TURNS: Check minimum conversation turns
|
||||||
|
* config: { minTurns: number }
|
||||||
|
*/
|
||||||
|
private checkStageMinTurns(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const minTurns = (config.minTurns as number) ?? 1;
|
||||||
|
const passed = context.turnCount >= minTurns;
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed,
|
||||||
|
message: passed
|
||||||
|
? undefined
|
||||||
|
: `当前轮次不足（${context.turnCount}/${minTurns}轮）`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CONVERSION_SIGNAL: Check for conversion-related content
|
||||||
|
* config: { keywords?: string[] }
|
||||||
|
*/
|
||||||
|
private checkConversionSignal(
|
||||||
|
config: Record<string, unknown>,
|
||||||
|
context: EvaluationContext,
|
||||||
|
): { passed: boolean; message?: string } {
|
||||||
|
const defaultKeywords = ['下一步', '预约', '联系', '咨询', '费用', '付款', '签约'];
|
||||||
|
const keywords = (config.keywords as string[]) || defaultKeywords;
|
||||||
|
|
||||||
|
const text = context.responseText;
|
||||||
|
const found = keywords.some(kw => text.includes(kw));
|
||||||
|
|
||||||
|
return {
|
||||||
|
passed: found,
|
||||||
|
message: found
|
||||||
|
? undefined
|
||||||
|
: '回复中缺少转化引导内容（如下一步行动、预约、联系方式等）',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// Feedback Builder
|
||||||
|
// ============================================================
|
||||||
|
|
||||||
|
private buildFeedback(
|
||||||
|
failedResults: RuleCheckResult[],
|
||||||
|
action: EvaluationFailureActionValue,
|
||||||
|
): string {
|
||||||
|
const lines = ['[系统评估] 回复未通过质量检查：'];
|
||||||
|
|
||||||
|
for (const result of failedResults) {
|
||||||
|
lines.push(`- [${result.ruleName}] ${result.message || '未通过'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (action === EvaluationFailureAction.RETRY) {
|
||||||
|
lines.push('请根据以上反馈重新生成回复。');
|
||||||
|
} else if (action === EvaluationFailureAction.SUPPLEMENT) {
|
||||||
|
lines.push('请在回复中补充缺失的信息后再回复用户。');
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -274,6 +274,12 @@ export interface AgentLoopParams {
|
||||||
currentTurnCount?: number;
|
currentTurnCount?: number;
|
||||||
/** 已经消耗的成本（递归时累加）*/
|
/** 已经消耗的成本（递归时累加）*/
|
||||||
currentCostUsd?: number;
|
currentCostUsd?: number;
|
||||||
|
/** 可选评估门控回调。提供时，在 yield 'end' 前调用检查回复质量。 */
|
||||||
|
evaluationGate?: (
|
||||||
|
responseText: string,
|
||||||
|
turnCount: number,
|
||||||
|
agentsUsed: string[],
|
||||||
|
) => Promise<import('../coordinator/evaluation-gate.service').GateResult>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Claude API 消息格式 */
|
/** Claude API 消息格式 */
|
||||||
|
|
|
||||||
|
|
@ -74,7 +74,7 @@ export interface AgentCompleteEvent extends BaseStreamEvent {
|
||||||
/** Coordinator 正在思考/编排 */
|
/** Coordinator 正在思考/编排 */
|
||||||
export interface CoordinatorThinkingEvent extends BaseStreamEvent {
|
export interface CoordinatorThinkingEvent extends BaseStreamEvent {
|
||||||
type: 'coordinator_thinking';
|
type: 'coordinator_thinking';
|
||||||
phase: 'analyzing' | 'orchestrating' | 'synthesizing';
|
phase: 'analyzing' | 'orchestrating' | 'synthesizing' | 'evaluating';
|
||||||
message?: string;
|
message?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -113,6 +113,20 @@ export interface EndStreamEvent extends BaseStreamEvent {
|
||||||
agentsUsed: SpecialistAgentType[];
|
agentsUsed: SpecialistAgentType[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 评估门控警告事件 */
|
||||||
|
export interface EvaluationWarningEvent extends BaseStreamEvent {
|
||||||
|
type: 'evaluation_warning';
|
||||||
|
results: Array<{
|
||||||
|
ruleId: string;
|
||||||
|
ruleName: string;
|
||||||
|
ruleType: string;
|
||||||
|
passed: boolean;
|
||||||
|
failureAction: string;
|
||||||
|
message?: string;
|
||||||
|
}>;
|
||||||
|
action: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** 错误事件 */
|
/** 错误事件 */
|
||||||
export interface ErrorStreamEvent extends BaseStreamEvent {
|
export interface ErrorStreamEvent extends BaseStreamEvent {
|
||||||
type: 'error';
|
type: 'error';
|
||||||
|
|
@ -137,6 +151,7 @@ export type StreamEvent =
|
||||||
| StateUpdateEvent
|
| StateUpdateEvent
|
||||||
| UsageEvent
|
| UsageEvent
|
||||||
| EndStreamEvent
|
| EndStreamEvent
|
||||||
|
| EvaluationWarningEvent
|
||||||
| ErrorStreamEvent;
|
| ErrorStreamEvent;
|
||||||
|
|
||||||
// ============================================================
|
// ============================================================
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
import {
|
||||||
|
Entity,
|
||||||
|
PrimaryGeneratedColumn,
|
||||||
|
Column,
|
||||||
|
CreateDateColumn,
|
||||||
|
UpdateDateColumn,
|
||||||
|
Index,
|
||||||
|
} from 'typeorm';
|
||||||
|
|
||||||
|
@Entity('evaluation_rules')
|
||||||
|
@Index('idx_evaluation_rules_tenant', ['tenantId'])
|
||||||
|
@Index('idx_evaluation_rules_tenant_stage', ['tenantId', 'stage', 'enabled'])
|
||||||
|
export class EvaluationRuleORM {
|
||||||
|
@PrimaryGeneratedColumn('uuid')
|
||||||
|
id: string;
|
||||||
|
|
||||||
|
@Column({ name: 'tenant_id', type: 'uuid', nullable: true })
|
||||||
|
tenantId: string | null;
|
||||||
|
|
||||||
|
@Column({ type: 'varchar', length: 50 })
|
||||||
|
stage: string;
|
||||||
|
|
||||||
|
@Column({ name: 'rule_type', type: 'varchar', length: 50 })
|
||||||
|
ruleType: string;
|
||||||
|
|
||||||
|
@Column({ type: 'varchar', length: 255 })
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
@Column({ type: 'text', nullable: true })
|
||||||
|
description: string | null;
|
||||||
|
|
||||||
|
@Column({ type: 'jsonb', default: '{}' })
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
|
||||||
|
@Column({ type: 'boolean', default: true })
|
||||||
|
enabled: boolean;
|
||||||
|
|
||||||
|
@Column({ type: 'int', default: 0 })
|
||||||
|
priority: number;
|
||||||
|
|
||||||
|
@Column({ name: 'failure_action', type: 'varchar', length: 20, default: 'WARN_AND_PASS' })
|
||||||
|
failureAction: string;
|
||||||
|
|
||||||
|
@Column({ name: 'created_by', type: 'uuid', nullable: true })
|
||||||
|
createdBy: string | null;
|
||||||
|
|
||||||
|
@Column({ name: 'updated_by', type: 'uuid', nullable: true })
|
||||||
|
updatedBy: string | null;
|
||||||
|
|
||||||
|
@CreateDateColumn({ name: 'created_at' })
|
||||||
|
createdAt: Date;
|
||||||
|
|
||||||
|
@UpdateDateColumn({ name: 'updated_at' })
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
|
import { Repository, IsNull } from 'typeorm';
|
||||||
|
import { EvaluationRuleORM } from '../entities/evaluation-rule.orm';
|
||||||
|
import {
|
||||||
|
IEvaluationRuleRepository,
|
||||||
|
} from '../../../../domain/repositories/evaluation-rule.repository.interface';
|
||||||
|
import {
|
||||||
|
EvaluationRuleEntity,
|
||||||
|
EvaluationRuleTypeValue,
|
||||||
|
EvaluationFailureActionValue,
|
||||||
|
} from '../../../../domain/entities/evaluation-rule.entity';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class EvaluationRulePostgresRepository implements IEvaluationRuleRepository {
|
||||||
|
constructor(
|
||||||
|
@InjectRepository(EvaluationRuleORM)
|
||||||
|
private readonly repo: Repository<EvaluationRuleORM>,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async findByTenantAndStage(
|
||||||
|
tenantId: string | null,
|
||||||
|
stage: string,
|
||||||
|
): Promise<EvaluationRuleEntity[]> {
|
||||||
|
const orms = await this.repo
|
||||||
|
.createQueryBuilder('r')
|
||||||
|
.where('(r.tenant_id = :tenantId OR r.tenant_id IS NULL)', { tenantId })
|
||||||
|
.andWhere('(r.stage = :stage OR r.stage = :wildcard)', { stage, wildcard: '*' })
|
||||||
|
.andWhere('r.enabled = true')
|
||||||
|
.orderBy('r.priority', 'ASC')
|
||||||
|
.getMany();
|
||||||
|
|
||||||
|
return orms.map(orm => this.toEntity(orm));
|
||||||
|
}
|
||||||
|
|
||||||
|
async findAllByTenant(tenantId: string | null): Promise<EvaluationRuleEntity[]> {
|
||||||
|
const orms = await this.repo.find({
|
||||||
|
where: { tenantId: tenantId ?? IsNull() },
|
||||||
|
order: { stage: 'ASC', priority: 'ASC' },
|
||||||
|
});
|
||||||
|
return orms.map(orm => this.toEntity(orm));
|
||||||
|
}
|
||||||
|
|
||||||
|
async findById(id: string): Promise<EvaluationRuleEntity | null> {
|
||||||
|
const orm = await this.repo.findOne({ where: { id } });
|
||||||
|
return orm ? this.toEntity(orm) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async save(rule: EvaluationRuleEntity): Promise<EvaluationRuleEntity> {
|
||||||
|
const orm = this.toORM(rule);
|
||||||
|
const saved = await this.repo.save(orm);
|
||||||
|
return this.toEntity(saved);
|
||||||
|
}
|
||||||
|
|
||||||
|
async update(rule: EvaluationRuleEntity): Promise<EvaluationRuleEntity> {
|
||||||
|
const orm = this.toORM(rule);
|
||||||
|
const saved = await this.repo.save(orm);
|
||||||
|
return this.toEntity(saved);
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(id: string): Promise<void> {
|
||||||
|
await this.repo.delete(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private toEntity(orm: EvaluationRuleORM): EvaluationRuleEntity {
|
||||||
|
return EvaluationRuleEntity.fromPersistence({
|
||||||
|
id: orm.id,
|
||||||
|
tenantId: orm.tenantId,
|
||||||
|
stage: orm.stage,
|
||||||
|
ruleType: orm.ruleType as EvaluationRuleTypeValue,
|
||||||
|
name: orm.name,
|
||||||
|
description: orm.description,
|
||||||
|
config: orm.config,
|
||||||
|
enabled: orm.enabled,
|
||||||
|
priority: orm.priority,
|
||||||
|
failureAction: orm.failureAction as EvaluationFailureActionValue,
|
||||||
|
createdBy: orm.createdBy,
|
||||||
|
updatedBy: orm.updatedBy,
|
||||||
|
createdAt: orm.createdAt,
|
||||||
|
updatedAt: orm.updatedAt,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private toORM(entity: EvaluationRuleEntity): EvaluationRuleORM {
|
||||||
|
const orm = new EvaluationRuleORM();
|
||||||
|
orm.id = entity.id;
|
||||||
|
orm.tenantId = entity.tenantId;
|
||||||
|
orm.stage = entity.stage;
|
||||||
|
orm.ruleType = entity.ruleType;
|
||||||
|
orm.name = entity.name;
|
||||||
|
orm.description = entity.description;
|
||||||
|
orm.config = entity.config;
|
||||||
|
orm.enabled = entity.enabled;
|
||||||
|
orm.priority = entity.priority;
|
||||||
|
orm.failureAction = entity.failureAction;
|
||||||
|
orm.createdBy = entity.createdBy;
|
||||||
|
orm.updatedBy = entity.updatedBy;
|
||||||
|
return orm;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue