feat(resilience): add circuit breaker for downstream services

- New CircuitBreaker class: CLOSED → OPEN → HALF_OPEN three-state model
- Zero external dependencies, ~90 lines, fail-open semantics
- KnowledgeClientService: threshold=5, cooldown=60s, protects all 9 endpoints
- PaymentClientService: threshold=3, cooldown=30s, protects all 7 endpoints
- Both services refactored to use protectedFetch() — cleaner code, fewer try-catch
- Replaces verbose per-method error handling with centralized circuit breaker
- When tripped: returns null/empty fallback instantly, no network call

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-02-08 04:21:30 -08:00
parent 2ebc8e6da6
commit 1a1573dda3
3 changed files with 285 additions and 318 deletions

View File

@ -0,0 +1,96 @@
/**
* Circuit Breaker
*
*
* CLOSED
* OPEN fallback failureThreshold
* HALF_OPEN 1
*
* Redis
*/
import { Logger } from '@nestjs/common';
export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
export interface CircuitBreakerOptions {
/** 服务名称(用于日志) */
name: string;
/** 连续失败触发熔断的阈值 */
failureThreshold: number;
/** 熔断持续时间ms过后进入 HALF_OPEN */
resetTimeoutMs: number;
}
export class CircuitBreaker {
private readonly logger: Logger;
private state: CircuitState = 'CLOSED';
private failureCount = 0;
private lastFailureTime = 0;
private readonly name: string;
private readonly failureThreshold: number;
private readonly resetTimeoutMs: number;
constructor(options: CircuitBreakerOptions) {
this.name = options.name;
this.failureThreshold = options.failureThreshold;
this.resetTimeoutMs = options.resetTimeoutMs;
this.logger = new Logger(`CircuitBreaker:${this.name}`);
}
/**
*
* @param fn
* @param fallback
*/
async execute<T>(fn: () => Promise<T>, fallback: T): Promise<T> {
// OPEN 状态:检查是否可以转为 HALF_OPEN
if (this.state === 'OPEN') {
const elapsed = Date.now() - this.lastFailureTime;
if (elapsed >= this.resetTimeoutMs) {
this.state = 'HALF_OPEN';
this.logger.log(`${this.name}: OPEN → HALF_OPEN (${elapsed}ms elapsed, probing...)`);
} else {
// 仍在熔断窗口内,直接返回 fallback
return fallback;
}
}
try {
const result = await fn();
// 成功:重置计数器
if (this.state === 'HALF_OPEN') {
this.logger.log(`${this.name}: HALF_OPEN → CLOSED (probe succeeded)`);
}
this.state = 'CLOSED';
this.failureCount = 0;
return result;
} catch (error) {
this.failureCount++;
this.lastFailureTime = Date.now();
if (this.failureCount >= this.failureThreshold) {
this.state = 'OPEN';
this.logger.warn(
`${this.name}: → OPEN (${this.failureCount} consecutive failures, cooldown ${this.resetTimeoutMs}ms)`,
);
}
this.logger.debug(
`${this.name}: failure ${this.failureCount}/${this.failureThreshold}${error}`,
);
return fallback;
}
}
/** 当前状态 */
getState(): CircuitState {
return this.state;
}
/** 连续失败次数 */
getFailureCount(): number {
return this.failureCount;
}
}

View File

@ -1,5 +1,6 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { Injectable, OnModuleInit, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { CircuitBreaker } from '../common/circuit-breaker';
/**
* RAG
@ -63,13 +64,36 @@ interface ApiResponse<T> {
*/
@Injectable()
export class KnowledgeClientService implements OnModuleInit {
private readonly logger = new Logger(KnowledgeClientService.name);
private baseUrl: string;
private readonly circuitBreaker: CircuitBreaker;
constructor(private configService: ConfigService) {}
constructor(private configService: ConfigService) {
this.circuitBreaker = new CircuitBreaker({
name: 'knowledge-service',
failureThreshold: 5,
resetTimeoutMs: 60_000,
});
}
onModuleInit() {
this.baseUrl = this.configService.get<string>('KNOWLEDGE_SERVICE_URL') || 'http://knowledge-service:3003';
console.log(`[KnowledgeClient] Initialized with base URL: ${this.baseUrl}`);
this.logger.log(`Initialized with base URL: ${this.baseUrl}`);
}
/**
* fetch 5 60s
* null
*/
private protectedFetch(url: string, init?: RequestInit): Promise<Response | null> {
return this.circuitBreaker.execute(
async () => {
const resp = await fetch(url, init);
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
return resp;
},
null,
);
}
/**
@ -82,24 +106,14 @@ export class KnowledgeClientService implements OnModuleInit {
includeMemories?: boolean;
includeExperiences?: boolean;
}): Promise<RAGResult | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/knowledge/retrieve`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] RAG retrieve failed: ${response.status}`);
return null;
}
const data = (await response.json()) as ApiResponse<RAGResult>;
return data.success ? data.data : null;
} catch (error) {
console.error('[KnowledgeClient] RAG retrieve error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/knowledge/retrieve`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<RAGResult>;
return data.success ? data.data : null;
}
/**
@ -110,48 +124,28 @@ export class KnowledgeClientService implements OnModuleInit {
userId?: string;
category?: string;
}): Promise<string | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/knowledge/retrieve/prompt`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] RAG retrieve/prompt failed: ${response.status}`);
return null;
}
const data = (await response.json()) as ApiResponse<{ context: string }>;
return data.success ? data.data.context : null;
} catch (error) {
console.error('[KnowledgeClient] RAG retrieve/prompt error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/knowledge/retrieve/prompt`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<{ context: string }>;
return data.success ? data.data.context : null;
}
/**
*
*/
async checkOffTopic(query: string): Promise<OffTopicResult> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/knowledge/check-off-topic`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query }),
});
if (!response.ok) {
console.error(`[KnowledgeClient] checkOffTopic failed: ${response.status}`);
return { isOffTopic: false, confidence: 0 };
}
const data = (await response.json()) as ApiResponse<OffTopicResult>;
return data.success ? data.data : { isOffTopic: false, confidence: 0 };
} catch (error) {
console.error('[KnowledgeClient] checkOffTopic error:', error);
return { isOffTopic: false, confidence: 0 };
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/knowledge/check-off-topic`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query }),
});
if (!resp) return { isOffTopic: false, confidence: 0 };
const data = (await resp.json()) as ApiResponse<OffTopicResult>;
return data.success ? data.data : { isOffTopic: false, confidence: 0 };
}
/**
@ -165,24 +159,14 @@ export class KnowledgeClientService implements OnModuleInit {
sourceConversationId?: string;
relatedCategory?: string;
}): Promise<UserMemory | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/memory/user`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] saveUserMemory failed: ${response.status}`);
return null;
}
const data = (await response.json()) as ApiResponse<UserMemory>;
return data.success ? data.data : null;
} catch (error) {
console.error('[KnowledgeClient] saveUserMemory error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/user`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<UserMemory>;
return data.success ? data.data : null;
}
/**
@ -193,44 +177,24 @@ export class KnowledgeClientService implements OnModuleInit {
query: string;
limit?: number;
}): Promise<UserMemory[]> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/memory/user/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] searchUserMemories failed: ${response.status}`);
return [];
}
const data = (await response.json()) as ApiResponse<UserMemory[]>;
return data.success ? data.data : [];
} catch (error) {
console.error('[KnowledgeClient] searchUserMemories error:', error);
return [];
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/user/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return [];
const data = (await resp.json()) as ApiResponse<UserMemory[]>;
return data.success ? data.data : [];
}
/**
*
*/
async getUserTopMemories(userId: string, limit = 5): Promise<UserMemory[]> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/memory/user/${userId}/top?limit=${limit}`);
if (!response.ok) {
console.error(`[KnowledgeClient] getUserTopMemories failed: ${response.status}`);
return [];
}
const data = (await response.json()) as ApiResponse<UserMemory[]>;
return data.success ? data.data : [];
} catch (error) {
console.error('[KnowledgeClient] getUserTopMemories error:', error);
return [];
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/user/${userId}/top?limit=${limit}`);
if (!resp) return [];
const data = (await resp.json()) as ApiResponse<UserMemory[]>;
return data.success ? data.data : [];
}
/**
@ -243,48 +207,31 @@ export class KnowledgeClientService implements OnModuleInit {
activeOnly?: boolean;
limit?: number;
}): Promise<SystemExperience[]> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/memory/experience/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] searchExperiences failed: ${response.status}`);
return [];
}
const data = (await response.json()) as ApiResponse<SystemExperience[]>;
return data.success ? data.data : [];
} catch (error) {
console.error('[KnowledgeClient] searchExperiences error:', error);
return [];
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/experience/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return [];
const data = (await resp.json()) as ApiResponse<SystemExperience[]>;
return data.success ? data.data : [];
}
/**
* Neo4j
*/
async initializeUser(userId: string): Promise<boolean> {
try {
// 通过保存一个初始记忆来初始化用户
const response = await fetch(`${this.baseUrl}/api/v1/memory/user`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
userId,
memoryType: 'FACT',
content: '用户首次访问系统',
importance: 10,
}),
});
return response.ok;
} catch (error) {
console.error('[KnowledgeClient] initializeUser error:', error);
return false;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/user`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
userId,
memoryType: 'FACT',
content: '用户首次访问系统',
importance: 10,
}),
});
return resp !== null;
}
/**
@ -299,24 +246,14 @@ export class KnowledgeClientService implements OnModuleInit {
confidence?: number;
relatedCategory?: string;
}): Promise<SystemExperience | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/memory/experience`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!response.ok) {
console.error(`[KnowledgeClient] saveExperience failed: ${response.status}`);
return null;
}
const data = (await response.json()) as ApiResponse<SystemExperience>;
return data.success ? data.data : null;
} catch (error) {
console.error('[KnowledgeClient] saveExperience error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/memory/experience`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(params),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<SystemExperience>;
return data.success ? data.data : null;
}
// ============================================================

View File

@ -1,5 +1,6 @@
import { Injectable, OnModuleInit } from '@nestjs/common';
import { Injectable, OnModuleInit, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { CircuitBreaker } from '../common/circuit-breaker';
/**
*
@ -50,16 +51,36 @@ interface ApiResponse<T> {
*/
@Injectable()
export class PaymentClientService implements OnModuleInit {
private readonly logger = new Logger(PaymentClientService.name);
private baseUrl: string;
private readonly circuitBreaker: CircuitBreaker;
constructor(private configService: ConfigService) {}
constructor(private configService: ConfigService) {
this.circuitBreaker = new CircuitBreaker({
name: 'payment-service',
failureThreshold: 3,
resetTimeoutMs: 30_000,
});
}
onModuleInit() {
this.baseUrl =
this.configService.get<string>('PAYMENT_SERVICE_URL') ||
'http://payment-service:3002';
console.log(
`[PaymentClient] Initialized with base URL: ${this.baseUrl}`,
this.logger.log(`Initialized with base URL: ${this.baseUrl}`);
}
/**
* fetch 3 30s
*/
private protectedFetch(url: string, init?: RequestInit): Promise<Response | null> {
return this.circuitBreaker.execute(
async () => {
const resp = await fetch(url, init);
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
return resp;
},
null,
);
}
@ -72,34 +93,21 @@ export class PaymentClientService implements OnModuleInit {
serviceCategory?: string;
conversationId?: string;
}): Promise<OrderInfo | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/orders`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-user-id': params.userId,
},
body: JSON.stringify({
serviceType: params.serviceType,
serviceCategory: params.serviceCategory,
conversationId: params.conversationId,
}),
});
if (!response.ok) {
const errText = await response.text();
console.error(
`[PaymentClient] createOrder failed: ${response.status} ${errText}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<OrderInfo>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] createOrder error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/orders`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-user-id': params.userId,
},
body: JSON.stringify({
serviceType: params.serviceType,
serviceCategory: params.serviceCategory,
conversationId: params.conversationId,
}),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<OrderInfo>;
return data.success ? data.data : null;
}
/**
@ -109,30 +117,17 @@ export class PaymentClientService implements OnModuleInit {
orderId: string;
method: string;
}): Promise<PaymentResult | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/payments`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
orderId: params.orderId,
method: params.method,
}),
});
if (!response.ok) {
const errText = await response.text();
console.error(
`[PaymentClient] createPayment failed: ${response.status} ${errText}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<PaymentResult>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] createPayment error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/payments`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
orderId: params.orderId,
method: params.method,
}),
});
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<PaymentResult>;
return data.success ? data.data : null;
}
/**
@ -141,27 +136,15 @@ export class PaymentClientService implements OnModuleInit {
async checkPaymentStatus(
paymentId: string,
): Promise<{ status: string; paidAt?: string } | null> {
try {
const response = await fetch(
`${this.baseUrl}/api/v1/payments/${paymentId}/status`,
);
if (!response.ok) {
console.error(
`[PaymentClient] checkPaymentStatus failed: ${response.status}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<{
status: string;
paidAt?: string;
}>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] checkPaymentStatus error:', error);
return null;
}
const resp = await this.protectedFetch(
`${this.baseUrl}/api/v1/payments/${paymentId}/status`,
);
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<{
status: string;
paidAt?: string;
}>;
return data.success ? data.data : null;
}
/**
@ -175,75 +158,39 @@ export class PaymentClientService implements OnModuleInit {
paidAt?: string;
completedAt?: string;
} | null> {
try {
const response = await fetch(
`${this.baseUrl}/api/v1/orders/${orderId}/status`,
);
if (!response.ok) {
console.error(
`[PaymentClient] getOrderStatus failed: ${response.status}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<{
orderId: string;
status: string;
paidAt?: string;
completedAt?: string;
}>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] getOrderStatus error:', error);
return null;
}
const resp = await this.protectedFetch(
`${this.baseUrl}/api/v1/orders/${orderId}/status`,
);
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<{
orderId: string;
status: string;
paidAt?: string;
completedAt?: string;
}>;
return data.success ? data.data : null;
}
/**
*
*/
async getUserOrders(userId: string): Promise<OrderInfo[]> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/orders`, {
headers: { 'x-user-id': userId },
});
if (!response.ok) {
console.error(
`[PaymentClient] getUserOrders failed: ${response.status}`,
);
return [];
}
const data = (await response.json()) as ApiResponse<OrderInfo[]>;
return data.success ? data.data : [];
} catch (error) {
console.error('[PaymentClient] getUserOrders error:', error);
return [];
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/orders`, {
headers: { 'x-user-id': userId },
});
if (!resp) return [];
const data = (await resp.json()) as ApiResponse<OrderInfo[]>;
return data.success ? data.data : [];
}
/**
*
*/
async getOrderDetail(orderId: string): Promise<OrderInfo | null> {
try {
const response = await fetch(`${this.baseUrl}/api/v1/orders/${orderId}`);
if (!response.ok) {
console.error(
`[PaymentClient] getOrderDetail failed: ${response.status}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<OrderInfo>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] getOrderDetail error:', error);
return null;
}
const resp = await this.protectedFetch(`${this.baseUrl}/api/v1/orders/${orderId}`);
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<OrderInfo>;
return data.success ? data.data : null;
}
/**
@ -252,28 +199,15 @@ export class PaymentClientService implements OnModuleInit {
async cancelOrder(
orderId: string,
): Promise<{ orderId: string; status: string } | null> {
try {
const response = await fetch(
`${this.baseUrl}/api/v1/orders/${orderId}/cancel`,
{ method: 'POST' },
);
if (!response.ok) {
const errText = await response.text();
console.error(
`[PaymentClient] cancelOrder failed: ${response.status} ${errText}`,
);
return null;
}
const data = (await response.json()) as ApiResponse<{
orderId: string;
status: string;
}>;
return data.success ? data.data : null;
} catch (error) {
console.error('[PaymentClient] cancelOrder error:', error);
return null;
}
const resp = await this.protectedFetch(
`${this.baseUrl}/api/v1/orders/${orderId}/cancel`,
{ method: 'POST' },
);
if (!resp) return null;
const data = (await resp.json()) as ApiResponse<{
orderId: string;
status: string;
}>;
return data.success ? data.data : null;
}
}