From 3cb9ebd4070cb670207f82ca7b6e0193cfe9d0c7 Mon Sep 17 00:00:00 2001 From: hailin Date: Mon, 23 Feb 2026 15:55:06 -0800 Subject: [PATCH] fix: release QueryRunner connections to prevent pool exhaustion TenantAwareRepository.getRepository() was calling createQueryRunner() without ever releasing it, causing database connection pool exhaustion. This caused ops-service (and eventually other services) to hang on all API requests once the pool filled up. Replaced getRepository() with withRepository() pattern that wraps operations in try/finally to always release the QueryRunner. Co-Authored-By: Claude Opus 4.6 --- .../repositories/session.repository.ts | 10 ++- .../repositories/task.repository.ts | 5 +- .../repositories/audit-log.repository.ts | 80 ++++++++++--------- .../repositories/contact.repository.ts | 5 +- .../escalation-policy.repository.ts | 13 +-- .../repositories/message.repository.ts | 10 ++- .../repositories/cluster.repository.ts | 5 +- .../repositories/credential.repository.ts | 5 +- .../repositories/server.repository.ts | 10 ++- .../repositories/alert-event.repository.ts | 10 ++- .../repositories/alert-rule.repository.ts | 5 +- .../health-check-result.repository.ts | 10 ++- .../metric-snapshot.repository.ts | 20 ++--- .../repositories/approval.repository.ts | 10 ++- .../repositories/runbook.repository.ts | 5 +- .../standing-order-execution.repository.ts | 5 +- .../repositories/standing-order.repository.ts | 5 +- .../repositories/task.repository.ts | 10 ++- .../database/src/tenant-aware.repository.ts | 23 +++--- 19 files changed, 137 insertions(+), 109 deletions(-) diff --git a/packages/services/agent-service/src/infrastructure/repositories/session.repository.ts b/packages/services/agent-service/src/infrastructure/repositories/session.repository.ts index 8468be9..5f85517 100644 --- a/packages/services/agent-service/src/infrastructure/repositories/session.repository.ts +++ b/packages/services/agent-service/src/infrastructure/repositories/session.repository.ts @@ -10,12 +10,14 @@ export class SessionRepository extends TenantAwareRepository { } async findByTenant(tenantId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { tenantId } as any }); + return this.withRepository((repo) => + repo.find({ where: { tenantId } as any }), + ); } async findByStatus(status: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { status } as any }); + return this.withRepository((repo) => + repo.find({ where: { status } as any }), + ); } } diff --git a/packages/services/agent-service/src/infrastructure/repositories/task.repository.ts b/packages/services/agent-service/src/infrastructure/repositories/task.repository.ts index 6cba08e..c1bd9f4 100644 --- a/packages/services/agent-service/src/infrastructure/repositories/task.repository.ts +++ b/packages/services/agent-service/src/infrastructure/repositories/task.repository.ts @@ -10,7 +10,8 @@ export class TaskRepository extends TenantAwareRepository { } async findBySessionId(sessionId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { sessionId } as any }); + return this.withRepository((repo) => + repo.find({ where: { sessionId } as any }), + ); } } diff --git a/packages/services/audit-service/src/infrastructure/repositories/audit-log.repository.ts b/packages/services/audit-service/src/infrastructure/repositories/audit-log.repository.ts index c9b06a6..5009d5e 100644 --- a/packages/services/audit-service/src/infrastructure/repositories/audit-log.repository.ts +++ b/packages/services/audit-service/src/infrastructure/repositories/audit-log.repository.ts @@ -21,58 +21,60 @@ export class AuditLogRepository extends TenantAwareRepository { } async queryLogs(filters: AuditLogFilters): Promise<{ data: AuditLog[]; total: number }> { - const repo = await this.getRepository(); - const qb = repo.createQueryBuilder('log'); + return this.withRepository(async (repo) => { + const qb = repo.createQueryBuilder('log'); - if (filters.actionType) { - qb.andWhere('log.actionType = :actionType', { actionType: filters.actionType }); - } + if (filters.actionType) { + qb.andWhere('log.actionType = :actionType', { actionType: filters.actionType }); + } - if (filters.actorType) { - qb.andWhere('log.actorType = :actorType', { actorType: filters.actorType }); - } + if (filters.actorType) { + qb.andWhere('log.actorType = :actorType', { actorType: filters.actorType }); + } - if (filters.actorId) { - qb.andWhere('log.actorId = :actorId', { actorId: filters.actorId }); - } + if (filters.actorId) { + qb.andWhere('log.actorId = :actorId', { actorId: filters.actorId }); + } - if (filters.resourceType) { - qb.andWhere('log.resourceType = :resourceType', { resourceType: filters.resourceType }); - } + if (filters.resourceType) { + qb.andWhere('log.resourceType = :resourceType', { resourceType: filters.resourceType }); + } - if (filters.from) { - qb.andWhere('log.createdAt >= :from', { from: new Date(filters.from) }); - } + if (filters.from) { + qb.andWhere('log.createdAt >= :from', { from: new Date(filters.from) }); + } - if (filters.to) { - qb.andWhere('log.createdAt <= :to', { to: new Date(filters.to) }); - } + if (filters.to) { + qb.andWhere('log.createdAt <= :to', { to: new Date(filters.to) }); + } - qb.orderBy('log.createdAt', 'DESC'); + qb.orderBy('log.createdAt', 'DESC'); - const page = filters.page ?? 1; - const limit = filters.limit ?? 50; - qb.skip((page - 1) * limit).take(limit); + const page = filters.page ?? 1; + const limit = filters.limit ?? 50; + qb.skip((page - 1) * limit).take(limit); - const [data, total] = await qb.getManyAndCount(); - return { data, total }; + const [data, total] = await qb.getManyAndCount(); + return { data, total }; + }); } async exportLogs(format: 'json' | 'csv'): Promise { - const repo = await this.getRepository(); - const logs = await repo.find({ order: { createdAt: 'DESC' } as any }); + return this.withRepository(async (repo) => { + const logs = await repo.find({ order: { createdAt: 'DESC' } as any }); - if (format === 'csv') { - const headers = ['id', 'tenantId', 'actionType', 'actorType', 'actorId', 'resourceType', 'resourceId', 'ipAddress', 'createdAt']; - const rows = logs.map(log => - headers.map(h => { - const value = (log as any)[h]; - return value !== undefined && value !== null ? String(value) : ''; - }).join(','), - ); - return [headers.join(','), ...rows].join('\n'); - } + if (format === 'csv') { + const headers = ['id', 'tenantId', 'actionType', 'actorType', 'actorId', 'resourceType', 'resourceId', 'ipAddress', 'createdAt']; + const rows = logs.map(log => + headers.map(h => { + const value = (log as any)[h]; + return value !== undefined && value !== null ? String(value) : ''; + }).join(','), + ); + return [headers.join(','), ...rows].join('\n'); + } - return logs; + return logs; + }); } } diff --git a/packages/services/comm-service/src/infrastructure/repositories/contact.repository.ts b/packages/services/comm-service/src/infrastructure/repositories/contact.repository.ts index d020a74..2d44c65 100644 --- a/packages/services/comm-service/src/infrastructure/repositories/contact.repository.ts +++ b/packages/services/comm-service/src/infrastructure/repositories/contact.repository.ts @@ -10,7 +10,8 @@ export class ContactRepository extends TenantAwareRepository { } async findByUserId(userId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { userId } as any }); + return this.withRepository((repo) => + repo.find({ where: { userId } as any }), + ); } } diff --git a/packages/services/comm-service/src/infrastructure/repositories/escalation-policy.repository.ts b/packages/services/comm-service/src/infrastructure/repositories/escalation-policy.repository.ts index ab4d404..de4f808 100644 --- a/packages/services/comm-service/src/infrastructure/repositories/escalation-policy.repository.ts +++ b/packages/services/comm-service/src/infrastructure/repositories/escalation-policy.repository.ts @@ -10,17 +10,18 @@ export class EscalationPolicyRepository extends TenantAwareRepository { - const repo = await this.getRepository(); - return repo.find({ where: { severity } as any }); + return this.withRepository((repo) => + repo.find({ where: { severity } as any }), + ); } async findDefault(): Promise { - const repo = await this.getRepository(); - return repo.findOne({ where: { isDefault: true } as any }); + return this.withRepository((repo) => + repo.findOne({ where: { isDefault: true } as any }), + ); } async delete(id: string): Promise { - const repo = await this.getRepository(); - await repo.delete(id); + await this.withRepository((repo) => repo.delete(id)); } } diff --git a/packages/services/comm-service/src/infrastructure/repositories/message.repository.ts b/packages/services/comm-service/src/infrastructure/repositories/message.repository.ts index a833172..7d33bbc 100644 --- a/packages/services/comm-service/src/infrastructure/repositories/message.repository.ts +++ b/packages/services/comm-service/src/infrastructure/repositories/message.repository.ts @@ -10,12 +10,14 @@ export class MessageRepository extends TenantAwareRepository { } async findByDirection(direction: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { direction } as any }); + return this.withRepository((repo) => + repo.find({ where: { direction } as any }), + ); } async findByContactId(contactId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { contactId } as any }); + return this.withRepository((repo) => + repo.find({ where: { contactId } as any }), + ); } } diff --git a/packages/services/inventory-service/src/infrastructure/repositories/cluster.repository.ts b/packages/services/inventory-service/src/infrastructure/repositories/cluster.repository.ts index 99b7688..0b56ac6 100644 --- a/packages/services/inventory-service/src/infrastructure/repositories/cluster.repository.ts +++ b/packages/services/inventory-service/src/infrastructure/repositories/cluster.repository.ts @@ -10,7 +10,8 @@ export class ClusterRepository extends TenantAwareRepository { } async findByEnvironment(env: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { environment: env } as any }); + return this.withRepository((repo) => + repo.find({ where: { environment: env } as any }), + ); } } diff --git a/packages/services/inventory-service/src/infrastructure/repositories/credential.repository.ts b/packages/services/inventory-service/src/infrastructure/repositories/credential.repository.ts index d5e67f2..ba586f1 100644 --- a/packages/services/inventory-service/src/infrastructure/repositories/credential.repository.ts +++ b/packages/services/inventory-service/src/infrastructure/repositories/credential.repository.ts @@ -10,7 +10,8 @@ export class CredentialRepository extends TenantAwareRepository { } async findByType(type: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { type } as any }); + return this.withRepository((repo) => + repo.find({ where: { type } as any }), + ); } } diff --git a/packages/services/inventory-service/src/infrastructure/repositories/server.repository.ts b/packages/services/inventory-service/src/infrastructure/repositories/server.repository.ts index 4f0d4f8..9b2e437 100644 --- a/packages/services/inventory-service/src/infrastructure/repositories/server.repository.ts +++ b/packages/services/inventory-service/src/infrastructure/repositories/server.repository.ts @@ -10,12 +10,14 @@ export class ServerRepository extends TenantAwareRepository { } async findByEnvironment(env: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { environment: env } as any }); + return this.withRepository((repo) => + repo.find({ where: { environment: env } as any }), + ); } async findByClusterId(clusterId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { clusterId } as any }); + return this.withRepository((repo) => + repo.find({ where: { clusterId } as any }), + ); } } diff --git a/packages/services/monitor-service/src/infrastructure/repositories/alert-event.repository.ts b/packages/services/monitor-service/src/infrastructure/repositories/alert-event.repository.ts index 56e3da8..f7b5b9d 100644 --- a/packages/services/monitor-service/src/infrastructure/repositories/alert-event.repository.ts +++ b/packages/services/monitor-service/src/infrastructure/repositories/alert-event.repository.ts @@ -10,12 +10,14 @@ export class AlertEventRepository extends TenantAwareRepository { } async findByStatus(status: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { status } as any }); + return this.withRepository((repo) => + repo.find({ where: { status } as any }), + ); } async findByRuleId(ruleId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { ruleId } as any }); + return this.withRepository((repo) => + repo.find({ where: { ruleId } as any }), + ); } } diff --git a/packages/services/monitor-service/src/infrastructure/repositories/alert-rule.repository.ts b/packages/services/monitor-service/src/infrastructure/repositories/alert-rule.repository.ts index db261f6..f97d843 100644 --- a/packages/services/monitor-service/src/infrastructure/repositories/alert-rule.repository.ts +++ b/packages/services/monitor-service/src/infrastructure/repositories/alert-rule.repository.ts @@ -10,7 +10,8 @@ export class AlertRuleRepository extends TenantAwareRepository { } async findActive(): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { isActive: true } as any }); + return this.withRepository((repo) => + repo.find({ where: { isActive: true } as any }), + ); } } diff --git a/packages/services/monitor-service/src/infrastructure/repositories/health-check-result.repository.ts b/packages/services/monitor-service/src/infrastructure/repositories/health-check-result.repository.ts index 55693d4..e3d43c0 100644 --- a/packages/services/monitor-service/src/infrastructure/repositories/health-check-result.repository.ts +++ b/packages/services/monitor-service/src/infrastructure/repositories/health-check-result.repository.ts @@ -10,12 +10,14 @@ export class HealthCheckResultRepository extends TenantAwareRepository { - const repo = await this.getRepository(); - return repo.find({ where: { serverId } as any, order: { checkedAt: 'DESC' } as any }); + return this.withRepository((repo) => + repo.find({ where: { serverId } as any, order: { checkedAt: 'DESC' } as any }), + ); } async findRecent(limit = 50): Promise { - const repo = await this.getRepository(); - return repo.find({ order: { checkedAt: 'DESC' } as any, take: limit }); + return this.withRepository((repo) => + repo.find({ order: { checkedAt: 'DESC' } as any, take: limit }), + ); } } diff --git a/packages/services/monitor-service/src/infrastructure/repositories/metric-snapshot.repository.ts b/packages/services/monitor-service/src/infrastructure/repositories/metric-snapshot.repository.ts index b275dcb..5789215 100644 --- a/packages/services/monitor-service/src/infrastructure/repositories/metric-snapshot.repository.ts +++ b/packages/services/monitor-service/src/infrastructure/repositories/metric-snapshot.repository.ts @@ -10,17 +10,19 @@ export class MetricSnapshotRepository extends TenantAwareRepository { - const repo = await this.getRepository(); - return repo.find({ where: { serverId } as any, order: { recordedAt: 'DESC' } as any, take: 100 }); + return this.withRepository((repo) => + repo.find({ where: { serverId } as any, order: { recordedAt: 'DESC' } as any, take: 100 }), + ); } async findRecent(serverId: string, metricType: string, since: Date): Promise { - const repo = await this.getRepository(); - const qb = repo.createQueryBuilder('ms'); - qb.where('ms.serverId = :serverId', { serverId }); - qb.andWhere('ms.metricType = :metricType', { metricType }); - qb.andWhere('ms.recordedAt >= :since', { since }); - qb.orderBy('ms.recordedAt', 'ASC'); - return qb.getMany(); + return this.withRepository((repo) => { + const qb = repo.createQueryBuilder('ms'); + qb.where('ms.serverId = :serverId', { serverId }); + qb.andWhere('ms.metricType = :metricType', { metricType }); + qb.andWhere('ms.recordedAt >= :since', { since }); + qb.orderBy('ms.recordedAt', 'ASC'); + return qb.getMany(); + }); } } diff --git a/packages/services/ops-service/src/infrastructure/repositories/approval.repository.ts b/packages/services/ops-service/src/infrastructure/repositories/approval.repository.ts index 772f745..253a645 100644 --- a/packages/services/ops-service/src/infrastructure/repositories/approval.repository.ts +++ b/packages/services/ops-service/src/infrastructure/repositories/approval.repository.ts @@ -10,12 +10,14 @@ export class ApprovalRepository extends TenantAwareRepository { } async findPending(): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { status: 'pending' } as any }); + return this.withRepository((repo) => + repo.find({ where: { status: 'pending' } as any }), + ); } async findByTaskId(taskId: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { taskId } as any }); + return this.withRepository((repo) => + repo.find({ where: { taskId } as any }), + ); } } diff --git a/packages/services/ops-service/src/infrastructure/repositories/runbook.repository.ts b/packages/services/ops-service/src/infrastructure/repositories/runbook.repository.ts index d0a8ba4..07eccdd 100644 --- a/packages/services/ops-service/src/infrastructure/repositories/runbook.repository.ts +++ b/packages/services/ops-service/src/infrastructure/repositories/runbook.repository.ts @@ -10,7 +10,8 @@ export class RunbookRepository extends TenantAwareRepository { } async findActive(): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { isActive: true } as any }); + return this.withRepository((repo) => + repo.find({ where: { isActive: true } as any }), + ); } } diff --git a/packages/services/ops-service/src/infrastructure/repositories/standing-order-execution.repository.ts b/packages/services/ops-service/src/infrastructure/repositories/standing-order-execution.repository.ts index f949f05..455ac50 100644 --- a/packages/services/ops-service/src/infrastructure/repositories/standing-order-execution.repository.ts +++ b/packages/services/ops-service/src/infrastructure/repositories/standing-order-execution.repository.ts @@ -10,7 +10,8 @@ export class StandingOrderExecutionRepository extends TenantAwareRepository { - const repo = await this.getRepository(); - return repo.find({ where: { orderId } as any }); + return this.withRepository((repo) => + repo.find({ where: { orderId } as any }), + ); } } diff --git a/packages/services/ops-service/src/infrastructure/repositories/standing-order.repository.ts b/packages/services/ops-service/src/infrastructure/repositories/standing-order.repository.ts index 0929e75..ea45a33 100644 --- a/packages/services/ops-service/src/infrastructure/repositories/standing-order.repository.ts +++ b/packages/services/ops-service/src/infrastructure/repositories/standing-order.repository.ts @@ -10,7 +10,8 @@ export class StandingOrderRepository extends TenantAwareRepository { - const repo = await this.getRepository(); - return repo.find({ where: { status } as any }); + return this.withRepository((repo) => + repo.find({ where: { status } as any }), + ); } } diff --git a/packages/services/ops-service/src/infrastructure/repositories/task.repository.ts b/packages/services/ops-service/src/infrastructure/repositories/task.repository.ts index a25f6ef..96274a1 100644 --- a/packages/services/ops-service/src/infrastructure/repositories/task.repository.ts +++ b/packages/services/ops-service/src/infrastructure/repositories/task.repository.ts @@ -10,12 +10,14 @@ export class TaskRepository extends TenantAwareRepository { } async findByStatus(status: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { status } as any }); + return this.withRepository((repo) => + repo.find({ where: { status } as any }), + ); } async findByCreatedBy(createdBy: string): Promise { - const repo = await this.getRepository(); - return repo.find({ where: { createdBy } as any }); + return this.withRepository((repo) => + repo.find({ where: { createdBy } as any }), + ); } } diff --git a/packages/shared/database/src/tenant-aware.repository.ts b/packages/shared/database/src/tenant-aware.repository.ts index 130056d..4dffe90 100644 --- a/packages/shared/database/src/tenant-aware.repository.ts +++ b/packages/shared/database/src/tenant-aware.repository.ts @@ -7,30 +7,31 @@ export abstract class TenantAwareRepository { protected readonly entity: EntityTarget, ) {} - protected async getRepository(): Promise> { + protected async withRepository(fn: (repo: Repository) => Promise): Promise { const schema = TenantContextService.getSchemaName(); const queryRunner = this.dataSource.createQueryRunner(); - await queryRunner.query(`SET search_path TO "${schema}", public`); - return queryRunner.manager.getRepository(this.entity); + try { + await queryRunner.query(`SET search_path TO "${schema}", public`); + const repo = queryRunner.manager.getRepository(this.entity); + return await fn(repo); + } finally { + await queryRunner.release(); + } } async findById(id: string): Promise { - const repo = await this.getRepository(); - return repo.findOneBy({ id } as any); + return this.withRepository((repo) => repo.findOneBy({ id } as any)); } async save(entity: T): Promise { - const repo = await this.getRepository(); - return repo.save(entity); + return this.withRepository((repo) => repo.save(entity)); } async findAll(): Promise { - const repo = await this.getRepository(); - return repo.find(); + return this.withRepository((repo) => repo.find()); } async remove(entity: T): Promise { - const repo = await this.getRepository(); - return repo.remove(entity); + return this.withRepository((repo) => repo.remove(entity)); } }