fix: add restart policy, global error handlers, and fix tenant schema bug

- Add restart: unless-stopped to all 12 Docker services
- Add process.on(unhandledRejection/uncaughtException) to all 7 service main.ts
- Fix handleEventTrigger using tenantId UUID as schema name instead of slug lookup
- Wrap Redis event subscription callbacks in try/catch

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-02-23 05:30:34 -08:00
parent 6dcfe7cd9a
commit 9a1ecf10ec
9 changed files with 205 additions and 60 deletions

View File

@ -5,6 +5,7 @@ services:
postgres: postgres:
image: postgres:16-alpine image: postgres:16-alpine
container_name: it0-postgres container_name: it0-postgres
restart: unless-stopped
environment: environment:
POSTGRES_USER: ${POSTGRES_USER:-it0} POSTGRES_USER: ${POSTGRES_USER:-it0}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-it0_dev} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-it0_dev}
@ -25,6 +26,7 @@ services:
redis: redis:
image: redis:7-alpine image: redis:7-alpine
container_name: it0-redis container_name: it0-redis
restart: unless-stopped
ports: ports:
- "16379:6379" - "16379:6379"
healthcheck: healthcheck:
@ -40,6 +42,7 @@ services:
build: build:
context: ../../packages/gateway context: ../../packages/gateway
container_name: it0-api-gateway container_name: it0-api-gateway
restart: unless-stopped
environment: environment:
- JWT_SECRET=${JWT_SECRET:-dev-jwt-secret} - JWT_SECRET=${JWT_SECRET:-dev-jwt-secret}
ports: ports:
@ -77,6 +80,7 @@ services:
SERVICE_NAME: auth-service SERVICE_NAME: auth-service
SERVICE_PORT: 3001 SERVICE_PORT: 3001
container_name: it0-auth-service container_name: it0-auth-service
restart: unless-stopped
ports: ports:
- "13001:3001" - "13001:3001"
environment: environment:
@ -111,6 +115,7 @@ services:
SERVICE_NAME: agent-service SERVICE_NAME: agent-service
SERVICE_PORT: 3002 SERVICE_PORT: 3002
container_name: it0-agent-service container_name: it0-agent-service
restart: unless-stopped
ports: ports:
- "13002:3002" - "13002:3002"
environment: environment:
@ -145,6 +150,7 @@ services:
SERVICE_NAME: ops-service SERVICE_NAME: ops-service
SERVICE_PORT: 3003 SERVICE_PORT: 3003
container_name: it0-ops-service container_name: it0-ops-service
restart: unless-stopped
ports: ports:
- "13003:3003" - "13003:3003"
environment: environment:
@ -177,6 +183,7 @@ services:
SERVICE_NAME: inventory-service SERVICE_NAME: inventory-service
SERVICE_PORT: 3004 SERVICE_PORT: 3004
container_name: it0-inventory-service container_name: it0-inventory-service
restart: unless-stopped
ports: ports:
- "13004:3004" - "13004:3004"
environment: environment:
@ -208,6 +215,7 @@ services:
SERVICE_NAME: monitor-service SERVICE_NAME: monitor-service
SERVICE_PORT: 3005 SERVICE_PORT: 3005
container_name: it0-monitor-service container_name: it0-monitor-service
restart: unless-stopped
ports: ports:
- "13005:3005" - "13005:3005"
environment: environment:
@ -238,6 +246,7 @@ services:
SERVICE_NAME: comm-service SERVICE_NAME: comm-service
SERVICE_PORT: 3006 SERVICE_PORT: 3006
container_name: it0-comm-service container_name: it0-comm-service
restart: unless-stopped
ports: ports:
- "13006:3006" - "13006:3006"
environment: environment:
@ -273,6 +282,7 @@ services:
SERVICE_NAME: audit-service SERVICE_NAME: audit-service
SERVICE_PORT: 3007 SERVICE_PORT: 3007
container_name: it0-audit-service container_name: it0-audit-service
restart: unless-stopped
ports: ports:
- "13007:3007" - "13007:3007"
environment: environment:
@ -299,6 +309,7 @@ services:
build: build:
context: ../../packages/services/voice-service context: ../../packages/services/voice-service
container_name: it0-voice-service container_name: it0-voice-service
restart: unless-stopped
ports: ports:
- "13008:3008" - "13008:3008"
environment: environment:
@ -323,6 +334,7 @@ services:
build: build:
context: ../../it0-web-admin context: ../../it0-web-admin
container_name: it0-web-admin container_name: it0-web-admin
restart: unless-stopped
ports: ports:
- "13000:3000" - "13000:3000"
environment: environment:

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { AgentModule } from './agent.module'; import { AgentModule } from './agent.module';
const logger = new Logger('AgentService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(AgentModule); const app = await NestFactory.create(AgentModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('AGENT_SERVICE_PORT', 3002); const port = config.get<number>('AGENT_SERVICE_PORT', 3002);
await app.listen(port); await app.listen(port);
console.log(`agent-service running on port ${port}`); logger.log(`agent-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start agent-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { AuditModule } from './audit.module'; import { AuditModule } from './audit.module';
const logger = new Logger('AuditService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(AuditModule); const app = await NestFactory.create(AuditModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('AUDIT_SERVICE_PORT', 3007); const port = config.get<number>('AUDIT_SERVICE_PORT', 3007);
await app.listen(port); await app.listen(port);
console.log(`audit-service running on port ${port}`); logger.log(`audit-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start audit-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { AuthModule } from './auth.module'; import { AuthModule } from './auth.module';
const logger = new Logger('AuthService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(AuthModule); const app = await NestFactory.create(AuthModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('AUTH_SERVICE_PORT', 3001); const port = config.get<number>('AUTH_SERVICE_PORT', 3001);
await app.listen(port); await app.listen(port);
console.log(`auth-service running on port ${port}`); logger.log(`auth-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start auth-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { CommModule } from './comm.module'; import { CommModule } from './comm.module';
const logger = new Logger('CommService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(CommModule); const app = await NestFactory.create(CommModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('COMM_SERVICE_PORT', 3006); const port = config.get<number>('COMM_SERVICE_PORT', 3006);
await app.listen(port); await app.listen(port);
console.log(`comm-service running on port ${port}`); logger.log(`comm-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start comm-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { InventoryModule } from './inventory.module'; import { InventoryModule } from './inventory.module';
const logger = new Logger('InventoryService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(InventoryModule); const app = await NestFactory.create(InventoryModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('INVENTORY_SERVICE_PORT', 3004); const port = config.get<number>('INVENTORY_SERVICE_PORT', 3004);
await app.listen(port); await app.listen(port);
console.log(`inventory-service running on port ${port}`); logger.log(`inventory-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start inventory-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { MonitorModule } from './monitor.module'; import { MonitorModule } from './monitor.module';
const logger = new Logger('MonitorService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(MonitorModule); const app = await NestFactory.create(MonitorModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('MONITOR_SERVICE_PORT', 3005); const port = config.get<number>('MONITOR_SERVICE_PORT', 3005);
await app.listen(port); await app.listen(port);
console.log(`monitor-service running on port ${port}`); logger.log(`monitor-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start monitor-service: ${err.message}`, err.stack);
process.exit(1);
});

View File

@ -33,25 +33,37 @@ export class StandingOrderExecutorService implements OnModuleInit {
async onModuleInit(): Promise<void> { async onModuleInit(): Promise<void> {
this.logger.log('Subscribing to event-driven standing order triggers...'); this.logger.log('Subscribing to event-driven standing order triggers...');
await this.eventBus.subscribe( try {
EventPatterns.ALERT_FIRED, await this.eventBus.subscribe(
'standing-order-executor', EventPatterns.ALERT_FIRED,
`executor-${crypto.randomUUID().slice(0, 8)}`, 'standing-order-executor',
async (event) => { `executor-${crypto.randomUUID().slice(0, 8)}`,
this.logger.debug(`Received ALERT_FIRED event: ${event.id}`); async (event) => {
await this.handleEventTrigger(EventPatterns.ALERT_FIRED, event); try {
}, this.logger.debug(`Received ALERT_FIRED event: ${event.id}`);
); await this.handleEventTrigger(EventPatterns.ALERT_FIRED, event);
} catch (err) {
this.logger.error(`ALERT_FIRED handler error: ${err}`);
}
},
);
await this.eventBus.subscribe( await this.eventBus.subscribe(
EventPatterns.STANDING_ORDER_TRIGGERED, EventPatterns.STANDING_ORDER_TRIGGERED,
'standing-order-executor', 'standing-order-executor',
`executor-${crypto.randomUUID().slice(0, 8)}`, `executor-${crypto.randomUUID().slice(0, 8)}`,
async (event) => { async (event) => {
this.logger.debug(`Received STANDING_ORDER_TRIGGERED event: ${event.id}`); try {
await this.handleEventTrigger(EventPatterns.STANDING_ORDER_TRIGGERED, event); this.logger.debug(`Received STANDING_ORDER_TRIGGERED event: ${event.id}`);
}, await this.handleEventTrigger(EventPatterns.STANDING_ORDER_TRIGGERED, event);
); } catch (err) {
this.logger.error(`STANDING_ORDER_TRIGGERED handler error: ${err}`);
}
},
);
} catch (err) {
this.logger.error(`Failed to subscribe to events: ${err}`);
}
} }
/** /**
@ -120,7 +132,7 @@ export class StandingOrderExecutorService implements OnModuleInit {
/** /**
* Handles an incoming event and matches it against event-triggered standing orders. * Handles an incoming event and matches it against event-triggered standing orders.
* Events from Redis carry tenantId wrap in tenant context. * Events from Redis carry tenantId look up slug for correct schema name.
*/ */
private async handleEventTrigger( private async handleEventTrigger(
eventType: string, eventType: string,
@ -132,35 +144,58 @@ export class StandingOrderExecutorService implements OnModuleInit {
return; return;
} }
await TenantContextService.run( let tenant: { id: string; name: string; slug: string } | undefined;
{ try {
tenantId, const rows = await this.dataSource.query(
tenantName: tenantId, `SELECT id, name, slug FROM public.tenants WHERE id = $1 AND status = 'active'`,
plan: 'pro', [tenantId],
schemaName: `it0_t_${tenantId}`, );
}, tenant = rows[0];
async () => { } catch (err) {
const activeOrders = this.logger.error(`Failed to look up tenant ${tenantId}: ${err}`);
await this.standingOrderRepo.findByStatus('active'); return;
const matchingOrders = activeOrders.filter( }
(order) =>
order.trigger.type === 'event' &&
order.trigger.eventType === eventType,
);
for (const order of matchingOrders) { if (!tenant) {
this.logger.log( this.logger.warn(`Tenant ${tenantId} not found or inactive, skipping event`);
`Event match for standing order "${order.name}" (${order.id}) on event ${eventType}`, return;
}
try {
await TenantContextService.run(
{
tenantId: tenant.id,
tenantName: tenant.name,
plan: 'pro',
schemaName: `it0_t_${tenant.slug}`,
},
async () => {
const activeOrders =
await this.standingOrderRepo.findByStatus('active');
const matchingOrders = activeOrders.filter(
(order) =>
order.trigger.type === 'event' &&
order.trigger.eventType === eventType,
); );
await this.executeOrder(order, {
triggerType: 'event', for (const order of matchingOrders) {
eventType, this.logger.log(
eventId: event.id, `Event match for standing order "${order.name}" (${order.id}) on event ${eventType}`,
eventPayload: event.payload, );
}); await this.executeOrder(order, {
} triggerType: 'event',
}, eventType,
); eventId: event.id,
eventPayload: event.payload,
});
}
},
);
} catch (err) {
this.logger.error(
`Event trigger error for tenant ${tenant.id}: ${err}`,
);
}
} }
/** /**

View File

@ -1,12 +1,26 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { OpsModule } from './ops.module'; import { OpsModule } from './ops.module';
const logger = new Logger('OpsService');
// Prevent process crash from unhandled errors
process.on('unhandledRejection', (reason) => {
logger.error(`Unhandled Rejection: ${reason}`);
});
process.on('uncaughtException', (error) => {
logger.error(`Uncaught Exception: ${error.message}`, error.stack);
});
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(OpsModule); const app = await NestFactory.create(OpsModule);
const config = app.get(ConfigService); const config = app.get(ConfigService);
const port = config.get<number>('OPS_SERVICE_PORT', 3003); const port = config.get<number>('OPS_SERVICE_PORT', 3003);
await app.listen(port); await app.listen(port);
console.log(`ops-service running on port ${port}`); logger.log(`ops-service running on port ${port}`);
} }
bootstrap(); bootstrap().catch((err) => {
logger.error(`Failed to start ops-service: ${err.message}`, err.stack);
process.exit(1);
});