fix(contribution-service): 修复CDC消息解析以支持Debezium扁平化格式

Debezium配置了ExtractNewRecordState转换,消息格式是扁平化的:
- 元数据字段使用__前缀(__op, __table, __source_ts_ms, __deleted)
- 业务数据字段直接在根级别
- 修改handleMessage方法正确解析扁平化格式
- 更新CDCEvent接口以匹配实际消息结构

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
hailin 2026-01-11 09:39:09 -08:00
parent 4b55c63e71
commit 5f76108579
1 changed files with 49 additions and 24 deletions

View File

@ -2,28 +2,32 @@ import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Kafka, Consumer, EachMessagePayload } from 'kafkajs';
/**
* CDC
*
* Debezium ExtractNewRecordState unwrap
* __
*
*
* {
* "order_id": 1,
* "tree_count": 1,
* "account_sequence": "D25122700015",
* "__op": "c",
* "__table": "planting_orders",
* "__source_ts_ms": 1767892060857,
* "__deleted": "false"
* }
*/
export interface CDCEvent {
schema: any;
// 为了兼容性,构造一个 payload 对象
payload: {
op: 'c' | 'u' | 'd' | 'r'; // create, update, delete, read (snapshot)
before: any | null;
after: any | null;
source: {
version: string;
connector: string;
name: string;
ts_ms: number;
snapshot: string;
db: string;
sequence: string;
schema: string;
table: string;
txId: number;
lsn: number;
xmin: number | null;
};
op: 'c' | 'u' | 'd' | 'r'; // create, update, delete, read (snapshot)
ts_ms: number;
transaction: any;
table: string;
source_ts_ms: number;
deleted: boolean;
};
// 内部使用Kafka offset 作为序列号
sequenceNum: bigint;
@ -133,21 +137,42 @@ export class CDCConsumerService implements OnModuleInit {
return;
}
const eventData = JSON.parse(message.value.toString());
const rawData = JSON.parse(message.value.toString());
// Debezium ExtractNewRecordState 转换后的扁平化格式
// 元数据字段: __op, __table, __source_ts_ms, __deleted
// 数据字段直接在根级别
const op = rawData.__op || rawData.op;
const table = rawData.__table;
const sourceTsMs = rawData.__source_ts_ms || 0;
const deleted = rawData.__deleted === 'true' || rawData.__deleted === true;
// 从原始数据中移除元数据字段,剩下的就是业务数据
const { __op, __table, __source_ts_ms, __deleted, ...businessData } = rawData;
// 构造兼容的 CDCEvent 对象
// 对于 create/update/read数据在 after对于 delete数据在 before
const event: CDCEvent = {
...eventData,
payload: {
op: op as 'c' | 'u' | 'd' | 'r',
before: op === 'd' ? businessData : null,
after: op !== 'd' ? businessData : null,
table: table,
source_ts_ms: sourceTsMs,
deleted: deleted,
},
sequenceNum: BigInt(message.offset),
};
// 从 topic 名称提取表名
// 格式通常是: dbserver1.schema.tablename
// 从 topic 名称提取表名作为备选
// 格式: cdc.<service>.public.<table_name>
const parts = topic.split('.');
const tableName = parts[parts.length - 1];
const tableName = table || parts[parts.length - 1];
const handler = this.handlers.get(tableName);
if (handler) {
await handler(event);
this.logger.debug(`Processed CDC event for table ${tableName}, op: ${event.payload.op}`);
this.logger.debug(`Processed CDC event for table ${tableName}, op: ${op}`);
} else {
this.logger.warn(`No handler registered for table: ${tableName}`);
}