本文是 Canal与Elasticsearch数据同步企业级实战 的分章阅读版。
全链路数据同步架构:
┌─────────┐ binlog ┌──────────┐ gRPC ┌──────────┐
│ MySQL │ ──────────→ │ Canal HA │ ──────→ │RocketMQ │
│ Master │ ROW模式 │(解析过滤) │ 投递 │ (解耦削峰)│
└─────────┘ └──────────┘ └─────┬────┘
│ 消费
┌──────▼──────┐
│ Sync Consumer│
│ (数据转换) │
└──────┬──────┘
┌─────────────┼─────────────┐
▼ ▼ ▼
┌────────┐ ┌────────┐ ┌────────┐
│ ES 8 │ │ Redis │ │ Click- │
│(全文搜索)│ │(缓存) │ │ House │
└────────┘ └────────┘ └────────┘
# canal.properties — MQ模式投递
canal.serverMode = rocketmq
canal.mq.servers = rocketmq-nameserver:9876
# instance.properties — 数据过滤和分区策略
canal.instance.filter.regex = house_db\\..*
# Topic策略:所有表投到一个Topic
canal.mq.topic = canal_house_sync
# 或者按表名动态Topic(推荐,每张表独立Topic)
canal.mq.dynamicTopic = house_db\\..*
# 分区策略:按主键hash,保证同ID的变更有序
canal.mq.partitionHash = house_db\\.house_data:id,house_db\\.house_price:id
# 分区数(等于RocketMQ Topic的Queue数量)
canal.mq.partitionsNum = 8
# 消息大小限制
canal.mq.flatMessage = true # 扁平化JSON消息
canal.mq.transaction = true # 开启事务消息
canal.mq.compressionType = LZ4 # 压缩(减少网络开销)
canal.mq.enableMessageTrace = true # 消息轨迹(排查用)
@Component
@RocketMQMessageListener(
topic = "canal_house_sync",
consumerGroup = "es-sync-consumer-group",
selectorExpression = "*",
consumeMode = ConsumeMode.ORDERLY, // 顺序消费(保证同ID有序)
maxReconsumeTimes = 5
)
@Slf4j
public class HouseSyncConsumer implements RocketMQListener<MessageExt> {
@Autowired private RestHighLevelClient esClient;
@Autowired private ObjectMapper objectMapper;
@Override
public void onMessage(MessageExt messageExt) {
String body = new String(messageExt.getBody(), StandardCharsets.UTF_8);
CanalMessage canalMsg;
try {
canalMsg = objectMapper.readValue(body, CanalMessage.class);
} catch (Exception e) {
log.error("Canal消息解析失败: {}", body, e);
return;
}
String type = canalMsg.getType();
String table = canalMsg.getTable();
List<Map<String, Object>> data = canalMsg.getData();
if (data == null || data.isEmpty()) return;
switch (table) {
case "house_data": syncHouseData(type, data); break;
case "house_price": syncHousePrice(type, data); break;
default: log.warn("未处理的表: {}", table);
}
}
private void syncHouseData(String type, List<Map<String, Object>> data) {
BulkRequest bulkRequest = new BulkRequest();
for (Map<String, Object> row : data) {
String docId = "house_" + row.get("id");
switch (type) {
case "DELETE":
bulkRequest.add(new DeleteRequest("house_index", docId));
break;
case "INSERT":
case "UPDATE":
Map<String, Object> doc = convertToEsDoc(row);
bulkRequest.add(new IndexRequest("house_index")
.id(docId).source(doc));
break;
}
}
try {
BulkResponse response = esClient.bulk(bulkRequest, RequestOptions.DEFAULT);
if (response.hasFailures()) {
log.error("ES Bulk部分失败: {}", response.buildFailureMessage());
saveFailedRecords(data, response.buildFailureMessage());
} else {
log.info("同步成功: table={}, type={}, count={}",
"house_data", type, data.size());
}
} catch (IOException e) {
log.error("ES Bulk异常", e);
throw new RuntimeException("ES写入异常,触发MQ重试", e);
}
}
private Map<String, Object> convertToEsDoc(Map<String, Object> row) {
Map<String, Object> doc = new HashMap<>();
doc.put("houseId", row.get("id"));
doc.put("title", row.get("title"));
doc.put("address", row.get("address"));
doc.put("district", row.get("district"));
doc.put("price", parseDouble(row.get("price")));
doc.put("area", parseDouble(row.get("area")));
doc.put("status", convertStatus(row.get("status")));
doc.put("updateTime", row.get("update_time"));
return doc;
}
}
{
"data": [
{
"id": "10086",
"title": "南山三室精装 南向通透 近地铁",
"address": "深圳市南山区科技园南路XX号",
"price": "5500000",
"area": "89.5",
"district": "nanshan",
"status": "1",
"create_time": "2026-06-12 10:30:00",
"update_time": "2026-06-12 10:30:00"
}
],
"database": "house_db",
"table": "house_data",
"type": "UPDATE",
"es": 1718162400000,
"ts": 1718162401000,
"old": [
{
"price": "5000000"
}
],
"pkNames": ["id"],
"sqlType": {"id": 4, "title": 12, "price": 8},
"isDdl": false
}
全链路延迟监控:
Canal位点延迟: Canal消费位点 vs MySQL当前binlog位点 < 1s
MQ消息堆积: RocketMQ Dashboard Topic Lag < 100
Consumer处理延迟: 消费时间戳 - 消息产生时间戳 < 500ms
ES写入延迟: Bulk请求平均耗时 < 200ms
端到端延迟: MySQL变更 → ES可搜索 < 2s (P99)
监控告警:
Canal延迟 > 30s → 预警
Canal延迟 > 5min → 严重
MQ堆积 > 10000 → 预警
ES Bulk拒绝 > 0 → 告警
面试话术: "全链路同步架构:MySQL binlog→Canal HA解析→RocketMQ解耦削峰→Java Consumer数据转换→ES Bulk写入。关键设计:MQ按主键hash分区保证同ID有序,Consumer顺序消费保证顺序性,Bulk失败重试+死信队列兜底,端到端延迟<2秒。"