Flink流计算与实时数据处理实战 - 三、Flink + Kafka整合实战
本文是 Flink流计算与实时数据处理实战 的分章阅读版。
三、Flink + Kafka整合实战
3.1 依赖配置
<!-- pom.xml -->
<dependency>
<groupId> org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>1.19.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka</artifactId>
<version>3.1.0-1.18</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-jdbc</artifactId>
<version>3.1.2-1.18</version>
</dependency>
3.2 Kafka Source - 实时消费订单数据
public class OrderRealTimeJob {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment
.getExecutionEnvironment();
// 开启Checkpoint - 精确一次语义
env.enableCheckpointing(60000); // 60秒一次Checkpoint
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(30000);
env.getCheckpointConfig().setCheckpointTimeout(120000);
// Kafka Source
KafkaSource<OrderEvent> kafkaSource = KafkaSource.<OrderEvent>builder()
.setBootstrapServers("kafka1:9092,kafka2:9092,kafka3:9092")
.setTopics("order-events")
.setGroupId("flink-order-analytics")
.setStartingOffsets(OffsetsInitializer.earliest())
.setValueOnlyDeserializer(new JsonDeserializationSchema<>(OrderEvent.class))
.setProperty("partition.discovery.interval", "30s")
.build();
DataStream<OrderEvent> orderStream = env.fromSource(
kafkaSource,
WatermarkStrategy.<OrderEvent>forBoundedOutOfOrderness(Duration.ofSeconds(5))
.withTimestampAssigner((event, ts) -> event.getEventTime()),
"kafka-order-source"
);
// 处理逻辑
processOrders(orderStream);
env.execute("Order Real-time Analytics");
}
}
3.3 实时聚合 - 按品类统计5分钟销售额
private static void processOrders(DataStream<OrderEvent> orderStream) {
// 1. 过滤有效订单
SingleOutputStreamOperator<OrderEvent> validOrders = orderStream
.filter(event -> "PAID".equals(event.getStatus()));
// 2. 按品类分组 + 5分钟滚动窗口
validOrders
.keyBy(OrderEvent::getCategory)
.window(TumblingEventTimeWindows.of(Time.minutes(5)))
.allowedLateness(Time.minutes(1)) // 允许1分钟迟到
.sideOutputLateData(lateOutputTag) // 迟到数据侧输出
.aggregate(new OrderAggFunction())
.addSink(new RedisSink()); // 写入Redis供Spring Boot查询
}
3.4 自定义聚合函数
/**
* 订单聚合函数: 统计每个品类5分钟内的销售额、订单量、客单价
*/
public class OrderAggFunction
implements AggregateFunction<
OrderEvent, // 输入类型
OrderAccumulator, // 累加器类型
CategoryStats> { // 输出类型
@Override
public OrderAccumulator createAccumulator() {
return new OrderAccumulator();
}
@Override
public OrderAccumulator add(OrderEvent order, OrderAccumulator acc) {
acc.category = order.getCategory();
acc.totalAmount += order.getAmount();
acc.orderCount += 1;
acc.windowEnd = Math.max(acc.windowEnd, order.getEventTime());
return acc;
}
@Override
public CategoryStats getResult(OrderAccumulator acc) {
CategoryStats stats = new CategoryStats();
stats.setCategory(acc.category);
stats.setTotalAmount(acc.totalAmount);
stats.setOrderCount(acc.orderCount);
stats.setAvgAmount(acc.orderCount > 0 ?
acc.totalAmount / acc.orderCount : 0);
stats.setWindowEnd(acc.windowEnd);
return stats;
}
@Override
public OrderAccumulator merge(OrderAccumulator a, OrderAccumulator b) {
a.totalAmount += b.totalAmount;
a.orderCount += b.orderCount;
return a;
}
}
// 累加器
@Data
class OrderAccumulator {
String category;
double totalAmount;
long orderCount;
long windowEnd;
}
3.5 Kafka Sink - 处理结果写入Kafka
// 结果写入Kafka供下游消费
KafkaRecordSerializationSchema<CategoryStats> serializer =
KafkaRecordSerializationSchema.<CategoryStats>builder()
.setTopic("order-category-stats")
.setValueSerializationSchema(new JsonSerializationSchema<>())
.build();
KafkaSink<CategoryStats> kafkaSink = KafkaSink.<CategoryStats>builder()
.setBootstrapServers("kafka1:9092,kafka2:9092")
.setRecordSerializer(serializer)
.setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
.build();
categoryStatsStream.sinkTo(kafkaSink);
相关页面