← 返回首页 📖 📖 Wiki知识库
📐 学习维度 本篇是 📖 📖 Wiki知识库 🏠 首页 🛤️ 学习路径 📚 全部文章

Flink流计算与实时数据处理实战 - 三、Flink + Kafka整合实战

本文是 Flink流计算与实时数据处理实战 的分章阅读版。

3.1 依赖配置


<!-- pom.xml -->
<dependency>
    <groupId> org.apache.flink</groupId>
    <artifactId>flink-streaming-java</artifactId>
    <version>1.19.0</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka</artifactId>
    <version>3.1.0-1.18</version>
</dependency>
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-jdbc</artifactId>
    <version>3.1.2-1.18</version>
</dependency>

3.2 Kafka Source - 实时消费订单数据


public class OrderRealTimeJob {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment
            .getExecutionEnvironment();

        // 开启Checkpoint - 精确一次语义
        env.enableCheckpointing(60000);  // 60秒一次Checkpoint
        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(30000);
        env.getCheckpointConfig().setCheckpointTimeout(120000);

        // Kafka Source
        KafkaSource<OrderEvent> kafkaSource = KafkaSource.<OrderEvent>builder()
            .setBootstrapServers("kafka1:9092,kafka2:9092,kafka3:9092")
            .setTopics("order-events")
            .setGroupId("flink-order-analytics")
            .setStartingOffsets(OffsetsInitializer.earliest())
            .setValueOnlyDeserializer(new JsonDeserializationSchema<>(OrderEvent.class))
            .setProperty("partition.discovery.interval", "30s")
            .build();

        DataStream<OrderEvent> orderStream = env.fromSource(
            kafkaSource,
            WatermarkStrategy.<OrderEvent>forBoundedOutOfOrderness(Duration.ofSeconds(5))
                .withTimestampAssigner((event, ts) -> event.getEventTime()),
            "kafka-order-source"
        );

        // 处理逻辑
        processOrders(orderStream);

        env.execute("Order Real-time Analytics");
    }
}

3.3 实时聚合 - 按品类统计5分钟销售额


private static void processOrders(DataStream<OrderEvent> orderStream) {

    // 1. 过滤有效订单
    SingleOutputStreamOperator<OrderEvent> validOrders = orderStream
        .filter(event -> "PAID".equals(event.getStatus()));

    // 2. 按品类分组 + 5分钟滚动窗口
    validOrders
        .keyBy(OrderEvent::getCategory)
        .window(TumblingEventTimeWindows.of(Time.minutes(5)))
        .allowedLateness(Time.minutes(1))  // 允许1分钟迟到
        .sideOutputLateData(lateOutputTag) // 迟到数据侧输出
        .aggregate(new OrderAggFunction())
        .addSink(new RedisSink());  // 写入Redis供Spring Boot查询
}

3.4 自定义聚合函数


/**
 * 订单聚合函数: 统计每个品类5分钟内的销售额、订单量、客单价
 */
public class OrderAggFunction
    implements AggregateFunction<
        OrderEvent,           // 输入类型
        OrderAccumulator,     // 累加器类型
        CategoryStats> {      // 输出类型

    @Override
    public OrderAccumulator createAccumulator() {
        return new OrderAccumulator();
    }

    @Override
    public OrderAccumulator add(OrderEvent order, OrderAccumulator acc) {
        acc.category = order.getCategory();
        acc.totalAmount += order.getAmount();
        acc.orderCount += 1;
        acc.windowEnd = Math.max(acc.windowEnd, order.getEventTime());
        return acc;
    }

    @Override
    public CategoryStats getResult(OrderAccumulator acc) {
        CategoryStats stats = new CategoryStats();
        stats.setCategory(acc.category);
        stats.setTotalAmount(acc.totalAmount);
        stats.setOrderCount(acc.orderCount);
        stats.setAvgAmount(acc.orderCount > 0 ?
            acc.totalAmount / acc.orderCount : 0);
        stats.setWindowEnd(acc.windowEnd);
        return stats;
    }

    @Override
    public OrderAccumulator merge(OrderAccumulator a, OrderAccumulator b) {
        a.totalAmount += b.totalAmount;
        a.orderCount += b.orderCount;
        return a;
    }
}

// 累加器
@Data
class OrderAccumulator {
    String category;
    double totalAmount;
    long orderCount;
    long windowEnd;
}

3.5 Kafka Sink - 处理结果写入Kafka


// 结果写入Kafka供下游消费
KafkaRecordSerializationSchema<CategoryStats> serializer =
    KafkaRecordSerializationSchema.<CategoryStats>builder()
        .setTopic("order-category-stats")
        .setValueSerializationSchema(new JsonSerializationSchema<>())
        .build();

KafkaSink<CategoryStats> kafkaSink = KafkaSink.<CategoryStats>builder()
    .setBootstrapServers("kafka1:9092,kafka2:9092")
    .setRecordSerializer(serializer)
    .setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
    .build();

categoryStatsStream.sinkTo(kafkaSink);


相关页面