Flink流计算与实时数据处理实战 - 四、Spring Boot微服务与Flink协作
本文是 Flink流计算与实时数据处理实战 的分章阅读版。
四、Spring Boot微服务与Flink协作
4.1 协作架构
+----------+ Kafka +----------+ +----------+
| 业务服务 | ---写入事件--→ | Kafka | | Spring |
| (Spring | | Topics | | Boot |
| Boot) | +----+-----+ | 查询服务 |
+----------+ | +----+-----+
| Flink消费 |
+------+-------+ 查询Redis/MySQL
| Flink Job | |
| (流计算) | |
+------+-------+ |
| | | |
+------+------++------+ +------+------+
| | | | |
+---+---+ +----+---+ +--+----+ +--+---+ +----+---+
| Redis | | MySQL | | Kafka | | HTTP | | WebSocket|
|(实时) | | (持久) | |(下游) | |回调 | | (推送) |
+-------+ +--------+ +-------+ +------+ +---------+
4.2 Flink结果写Redis - Spring Boot实时查询
// Flink端: 自定义Redis Sink
public class RedisCategorySink extends RichSinkFunction<CategoryStats> {
private JedisPool jedisPool;
@Override
public void open(Configuration parameters) {
jedisPool = new JedisPool("redis", 6379);
}
@Override
public void invoke(CategoryStats stats, Context context) {
try (Jedis jedis = jedisPool.getResource()) {
// 写入Hash: key=category:stats, field=category, value=json
jedis.hset("category:stats", stats.getCategory(),
JsonUtils.toJson(stats));
// 写入Sorted Set: 实时排行榜
jedis.zadd("category:ranking", stats.getTotalAmount(),
stats.getCategory());
// 设置过期时间(窗口统计5分钟过期)
jedis.expire("category:stats", 300);
}
}
@Override
public void close() {
if (jedisPool != null) jedisPool.close();
}
}
// Spring Boot端: 查询Flink计算结果
@RestController
@RequestMapping("/api/realtime")
@RequiredArgsConstructor
public class RealtimeStatsController {
private final StringRedisTemplate redisTemplate;
/**
* 查询品类实时销售额
*/
@GetMapping("/category/{category}")
public CategoryStats getCategoryStats(@PathVariable String category) {
String json = (String) redisTemplate.opsForHash()
.get("category:stats", category);
return JsonUtils.fromJson(json, CategoryStats.class);
}
/**
* 品类销售额排行榜TOP10
*/
@GetMapping("/category/ranking")
public List<CategoryStats> getCategoryRanking() {
Set<ZSetOperations.TypedTuple<String>> tuples =
redisTemplate.opsForZSet()
.reverseRangeWithScores("category:ranking", 0, 9);
return tuples.stream()
.map(t -> new CategoryStats(
t.getValue(),
t.getScore()))
.collect(Collectors.toList());
}
}
4.3 Flink结果写MySQL - Spring Boot查询报表
// Flink端: JDBC Sink
JdbcSink.sink(
"INSERT INTO category_stats (category, total_amount, order_count, window_end) " +
"VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE " +
"total_amount = VALUES(total_amount), order_count = VALUES(order_count)",
(ps, stats) -> {
ps.setString(1, stats.getCategory());
ps.setDouble(2, stats.getTotalAmount());
ps.setLong(3, stats.getOrderCount());
ps.setTimestamp(4, new Timestamp(stats.getWindowEnd()));
},
JdbcExecutionOptions.builder()
.withBatchSize(1000)
.withBatchIntervalMs(5000)
.withMaxRetries(3)
.build(),
new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
.withUrl("jdbc:mysql://mysql:3306/analytics")
.withDriverName("com.mysql.cj.jdbc.Driver")
.withUsername("root")
.withPassword("password")
.build()
);
相关页面