From df7f1a46f9ce084864d0f30fe61c65ee1f278642 Mon Sep 17 00:00:00 2001 From: "terrance.lzm" Date: Wed, 22 Jul 2026 17:21:42 +0800 Subject: [PATCH] [ISSUE #135] feat: add gRPC connector on LiteSimpleConsumer with downstream ack and sub-topic throttling Add the flink-connector-rocketmq-grpc module built on rocketmq-client-java 5.2.1+ LiteSimpleConsumer (Pop model): - Source binds one main lite topic with a wildcard subscription and never acks; each subtask shares one consumer across fetch-concurrency long-poll workers. - Emits AckableMessage carrying a serializable, credential-free RocketMQReceiptHandle that can cross shuffle boundaries. - Downstream ack via a per-TaskManager reference-counted client pool, with two wirings: RocketMQAckProcessFunction (direct ack) and MessageThrottlePolicy + RocketMQThrottleProcessFunction (policy-based fair throttling with DLQ safety valves). - Optional invisible-duration renewal prevents premature redelivery under backpressure. - gRPC producer sink and rocketmq-grpc SQL connector; design documented in docs/grpc-connector.md. --- README.md | 19 +- docs/grpc-connector.md | 241 ++++++++++- docs/sql-connector.md | 4 +- flink-connector-rocketmq-grpc/pom.xml | 153 +++++++ .../rocketmq/grpc/RocketMQGrpcOptions.java | 105 +++++ .../rocketmq/grpc/ack/AckableMessage.java | 75 ++++ .../grpc/ack/AckableMessageTypeInfo.java | 264 ++++++++++++ .../grpc/ack/MessageThrottlePolicy.java | 46 ++ .../grpc/ack/RocketMQAckProcessFunction.java | 115 +++++ .../grpc/ack/RocketMQLiteAckClient.java | 398 +++++++++++++++++ .../grpc/ack/RocketMQReceiptHandle.java | 299 +++++++++++++ .../grpc/ack/RocketMQReceiptHandleCodec.java | 162 +++++++ .../ack/RocketMQThrottleProcessFunction.java | 104 +++++ .../common/ClientConfigurationProvider.java | 79 ++++ .../grpc/common/CredentialsResolver.java | 67 +++ .../grpc/common/CredentialsResolvers.java | 99 +++++ .../grpc/common/FilterExpressionParser.java | 64 +++ .../rocketmq/grpc/sink/ProducerProvider.java | 51 +++ .../rocketmq/grpc/sink/RocketMQGrpcSink.java | 70 +++ .../grpc/sink/RocketMQGrpcSinkBuilder.java | 99 +++++ .../grpc/sink/RocketMQGrpcSinkOptions.java | 68 +++ .../grpc/sink/RocketMQGrpcSinkWriter.java | 83 ++++ .../RocketMQGrpcSerializationSchema.java | 68 +++ ...ocketMQGrpcSerializationSchemaWrapper.java | 64 +++ .../InvisibleDurationRenewalPolicies.java | 69 +++ .../InvisibleDurationRenewalPolicy.java | 69 +++ .../grpc/source/RocketMQGrpcSource.java | 163 +++++++ .../source/RocketMQGrpcSourceBuilder.java | 144 +++++++ .../source/RocketMQGrpcSourceOptions.java | 118 +++++ .../RocketMQGrpcDeserializationSchema.java | 62 +++ ...ketMQGrpcDeserializationSchemaWrapper.java | 62 +++ .../RocketMQGrpcSourceEnumState.java | 28 ++ ...RocketMQGrpcSourceEnumStateSerializer.java | 46 ++ .../RocketMQGrpcSourceEnumerator.java | 86 ++++ .../reader/LiteSimpleConsumerProvider.java | 58 +++ .../grpc/source/reader/MessageView.java | 54 +++ .../grpc/source/reader/MessageViewImpl.java | 128 ++++++ .../RocketMQGrpcSourceFetcherManager.java | 45 ++ .../reader/RocketMQGrpcSourceReader.java | 78 ++++ .../RocketMQGrpcSourceRecordEmitter.java | 99 +++++ .../reader/RocketMQGrpcSourceSplitReader.java | 325 ++++++++++++++ .../source/split/RocketMQGrpcSourceSplit.java | 65 +++ .../RocketMQGrpcSourceSplitSerializer.java | 46 ++ .../split/RocketMQGrpcSourceSplitState.java | 48 +++ .../table/RocketMQGrpcConnectorOptions.java | 188 ++++++++ .../RocketMQGrpcDynamicTableFactory.java | 245 +++++++++++ .../table/RocketMQGrpcDynamicTableSink.java | 109 +++++ .../table/RocketMQGrpcDynamicTableSource.java | 197 +++++++++ .../table/RocketMQGrpcReadableMetadata.java | 118 +++++ .../table/RocketMQGrpcRowDataConverter.java | 181 ++++++++ .../org.apache.flink.table.factories.Factory | 16 + .../rocketmq/grpc/RocketMQGrpcITCase.java | 405 ++++++++++++++++++ .../grpc/RocketMQGrpcLiteE2EVerify.java | 346 +++++++++++++++ .../ack/AckableMessageSerializerTest.java | 75 ++++ .../grpc/ack/RocketMQLiteAckClientTest.java | 82 ++++ .../ack/RocketMQReceiptHandleCodecTest.java | 110 +++++ .../RocketMQReceiptHandleSerializerTest.java | 73 ++++ .../RocketMQThrottleProcessFunctionTest.java | 149 +++++++ .../grpc/common/CredentialsResolversTest.java | 153 +++++++ .../sink/RocketMQGrpcSinkBuilderTest.java | 90 ++++ .../InvisibleDurationRenewalPoliciesTest.java | 95 ++++ .../source/RocketMQGrpcSourceBuilderTest.java | 94 ++++ ...etMQGrpcSourceEnumStateSerializerTest.java | 42 ++ ...RocketMQGrpcSourceSplitSerializerTest.java | 44 ++ .../source/reader/MessageViewImplTest.java | 141 ++++++ .../RocketMQGrpcDynamicTableFactoryTest.java | 83 ++++ .../RocketMQGrpcRowDataConverterTest.java | 172 ++++++++ .../config/RocketMQConfigValidator.java | 6 +- .../connector/rocketmq/sink/RocketMQSink.java | 2 +- .../rocketmq/sink/RocketMQSinkBuilder.java | 2 +- .../rocketmq/sink/RocketMQSinkOptions.java | 4 +- .../rocketmq/sink/writer/RocketMQWriter.java | 2 +- .../rocketmq/source/InnerConsumerImpl.java | 14 +- .../rocketmq/source/RocketMQSource.java | 1 - .../source/RocketMQSourceOptions.java | 2 +- .../enumerator/offset/OffsetsSelector.java | 2 +- .../reader/RocketMQSourceFetcherManager.java | 2 +- .../RocketMQRowDeserializationSchema.java | 3 +- .../connectors/rocketmq/RocketMQSink.java | 4 +- .../rocketmq/RocketMQSourceFunction.java | 26 +- .../table/RocketMQDynamicTableSink.java | 2 +- .../RocketMQDynamicTableSourceFactory.java | 2 +- .../RocketMQRowDataSerializationSchema.java | 4 +- .../rocketmq/table/RocketMQRowDataSink.java | 2 +- .../table/RocketMQScanTableSource.java | 8 +- .../rocketmq/catalog/RocketMQCatalogTest.java | 15 +- .../example/ConnectorIntegrationTest.java | 19 +- .../rocketmq/example/SqlIntegrationTest.java | 3 +- .../TransactionSinkIntegrationTest.java | 4 +- .../rocketmq/sink/InnerProducerImplTest.java | 3 +- .../rocketmq/RocketMQSourceTest.java | 2 +- .../example/LegacyConnectorExample.java | 12 +- ...MQDynamicTableSourceFactoryOffsetTest.java | 1 - .../RocketMQScanTableSourceFilterTest.java | 6 +- .../table/RocketMQScanTableSourceTest.java | 1 - flink-sql-connector-rocketmq-grpc/pom.xml | 144 +++++++ pom.xml | 2 + 97 files changed, 7995 insertions(+), 103 deletions(-) create mode 100644 flink-connector-rocketmq-grpc/pom.xml create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java create mode 100644 flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java create mode 100644 flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java create mode 100644 flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java create mode 100644 flink-sql-connector-rocketmq-grpc/pom.xml diff --git a/README.md b/README.md index 37084b2..a2a91d4 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,10 @@ Two connector tracks are shipped: | Track | Module | RocketMQ client | Best for | | --- | --- | --- | --- | | **Remoting** | `flink-connector-rocketmq` | `rocketmq-client` (remoting protocol) | Classic topics, FLIP-27 source + SinkV2 sink, SQL | +| **gRPC** | `flink-connector-rocketmq-grpc` | `rocketmq-client-java` (gRPC protocol) | RocketMQ 5.x lite topics, downstream ack / fair throttling | -The SQL fat-jar is packaged by `flink-sql-connector-rocketmq`. +The SQL fat-jars are packaged by `flink-sql-connector-rocketmq` and +`flink-sql-connector-rocketmq-grpc`. ## Apache Flink @@ -93,6 +95,20 @@ CREATE TABLE rocketmq_source ( ); ``` +### gRPC lite consumer (RocketMQ 5.x) + +```java +RocketMQGrpcSource source = RocketMQGrpcSource.builder() + .setEndpoints("127.0.0.1:8081") + .setConsumerGroup("GID-lite") + .setMainTopic("LiteMainTopic") // wildcard-subscribes all lite topics under it + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); +``` + +The gRPC source emits `AckableMessage` and never acks by itself: the downstream operators own +the ack / throttle decision. See the gRPC connector doc for the ack operator wirings. + ## Documentation Connector documentation is located in the `docs/` directory of this repository: @@ -100,6 +116,7 @@ Connector documentation is located in the `docs/` directory of this repository: | Document | Content | | --- | --- | | [docs/remoting-connector.md](docs/remoting-connector.md) | Remoting DataStream connector: usage and all configuration options | +| [docs/grpc-connector.md](docs/grpc-connector.md) | gRPC lite connector: prerequisites, downstream ack / throttling, options | | [docs/sql-connector.md](docs/sql-connector.md) | Table/SQL connector: DDL, metadata columns, fat-jar notes | | [docs/legacy-connector.md](docs/legacy-connector.md) | Legacy `RocketMQSourceFunction` / `RocketMQSink` (deprecated) | | [docs/connector-overview.md](docs/connector-overview.md) | Feature comparison with other messaging connectors | diff --git a/docs/grpc-connector.md b/docs/grpc-connector.md index a799a67..017b176 100644 --- a/docs/grpc-connector.md +++ b/docs/grpc-connector.md @@ -4,7 +4,215 @@ The gRPC connector (`flink-connector-rocketmq-grpc`) targets RocketMQ 5.x **lite `rocketmq-client-java` 5.2.1+. The source binds one main lite topic with a wildcard subscription, never acks by itself, and hands the ack / throttle decision to downstream operators. -## Prerequisites +This document covers both the design (architecture, semantics, decisions and trade-offs) and the +usage of the connector. + +--- + +## 1. Motivation + +The target scenario is a **main lite topic with a large number of sub (lite) topics** underneath, +e.g. AI workloads where every tenant/session gets its own sub topic: + +- The source subscribes to the main topic with a **wildcard (generalized) subscription**; the + broker funnels messages of *all* sub topics through one `receive()` stream, so the reader never + has to enumerate or track sub topics. +- Whether a message should be **acknowledged** or **deferred (throttled)** is a business decision + that is only known *after* downstream processing — possibly on the other side of a + keyBy/shuffle. A hot sub topic that consumes too many resources should be slowed down without + starving the other sub topics. + +This leads to the two defining properties of the connector: + +1. **The source never acknowledges messages.** Acknowledgement is deferred to downstream + operators, which receive a self-contained, serializable receipt handle with every record. +2. **Throttling is expressed per message via `changeInvisibleDuration`.** Deferring the + redelivery of a hot sub topic's messages naturally yields fair sharing of the fetch budget: + +``` + sub topic t_hot floods messages ─┐ + ├─► each one gets changeInvisibleDuration(+20s) + │ → invisible to the group for the next 20s + ▼ + reader.receive() only returns currently-visible messages: + ├─ t1, t2, t3 ... normal messages (visible) ← keep flowing + └─ t_hot messages mostly invisible ← suppressed, trickle back on expiry + + ⇒ the hot sub topic is throttled, the others are not starved = fairness +``` + +## 2. Consumption model + +RocketMQ 5.x offers a push client (`PushConsumer`) and pull/Pop clients +(`SimpleConsumer` / `LiteSimpleConsumer`). The connector uses **`LiteSimpleConsumer`** +(available since `rocketmq-client-java` 5.2.1), whose consumption is split into three explicit +phases that map cleanly onto a Flink pipeline: + +1. **Receive** — `receive(maxMessageNum, invisibleDuration)` long-polls the proxy; returned + messages become *invisible* to the rest of the consumer group for `invisibleDuration`. +2. **Process** — the caller (here: the Flink job) processes the messages. +3. **Ack / changeInvisibleDuration** — `ack()` commits a message; + `changeInvisibleDuration()` extends/renews its invisibility. **Un-acked messages are + redelivered by the broker once their invisible duration expires** — this is the source of the + at-least-once guarantee. + +Key SDK facts the design relies on (verified against the SDK sources): + +- **Wildcard subscription = `bindTopic` only.** Binding the main topic without calling + `subscribeLite` puts the consumer into generalized mode: it receives from all sub topics. + The consumer group must carry the broker-side attribute `lite.sub.wildcard=true` + (a **consumer group** attribute — see Prerequisites). +- **Pop receipt handles are self-contained.** The server-side handle encodes its own routing + (broker, queue, offsets, timestamps) as a string; the proxy routes an ack by the handle alone. + Any consumer of the **same group** can ack a message, regardless of which consumer received it. +- **`SimpleConsumer`/`LiteSimpleConsumer` are thread-safe.** Multiple threads may block in + `receive()` on one consumer instance concurrently. +- **`OffsetOption` (start-position control) exists only for exact `subscribeLite`**, therefore + the wildcard-mode source has no startup-offset capability. + +## 3. Architecture + +### 3.1 Source reader + +Because the broker performs **message-level load balancing** across the consumer group, the +source does not partition topics into splits. Every subtask binds the same main topic and simply +receives whatever the broker hands out; a single placeholder split only triggers the reader to +start. Per subtask: + +``` +subtask + └─ SourceReader (mailbox thread) + └─ SingleThreadFetcherManager → 1 fetcher thread + └─ SplitReader + ├─ 1 shared LiteSimpleConsumer (thread-safe) + ├─ N ReceiveWorker threads (N = rocketmq.source.fetch-concurrency) + │ loop: consumer.receive(maxMessageNum, invisibleDuration) + │ → put into a bounded in-memory queue (capacity = maxMessageNum × N) + └─ fetch(): drain the queue in batches for the record emitter +``` + +- **One consumer, many workers.** A blocking `receive()` long poll only occupies its calling + thread, so `fetch-concurrency` worker threads sharing one consumer keep that many long polls in + flight — raising a single subtask's throughput without extra clientIds, heartbeats, connections + or route caches. Total throughput scales with + `parallelism × fetch-concurrency × maxMessageNum / receive-latency`. +- **Backpressure = stop receiving.** When the bounded queue is full, workers block on `put()`; + the pull model has no explicit pause API and does not need one. +- **Receive failures back off exponentially** (100 ms doubling up to 30 s) instead of + hot-looping. + +### 3.2 Receipt handle and record type + +The source emits `AckableMessage = { value, RocketMQReceiptHandle }`. + +`RocketMQReceiptHandle` is a small, immutable, `Serializable` value object made of strings and an +int: + +| Field group | Fields | Purpose | +| --- | --- | --- | +| Routing triple | `endpoint`, `namespace`, `consumerGroup` | Selects which pooled consumer must issue the ack RPC. Records from multiple sources (different clusters/namespaces/groups) can coexist in one stream. | +| Message identity | `topic`, `liteTopic`, `messageId`, `receiptHandle`, `deliveryAttempt` | Rebuilds the minimal SDK message view required by `ack` / `changeInvisibleDuration`. | + +It deliberately contains **no credentials, no protobuf blob and no message body**, so it is safe +and cheap to ship across keyBy/shuffle boundaries. Dedicated `TypeInformation`/`TypeSerializer` +implementations (`AckableMessageTypeInfo` and the nested serializers) keep the type off the Kryo +fallback path and give it a stable wire format. + +The single place that touches SDK-internal classes (`org.apache.rocketmq.client.java.*`) is +`RocketMQReceiptHandleCodec`: the public `apis` package exposes no receipt handle, so extracting +it on the source side and rebuilding a minimal message view on the ack side requires internal +types. Confining that (experimental) coupling to one `@Internal` adapter keeps the rest of the +connector SDK-clean. + +### 3.3 Downstream acknowledgement + +Downstream operators obtain a shared, credential-free ack client: + +- **Pooling.** `RocketMQLiteAckClient` is shared per TaskManager JVM via reference-counted + `acquire`/`release` keyed by the client configuration. Internally it lazily keeps one + same-group `LiteSimpleConsumer` per routing triple `(endpoint, namespace, consumerGroup)` + carried by the incoming handles. The consumer built for a handle uses the *handle's* endpoints + and namespace (the SDK stamps ack requests with the consumer's namespace) and the *operator + configuration's* credentials/TLS/timeout. +- **Retry & tolerance.** Ack RPCs retry with bounded exponential backoff. An + `INVALID_RECEIPT_HANDLE` response (expired handle) is tolerated as a warning: the message will + simply be redelivered, which is preferable to failing the job under at-least-once semantics. +- **Credentials never travel in the stream.** They are configured on the ack operator and + resolved locally on each TaskManager — either static access/secret keys, or a pluggable + per-endpoint `CredentialsResolver` (e.g. environment variables, mounted secrets, external KMS) + that also keeps plaintext secrets out of the job graph. + +Two wirings are provided on top of the client (DataStream API only): + +1. **Client injection** — extend the abstract `RocketMQAckProcessFunction` and call the + protected `ack()` / `changeInvisibleDuration()` from business logic. No extra edge in the job + graph, lowest latency. +2. **Policy-based throttling** — implement `MessageThrottlePolicy` + (`Optional onMessage(value)`: empty = ack, duration = defer) and run it with the + one-stop `RocketMQThrottleProcessFunction`. + +### 3.4 Throttling semantics and safety valves + +Using `changeInvisibleDuration` for throttling is a deliberate, documented trade-off: it is a +**per-message defer**, not a topic-level switch. Consequences and mitigations: + +- Every deferred message costs one extra RPC, and deferring reorders the stream — downstream must + already be idempotent and order-tolerant under at-least-once. +- Each redelivery increments `deliveryAttempt`; endless deferring would eventually push a message + into the dead-letter queue. `RocketMQThrottleProcessFunction` therefore enforces two safety + valves: once `deliveryAttempt` reaches `maxDeliveryAttempt` (default 16) the message is acked + instead of deferred, and every requested delay is capped at `maxInvisibleDuration` + (default 30 min). +- The broker slows down dispatch for lite topics with a large un-acked backlog; spreading load + over many sub topics is part of the intended usage pattern. + +### 3.5 Invisible-duration renewal + +Under backpressure, messages can sit in the reader's internal queue long enough for their +invisible duration to expire, causing spurious redelivery. An optional +`InvisibleDurationRenewalPolicy` renews queue-resident messages `renewal-ahead-time` before +expiry. Messages are **frozen once emitted** to the record emitter: renewing refreshes the +receipt handle, which would invalidate the handle already travelling downstream. + +## 4. Design decisions and alternatives + +**Why downstream ack at all?** Mainstream Flink connectors with a Pop/lease-like model (GCP +Pub/Sub, Amazon SQS) ack **on the source side**, typically on `notifyCheckpointComplete`. That +only works when the ack decision is available in the source subtask. Here the requirement is to +decide *after* arbitrary downstream processing (possibly across shuffles), so the handle must +travel with the record and the ack must happen downstream. + +Alternatives that were evaluated and rejected: + +| Alternative | Why rejected | +| --- | --- | +| Source-side ack on checkpoint (Pub/Sub / SQS style) | Decision must be co-located with the source subtask; does not meet the requirement. | +| Feeding ack commands back to the reader | Flink's DAG is acyclic: DataStream Iterations are deprecated and checkpoint-incompatible, `OperatorCoordinator` does not connect different operators, and an external command topic would need credentials downstream anyway. | +| Carrying credentials in the record stream | Security red line. Credentials stay in operator configuration, resolved locally per TaskManager. | +| Serializing the SDK protobuf receipt blob | Unnecessary — the server-side handle string self-encodes routing; a handful of strings suffice. | + +Other decisions: + +- **Commit point is user-driven.** The connector does not align acks with checkpoints; the user + acks when processing is complete. This gives at-least-once with user-controlled granularity. +- **One main topic per source** (SDK `bindTopic` limitation). Use multiple sources for multiple + main topics; the routing triple in the handle lets one downstream ack operator serve all of + them. +- **Ack/throttle APIs are DataStream-only.** The SQL/Table path consumes values only. + +## 5. Delivery semantics + +- **At-least-once.** The source never acks; un-acked messages are redelivered after + `invisible-duration` expires. Downstream must be idempotent. +- `invisible-duration` must cover the full downstream processing time of a message (including + shuffles and slow operators such as model inference); use the renewal policy when queueing + time is unpredictable. +- Exactly-once is out of scope for the Pop model: there is no offset the connector could + checkpoint-align, and acks are user-driven by design. + +--- + +## 6. Prerequisites - RocketMQ 5.x cluster with lite topic support enabled on the broker (`enableMultiDispatch`, `enableLmq`, etc. — preset by the official Helm chart). @@ -19,7 +227,7 @@ mqadmin updateSubGroup -n -c -g GID-lite \ `lite.sub.wildcard` is a **consumer group** attribute, not a topic attribute. Without it the wildcard subscription receives nothing. -## Source +## 7. Source ```java RocketMQGrpcSource source = RocketMQGrpcSource.builder() @@ -41,13 +249,11 @@ handle can cross keyBy/shuffle boundaries; credentials never travel with it. ### Downstream ack / throttling Unacked messages are redelivered by the broker after `invisible-duration` expires -(at-least-once — downstream must be idempotent). Three wirings are provided: +(at-least-once — downstream must be idempotent). Two wirings are provided: -1. **Standalone ack operator** — user operators emit `AckCommand{handle, action, delay}`, - `RocketMQAckOperator` executes them. Users never touch the SDK. -2. **Client injection** — extend `RocketMQAckProcessFunction` and call `ack()` / +1. **Client injection** — extend `RocketMQAckProcessFunction` and call `ack()` / `changeInvisibleDuration()` directly. -3. **Policy-based throttling** — implement `MessageThrottlePolicy` +2. **Policy-based throttling** — implement `MessageThrottlePolicy` (`Optional onMessage(value)`: empty = ack, duration = defer via `changeInvisibleDuration`) and run it with `RocketMQThrottleProcessFunction`. Built-in safety valves: maxDeliveryAttempt=16, maxInvisibleDuration=30min (avoids DLQ from @@ -67,7 +273,7 @@ builder.setRenewalPolicyClass("com.example.MyRenewalPolicy") .setRenewalAheadTime(Duration.ofSeconds(5)); ``` -## Sink +## 8. Sink ```java RocketMQGrpcSink sink = RocketMQGrpcSink.builder() @@ -78,7 +284,7 @@ RocketMQGrpcSink sink = RocketMQGrpcSink.builder() .build(); ``` -## Options +## 9. Options ### Client (`rocketmq.client.*`) @@ -98,7 +304,7 @@ RocketMQGrpcSink sink = RocketMQGrpcSink.builder() | `rocketmq.source.main-topic` | String | (none) | Main lite topic to bind, required | | `rocketmq.source.consumer-group` | String | (none) | Consumer group, required | | `rocketmq.source.fetch-concurrency` | Integer | 1 | Concurrent long-polling receive workers | -| `rocketmq.source.await-duration` | Duration | 30s | Long-polling await time | +| `rocketmq.source.await-duration` | Duration | 20s | Long-polling await time | | `rocketmq.source.invisible-duration` | Duration | 60s | Invisible time per receive (min 10s) | | `rocketmq.source.max-message-num` | Integer | 32 | Max messages per receive | | `rocketmq.source.renewal-policy-class` | String | (none) | `InvisibleDurationRenewalPolicy` implementation | @@ -112,8 +318,19 @@ RocketMQGrpcSink sink = RocketMQGrpcSink.builder() | `rocketmq.sink.lite-topic` | String | (none) | Lite topic within the main topic | | `rocketmq.sink.max-attempts` | Integer | 3 | Producer send attempts | -## SQL +## 10. SQL The SQL connector identifier is `rocketmq-grpc` (fat-jar module `flink-sql-connector-rocketmq-grpc`). It exposes the same option keys as above, plus -`rocketmq.source.fetch-concurrency`. +`rocketmq.source.fetch-concurrency`. The SQL/Table path consumes message values only; the +downstream ack / throttling APIs are DataStream-only. + +## 11. Known limitations + +- `LiteSimpleConsumer` is experimental in the SDK (5.2.1+) and offers synchronous APIs only. +- One main lite topic per source; no startup-offset control in wildcard-subscription mode + (`OffsetOption` is limited to exact `subscribeLite`). +- Throttling defers messages per message (extra RPC per defer, reordering, `deliveryAttempt` + growth); rely on the built-in safety valves and idempotent downstream processing. +- The broker throttles dispatch for lite topics with large un-acked backlogs; design the + workload to spread across sub topics. diff --git a/docs/sql-connector.md b/docs/sql-connector.md index 9f0b015..4a65649 100644 --- a/docs/sql-connector.md +++ b/docs/sql-connector.md @@ -1,7 +1,9 @@ # Table / SQL Connector The remoting SQL connector is packaged as the fat-jar `flink-sql-connector-rocketmq` -(identifier `rocketmq`). +(identifier `rocketmq`). The gRPC SQL connector is `flink-sql-connector-rocketmq-grpc` +(identifier `rocketmq-grpc`); both fat-jars relocate their dependencies and can be deployed on +the same classpath. ## Creating tables diff --git a/flink-connector-rocketmq-grpc/pom.xml b/flink-connector-rocketmq-grpc/pom.xml new file mode 100644 index 0000000..512ddce --- /dev/null +++ b/flink-connector-rocketmq-grpc/pom.xml @@ -0,0 +1,153 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connector-rocketmq-parent + 1.0.0-SNAPSHOT + + + flink-connector-rocketmq-grpc + Flink : Connectors : RocketMQ gRPC + jar + + + + + + org.apache.flink + flink-streaming-java + provided + + + + org.apache.flink + flink-connector-base + provided + + + + + + org.apache.flink + flink-table-common + provided + true + + + + org.apache.flink + flink-table-api-java-bridge + provided + true + + + + + + org.apache.rocketmq + rocketmq-client-java + + + + + + org.apache.flink + flink-clients + test + + + + org.apache.flink + flink-test-utils + test + + + + org.apache.flink + flink-connector-test-utils + test + + + + org.apache.flink + flink-table-common + ${flink.version} + test + test-jar + + + + org.apache.flink + flink-core + test + test-jar + + + + org.apache.flink + flink-streaming-java + test + test-jar + + + + org.apache.flink + flink-connector-base + test + test-jar + + + + org.apache.flink + flink-runtime + test + test-jar + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze-deps + + analyze + + verify + + + org.apache.flink:flink-table-api-java-bridge + + + + + + + + diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java new file mode 100644 index 0000000..6948218 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.time.Duration; + +/** + * Shared configuration options for the RocketMQ gRPC connector. These options describe how the + * underlying {@code rocketmq-client-java} SDK connects to the RocketMQ proxy (endpoints, + * credentials, TLS and namespace) and are consumed by both the source and the sink. + * + *

These are the programmatic/SDK-facing options used by the source and sink builders and their + * runtime; every key carries the {@link #CLIENT_CONFIG_PREFIX} prefix. The SQL DDL keys are defined + * separately in {@code RocketMQGrpcConnectorOptions}. + */ +@PublicEvolving +public class RocketMQGrpcOptions { + + private RocketMQGrpcOptions() {} + + /** Prefix for the shared RocketMQ gRPC client options. */ + public static final String CLIENT_CONFIG_PREFIX = "rocketmq.client."; + + public static final ConfigOption ENDPOINTS = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "endpoints") + .stringType() + .noDefaultValue() + .withDescription( + "The access point (proxy) endpoints the gRPC SDK communicates with, " + + "for example '127.0.0.1:8080'."); + + public static final ConfigOption NAMESPACE = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "namespace") + .stringType() + .defaultValue("") + .withDescription("The resource namespace of the RocketMQ instance."); + + public static final ConfigOption ACCESS_KEY = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "access-key") + .stringType() + .noDefaultValue() + .withDescription("The access key used for static session credentials."); + + public static final ConfigOption SECRET_KEY = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "secret-key") + .stringType() + .noDefaultValue() + .withDescription("The secret key used for static session credentials."); + + /** + * The fully qualified class name of a {@link + * org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolver} implementation (public, + * with a public no-argument constructor). + * + *

The resolver is instantiated reflectively on each TaskManager (never serialized) and + * resolves session credentials per proxy endpoint locally, e.g. from environment variables, + * mounted secret files or an external KMS. This keeps plaintext secrets out of the job graph + * and lets one downstream ack operator serve handles from multiple clusters with different + * credentials. When set, it takes precedence over {@link #ACCESS_KEY} / {@link #SECRET_KEY}; + * returning {@code null} for an endpoint means no authentication is required. + */ + public static final ConfigOption CREDENTIALS_RESOLVER_CLASS = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "credentials-resolver-class") + .stringType() + .noDefaultValue() + .withDescription( + "The fully qualified class name of a CredentialsResolver that resolves " + + "session credentials per endpoint locally on the TaskManager " + + "(for example from environment variables, mounted secret files " + + "or an external KMS). When set, it takes precedence over the " + + "static access-key/secret-key options and keeps plaintext " + + "credentials out of the job graph."); + + public static final ConfigOption TLS_ENABLED = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tls-enabled") + .booleanType() + .defaultValue(false) + .withDescription("Whether TLS is enabled for the gRPC transport."); + + public static final ConfigOption REQUEST_TIMEOUT = + ConfigOptions.key(CLIENT_CONFIG_PREFIX + "request-timeout") + .durationType() + .defaultValue(Duration.ofSeconds(3)) + .withDescription("The request timeout for a single gRPC invocation."); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java new file mode 100644 index 0000000..2ab5c4d --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.PublicEvolving; + +import java.util.Objects; + +/** + * The element type produced by the RocketMQ gRPC source when running in downstream-acknowledgement + * mode. It pairs the deserialized record {@code value} with the credential-free {@link + * RocketMQReceiptHandle} needed to acknowledge (or re-schedule) the originating message from any + * downstream operator. + * + * @param the deserialized record type. + */ +@PublicEvolving +public final class AckableMessage { + + private final T value; + private final RocketMQReceiptHandle handle; + + public AckableMessage(T value, RocketMQReceiptHandle handle) { + this.value = value; + this.handle = Objects.requireNonNull(handle, "handle should not be null"); + } + + /** The deserialized record value. */ + public T getValue() { + return value; + } + + /** The receipt handle used to acknowledge or re-schedule the originating message. */ + public RocketMQReceiptHandle getHandle() { + return handle; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AckableMessage that = (AckableMessage) o; + return Objects.equals(value, that.value) && handle.equals(that.handle); + } + + @Override + public int hashCode() { + return Objects.hash(value, handle); + } + + @Override + public String toString() { + return "AckableMessage{value=" + value + ", handle=" + handle + '}'; + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java new file mode 100644 index 0000000..9e81050 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; +import java.util.Objects; + +/** + * The {@link TypeInformation} for {@link AckableMessage}. It carries the value {@link + * TypeInformation} so that Flink can derive a serializer for {@code T} while the receipt handle is + * serialized by the dedicated {@link RocketMQReceiptHandle.Serializer}. + * + * @param the deserialized record type. + */ +@PublicEvolving +public final class AckableMessageTypeInfo extends TypeInformation> { + + private static final long serialVersionUID = 1L; + + private final TypeInformation valueTypeInfo; + + public AckableMessageTypeInfo(TypeInformation valueTypeInfo) { + this.valueTypeInfo = Objects.requireNonNull(valueTypeInfo, "valueTypeInfo"); + } + + @Override + public boolean isBasicType() { + return false; + } + + @Override + public boolean isTupleType() { + return false; + } + + @Override + public int getArity() { + return 2; + } + + @Override + public int getTotalFields() { + return valueTypeInfo.getTotalFields() + 1; + } + + @Override + @SuppressWarnings("unchecked") + public Class> getTypeClass() { + return (Class>) (Class) AckableMessage.class; + } + + @Override + public boolean isKeyType() { + return false; + } + + @Override + public TypeSerializer> createSerializer(ExecutionConfig config) { + return new Serializer<>(valueTypeInfo.createSerializer(config)); + } + + @Override + public String toString() { + return "AckableMessageTypeInfo<" + valueTypeInfo + '>'; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + AckableMessageTypeInfo that = (AckableMessageTypeInfo) obj; + return valueTypeInfo.equals(that.valueTypeInfo); + } + + @Override + public int hashCode() { + return valueTypeInfo.hashCode(); + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof AckableMessageTypeInfo; + } + + /** + * A {@link TypeSerializer} for {@link AckableMessage} that composes a caller-supplied value + * serializer with the {@link RocketMQReceiptHandle.Serializer}. A {@code null} value is + * supported via a boolean flag so that the value serializer never has to encode {@code null}. + * + * @param the deserialized record type. + */ + @Internal + public static final class Serializer extends TypeSerializer> { + + private static final long serialVersionUID = 1L; + + private final TypeSerializer valueSerializer; + private final TypeSerializer handleSerializer; + + public Serializer(TypeSerializer valueSerializer) { + this(valueSerializer, RocketMQReceiptHandle.Serializer.INSTANCE); + } + + Serializer( + TypeSerializer valueSerializer, + TypeSerializer handleSerializer) { + this.valueSerializer = valueSerializer; + this.handleSerializer = handleSerializer; + } + + @Override + public boolean isImmutableType() { + return valueSerializer.isImmutableType(); + } + + @Override + public TypeSerializer> duplicate() { + final TypeSerializer duplicatedValue = valueSerializer.duplicate(); + if (duplicatedValue == valueSerializer) { + return this; + } + return new Serializer<>(duplicatedValue, handleSerializer.duplicate()); + } + + @Override + public AckableMessage createInstance() { + return null; + } + + @Override + public AckableMessage copy(AckableMessage from) { + final T value = from.getValue(); + final T copiedValue = value == null ? null : valueSerializer.copy(value); + return new AckableMessage<>(copiedValue, from.getHandle()); + } + + @Override + public AckableMessage copy(AckableMessage from, AckableMessage reuse) { + return copy(from); + } + + @Override + public int getLength() { + return -1; + } + + @Override + public void serialize(AckableMessage record, DataOutputView target) throws IOException { + final T value = record.getValue(); + if (value == null) { + target.writeBoolean(false); + } else { + target.writeBoolean(true); + valueSerializer.serialize(value, target); + } + handleSerializer.serialize(record.getHandle(), target); + } + + @Override + public AckableMessage deserialize(DataInputView source) throws IOException { + final T value = source.readBoolean() ? valueSerializer.deserialize(source) : null; + final RocketMQReceiptHandle handle = handleSerializer.deserialize(source); + return new AckableMessage<>(value, handle); + } + + @Override + public AckableMessage deserialize(AckableMessage reuse, DataInputView source) + throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + serialize(deserialize(source), target); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Serializer that = (Serializer) obj; + return valueSerializer.equals(that.valueSerializer) + && handleSerializer.equals(that.handleSerializer); + } + + @Override + public int hashCode() { + return Objects.hash(valueSerializer, handleSerializer); + } + + @Override + public TypeSerializerSnapshot> snapshotConfiguration() { + return new SerializerSnapshot<>(this); + } + } + + /** Serializer snapshot that tracks the compatibility of the nested value serializer. */ + public static final class SerializerSnapshot + extends CompositeTypeSerializerSnapshot, Serializer> { + + private static final int CURRENT_VERSION = 1; + + public SerializerSnapshot() {} + + public SerializerSnapshot(Serializer serializer) { + super(serializer); + } + + @Override + protected int getCurrentOuterSnapshotVersion() { + return CURRENT_VERSION; + } + + @Override + protected TypeSerializer[] getNestedSerializers(Serializer outerSerializer) { + return new TypeSerializer[] { + outerSerializer.valueSerializer, outerSerializer.handleSerializer + }; + } + + @Override + @SuppressWarnings("unchecked") + protected Serializer createOuterSerializerWithNestedSerializers( + TypeSerializer[] nestedSerializers) { + return new Serializer<>( + (TypeSerializer) nestedSerializers[0], + (TypeSerializer) nestedSerializers[1]); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java new file mode 100644 index 0000000..1d45c6e --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.PublicEvolving; + +import java.io.Serializable; +import java.time.Duration; +import java.util.Optional; + +/** + * A per-message throttling policy used by {@link RocketMQThrottleProcessFunction}. For each value + * it decides whether the message should be acknowledged normally or deferred (throttled) by + * extending its invisible duration. + * + * @param the value type carried by the {@code AckableMessage}. + */ +@PublicEvolving +@FunctionalInterface +public interface MessageThrottlePolicy extends Serializable { + + /** + * Decide how to handle the given value. + * + * @param value the deserialized message value. + * @return an empty {@link Optional} to acknowledge the message normally, or a positive {@link + * Duration} to defer its re-delivery by that amount (throttle). + */ + Optional onMessage(T value); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java new file mode 100644 index 0000000..29330af --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.functions.OpenContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.Counter; +import org.apache.flink.streaming.api.functions.ProcessFunction; + +import java.time.Duration; +import java.util.Objects; + +/** + * An abstract {@link ProcessFunction} that gives subclasses a ready-to-use {@link + * RocketMQLiteAckClient} so they can acknowledge or re-schedule RocketMQ Pop messages directly from + * their own business logic via {@link #ack} and {@link #changeInvisibleDuration}. + * + *

The client is shared per TaskManager through {@link RocketMQLiteAckClient#acquire}: it is + * acquired in {@link #open(OpenContext)} and released in {@link #close()}. The RocketMQ client + * options (endpoints, namespace, credentials, TLS, timeout) are provided through the {@link + * Configuration} passed to the constructor and never travel in the data stream. + * + * @param the input record type, typically {@code AckableMessage}. + * @param the output record type. + */ +@PublicEvolving +public abstract class RocketMQAckProcessFunction extends ProcessFunction { + + private static final long serialVersionUID = 1L; + + private final Configuration configuration; + + private transient RocketMQLiteAckClient ackClient; + private transient boolean acquired; + private transient Counter numAcksSucceeded; + private transient Counter numAcksFailed; + private transient Counter numInvisibleDurationChangesSucceeded; + private transient Counter numInvisibleDurationChangesFailed; + + protected RocketMQAckProcessFunction(Configuration configuration) { + this.configuration = + new Configuration( + Objects.requireNonNull(configuration, "configuration should not be null")); + } + + @Override + public void open(OpenContext openContext) throws Exception { + super.open(openContext); + this.ackClient = RocketMQLiteAckClient.acquire(configuration); + this.acquired = true; + this.numAcksSucceeded = getRuntimeContext().getMetricGroup().counter("numAcksSucceeded"); + this.numAcksFailed = getRuntimeContext().getMetricGroup().counter("numAcksFailed"); + this.numInvisibleDurationChangesSucceeded = + getRuntimeContext() + .getMetricGroup() + .counter("numInvisibleDurationChangesSucceeded"); + this.numInvisibleDurationChangesFailed = + getRuntimeContext().getMetricGroup().counter("numInvisibleDurationChangesFailed"); + } + + @Override + public void close() throws Exception { + // Only release when open() actually acquired the client; Flink calls close() even after a + // failed open(), and a stray release would decrement another operator's reference. + if (acquired) { + RocketMQLiteAckClient.release(configuration); + acquired = false; + } + this.ackClient = null; + super.close(); + } + + /** Acknowledge the message described by the handle so it is not re-delivered. */ + protected void ack(RocketMQReceiptHandle handle) { + try { + ackClient.ack(handle); + numAcksSucceeded.inc(); + } catch (RuntimeException e) { + numAcksFailed.inc(); + throw e; + } + } + + /** + * Change the invisible duration of the message described by the handle, deferring its next + * re-delivery by {@code invisibleDuration}. + */ + protected void changeInvisibleDuration( + RocketMQReceiptHandle handle, Duration invisibleDuration) { + try { + ackClient.changeInvisibleDuration(handle, invisibleDuration); + numInvisibleDurationChangesSucceeded.inc(); + } catch (RuntimeException e) { + numInvisibleDurationChangesFailed.inc(); + throw e; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java new file mode 100644 index 0000000..f6754c4 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider; +import org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolver; +import org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolvers; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.StringUtils; + +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.ClientServiceProvider; +import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A credential-free acknowledgement client that a downstream operator uses to acknowledge or + * re-schedule RocketMQ Pop messages described by a {@link RocketMQReceiptHandle}. The credentials + * used to talk to the RocketMQ proxy are configured on the operator that owns the client and are + * never carried in the data stream. + * + *

The client keeps a lazily populated pool of same-group {@link LiteSimpleConsumer}s, one per + * {@link ConsumerKey routing triple} carried by the incoming handles, and issues the {@code ack} / + * {@code changeInvisibleDuration} RPC through the consumer that matches the handle. The pooled + * consumer for a handle is built with a {@link ClientConfiguration} whose endpoints and namespace + * come from the handle (so the ack RPC is routed to the right proxy and resource) and whose + * credentials/TLS/timeout come from the operator {@link Configuration}. This is required because + * the SDK stamps the ack request with the consumer's namespace, so a pooled consumer must + * share the handle's namespace. + * + *

Instances are shared per TaskManager JVM: {@link #acquire(Configuration)} reference-counts a + * client per distinct client configuration and {@link #release(Configuration)} closes it once no + * operator instance references it anymore, so callers must not close a client directly. + */ +@Internal +public final class RocketMQLiteAckClient { + + private static final Logger LOG = LoggerFactory.getLogger(RocketMQLiteAckClient.class); + + /** + * The await duration is only consulted by {@code receive}, which this ack-only client never + * calls; a small non-null value is enough to satisfy the builder. + */ + private static final Duration ACK_ONLY_AWAIT_DURATION = Duration.ofSeconds(5); + + private static final int MAX_RPC_ATTEMPTS = 3; + private static final long RPC_RETRY_BACKOFF_INITIAL_MS = 100L; + + /** {@code apache.rocketmq.v2.Code.INVALID_RECEIPT_HANDLE}, embedded in exception messages. */ + private static final String INVALID_RECEIPT_HANDLE_MARKER = "response-code=40013"; + + private static final Map CLIENTS = new HashMap<>(); + + private final Configuration configuration; + @Nullable private final CredentialsResolver credentialsResolver; + private final Map consumers = new ConcurrentHashMap<>(); + + private volatile boolean closed = false; + + private RocketMQLiteAckClient(Configuration configuration) { + this.configuration = + Objects.requireNonNull(configuration, "configuration should not be null"); + this.credentialsResolver = CredentialsResolvers.createFromConfiguration(configuration); + } + + /** + * Acquire the shared ack client for the given client configuration, creating it if necessary + * and incrementing its reference count. + * + * @param configuration the operator configuration carrying the RocketMQ client options. + * @return the shared ack client; callers must not close it directly. + */ + public static synchronized RocketMQLiteAckClient acquire(Configuration configuration) { + Objects.requireNonNull(configuration, "configuration should not be null"); + final String key = keyOf(configuration); + RefCounted ref = CLIENTS.get(key); + if (ref == null) { + ref = new RefCounted(new RocketMQLiteAckClient(configuration)); + CLIENTS.put(key, ref); + } + ref.count++; + return ref.client; + } + + /** + * Release a previously {@link #acquire(Configuration) acquired} client, decrementing its + * reference count and closing it once no operator instance references it anymore. + * + *

The client is closed outside the registry lock so that a slow close (it issues network + * calls) cannot block concurrent {@code acquire}/{@code release} calls of other operators. + * + * @param configuration the same configuration that was passed to {@link + * #acquire(Configuration)}. + */ + public static void release(Configuration configuration) { + Objects.requireNonNull(configuration, "configuration should not be null"); + final String key = keyOf(configuration); + RocketMQLiteAckClient toClose = null; + synchronized (RocketMQLiteAckClient.class) { + final RefCounted ref = CLIENTS.get(key); + if (ref == null) { + return; + } + ref.count--; + if (ref.count <= 0) { + CLIENTS.remove(key); + toClose = ref.client; + } + } + if (toClose != null) { + toClose.close(); + } + } + + @VisibleForTesting + static synchronized int getReferenceCount(Configuration configuration) { + final RefCounted ref = CLIENTS.get(keyOf(configuration)); + return ref == null ? 0 : ref.count; + } + + /** + * Build a stable pool key from the credential-bearing client options. Distinct credentials, + * endpoints, namespaces, TLS settings or timeouts each get their own shared client. The secret + * key is hashed so the plaintext secret is never held in the registry key. + */ + private static String keyOf(Configuration configuration) { + return configuration.get(RocketMQGrpcOptions.ENDPOINTS) + + '|' + + configuration.get(RocketMQGrpcOptions.NAMESPACE) + + '|' + + configuration.get(RocketMQGrpcOptions.ACCESS_KEY) + + '|' + + sha256(configuration.get(RocketMQGrpcOptions.SECRET_KEY)) + + '|' + + configuration.get(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS) + + '|' + + configuration.get(RocketMQGrpcOptions.TLS_ENABLED) + + '|' + + configuration.get(RocketMQGrpcOptions.REQUEST_TIMEOUT); + } + + private static String sha256(@Nullable String value) { + if (value == null) { + return ""; + } + try { + final MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return StringUtils.byteToHexString( + digest.digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new FlinkRuntimeException("SHA-256 is not available.", e); + } + } + + /** + * Acknowledge the message described by the handle, removing it from the Pop invisible set so it + * is not re-delivered. + * + * @param handle the credential-free receipt handle of the message to acknowledge. + */ + public void ack(RocketMQReceiptHandle handle) { + Objects.requireNonNull(handle, "handle should not be null"); + final LiteSimpleConsumer consumer = getOrCreateConsumer(handle); + invokeWithRetry( + "acknowledge", + handle, + () -> consumer.ack(RocketMQReceiptHandleCodec.toAckable(handle))); + } + + /** + * Change the invisible duration of the message described by the handle. Extending the invisible + * duration defers the next possible re-delivery of the message, which downstream operators use + * to throttle a (sub) topic without dropping the message. + * + * @param handle the credential-free receipt handle of the message. + * @param invisibleDuration the new invisible duration counted from now. + */ + public void changeInvisibleDuration(RocketMQReceiptHandle handle, Duration invisibleDuration) { + Objects.requireNonNull(handle, "handle should not be null"); + Objects.requireNonNull(invisibleDuration, "invisibleDuration should not be null"); + final LiteSimpleConsumer consumer = getOrCreateConsumer(handle); + invokeWithRetry( + "change the invisible duration of", + handle, + () -> + consumer.changeInvisibleDuration( + RocketMQReceiptHandleCodec.toAckable(handle), invisibleDuration)); + } + + /** + * Run the RPC with a bounded exponential-backoff retry. An expired/invalid receipt handle is + * tolerated: the message will simply be redelivered by the broker (at-least-once), so failing + * the job for it would be worse than the duplicate. + */ + private void invokeWithRetry(String action, RocketMQReceiptHandle handle, AckRpc rpc) { + long backoffMs = RPC_RETRY_BACKOFF_INITIAL_MS; + for (int attempt = 1; ; attempt++) { + try { + rpc.run(); + return; + } catch (ClientException e) { + if (isInvalidReceiptHandle(e)) { + LOG.warn( + "The receipt handle of message {} is invalid or expired; the message " + + "will be redelivered by the broker.", + handle.getMessageId(), + e); + return; + } + if (attempt >= MAX_RPC_ATTEMPTS) { + throw new FlinkRuntimeException( + "Failed to " + + action + + " RocketMQ message " + + handle.getMessageId() + + " after " + + attempt + + " attempts", + e); + } + LOG.warn( + "Failed to {} RocketMQ message {} (attempt {}/{}), retrying in {} ms", + action, + handle.getMessageId(), + attempt, + MAX_RPC_ATTEMPTS, + backoffMs, + e); + try { + Thread.sleep(backoffMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + final FlinkRuntimeException interrupted = + new FlinkRuntimeException( + "Interrupted while retrying to " + + action + + " RocketMQ message " + + handle.getMessageId(), + ie); + interrupted.addSuppressed(e); + throw interrupted; + } + backoffMs *= 2; + } + } + } + + private static boolean isInvalidReceiptHandle(ClientException e) { + for (Throwable t = e; t != null; t = t.getCause()) { + final String message = t.getMessage(); + if (message != null && message.contains(INVALID_RECEIPT_HANDLE_MARKER)) { + return true; + } + } + return false; + } + + /** An ack-related RPC over a pooled consumer. */ + @FunctionalInterface + private interface AckRpc { + void run() throws ClientException; + } + + /** Close all pooled consumers. Invoked by {@link #release(Configuration)} on last release. */ + private void close() { + closed = true; + for (Map.Entry entry : consumers.entrySet()) { + try { + entry.getValue().close(); + } catch (Exception e) { + LOG.warn("Failed to close the ack consumer for {}.", entry.getKey(), e); + } + } + consumers.clear(); + } + + private LiteSimpleConsumer getOrCreateConsumer(RocketMQReceiptHandle handle) { + if (closed) { + throw new IllegalStateException("The ack client has been closed."); + } + final ConsumerKey key = + new ConsumerKey( + handle.getEndpoint(), handle.getNamespace(), handle.getConsumerGroup()); + return consumers.computeIfAbsent(key, k -> createConsumer(k, handle.getTopic())); + } + + private LiteSimpleConsumer createConsumer(ConsumerKey key, String bindTopic) { + try { + final ClientServiceProvider provider = ClientServiceProvider.loadService(); + return provider.newLiteSimpleConsumerBuilder() + .setClientConfiguration( + ClientConfigurationProvider.getClientConfiguration( + configuration, + key.endpoint, + key.namespace, + credentialsResolver)) + .setConsumerGroup(key.consumerGroup) + .setAwaitDuration(ACK_ONLY_AWAIT_DURATION) + .bindTopic(bindTopic) + .build(); + } catch (ClientException e) { + throw new FlinkRuntimeException("Failed to create the ack consumer for " + key, e); + } + } + + private static final class RefCounted { + private final RocketMQLiteAckClient client; + private int count; + + private RefCounted(RocketMQLiteAckClient client) { + this.client = client; + } + } + + /** + * The identity of a pooled ack consumer: a message can only be acknowledged by a consumer in + * the same group, pointing at the same proxy endpoint(s) and resource namespace. + */ + private static final class ConsumerKey { + + private final String endpoint; + private final String namespace; + private final String consumerGroup; + + private ConsumerKey(String endpoint, String namespace, String consumerGroup) { + this.endpoint = endpoint; + this.namespace = namespace; + this.consumerGroup = consumerGroup; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ConsumerKey that = (ConsumerKey) o; + return endpoint.equals(that.endpoint) + && namespace.equals(that.namespace) + && consumerGroup.equals(that.consumerGroup); + } + + @Override + public int hashCode() { + return Objects.hash(endpoint, namespace, consumerGroup); + } + + @Override + public String toString() { + return "ConsumerKey{" + + "endpoint='" + + endpoint + + '\'' + + ", namespace='" + + namespace + + '\'' + + ", consumerGroup='" + + consumerGroup + + '\'' + + '}'; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java new file mode 100644 index 0000000..eb8c43c --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.typeutils.SimpleTypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Objects; + +/** + * A self-contained, serializable descriptor of a RocketMQ Pop message that carries everything a + * downstream operator needs to acknowledge (or extend the invisible duration of) the message + * without holding on to the SDK message object or any credentials. + * + *

The RocketMQ 5.x Pop model lets any consumer in the same group acknowledge a message given its + * receipt handle and routing information; the message does not have to be acked by the consumer + * that originally received it. This class captures exactly that routing information so that the + * acknowledgement decision can be moved to any downstream operator: + * + *

    + *
  • {@code endpoint}, {@code namespace}, {@code consumerGroup} — the routing triple that + * selects which pooled consumer must issue the ack RPC. Multiple sources (possibly pointing + * at different clusters/namespaces/groups) can coexist because each message carries its own + * triple. + *
  • {@code topic}, {@code liteTopic}, {@code messageId}, {@code receiptHandle}, {@code + * deliveryAttempt} — the per-message fields required to rebuild the SDK message view used by + * the ack/changeInvisibleDuration calls. + *
+ * + *

This type intentionally contains no credentials, no protobuf blob and no message body, + * so it is safe to ship across the Flink data stream. + */ +@PublicEvolving +public final class RocketMQReceiptHandle implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String endpoint; + private final String namespace; + private final String consumerGroup; + private final String topic; + /** The lite (sub) topic, or {@code null} when the message was not received via a lite topic. */ + private final String liteTopic; + + private final String messageId; + private final String receiptHandle; + private final int deliveryAttempt; + + public RocketMQReceiptHandle( + String endpoint, + String namespace, + String consumerGroup, + String topic, + String liteTopic, + String messageId, + String receiptHandle, + int deliveryAttempt) { + this.endpoint = Objects.requireNonNull(endpoint, "endpoint should not be null"); + this.namespace = Objects.requireNonNull(namespace, "namespace should not be null"); + this.consumerGroup = + Objects.requireNonNull(consumerGroup, "consumerGroup should not be null"); + this.topic = Objects.requireNonNull(topic, "topic should not be null"); + this.liteTopic = liteTopic; + this.messageId = Objects.requireNonNull(messageId, "messageId should not be null"); + this.receiptHandle = + Objects.requireNonNull(receiptHandle, "receiptHandle should not be null"); + this.deliveryAttempt = deliveryAttempt; + } + + /** + * The proxy endpoint(s) the message came from, as a {@code host:port[;host:port...]} string + * that can be parsed back into the SDK endpoints. + */ + public String getEndpoint() { + return endpoint; + } + + /** The resource namespace of the RocketMQ instance ({@code ""} when none is configured). */ + public String getNamespace() { + return namespace; + } + + /** + * The consumer group the source used; the ack must be issued by a consumer of the same group. + */ + public String getConsumerGroup() { + return consumerGroup; + } + + /** The (physical) topic the message belongs to. */ + public String getTopic() { + return topic; + } + + /** The lite (sub) topic, or {@code null} when the message was not received via a lite topic. */ + public String getLiteTopic() { + return liteTopic; + } + + /** The unique message id. */ + public String getMessageId() { + return messageId; + } + + /** The Pop receipt handle used to acknowledge or re-schedule the message. */ + public String getReceiptHandle() { + return receiptHandle; + } + + /** The number of times the message has been delivered. */ + public int getDeliveryAttempt() { + return deliveryAttempt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RocketMQReceiptHandle that = (RocketMQReceiptHandle) o; + return deliveryAttempt == that.deliveryAttempt + && endpoint.equals(that.endpoint) + && namespace.equals(that.namespace) + && consumerGroup.equals(that.consumerGroup) + && topic.equals(that.topic) + && Objects.equals(liteTopic, that.liteTopic) + && messageId.equals(that.messageId) + && receiptHandle.equals(that.receiptHandle); + } + + @Override + public int hashCode() { + return Objects.hash( + endpoint, + namespace, + consumerGroup, + topic, + liteTopic, + messageId, + receiptHandle, + deliveryAttempt); + } + + @Override + public String toString() { + return "RocketMQReceiptHandle{" + + "endpoint='" + + endpoint + + '\'' + + ", namespace='" + + namespace + + '\'' + + ", consumerGroup='" + + consumerGroup + + '\'' + + ", topic='" + + topic + + '\'' + + ", liteTopic='" + + liteTopic + + '\'' + + ", messageId='" + + messageId + + '\'' + + ", deliveryAttempt=" + + deliveryAttempt + + '}'; + } + + /** + * A stateless {@link org.apache.flink.api.common.typeutils.TypeSerializer} for {@link + * RocketMQReceiptHandle}. It writes the eight self-contained fields directly; the nullable + * {@code liteTopic} is guarded by a boolean flag. + */ + @Internal + public static final class Serializer extends TypeSerializerSingleton { + + private static final long serialVersionUID = 1L; + + public static final Serializer INSTANCE = new Serializer(); + + @Override + public boolean isImmutableType() { + return true; + } + + @Override + public RocketMQReceiptHandle createInstance() { + return null; + } + + @Override + public RocketMQReceiptHandle copy(RocketMQReceiptHandle from) { + // RocketMQReceiptHandle is immutable. + return from; + } + + @Override + public RocketMQReceiptHandle copy(RocketMQReceiptHandle from, RocketMQReceiptHandle reuse) { + return from; + } + + @Override + public int getLength() { + return -1; + } + + @Override + public void serialize(RocketMQReceiptHandle record, DataOutputView target) + throws IOException { + target.writeUTF(record.getEndpoint()); + target.writeUTF(record.getNamespace()); + target.writeUTF(record.getConsumerGroup()); + target.writeUTF(record.getTopic()); + final String liteTopic = record.getLiteTopic(); + if (liteTopic == null) { + target.writeBoolean(false); + } else { + target.writeBoolean(true); + target.writeUTF(liteTopic); + } + target.writeUTF(record.getMessageId()); + target.writeUTF(record.getReceiptHandle()); + target.writeInt(record.getDeliveryAttempt()); + } + + @Override + public RocketMQReceiptHandle deserialize(DataInputView source) throws IOException { + final String endpoint = source.readUTF(); + final String namespace = source.readUTF(); + final String consumerGroup = source.readUTF(); + final String topic = source.readUTF(); + final String liteTopic = source.readBoolean() ? source.readUTF() : null; + final String messageId = source.readUTF(); + final String receiptHandle = source.readUTF(); + final int deliveryAttempt = source.readInt(); + return new RocketMQReceiptHandle( + endpoint, + namespace, + consumerGroup, + topic, + liteTopic, + messageId, + receiptHandle, + deliveryAttempt); + } + + @Override + public RocketMQReceiptHandle deserialize(RocketMQReceiptHandle reuse, DataInputView source) + throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + serialize(deserialize(source), target); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new SerializerSnapshot(); + } + + /** Serializer snapshot for {@link Serializer}. */ + public static final class SerializerSnapshot + extends SimpleTypeSerializerSnapshot { + + public SerializerSnapshot() { + super(() -> INSTANCE); + } + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java new file mode 100644 index 0000000..8f15bf2 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.Internal; + +import org.apache.rocketmq.client.apis.message.MessageId; +import org.apache.rocketmq.client.apis.message.MessageView; +import org.apache.rocketmq.client.java.message.MessageIdCodec; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.route.Address; +import org.apache.rocketmq.client.java.route.Endpoints; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; + +import java.util.Collections; + +/** + * The single adapter that bridges the connector's credential-free {@link RocketMQReceiptHandle} and + * the {@code rocketmq-client-java} internal implementation classes. It is deliberately the + * only class in the connector that depends on the SDK {@code + * org.apache.rocketmq.client.java.*} internals, so that the (experimental) coupling is confined to + * one place. + * + *

{@link #extract} runs on the source side and pulls the routing/ack fields out of an SDK + * message view. {@link #toAckable} runs on the downstream ack side and rebuilds the minimal SDK + * message view that {@code LiteSimpleConsumer#ack} / {@code changeInvisibleDuration} require (both + * perform an {@code instanceof MessageViewImpl} downcast internally and only read the topic, + * message id, receipt handle, lite topic and endpoints). + */ +@Internal +public final class RocketMQReceiptHandleCodec { + + private RocketMQReceiptHandleCodec() {} + + /** + * Extract a credential-free receipt handle from an SDK message view. + * + * @param view the SDK message view returned by the consumer. + * @param namespace the resource namespace configured on the source. + * @param consumerGroup the consumer group configured on the source. + * @return a self-contained {@link RocketMQReceiptHandle}. + * @throws IllegalStateException if the view is not the expected internal implementation type. + */ + public static RocketMQReceiptHandle extract( + MessageView view, String namespace, String consumerGroup) { + if (!(view instanceof MessageViewImpl)) { + throw new IllegalStateException( + "Expected a " + + MessageViewImpl.class.getName() + + " but got " + + (view == null ? "null" : view.getClass().getName()) + + "; the RocketMQ SDK internal message type has changed."); + } + final MessageViewImpl impl = (MessageViewImpl) view; + final Endpoints endpoints = impl.getEndpoints(); + if (endpoints == null) { + throw new IllegalStateException( + "The message view does not carry endpoints, cannot build a receipt handle for " + + impl.getMessageId()); + } + return new RocketMQReceiptHandle( + toEndpointString(endpoints), + namespace, + consumerGroup, + impl.getTopic(), + impl.getLiteTopic().orElse(null), + impl.getMessageId().toString(), + impl.getReceiptHandle(), + impl.getDeliveryAttempt()); + } + + /** + * Render the endpoints as a {@code host:port[;host:port...]} string that round-trips through + * {@code new Endpoints(String)}. The {@code Endpoints#getFacade()} form is not used because it + * carries a scheme prefix ({@code ipv4:}/{@code ipv6:}/{@code dns:}) that the string + * constructor cannot parse back. + */ + private static String toEndpointString(Endpoints endpoints) { + final StringBuilder builder = new StringBuilder(); + for (Address address : endpoints.getAddresses()) { + if (builder.length() > 0) { + builder.append(';'); + } + builder.append(address.getAddress()); + } + return builder.toString(); + } + + /** + * Rebuild the minimal SDK message view required to acknowledge or re-schedule the message + * described by the given handle. The reconstructed view carries an empty body and no user + * properties; only the fields consulted by the ack / changeInvisibleDuration RPC are populated. + * + * @param handle the credential-free receipt handle. + * @return an SDK {@link MessageView} suitable for {@code ack} / {@code + * changeInvisibleDuration}. + */ + public static MessageView toAckable(RocketMQReceiptHandle handle) { + final MessageId messageId = MessageIdCodec.getInstance().decode(handle.getMessageId()); + final MessageQueueImpl messageQueue = buildMessageQueue(handle); + return new MessageViewImpl( + messageId, + handle.getTopic(), + new byte[0], + null, + null, + handle.getLiteTopic(), + null, + null, + Collections.emptyList(), + Collections.emptyMap(), + "", + 0L, + handle.getDeliveryAttempt(), + messageQueue, + handle.getReceiptHandle(), + 0L, + false, + null); + } + + private static MessageQueueImpl buildMessageQueue(RocketMQReceiptHandle handle) { + final apache.rocketmq.v2.Endpoints endpoints = + new Endpoints(handle.getEndpoint()).toProtobuf(); + final apache.rocketmq.v2.Broker broker = + apache.rocketmq.v2.Broker.newBuilder() + .setName("") + .setId(0) + .setEndpoints(endpoints) + .build(); + final apache.rocketmq.v2.Resource topic = + apache.rocketmq.v2.Resource.newBuilder() + .setResourceNamespace(handle.getNamespace()) + .setName(handle.getTopic()) + .build(); + // A concrete permission is required: MessageQueueImpl rejects PERMISSION_UNSPECIFIED. + final apache.rocketmq.v2.MessageQueue messageQueue = + apache.rocketmq.v2.MessageQueue.newBuilder() + .setTopic(topic) + .setId(0) + .setPermission(apache.rocketmq.v2.Permission.READ) + .setBroker(broker) + .build(); + return new MessageQueueImpl(messageQueue); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java new file mode 100644 index 0000000..0f0891b --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.ProcessFunction; +import org.apache.flink.util.Collector; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * A convenience one-stop throttle operator. For every {@link AckableMessage} it consults a {@link + * MessageThrottlePolicy}: when the policy returns a delay it {@linkplain + * RocketMQAckProcessFunction#changeInvisibleDuration changes the invisible duration} (deferring the + * message to throttle its (sub) topic); otherwise it {@linkplain RocketMQAckProcessFunction#ack + * acknowledges} the message. The underlying value is always forwarded downstream unchanged. + * + *

Two safeguards prevent a persistently throttled message from being deferred forever and + * eventually pushed into the dead-letter queue once its delivery attempts are exhausted: + * + *

    + *
  • {@code maxDeliveryAttempt} — once the message's delivery attempt reaches this bound the + * message is acknowledged instead of deferred again. + *
  • {@code maxInvisibleDuration} — every requested delay is capped to this value. + *
+ * + * @param the value type carried by the {@code AckableMessage}. + */ +@PublicEvolving +public class RocketMQThrottleProcessFunction + extends RocketMQAckProcessFunction, T> { + + private static final long serialVersionUID = 1L; + + /** Default cap on the number of times a message may be deferred before it is acknowledged. */ + public static final int DEFAULT_MAX_DELIVERY_ATTEMPT = 16; + + /** Default cap on a single requested invisible duration. */ + public static final Duration DEFAULT_MAX_INVISIBLE_DURATION = Duration.ofMinutes(30); + + private final MessageThrottlePolicy policy; + private final int maxDeliveryAttempt; + private final Duration maxInvisibleDuration; + + public RocketMQThrottleProcessFunction( + Configuration configuration, MessageThrottlePolicy policy) { + this(configuration, policy, DEFAULT_MAX_DELIVERY_ATTEMPT, DEFAULT_MAX_INVISIBLE_DURATION); + } + + public RocketMQThrottleProcessFunction( + Configuration configuration, + MessageThrottlePolicy policy, + int maxDeliveryAttempt, + Duration maxInvisibleDuration) { + super(configuration); + this.policy = Objects.requireNonNull(policy, "policy should not be null"); + this.maxDeliveryAttempt = maxDeliveryAttempt; + this.maxInvisibleDuration = + Objects.requireNonNull( + maxInvisibleDuration, "maxInvisibleDuration should not be null"); + } + + @Override + public void processElement( + AckableMessage message, + ProcessFunction, T>.Context context, + Collector out) { + final RocketMQReceiptHandle handle = message.getHandle(); + final Optional delay = policy.onMessage(message.getValue()); + + if (delay.isPresent() && handle.getDeliveryAttempt() < maxDeliveryAttempt) { + changeInvisibleDuration(handle, capDelay(delay.get())); + } else { + // Either the policy accepted the message, or the defer safeguard tripped: ack it so it + // is not eventually pushed into the dead-letter queue. + ack(handle); + } + + out.collect(message.getValue()); + } + + private Duration capDelay(Duration requested) { + return requested.compareTo(maxInvisibleDuration) > 0 ? maxInvisibleDuration : requested; + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java new file mode 100644 index 0000000..d0f9bb6 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.common; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.util.StringUtils; + +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientConfigurationBuilder; + +import javax.annotation.Nullable; + +/** + * Builds an SDK {@link ClientConfiguration} from a Flink {@link Configuration}. The produced + * configuration is shared by the source ({@code SimpleConsumer}) and the sink ({@code Producer}). + */ +@Internal +public class ClientConfigurationProvider { + + private ClientConfigurationProvider() {} + + /** Build a {@link ClientConfiguration} from the given Flink configuration. */ + public static ClientConfiguration getClientConfiguration(Configuration configuration) { + final String endpoints = configuration.get(RocketMQGrpcOptions.ENDPOINTS); + if (StringUtils.isNullOrWhitespaceOnly(endpoints)) { + throw new IllegalArgumentException( + "The endpoints of the RocketMQ gRPC connector must be configured."); + } + return getClientConfiguration( + configuration, + endpoints, + configuration.get(RocketMQGrpcOptions.NAMESPACE), + CredentialsResolvers.createFromConfiguration(configuration)); + } + + /** + * Build a {@link ClientConfiguration} whose endpoints and namespace are supplied by the caller + * (e.g. taken from a receipt handle), while the credentials, TLS and request timeout come from + * the Flink configuration. + */ + public static ClientConfiguration getClientConfiguration( + Configuration configuration, + String endpoints, + @Nullable String namespace, + @Nullable CredentialsResolver credentialsResolver) { + final ClientConfigurationBuilder builder = + ClientConfiguration.newBuilder() + .setEndpoints(endpoints) + .setRequestTimeout(configuration.get(RocketMQGrpcOptions.REQUEST_TIMEOUT)) + .enableSsl(configuration.get(RocketMQGrpcOptions.TLS_ENABLED)); + + if (!StringUtils.isNullOrWhitespaceOnly(namespace)) { + builder.setNamespace(namespace); + } + + CredentialsResolvers.applyCredentials( + builder, endpoints, configuration, credentialsResolver); + + return builder.build(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java new file mode 100644 index 0000000..1099820 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.common; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.Configuration; + +import org.apache.rocketmq.client.apis.SessionCredentialsProvider; + +import javax.annotation.Nullable; + +/** + * Resolves RocketMQ session credentials for a given proxy endpoint on the TaskManager, so that + * plaintext access/secret keys never need to be placed in the job graph or the Flink {@link + * Configuration}. + * + *

Implementations are instantiated reflectively from the class name configured under {@code + * rocketmq.client.credentials-resolver-class} and therefore must be public and provide a public + * no-argument constructor. The instance is created locally on each TaskManager (never serialized), + * which makes it safe to read credentials from environment variables, mounted secret files or an + * external KMS. + * + *

Because the resolver is keyed by endpoint, a single downstream acknowledgement operator can + * serve receipt handles originating from multiple sources that point at different clusters with + * different credentials. + * + *

When a resolver class is configured, it takes precedence over the static {@code + * rocketmq.client.access-key} / {@code rocketmq.client.secret-key} options. + */ +@PublicEvolving +public interface CredentialsResolver { + + /** + * Called once right after instantiation with the operator configuration, before any {@link + * #resolve(String)} call. Implementations may read custom options from it. + * + * @param configuration the operator configuration. + */ + default void configure(Configuration configuration) {} + + /** + * Resolve the session credentials for the given proxy endpoint. + * + * @param endpoint the proxy endpoint the SDK client will connect to, e.g. {@code + * "127.0.0.1:8081"}.x + * @return the credentials provider for the endpoint, or {@code null} if the endpoint requires + * no authentication. + */ + @Nullable + SessionCredentialsProvider resolve(String endpoint); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java new file mode 100644 index 0000000..e46915c --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.common; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.StringUtils; + +import org.apache.rocketmq.client.apis.ClientConfigurationBuilder; +import org.apache.rocketmq.client.apis.SessionCredentialsProvider; +import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider; + +import javax.annotation.Nullable; + +/** Instantiates {@link CredentialsResolver}s and applies credentials to SDK client builders. */ +@Internal +public final class CredentialsResolvers { + + private CredentialsResolvers() {} + + /** + * Create the {@link CredentialsResolver} configured under {@link + * RocketMQGrpcOptions#CREDENTIALS_RESOLVER_CLASS}, or return {@code null} if none is + * configured. The resolver is instantiated reflectively and {@link + * CredentialsResolver#configure(Configuration) configured} before being returned. + */ + @Nullable + public static CredentialsResolver createFromConfiguration(Configuration configuration) { + final String className = configuration.get(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS); + if (StringUtils.isNullOrWhitespaceOnly(className)) { + return null; + } + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = CredentialsResolvers.class.getClassLoader(); + } + try { + final CredentialsResolver resolver = + Class.forName(className, true, classLoader) + .asSubclass(CredentialsResolver.class) + .getDeclaredConstructor() + .newInstance(); + resolver.configure(configuration); + return resolver; + } catch (ReflectiveOperationException | ClassCastException e) { + throw new FlinkRuntimeException( + "Failed to instantiate the credentials resolver '" + + className + + "'. It must be a public implementation of " + + CredentialsResolver.class.getName() + + " with a public no-argument constructor.", + e); + } + } + + /** + * Apply credentials for the given endpoint to the SDK client builder. A configured {@link + * CredentialsResolver} takes precedence; otherwise the static access/secret key options are + * used when both are present. + */ + public static void applyCredentials( + ClientConfigurationBuilder builder, + String endpoint, + Configuration configuration, + @Nullable CredentialsResolver resolver) { + if (resolver != null) { + final SessionCredentialsProvider provider = resolver.resolve(endpoint); + if (provider != null) { + builder.setCredentialProvider(provider); + } + return; + } + final String accessKey = configuration.get(RocketMQGrpcOptions.ACCESS_KEY); + final String secretKey = configuration.get(RocketMQGrpcOptions.SECRET_KEY); + if (!StringUtils.isNullOrWhitespaceOnly(accessKey) + && !StringUtils.isNullOrWhitespaceOnly(secretKey)) { + builder.setCredentialProvider( + new StaticSessionCredentialsProvider(accessKey, secretKey)); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java new file mode 100644 index 0000000..3ee0596 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.common; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.StringUtils; + +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.apis.consumer.FilterExpressionType; + +/** + * Parses a filter definition into an SDK {@link FilterExpression}. A {@code null} or blank + * expression matches all messages ({@link FilterExpression#SUB_ALL}). + */ +@Internal +public class FilterExpressionParser { + + private FilterExpressionParser() {} + + /** + * Build a {@link FilterExpression} of the given type. + * + * @param expression the tag expression (e.g. {@code "tagA||tagB"}) or SQL92 expression; a blank + * value subscribes to all messages. + * @param type the filter type; a blank value defaults to {@link FilterExpressionType#TAG}. + */ + public static FilterExpression parse(String expression, String type) { + if (StringUtils.isNullOrWhitespaceOnly(expression) || "*".equals(expression.trim())) { + return FilterExpression.SUB_ALL; + } + final FilterExpressionType filterType = parseType(type); + return new FilterExpression(expression.trim(), filterType); + } + + private static FilterExpressionType parseType(String type) { + if (StringUtils.isNullOrWhitespaceOnly(type)) { + return FilterExpressionType.TAG; + } + switch (type.trim().toUpperCase()) { + case "SQL92": + return FilterExpressionType.SQL92; + case "TAG": + return FilterExpressionType.TAG; + default: + throw new IllegalArgumentException("Unsupported filter expression type: " + type); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java new file mode 100644 index 0000000..5c59c58 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider; + +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.ClientServiceProvider; +import org.apache.rocketmq.client.apis.producer.Producer; +import org.apache.rocketmq.client.apis.producer.ProducerBuilder; + +/** A factory that builds a gRPC {@link Producer} from a Flink configuration. */ +@Internal +public class ProducerProvider { + + private ProducerProvider() {} + + /** Build a {@link Producer} using the given service provider and configuration. */ + public static Producer create(ClientServiceProvider provider, Configuration configuration) + throws ClientException { + final ProducerBuilder builder = + provider.newProducerBuilder() + .setClientConfiguration( + ClientConfigurationProvider.getClientConfiguration(configuration)) + .setMaxAttempts(configuration.get(RocketMQGrpcSinkOptions.MAX_ATTEMPTS)); + + final String topic = configuration.get(RocketMQGrpcSinkOptions.TOPIC); + if (topic != null && !topic.isEmpty()) { + builder.setTopics(topic); + } + return builder.build(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java new file mode 100644 index 0000000..67259ec --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema; + +import java.io.IOException; + +/** + * The gRPC ({@code rocketmq-client-java}) implementation of a RocketMQ {@link Sink}. It provides + * at-least-once semantics using the synchronous {@code Producer.send()}; there is no committer and + * no two-phase transaction commit. + */ +@PublicEvolving +public class RocketMQGrpcSink implements Sink { + + private static final long serialVersionUID = 1L; + + private final Configuration configuration; + private final RocketMQGrpcSerializationSchema serializationSchema; + + RocketMQGrpcSink( + Configuration configuration, RocketMQGrpcSerializationSchema serializationSchema) { + this.configuration = configuration; + this.serializationSchema = serializationSchema; + } + + /** Create a {@link RocketMQGrpcSinkBuilder} to construct a new {@link RocketMQGrpcSink}. */ + public static RocketMQGrpcSinkBuilder builder() { + return new RocketMQGrpcSinkBuilder<>(); + } + + @Override + public SinkWriter createWriter(WriterInitContext context) throws IOException { + return new RocketMQGrpcSinkWriter<>( + configuration, + serializationSchema, + context.asSerializationSchemaInitializationContext()); + } + + @Deprecated + @Override + public SinkWriter createWriter(InitContext context) throws IOException { + return new RocketMQGrpcSinkWriter<>( + configuration, + serializationSchema, + context.asSerializationSchemaInitializationContext()); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java new file mode 100644 index 0000000..5b53c38 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** A fluent builder to construct a {@link RocketMQGrpcSink}. */ +@PublicEvolving +public class RocketMQGrpcSinkBuilder { + + private final Configuration configuration; + private RocketMQGrpcSerializationSchema serializationSchema; + + public RocketMQGrpcSinkBuilder() { + this.configuration = new Configuration(); + } + + /** Set the gRPC access point (proxy) endpoints. */ + public RocketMQGrpcSinkBuilder setEndpoints(String endpoints) { + return setConfig(RocketMQGrpcOptions.ENDPOINTS, endpoints); + } + + /** Set the parent topic for value-only serialization. */ + public RocketMQGrpcSinkBuilder setTopic(String topic) { + return setConfig(RocketMQGrpcSinkOptions.TOPIC, topic); + } + + /** Set the lite (sub) topic attached to every record for value-only serialization. */ + public RocketMQGrpcSinkBuilder setLiteTopic(String liteTopic) { + return setConfig(RocketMQGrpcSinkOptions.LITE_TOPIC, liteTopic); + } + + /** Set the {@link RocketMQGrpcSerializationSchema}. */ + public RocketMQGrpcSinkBuilder setSerializer( + RocketMQGrpcSerializationSchema serializationSchema) { + this.serializationSchema = checkNotNull(serializationSchema); + return this; + } + + /** + * Set a value-only serializer that encodes the record into the message body and publishes it to + * the configured parent {@link RocketMQGrpcSinkOptions#TOPIC} carrying the configured {@link + * RocketMQGrpcSinkOptions#LITE_TOPIC}. + */ + public RocketMQGrpcSinkBuilder setValueOnlySerializer( + SerializationSchema serializationSchema) { + final String topic = configuration.get(RocketMQGrpcSinkOptions.TOPIC); + checkNotNull(topic, "topic must be configured before setting a value-only serializer"); + final String liteTopic = configuration.get(RocketMQGrpcSinkOptions.LITE_TOPIC); + checkNotNull( + liteTopic, "lite topic must be configured before setting a value-only serializer"); + this.serializationSchema = + RocketMQGrpcSerializationSchema.flinkSchema(topic, liteTopic, serializationSchema); + return this; + } + + /** Set an arbitrary configuration option. */ + public RocketMQGrpcSinkBuilder setConfig(ConfigOption key, T value) { + configuration.set(key, value); + return this; + } + + /** Add arbitrary configuration options. */ + public RocketMQGrpcSinkBuilder setConfig(Configuration config) { + configuration.addAll(config); + return this; + } + + /** Build the {@link RocketMQGrpcSink}. */ + public RocketMQGrpcSink build() { + checkNotNull( + configuration.get(RocketMQGrpcOptions.ENDPOINTS), "endpoints must be configured"); + checkNotNull(serializationSchema, "serializer must be configured"); + return new RocketMQGrpcSink<>(configuration, serializationSchema); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java new file mode 100644 index 0000000..825d80b --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +/** + * Configuration options for the RocketMQ gRPC {@code Producer} based sink. These options are + * combined with the shared {@link org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}. + * The sink provides at-least-once semantics via synchronous {@code Producer.send()}. + * + *

These are the programmatic/SDK-facing options used by the sink builder and its runtime; every + * key carries the {@link #PRODUCER_CONFIG_PREFIX} prefix. The SQL DDL keys are defined separately + * in {@code RocketMQGrpcConnectorOptions}. + */ +@PublicEvolving +public class RocketMQGrpcSinkOptions { + + private RocketMQGrpcSinkOptions() {} + + /** Prefix for the RocketMQ gRPC sink options. */ + public static final String PRODUCER_CONFIG_PREFIX = "rocketmq.sink."; + + public static final ConfigOption TOPIC = + ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "topic") + .stringType() + .noDefaultValue() + .withDescription( + "The parent topic to send records to when using a value-only " + + "serializer."); + + public static final ConfigOption LITE_TOPIC = + ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "lite-topic") + .stringType() + .noDefaultValue() + .withDescription( + "The lite (sub) topic to attach to every record when using a " + + "value-only serializer. The message is published to the " + + "parent topic configured via '" + + PRODUCER_CONFIG_PREFIX + + "topic' and carries this lite topic, so that a " + + "LiteSimpleConsumer bound to the parent topic can receive " + + "it."); + + public static final ConfigOption MAX_ATTEMPTS = + ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "max-attempts") + .intType() + .defaultValue(3) + .withDescription("The maximum number of send attempts for a message."); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java new file mode 100644 index 0000000..041be68 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema; + +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.ClientServiceProvider; +import org.apache.rocketmq.client.apis.message.Message; +import org.apache.rocketmq.client.apis.producer.Producer; + +import java.io.IOException; + +/** + * The {@link SinkWriter} that synchronously sends records to RocketMQ using the gRPC {@code + * Producer}. Each {@code send()} is a synchronous call, providing at-least-once semantics. + */ +@Internal +public class RocketMQGrpcSinkWriter implements SinkWriter { + + private final ClientServiceProvider provider; + private final Producer producer; + private final RocketMQGrpcSerializationSchema serializationSchema; + + public RocketMQGrpcSinkWriter( + Configuration configuration, + RocketMQGrpcSerializationSchema serializationSchema, + SerializationSchema.InitializationContext initializationContext) + throws IOException { + this.provider = ClientServiceProvider.loadService(); + this.serializationSchema = serializationSchema; + try { + this.serializationSchema.open(initializationContext); + this.producer = ProducerProvider.create(provider, configuration); + } catch (Exception e) { + throw new IOException("Failed to initialize the RocketMQ gRPC producer.", e); + } + } + + @Override + public void write(IN element, Context context) throws IOException { + final Long timestamp = context.timestamp(); + final Message message = + serializationSchema.serialize(element, provider.newMessageBuilder(), timestamp); + try { + producer.send(message); + } catch (ClientException e) { + throw new IOException("Failed to send message to RocketMQ.", e); + } + } + + @Override + public void flush(boolean endOfInput) { + // Messages are sent synchronously, so there is nothing to flush. + } + + @Override + public void close() throws Exception { + if (producer != null) { + producer.close(); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java new file mode 100644 index 0000000..1e13f25 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink.serialization; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.SerializationSchema; + +import org.apache.rocketmq.client.apis.message.Message; +import org.apache.rocketmq.client.apis.message.MessageBuilder; + +import java.io.Serializable; + +/** + * A serialization schema that converts a value of type {@code T} into a RocketMQ gRPC {@link + * Message} using the given {@link MessageBuilder}. + * + * @param the type of values being serialized + */ +@PublicEvolving +public interface RocketMQGrpcSerializationSchema extends Serializable { + + /** + * Initialization method for the schema. It is called before the actual working method {@link + * #serialize} and thus suitable for one time setup work. + * + * @param context Contextual information that can be used during initialization. + */ + default void open(SerializationSchema.InitializationContext context) throws Exception { + // Nothing to do here for the default implementation. + } + + /** + * Serialize the given element into a {@link Message}. + * + * @param element the element to serialize. + * @param messageBuilder a fresh message builder to construct the message with. + * @param timestamp the (nullable) event timestamp of the element. + * @return the RocketMQ {@link Message} to send. + */ + Message serialize(T element, MessageBuilder messageBuilder, Long timestamp); + + /** + * Create a {@link RocketMQGrpcSerializationSchema} by wrapping a Flink {@link + * SerializationSchema}. The value is serialized into the message body and published to the + * given parent {@code topic} carrying the given {@code liteTopic}, so that a {@code + * LiteSimpleConsumer} bound to the parent topic can receive it. + */ + static RocketMQGrpcSerializationSchema flinkSchema( + String topic, String liteTopic, SerializationSchema serializationSchema) { + return new RocketMQGrpcSerializationSchemaWrapper<>(topic, liteTopic, serializationSchema); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java new file mode 100644 index 0000000..061408b --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink.serialization; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.SerializationSchema; + +import org.apache.rocketmq.client.apis.message.Message; +import org.apache.rocketmq.client.apis.message.MessageBuilder; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A {@link RocketMQGrpcSerializationSchema} that adapts a Flink {@link SerializationSchema} by + * serializing the value into the message body and publishing it to a fixed parent topic carrying a + * fixed lite (sub) topic. + */ +@Internal +public class RocketMQGrpcSerializationSchemaWrapper + implements RocketMQGrpcSerializationSchema { + + private static final long serialVersionUID = 1L; + + private final String topic; + private final String liteTopic; + private final SerializationSchema serializationSchema; + + public RocketMQGrpcSerializationSchemaWrapper( + String topic, String liteTopic, SerializationSchema serializationSchema) { + this.topic = checkNotNull(topic, "topic must not be null"); + this.liteTopic = checkNotNull(liteTopic, "lite topic must not be null"); + this.serializationSchema = checkNotNull(serializationSchema); + } + + @Override + public void open(SerializationSchema.InitializationContext context) throws Exception { + serializationSchema.open(context); + } + + @Override + public Message serialize(T element, MessageBuilder messageBuilder, Long timestamp) { + return messageBuilder + .setTopic(topic) + .setLiteTopic(liteTopic) + .setBody(serializationSchema.serialize(element)) + .build(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java new file mode 100644 index 0000000..a85891f --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.FlinkRuntimeException; +import org.apache.flink.util.StringUtils; + +import javax.annotation.Nullable; + +/** Instantiates {@link InvisibleDurationRenewalPolicy} implementations reflectively. */ +@Internal +public final class InvisibleDurationRenewalPolicies { + + private InvisibleDurationRenewalPolicies() {} + + /** + * Create the {@link InvisibleDurationRenewalPolicy} configured under {@link + * RocketMQGrpcSourceOptions#RENEWAL_POLICY_CLASS}, or return {@code null} if none is + * configured. The policy is instantiated reflectively and {@link + * InvisibleDurationRenewalPolicy#configure(Configuration) configured} before being returned. + */ + @Nullable + public static InvisibleDurationRenewalPolicy createFromConfiguration( + Configuration configuration) { + final String className = configuration.get(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS); + if (StringUtils.isNullOrWhitespaceOnly(className)) { + return null; + } + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = InvisibleDurationRenewalPolicies.class.getClassLoader(); + } + try { + final InvisibleDurationRenewalPolicy policy = + Class.forName(className, true, classLoader) + .asSubclass(InvisibleDurationRenewalPolicy.class) + .getDeclaredConstructor() + .newInstance(); + policy.configure(configuration); + return policy; + } catch (ReflectiveOperationException | ClassCastException e) { + throw new FlinkRuntimeException( + "Failed to instantiate the invisible duration renewal policy '" + + className + + "'. It must be a public implementation of " + + InvisibleDurationRenewalPolicy.class.getName() + + " with a public no-argument constructor.", + e); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java new file mode 100644 index 0000000..60ead32 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.time.Duration; + +/** + * A user-provided policy that decides whether a received message that is still buffered inside the + * source (i.e. not yet emitted downstream) should have its invisible duration renewed. + * + *

The source invokes {@link #renew(MessageView, int)} {@link + * RocketMQGrpcSourceOptions#RENEWAL_AHEAD_TIME} ahead of the moment the message would become + * visible again (for example with a 60s invisible duration and a 5s ahead time, the policy is + * consulted 55s after the message was received). Returning a positive duration triggers a {@code + * changeInvisibleDuration} call with that duration and schedules the next consultation; returning + * {@code null} (or a non-positive duration) stops renewing so the message becomes visible again + * after the current invisible duration elapses. + * + *

Messages that have already been handed to the record emitter are never renewed, because + * renewing refreshes the receipt handle and would invalidate the handle travelling downstream. + * + *

Implementations are instantiated reflectively from {@link + * RocketMQGrpcSourceOptions#RENEWAL_POLICY_CLASS} and therefore need a public no-argument + * constructor. One instance is created per split reader; invocations happen on a single renewal + * thread. + */ +@PublicEvolving +public interface InvisibleDurationRenewalPolicy extends Serializable { + + /** + * Configure this policy with the source configuration. Called once directly after + * instantiation. + */ + default void configure(Configuration configuration) {} + + /** + * Decide whether to renew the invisible duration of the given message. + * + * @param messageView the message still buffered inside the source. + * @param renewalCount how many times this message has already been renewed; {@code 0} on the + * first consultation. + * @return the new invisible duration, or {@code null} to stop renewing. + */ + @Nullable + Duration renew(MessageView messageView, int renewalCount); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java new file mode 100644 index 0000000..1f21b86 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.Source; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage; +import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessageTypeInfo; +import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema; +import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumState; +import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumStateSerializer; +import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumerator; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageViewImpl; +import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceFetcherManager; +import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceReader; +import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceRecordEmitter; +import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceSplitReader; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitSerializer; +import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.util.UserCodeClassLoader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Supplier; + +/** The gRPC ({@code rocketmq-client-java}) implementation of a FLIP-27 RocketMQ {@link Source}. */ +@PublicEvolving +public class RocketMQGrpcSource + implements Source< + AckableMessage, RocketMQGrpcSourceSplit, RocketMQGrpcSourceEnumState>, + ResultTypeQueryable> { + + private static final long serialVersionUID = 1L; + + private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSource.class); + + private final Configuration configuration; + private final Boundedness boundedness; + private final RocketMQGrpcDeserializationSchema deserializationSchema; + + RocketMQGrpcSource( + Configuration configuration, + Boundedness boundedness, + RocketMQGrpcDeserializationSchema deserializationSchema) { + this.configuration = configuration; + this.boundedness = boundedness; + this.deserializationSchema = deserializationSchema; + } + + /** Get a {@link RocketMQGrpcSourceBuilder} to build a {@link RocketMQGrpcSource}. */ + public static RocketMQGrpcSourceBuilder builder() { + return new RocketMQGrpcSourceBuilder<>(); + } + + @Override + public Boundedness getBoundedness() { + return boundedness; + } + + @Override + public SourceReader, RocketMQGrpcSourceSplit> createReader( + SourceReaderContext readerContext) throws Exception { + final FutureCompletingBlockingQueue> elementsQueue = + new FutureCompletingBlockingQueue<>(); + + deserializationSchema.open( + new DeserializationSchema.InitializationContext() { + @Override + public MetricGroup getMetricGroup() { + return readerContext.metricGroup().addGroup("deserializer"); + } + + @Override + public UserCodeClassLoader getUserCodeClassLoader() { + return readerContext.getUserCodeClassLoader(); + } + }); + + final RocketMQGrpcSourceSplitReader splitReader = + new RocketMQGrpcSourceSplitReader(configuration); + final Supplier> splitReaderSupplier = + () -> splitReader; + + final RocketMQGrpcSourceFetcherManager fetcherManager = + new RocketMQGrpcSourceFetcherManager(elementsQueue, splitReaderSupplier); + + final RocketMQGrpcSourceRecordEmitter recordEmitter = + new RocketMQGrpcSourceRecordEmitter<>( + deserializationSchema, + configuration.get(RocketMQGrpcOptions.NAMESPACE), + configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP)); + + final RocketMQGrpcSourceReader reader = + new RocketMQGrpcSourceReader<>( + elementsQueue, fetcherManager, recordEmitter, configuration, readerContext); + + return reader; + } + + @Override + public SplitEnumerator createEnumerator( + SplitEnumeratorContext enumContext) { + return new RocketMQGrpcSourceEnumerator(enumContext); + } + + @Override + public SplitEnumerator restoreEnumerator( + SplitEnumeratorContext enumContext, + RocketMQGrpcSourceEnumState checkpoint) { + LOG.info( + "Restoring RocketMQ gRPC source enumerator from checkpoint; the Pop-based " + + "enumerator is stateless, so no split assignment state is recovered."); + return new RocketMQGrpcSourceEnumerator(enumContext); + } + + @Override + public SimpleVersionedSerializer getSplitSerializer() { + return new RocketMQGrpcSourceSplitSerializer(); + } + + @Override + public SimpleVersionedSerializer + getEnumeratorCheckpointSerializer() { + return new RocketMQGrpcSourceEnumStateSerializer(); + } + + @Override + public TypeInformation> getProducedType() { + return new AckableMessageTypeInfo<>(deserializationSchema.getProducedType()); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java new file mode 100644 index 0000000..f38dcf8 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema; + +import java.time.Duration; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** A fluent builder to construct a {@link RocketMQGrpcSource}. */ +@PublicEvolving +public class RocketMQGrpcSourceBuilder { + + private final Configuration configuration; + private String mainTopic; + private Boundedness boundedness; + private RocketMQGrpcDeserializationSchema deserializationSchema; + + public RocketMQGrpcSourceBuilder() { + this.configuration = new Configuration(); + this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED; + } + + /** Set the gRPC access point (proxy) endpoints. */ + public RocketMQGrpcSourceBuilder setEndpoints(String endpoints) { + return setConfig(RocketMQGrpcOptions.ENDPOINTS, endpoints); + } + + /** Set the consumer group of the LiteSimpleConsumer. */ + public RocketMQGrpcSourceBuilder setConsumerGroup(String consumerGroup) { + return setConfig(RocketMQGrpcSourceOptions.CONSUMER_GROUP, consumerGroup); + } + + /** Set the main lite topic bound by the LiteSimpleConsumer. Every subtask binds this topic. */ + public RocketMQGrpcSourceBuilder setMainTopic(String mainTopic) { + checkArgument( + mainTopic != null && !mainTopic.trim().isEmpty(), + "main topic must not be null or blank"); + this.mainTopic = mainTopic; + return this; + } + + /** + * Set the fetch concurrency of each subtask, i.e. the number of concurrent fetch requests it + * issues. The requests are served by worker threads sharing a single thread-safe {@code + * SimpleConsumer}. Defaults to {@code 1}. + */ + public RocketMQGrpcSourceBuilder setFetchConcurrency(int fetchConcurrency) { + return setConfig(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY, fetchConcurrency); + } + + /** + * Set the fully qualified class name of an {@link InvisibleDurationRenewalPolicy}. When + * configured, the source consults the policy shortly before a still-buffered message would + * become visible again, and the policy decides whether to extend its invisible duration. + */ + public RocketMQGrpcSourceBuilder setRenewalPolicyClass(String renewalPolicyClass) { + return setConfig(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, renewalPolicyClass); + } + + /** + * Set how long before a buffered message becomes visible again the renewal policy is consulted. + * Defaults to 5 seconds, i.e. with a 60s invisible duration the policy runs 55s after the + * message was received. + */ + public RocketMQGrpcSourceBuilder setRenewalAheadTime(Duration aheadTime) { + return setConfig(RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME, aheadTime); + } + + /** Set the boundedness of this source. Defaults to {@link Boundedness#CONTINUOUS_UNBOUNDED}. */ + public RocketMQGrpcSourceBuilder setBoundedness(Boundedness boundedness) { + this.boundedness = checkNotNull(boundedness); + return this; + } + + /** Set the {@link RocketMQGrpcDeserializationSchema}. */ + public RocketMQGrpcSourceBuilder setDeserializer( + RocketMQGrpcDeserializationSchema deserializationSchema) { + this.deserializationSchema = checkNotNull(deserializationSchema); + return this; + } + + /** Set a value-only deserializer that decodes the message body using a Flink schema. */ + public RocketMQGrpcSourceBuilder setValueOnlyDeserializer( + DeserializationSchema deserializationSchema) { + this.deserializationSchema = + RocketMQGrpcDeserializationSchema.flinkSchema(deserializationSchema); + return this; + } + + /** Set an arbitrary configuration option. */ + public RocketMQGrpcSourceBuilder setConfig(ConfigOption key, T value) { + configuration.set(key, value); + return this; + } + + /** Add arbitrary configuration options. */ + public RocketMQGrpcSourceBuilder setConfig(Configuration config) { + configuration.addAll(config); + return this; + } + + /** Build the {@link RocketMQGrpcSource}. */ + public RocketMQGrpcSource build() { + checkNotNull( + configuration.get(RocketMQGrpcOptions.ENDPOINTS), "endpoints must be configured"); + checkNotNull( + configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP), + "consumer group must be configured"); + checkArgument( + mainTopic != null && !mainTopic.trim().isEmpty(), + "the main topic must be configured"); + checkArgument( + configuration.get(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY) >= 1, + "fetch concurrency must be at least 1"); + checkNotNull(deserializationSchema, "deserializer must be configured"); + configuration.set(RocketMQGrpcSourceOptions.MAIN_TOPIC, mainTopic); + return new RocketMQGrpcSource<>(configuration, boundedness, deserializationSchema); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java new file mode 100644 index 0000000..94b1e5a --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.time.Duration; + +/** + * Configuration options for the RocketMQ gRPC {@code SimpleConsumer} based source. These options + * are combined with the shared {@link + * org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}. + * + *

These are the programmatic/SDK-facing options used by the source builder and its runtime; + * every key carries the {@link #CONSUMER_CONFIG_PREFIX} prefix. The SQL DDL keys are defined + * separately in {@code RocketMQGrpcConnectorOptions}. + */ +@PublicEvolving +public class RocketMQGrpcSourceOptions { + + private RocketMQGrpcSourceOptions() {} + + /** Prefix for the RocketMQ gRPC source options. */ + public static final String CONSUMER_CONFIG_PREFIX = "rocketmq.source."; + + public static final ConfigOption CONSUMER_GROUP = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "consumer-group") + .stringType() + .noDefaultValue() + .withDescription("The consumer group of the SimpleConsumer."); + + public static final ConfigOption MAIN_TOPIC = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "main-topic") + .stringType() + .noDefaultValue() + .withDescription( + "The main lite topic bound by the LiteSimpleConsumer. Every subtask " + + "binds this topic; the broker performs message-level load " + + "balancing across the consumer group."); + + public static final ConfigOption FETCH_CONCURRENCY = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "fetch-concurrency") + .intType() + .defaultValue(1) + .withDescription( + "The fetch concurrency of each subtask, i.e. the number of concurrent " + + "fetch requests it issues. The requests are served by worker " + + "threads sharing a single thread-safe SimpleConsumer, so " + + "increasing this raises a single subtask's fetch throughput " + + "without changing the Flink parallelism."); + + public static final ConfigOption AWAIT_DURATION = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "await-duration") + .durationType() + .defaultValue(Duration.ofSeconds(20)) + .withDescription( + "The long-polling await duration of a single receive() invocation."); + + public static final ConfigOption INVISIBLE_DURATION = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "invisible-duration") + .durationType() + .defaultValue(Duration.ofSeconds(60)) + .withDescription( + "The invisible duration of a received message. Because messages are " + + "acknowledged by a downstream operator, it must be larger " + + "than the full downstream processing time of a message so " + + "that a message is not prematurely redelivered while still " + + "in flight; un-acked messages are redelivered after this " + + "duration, which provides the at-least-once guarantee."); + + public static final ConfigOption MAX_MESSAGE_NUM = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "max-message-num") + .intType() + .defaultValue(32) + .withDescription("The maximum number of messages returned by receive()."); + + public static final ConfigOption RENEWAL_POLICY_CLASS = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "renewal-policy-class") + .stringType() + .noDefaultValue() + .withDescription( + "The fully qualified class name of an InvisibleDurationRenewalPolicy " + + "implementation. When configured, the source consults the " + + "policy 'renewal-ahead-time' before a still-buffered message " + + "would become visible again, and the policy decides whether " + + "to extend its invisible duration. When absent, no renewal " + + "is performed."); + + public static final ConfigOption RENEWAL_AHEAD_TIME = + ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "renewal-ahead-time") + .durationType() + .defaultValue(Duration.ofSeconds(5)) + .withDescription( + "How long before a buffered message becomes visible again the renewal " + + "policy is consulted. For example with a 60s invisible " + + "duration and a 5s ahead time the policy runs 55s after the " + + "message was received. Must be positive and smaller than " + + "the invisible duration. Only effective when " + + "'renewal-policy-class' is configured."); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java new file mode 100644 index 0000000..d9dbf07 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.deserialization; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.util.Collector; + +import java.io.IOException; +import java.io.Serializable; + +/** An interface for the deserialization of RocketMQ gRPC messages. */ +@PublicEvolving +public interface RocketMQGrpcDeserializationSchema extends Serializable, ResultTypeQueryable { + + /** + * Initialization method for the schema. It is called before the actual working method {@link + * #deserialize} and thus suitable for one time setup work. + * + * @param context Contextual information that can be used during initialization. + */ + default void open(DeserializationSchema.InitializationContext context) throws Exception { + // Nothing to do here for the default implementation. + } + + /** + * Deserializes a {@link MessageView} and outputs zero or more records through the {@link + * Collector}. + * + * @param messageView The MessageView to deserialize. + * @param out The collector to put the resulting records. + */ + void deserialize(MessageView messageView, Collector out) throws IOException; + + /** + * Create a {@link RocketMQGrpcDeserializationSchema} by wrapping a Flink {@link + * DeserializationSchema}. The message body is deserialized using the given schema; the other + * fields such as key, tag and properties are ignored. + */ + static RocketMQGrpcDeserializationSchema flinkSchema( + DeserializationSchema deserializationSchema) { + return new RocketMQGrpcDeserializationSchemaWrapper<>(deserializationSchema); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java new file mode 100644 index 0000000..191cd29 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.deserialization; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.util.Collector; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A {@link RocketMQGrpcDeserializationSchema} that adapts a Flink {@link DeserializationSchema} by + * deserializing the message body only. + */ +@Internal +public class RocketMQGrpcDeserializationSchemaWrapper + implements RocketMQGrpcDeserializationSchema { + + private static final long serialVersionUID = 1L; + + private final DeserializationSchema deserializationSchema; + + public RocketMQGrpcDeserializationSchemaWrapper( + DeserializationSchema deserializationSchema) { + this.deserializationSchema = checkNotNull(deserializationSchema); + } + + @Override + public void open(DeserializationSchema.InitializationContext context) throws Exception { + deserializationSchema.open(context); + } + + @Override + public void deserialize(MessageView messageView, Collector out) throws IOException { + deserializationSchema.deserialize(messageView.getBody(), out); + } + + @Override + public TypeInformation getProducedType() { + return deserializationSchema.getProducedType(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java new file mode 100644 index 0000000..2f9422d --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.enumerator; + +/** + * The checkpointed state of the {@link RocketMQGrpcSourceEnumerator}. + * + *

The Pop model needs no enumerator state: splits carry no offset and every reader is simply + * assigned a placeholder split so it starts consuming all topics. This type only exists because the + * FLIP-27 {@code Source} API requires an enumerator checkpoint type, so it is a stateless marker. + */ +public class RocketMQGrpcSourceEnumState {} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java new file mode 100644 index 0000000..7e4faf6 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.enumerator; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +/** + * The {@link SimpleVersionedSerializer} for {@link RocketMQGrpcSourceEnumState}. The state is a + * stateless marker, so serialization is empty. + */ +public class RocketMQGrpcSourceEnumStateSerializer + implements SimpleVersionedSerializer { + + private static final int CURRENT_VERSION = 2; + + @Override + public int getVersion() { + return CURRENT_VERSION; + } + + @Override + public byte[] serialize(RocketMQGrpcSourceEnumState state) { + return new byte[0]; + } + + @Override + public RocketMQGrpcSourceEnumState deserialize(int version, byte[] serialized) { + return new RocketMQGrpcSourceEnumState(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java new file mode 100644 index 0000000..20c12a3 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.enumerator; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.List; + +/** + * The split enumerator for the RocketMQ gRPC source. + * + *

The Pop model performs message-level load balancing on the broker side, so there is nothing to + * distribute: every reader consumes all configured topics and the broker spreads messages across + * the consumer group. The enumerator is therefore stateless and simply hands each registered reader + * a single placeholder split, which is what triggers that reader to start its pop loop. + */ +@Internal +public class RocketMQGrpcSourceEnumerator + implements SplitEnumerator { + + private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSourceEnumerator.class); + + private final SplitEnumeratorContext context; + + public RocketMQGrpcSourceEnumerator(SplitEnumeratorContext context) { + this.context = context; + } + + @Override + public void start() { + // No topic discovery or split distribution is required for the Pop model. + } + + @Override + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + // Splits are assigned proactively when a reader registers. + } + + @Override + public void addReader(int subtaskId) { + context.assignSplit(new RocketMQGrpcSourceSplit(), subtaskId); + context.signalNoMoreSplits(subtaskId); + LOG.info("Assigned the placeholder split to reader {}", subtaskId); + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + // The placeholder split is stateless and is re-assigned when the reader re-registers, so + // returned splits can be dropped. + LOG.info("RocketMQ gRPC source dropped returned placeholder splits: {}", splits); + } + + @Override + public RocketMQGrpcSourceEnumState snapshotState(long checkpointId) { + return new RocketMQGrpcSourceEnumState(); + } + + @Override + public void close() { + // Nothing to close. + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java new file mode 100644 index 0000000..82b0c8d --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions; + +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.ClientServiceProvider; +import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer; + +/** + * A factory that builds a gRPC {@link LiteSimpleConsumer} (Pop model) from a Flink configuration. + * + *

The consumer binds the configured main topic and relies on its wildcard (generalized) + * subscription to receive messages from all of its sub topics; the consumer group must carry the + * {@code lite.sub.wildcard=true} attribute on the broker. + */ +@Internal +public class LiteSimpleConsumerProvider { + + private LiteSimpleConsumerProvider() {} + + /** + * Build and start a {@link LiteSimpleConsumer} from the given Flink configuration. + * + * @param configuration the Flink configuration. + * @return a started {@link LiteSimpleConsumer}. + */ + public static LiteSimpleConsumer create(Configuration configuration) throws ClientException { + final ClientServiceProvider provider = ClientServiceProvider.loadService(); + return provider.newLiteSimpleConsumerBuilder() + .setClientConfiguration( + ClientConfigurationProvider.getClientConfiguration(configuration)) + .setConsumerGroup(configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP)) + .setAwaitDuration(configuration.get(RocketMQGrpcSourceOptions.AWAIT_DURATION)) + .bindTopic(configuration.get(RocketMQGrpcSourceOptions.MAIN_TOPIC)) + .build(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java new file mode 100644 index 0000000..747a1c7 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import java.util.Collection; +import java.util.Map; + +/** + * A read-only view over a RocketMQ message returned by the gRPC {@code SimpleConsumer}. It exposes + * the message attributes needed for deserialization while hiding the SDK receipt handle used for + * acknowledgement. + */ +public interface MessageView { + + /** Get the unique message ID. */ + String getMessageId(); + + /** Get the topic that the message belongs to. */ + String getTopic(); + + /** Get the tag of the message, or {@code null} if the message has no tag. */ + String getTag(); + + /** Get the keys of the message. */ + Collection getKeys(); + + /** Get the body of the message. */ + byte[] getBody(); + + /** Get the number of times the message has been delivered. */ + int getDeliveryAttempt(); + + /** Get the born timestamp of the message, used as the event time. */ + long getEventTime(); + + /** Get the user-defined properties of the message. */ + Map getProperties(); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java new file mode 100644 index 0000000..d335139 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; + +import org.apache.rocketmq.client.apis.message.MessageId; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.Future; + +/** + * The {@link MessageView} implementation backed by an SDK {@link + * org.apache.rocketmq.client.apis.message.MessageView}. It retains the underlying SDK message so + * that the reader can acknowledge it after a checkpoint completes. + * + *

It additionally tracks whether the message has been handed to the record emitter. Renewing the + * invisible duration refreshes the SDK receipt handle, so a renewal and the handle extraction at + * emit time must be mutually exclusive: both happen under the monitor of this instance, and once + * {@link #markEmitted()} has been called no further renewal is attempted. + */ +@Internal +public class MessageViewImpl implements MessageView { + + private final org.apache.rocketmq.client.apis.message.MessageView messageView; + private final byte[] body; + + private boolean emitted; + private Future renewalFuture; + + public MessageViewImpl(org.apache.rocketmq.client.apis.message.MessageView messageView) { + this.messageView = messageView; + final ByteBuffer buffer = messageView.getBody(); + this.body = new byte[buffer.remaining()]; + buffer.get(this.body); + } + + /** The underlying SDK message, used for acknowledgement via the {@code SimpleConsumer}. */ + public org.apache.rocketmq.client.apis.message.MessageView getMessageView() { + return messageView; + } + + /** + * Mark this message as handed to the record emitter and cancel any pending renewal. Blocks + * while a renewal RPC for this message is in flight so that the receipt handle extracted + * afterwards is stable. + */ + public synchronized void markEmitted() { + emitted = true; + if (renewalFuture != null) { + renewalFuture.cancel(false); + renewalFuture = null; + } + } + + /** Whether this message has been handed to the record emitter. */ + public synchronized boolean isEmitted() { + return emitted; + } + + /** Track the pending renewal task so that {@link #markEmitted()} can cancel it. */ + public synchronized void setRenewalFuture(Future future) { + if (emitted) { + future.cancel(false); + } else { + this.renewalFuture = future; + } + } + + @Override + public String getMessageId() { + final MessageId messageId = messageView.getMessageId(); + return messageId == null ? null : messageId.toString(); + } + + @Override + public String getTopic() { + return messageView.getTopic(); + } + + @Override + public String getTag() { + return messageView.getTag().orElse(null); + } + + @Override + public Collection getKeys() { + return messageView.getKeys(); + } + + @Override + public byte[] getBody() { + return body; + } + + @Override + public int getDeliveryAttempt() { + return messageView.getDeliveryAttempt(); + } + + @Override + public long getEventTime() { + return messageView.getBornTimestamp(); + } + + @Override + public Map getProperties() { + return messageView.getProperties(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java new file mode 100644 index 0000000..10ebd61 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; + +import java.util.function.Supplier; + +/** + * The {@link SingleThreadFetcherManager} for the RocketMQ gRPC source. The Pop source does not + * acknowledge messages from the source side (acknowledgement is deferred to a downstream operator), + * so this manager only provides a single fetcher thread that receives messages. + */ +@Internal +public class RocketMQGrpcSourceFetcherManager + extends SingleThreadFetcherManager { + + public RocketMQGrpcSourceFetcherManager( + FutureCompletingBlockingQueue> elementsQueue, + Supplier> splitReaderSupplier) { + super(elementsQueue, splitReaderSupplier, new Configuration()); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java new file mode 100644 index 0000000..63fd328 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; +import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; +import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitState; + +import java.util.Map; + +/** + * The source reader for the RocketMQ gRPC connector. It emits {@link AckableMessage} records and + * does not acknowledge messages itself; acknowledgement is performed by a downstream operator using + * the self-contained receipt handle carried by each emitted record. Un-acked messages are + * redelivered by the broker after their invisible duration expires, providing the at-least-once + * guarantee. + */ +@Internal +public class RocketMQGrpcSourceReader + extends SingleThreadMultiplexSourceReaderBase< + MessageViewImpl, + AckableMessage, + RocketMQGrpcSourceSplit, + RocketMQGrpcSourceSplitState> { + + public RocketMQGrpcSourceReader( + FutureCompletingBlockingQueue> elementsQueue, + RocketMQGrpcSourceFetcherManager fetcherManager, + RocketMQGrpcSourceRecordEmitter recordEmitter, + Configuration config, + SourceReaderContext context) { + super(elementsQueue, fetcherManager, recordEmitter, config, context); + } + + @Override + protected void onSplitFinished(Map finishedSplitIds) { + // The Pop model is unbounded; splits do not finish. + } + + @Override + protected RocketMQGrpcSourceSplitState initializedState(RocketMQGrpcSourceSplit split) { + return new RocketMQGrpcSourceSplitState(split); + } + + @Override + protected RocketMQGrpcSourceSplit toSplitType( + String splitId, RocketMQGrpcSourceSplitState splitState) { + return splitState.toRocketMQGrpcSourceSplit(); + } + + @VisibleForTesting + int getNumAliveFetchers() { + return splitFetcherManager.getNumAliveFetchers(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java new file mode 100644 index 0000000..2dae65e --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.connector.source.SourceOutput; +import org.apache.flink.connector.base.source.reader.RecordEmitter; +import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage; +import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQReceiptHandle; +import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQReceiptHandleCodec; +import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitState; +import org.apache.flink.util.Collector; + +import java.io.IOException; + +/** + * The {@link RecordEmitter} implementation for the RocketMQ gRPC source. It deserializes the + * message body into the value type {@code T}, extracts the credential-free receipt handle from the + * SDK message view and emits an {@link AckableMessage} pairing the two so that a downstream + * operator can acknowledge the message. + */ +@Internal +public class RocketMQGrpcSourceRecordEmitter + implements RecordEmitter, RocketMQGrpcSourceSplitState> { + + private final RocketMQGrpcDeserializationSchema deserializationSchema; + private final String namespace; + private final String consumerGroup; + private final AckableCollector collector = new AckableCollector<>(); + + public RocketMQGrpcSourceRecordEmitter( + RocketMQGrpcDeserializationSchema deserializationSchema, + String namespace, + String consumerGroup) { + this.deserializationSchema = deserializationSchema; + this.namespace = namespace; + this.consumerGroup = consumerGroup; + } + + @Override + public void emitRecord( + MessageViewImpl element, + SourceOutput> output, + RocketMQGrpcSourceSplitState splitState) + throws IOException { + try { + final RocketMQReceiptHandle handle = + RocketMQReceiptHandleCodec.extract( + element.getMessageView(), namespace, consumerGroup); + collector.reset(output, element.getEventTime(), handle); + deserializationSchema.deserialize(element, collector); + splitState.incrementProcessedRecords(); + } catch (Exception e) { + throw new IOException("Failed to deserialize message due to", e); + } + } + + /** A collector that wraps each deserialized value into an {@link AckableMessage}. */ + private static class AckableCollector implements Collector { + + private SourceOutput> sourceOutput; + private long timestamp; + private RocketMQReceiptHandle handle; + + @Override + public void collect(T record) { + sourceOutput.collect(new AckableMessage<>(record, handle), timestamp); + } + + @Override + public void close() {} + + private void reset( + SourceOutput> sourceOutput, + long timestamp, + RocketMQReceiptHandle handle) { + this.sourceOutput = sourceOutput; + this.timestamp = timestamp; + this.handle = handle; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java new file mode 100644 index 0000000..570269a --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.base.source.reader.RecordsBySplits; +import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; +import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; +import org.apache.flink.connector.base.source.reader.splitreader.SplitsChange; +import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicies; +import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicy; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; +import org.apache.flink.util.FlinkRuntimeException; + +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.flink.util.Preconditions.checkArgument; + +/** + * The {@link SplitReader} implementation for the RocketMQ gRPC Pop model. + * + *

Because the broker performs message-level load balancing across the consumer group, this + * reader does not partition topics. Instead every subtask binds the same main lite topic. To raise + * a single subtask's throughput it runs {@code fetch-concurrency} worker threads that share a + * single {@link LiteSimpleConsumer}, each issuing a blocking {@code receive()} long poll. Received + * messages are handed to the (single) fetcher thread through an internal queue. + * + *

This reader never acknowledges messages: acknowledgement is deferred to a downstream operator + * that receives the {@code AckableMessage} records carrying a self-contained receipt handle. + * Messages that are never acked become visible again after their invisible duration and are + * redelivered by the broker, which provides the at-least-once guarantee. + * + *

When an {@link InvisibleDurationRenewalPolicy} is configured, messages that are still buffered + * in the internal queue shortly before they would become visible again are offered to the policy, + * which may extend their invisible duration. Messages already handed to the record emitter are + * never renewed because renewing refreshes the receipt handle carried downstream. + */ +@Internal +public class RocketMQGrpcSourceSplitReader + implements SplitReader { + + private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSourceSplitReader.class); + + private static final long RECEIVE_BACKOFF_INITIAL_MS = 100L; + private static final long RECEIVE_BACKOFF_MAX_MS = 30_000L; + + private final Configuration configuration; + private final int fetchConcurrency; + private final int maxMessageNum; + private final Duration invisibleDuration; + private final Duration renewalAheadTime; + @Nullable private final InvisibleDurationRenewalPolicy renewalPolicy; + + private final BlockingQueue elementQueue; + private final AtomicBoolean wakeup = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); + + private volatile boolean closed = false; + private ReceiveWorker[] workers; + private LiteSimpleConsumer consumer; + @Nullable private ScheduledThreadPoolExecutor renewalExecutor; + + public RocketMQGrpcSourceSplitReader(Configuration configuration) { + this.configuration = configuration; + this.fetchConcurrency = configuration.get(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY); + this.maxMessageNum = configuration.get(RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM); + this.invisibleDuration = configuration.get(RocketMQGrpcSourceOptions.INVISIBLE_DURATION); + this.renewalAheadTime = configuration.get(RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME); + this.renewalPolicy = + InvisibleDurationRenewalPolicies.createFromConfiguration(configuration); + if (renewalPolicy != null) { + checkArgument( + !renewalAheadTime.isNegative() && !renewalAheadTime.isZero(), + "renewal-ahead-time must be positive"); + checkArgument( + renewalAheadTime.compareTo(invisibleDuration) < 0, + "renewal-ahead-time (%s) must be smaller than the invisible duration (%s)", + renewalAheadTime, + invisibleDuration); + } + this.elementQueue = new ArrayBlockingQueue<>(maxMessageNum * fetchConcurrency); + } + + @Override + public RecordsWithSplitIds fetch() { + if (wakeup.compareAndSet(true, false) || closed) { + return new RecordsBySplits.Builder().build(); + } + + final MessageViewImpl first; + try { + first = elementQueue.poll(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new RecordsBySplits.Builder().build(); + } + if (first == null) { + return new RecordsBySplits.Builder().build(); + } + + final List batch = new ArrayList<>(maxMessageNum); + batch.add(first); + elementQueue.drainTo(batch, maxMessageNum - 1); + + final RecordsBySplits.Builder builder = new RecordsBySplits.Builder<>(); + for (MessageViewImpl message : batch) { + // From here on the receipt handle travels downstream, so freeze it: waits for an + // in-flight renewal to finish and prevents any further renewal. + message.markEmitted(); + builder.add(RocketMQGrpcSourceSplit.SPLIT_ID, message); + } + return builder.build(); + } + + @Override + public void handleSplitsChanges(SplitsChange splitsChanges) { + // The placeholder split simply triggers the reader to start consuming. + if (!splitsChanges.splits().isEmpty()) { + startWorkers(); + } + } + + private void startWorkers() { + if (!started.compareAndSet(false, true)) { + return; + } + try { + consumer = LiteSimpleConsumerProvider.create(configuration); + } catch (ClientException e) { + started.set(false); + throw new FlinkRuntimeException("Failed to create RocketMQ gRPC LiteSimpleConsumer", e); + } + if (renewalPolicy != null) { + renewalExecutor = + new ScheduledThreadPoolExecutor( + 1, + runnable -> { + final Thread thread = new Thread(runnable, "rocketmq-grpc-renewal"); + thread.setDaemon(true); + return thread; + }); + renewalExecutor.setRemoveOnCancelPolicy(true); + } + workers = new ReceiveWorker[fetchConcurrency]; + for (int i = 0; i < fetchConcurrency; i++) { + final ReceiveWorker worker = new ReceiveWorker(i); + workers[i] = worker; + worker.start(); + } + LOG.info( + "Started {} receive worker(s) sharing one lite consumer bound to topic {}", + fetchConcurrency, + configuration.get(RocketMQGrpcSourceOptions.MAIN_TOPIC)); + } + + @Override + public void wakeUp() { + wakeup.compareAndSet(false, true); + } + + @Override + public void close() throws Exception { + closed = true; + if (renewalExecutor != null) { + renewalExecutor.shutdownNow(); + } + if (workers != null) { + for (ReceiveWorker worker : workers) { + worker.shutdown(); + } + } + // Close the consumer before joining the workers so that a worker blocked in a receive() + // long poll fails fast instead of holding its thread until the await duration elapses. + if (consumer != null) { + consumer.close(); + } + if (workers != null) { + for (ReceiveWorker worker : workers) { + worker.join(TimeUnit.SECONDS.toMillis(10)); + } + } + } + + private void scheduleRenewal(MessageViewImpl message, int renewalCount, Duration current) { + if (renewalExecutor == null || closed) { + return; + } + final long delayMs = Math.max(current.minus(renewalAheadTime).toMillis(), 0L); + try { + final ScheduledFuture future = + renewalExecutor.schedule( + () -> renew(message, renewalCount), delayMs, TimeUnit.MILLISECONDS); + message.setRenewalFuture(future); + } catch (java.util.concurrent.RejectedExecutionException e) { + // The reader is closing; the message will simply become visible again. + } + } + + private void renew(MessageViewImpl message, int renewalCount) { + synchronized (message) { + if (closed || message.isEmitted()) { + return; + } + final Duration next; + try { + next = renewalPolicy.renew(message, renewalCount); + } catch (Exception e) { + LOG.warn( + "The renewal policy failed for message {}; it will become visible again", + message.getMessageId(), + e); + return; + } + if (next == null || next.isZero() || next.isNegative()) { + return; + } + try { + consumer.changeInvisibleDuration(message.getMessageView(), next); + } catch (ClientException e) { + LOG.warn( + "Failed to renew the invisible duration of message {}; it will become " + + "visible again", + message.getMessageId(), + e); + return; + } + scheduleRenewal(message, renewalCount + 1, next); + } + } + + /** + * A single receive loop over the shared {@link LiteSimpleConsumer}. Because the consumer is + * thread-safe, several workers can call {@code receive()} on it concurrently; a blocking {@code + * receive()} only occupies its calling thread, which is why several threads are used to keep + * that many long polls in flight. + */ + private class ReceiveWorker extends Thread { + + private final int index; + + private volatile boolean running = true; + + private ReceiveWorker(int index) { + super("rocketmq-grpc-receive-worker-" + index); + this.index = index; + } + + private void shutdown() { + running = false; + interrupt(); + } + + @Override + public void run() { + long backoffMs = RECEIVE_BACKOFF_INITIAL_MS; + while (running && !closed) { + try { + final List messages = + consumer.receive(maxMessageNum, invisibleDuration); + backoffMs = RECEIVE_BACKOFF_INITIAL_MS; + if (!running || closed) { + // Dropped messages are redelivered after their invisible duration. + break; + } + for (org.apache.rocketmq.client.apis.message.MessageView message : messages) { + final MessageViewImpl view = new MessageViewImpl(message); + scheduleRenewal(view, 0, invisibleDuration); + elementQueue.put(view); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } catch (ClientException e) { + if (running && !closed) { + LOG.warn( + "Receive worker {} failed to receive messages, retrying in {} ms", + index, + backoffMs, + e); + try { + Thread.sleep(backoffMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + backoffMs = Math.min(backoffMs * 2, RECEIVE_BACKOFF_MAX_MS); + } + } + } + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java new file mode 100644 index 0000000..9ec4d8e --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.split; + +import org.apache.flink.api.connector.source.SourceSplit; + +import java.io.Serializable; + +/** + * A placeholder {@link SourceSplit} for the RocketMQ gRPC Pop consumption model. + * + *

Unlike a pull-based connector, the Pop model performs message-level load balancing on the + * broker side: every subtask subscribes to all configured topics and the broker distributes + * messages across the consumer group. There is therefore no per-topic, per-queue or offset state to + * distribute. This split carries no data; it merely exists because the FLIP-27 {@code Source} API + * requires a split type, and assigning a single instance to a reader is what triggers that reader + * to start consuming. + */ +public class RocketMQGrpcSourceSplit implements SourceSplit, Serializable { + + private static final long serialVersionUID = 2L; + + /** The single, constant split id used by every reader. */ + public static final String SPLIT_ID = "rocketmq-grpc-pop"; + + public static final RocketMQGrpcSourceSplit INSTANCE = new RocketMQGrpcSourceSplit(); + + public RocketMQGrpcSourceSplit() {} + + @Override + public String splitId() { + return SPLIT_ID; + } + + @Override + public String toString() { + return "RocketMQGrpcSourceSplit(" + SPLIT_ID + ")"; + } + + @Override + public int hashCode() { + return SPLIT_ID.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof RocketMQGrpcSourceSplit; + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java new file mode 100644 index 0000000..01dd709 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.split; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +/** + * The {@link SimpleVersionedSerializer serializer} for the placeholder {@link + * RocketMQGrpcSourceSplit}. The split carries no data, so serialization is empty. + */ +public class RocketMQGrpcSourceSplitSerializer + implements SimpleVersionedSerializer { + + private static final int CURRENT_VERSION = 2; + + @Override + public int getVersion() { + return CURRENT_VERSION; + } + + @Override + public byte[] serialize(RocketMQGrpcSourceSplit split) { + return new byte[0]; + } + + @Override + public RocketMQGrpcSourceSplit deserialize(int version, byte[] serialized) { + return RocketMQGrpcSourceSplit.INSTANCE; + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java new file mode 100644 index 0000000..e88ed77 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.split; + +/** + * The mutable runtime state of the placeholder {@link RocketMQGrpcSourceSplit}. The Pop model has + * no offset state to track, so this simply counts the number of messages processed, which is useful + * for metrics and debugging. + */ +public class RocketMQGrpcSourceSplitState { + + private final RocketMQGrpcSourceSplit split; + private long processedRecords; + + public RocketMQGrpcSourceSplitState(RocketMQGrpcSourceSplit split) { + this.split = split; + this.processedRecords = 0L; + } + + public long getProcessedRecords() { + return processedRecords; + } + + public void incrementProcessedRecords() { + this.processedRecords++; + } + + /** Convert back to an immutable {@link RocketMQGrpcSourceSplit} for checkpointing. */ + public RocketMQGrpcSourceSplit toRocketMQGrpcSourceSplit() { + return split; + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java new file mode 100644 index 0000000..03b3d36 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.time.Duration; + +/** + * Config options that are used to configure the RocketMQ gRPC SQL connector. These keys are the + * user-facing option names in a {@code CREATE TABLE ... WITH (...)} statement. + * + *

Connection/identity options are top-level (mirroring the {@code service-url}/{@code topics} + * convention of the Pulsar and Kafka SQL connectors); functional options are grouped by role under + * the {@code source.} and {@code sink.} prefixes. The programmatic/SDK-facing options that the + * source and sink builders and their runtime consume are defined separately in {@link + * org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}, {@link + * org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions} and {@link + * org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSinkOptions}. + */ +@PublicEvolving +public class RocketMQGrpcConnectorOptions { + + private RocketMQGrpcConnectorOptions() {} + + // -------------------------------------------------------------------------------------------- + // Connection options + // -------------------------------------------------------------------------------------------- + + public static final ConfigOption ENDPOINTS = + ConfigOptions.key("endpoints") + .stringType() + .noDefaultValue() + .withDescription( + "The access point (proxy) endpoints the gRPC SDK communicates with, " + + "for example '127.0.0.1:8080'."); + + public static final ConfigOption NAMESPACE = + ConfigOptions.key("namespace") + .stringType() + .defaultValue("") + .withDescription("The resource namespace of the RocketMQ instance."); + + public static final ConfigOption ACCESS_KEY = + ConfigOptions.key("access-key") + .stringType() + .noDefaultValue() + .withDescription("The access key used for static session credentials."); + + public static final ConfigOption SECRET_KEY = + ConfigOptions.key("secret-key") + .stringType() + .noDefaultValue() + .withDescription("The secret key used for static session credentials."); + + public static final ConfigOption CREDENTIALS_RESOLVER_CLASS = + ConfigOptions.key("credentials-resolver-class") + .stringType() + .noDefaultValue() + .withDescription( + "The fully qualified class name of a CredentialsResolver that resolves " + + "the session credentials per endpoint on the TaskManager. " + + "When set, it takes precedence over 'access-key' and " + + "'secret-key'."); + + public static final ConfigOption TLS_ENABLED = + ConfigOptions.key("tls-enabled") + .booleanType() + .defaultValue(false) + .withDescription("Whether TLS is enabled for the gRPC transport."); + + public static final ConfigOption REQUEST_TIMEOUT = + ConfigOptions.key("request-timeout") + .durationType() + .defaultValue(Duration.ofSeconds(3)) + .withDescription("The request timeout for a single gRPC invocation."); + + public static final ConfigOption TOPIC = + ConfigOptions.key("topic") + .stringType() + .noDefaultValue() + .withDescription( + "The default topic to send records to when the table is used as a " + + "sink with a value-only serializer."); + + public static final ConfigOption LITE_TOPIC = + ConfigOptions.key("sink.lite-topic") + .stringType() + .noDefaultValue() + .withDescription( + "The lite (sub) topic attached to every record when the table is used " + + "as a sink. The message is published to the parent topic " + + "configured via 'topic' and carries this lite topic, so that " + + "a LiteSimpleConsumer bound to the parent topic can receive " + + "it. Required for sinks."); + + // -------------------------------------------------------------------------------------------- + // Source options + // -------------------------------------------------------------------------------------------- + + public static final ConfigOption CONSUMER_GROUP = + ConfigOptions.key("source.consumer-group") + .stringType() + .noDefaultValue() + .withDescription("The consumer group of the SimpleConsumer."); + + public static final ConfigOption AWAIT_DURATION = + ConfigOptions.key("source.await-duration") + .durationType() + .defaultValue(Duration.ofSeconds(30)) + .withDescription( + "The long-polling await duration of a single receive() invocation."); + + public static final ConfigOption INVISIBLE_DURATION = + ConfigOptions.key("source.invisible-duration") + .durationType() + .defaultValue(Duration.ofSeconds(60)) + .withDescription( + "The invisible duration of a received message. It must be larger than " + + "the checkpoint interval plus the checkpoint timeout so that " + + "un-acked messages are redelivered after a failure."); + + public static final ConfigOption MAX_MESSAGE_NUM = + ConfigOptions.key("source.max-message-num") + .intType() + .defaultValue(32) + .withDescription("The maximum number of messages returned by receive()."); + + public static final ConfigOption FETCH_CONCURRENCY = + ConfigOptions.key("source.fetch-concurrency") + .intType() + .defaultValue(1) + .withDescription( + "The fetch concurrency of each subtask, i.e. the number of concurrent " + + "fetch requests it issues. The requests are served by worker " + + "threads sharing a single thread-safe SimpleConsumer, so " + + "increasing this raises a single subtask's fetch throughput " + + "without changing the Flink parallelism."); + + public static final ConfigOption RENEWAL_POLICY_CLASS = + ConfigOptions.key("source.renewal-policy-class") + .stringType() + .noDefaultValue() + .withDescription( + "The fully qualified class name of an InvisibleDurationRenewalPolicy " + + "implementation consulted shortly before a still-buffered " + + "message would become visible again; the policy decides " + + "whether to extend its invisible duration. When absent, no " + + "renewal is performed."); + + public static final ConfigOption RENEWAL_AHEAD_TIME = + ConfigOptions.key("source.renewal-ahead-time") + .durationType() + .defaultValue(Duration.ofSeconds(5)) + .withDescription( + "How long before a buffered message becomes visible again the renewal " + + "policy is consulted. Must be positive and smaller than the " + + "invisible duration. Only effective when " + + "'source.renewal-policy-class' is configured."); + + // -------------------------------------------------------------------------------------------- + // Sink options + // -------------------------------------------------------------------------------------------- + + public static final ConfigOption MAX_ATTEMPTS = + ConfigOptions.key("sink.max-attempts") + .intType() + .defaultValue(3) + .withDescription("The maximum number of send attempts for a message."); +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java new file mode 100644 index 0000000..d6968f9 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSinkOptions; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DeserializationFormatFactory; +import org.apache.flink.table.factories.DynamicTableSinkFactory; +import org.apache.flink.table.factories.DynamicTableSourceFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.DataType; + +import java.util.HashSet; +import java.util.Set; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * The {@link DynamicTableSourceFactory} and {@link DynamicTableSinkFactory} for the RocketMQ gRPC + * connector. It is registered under the {@code rocketmq-grpc} identifier. + */ +@Internal +public class RocketMQGrpcDynamicTableFactory + implements DynamicTableSourceFactory, DynamicTableSinkFactory { + + public static final String IDENTIFIER = "rocketmq-grpc"; + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + final Set> options = new HashSet<>(); + options.add(RocketMQGrpcConnectorOptions.ENDPOINTS); + options.add(RocketMQGrpcConnectorOptions.TOPIC); + options.add(FactoryUtil.FORMAT); + return options; + } + + @Override + public Set> optionalOptions() { + final Set> options = new HashSet<>(); + options.add(RocketMQGrpcConnectorOptions.NAMESPACE); + options.add(RocketMQGrpcConnectorOptions.ACCESS_KEY); + options.add(RocketMQGrpcConnectorOptions.SECRET_KEY); + options.add(RocketMQGrpcConnectorOptions.CREDENTIALS_RESOLVER_CLASS); + options.add(RocketMQGrpcConnectorOptions.TLS_ENABLED); + options.add(RocketMQGrpcConnectorOptions.REQUEST_TIMEOUT); + options.add(RocketMQGrpcConnectorOptions.LITE_TOPIC); + options.add(RocketMQGrpcConnectorOptions.CONSUMER_GROUP); + options.add(RocketMQGrpcConnectorOptions.AWAIT_DURATION); + options.add(RocketMQGrpcConnectorOptions.INVISIBLE_DURATION); + options.add(RocketMQGrpcConnectorOptions.MAX_MESSAGE_NUM); + options.add(RocketMQGrpcConnectorOptions.FETCH_CONCURRENCY); + options.add(RocketMQGrpcConnectorOptions.RENEWAL_POLICY_CLASS); + options.add(RocketMQGrpcConnectorOptions.RENEWAL_AHEAD_TIME); + options.add(RocketMQGrpcConnectorOptions.MAX_ATTEMPTS); + return options; + } + + @Override + public DynamicTableSource createDynamicTableSource(Context context) { + final FactoryUtil.TableFactoryHelper helper = + FactoryUtil.createTableFactoryHelper(this, context); + final DecodingFormat> decodingFormat = + helper.discoverDecodingFormat( + DeserializationFormatFactory.class, FactoryUtil.FORMAT); + helper.validate(); + + final ReadableConfig options = helper.getOptions(); + checkNotNull( + options.get(RocketMQGrpcConnectorOptions.CONSUMER_GROUP), + "'%s' is required for the RocketMQ gRPC table source.", + RocketMQGrpcConnectorOptions.CONSUMER_GROUP.key()); + + final DataType physicalDataType = context.getPhysicalRowDataType(); + return new RocketMQGrpcDynamicTableSource( + buildConfiguration(options), + options.get(RocketMQGrpcConnectorOptions.TOPIC), + Boundedness.CONTINUOUS_UNBOUNDED, + physicalDataType, + decodingFormat); + } + + @Override + public DynamicTableSink createDynamicTableSink(Context context) { + final FactoryUtil.TableFactoryHelper helper = + FactoryUtil.createTableFactoryHelper(this, context); + final EncodingFormat> encodingFormat = + helper.discoverEncodingFormat(SerializationFormatFactory.class, FactoryUtil.FORMAT); + helper.validate(); + + final ReadableConfig options = helper.getOptions(); + final String liteTopic = options.get(RocketMQGrpcConnectorOptions.LITE_TOPIC); + checkNotNull( + liteTopic, + "'%s' is required for the RocketMQ gRPC table sink.", + RocketMQGrpcConnectorOptions.LITE_TOPIC.key()); + final DataType physicalDataType = context.getPhysicalRowDataType(); + return new RocketMQGrpcDynamicTableSink( + buildConfiguration(options), + options.get(RocketMQGrpcConnectorOptions.TOPIC), + liteTopic, + physicalDataType, + encodingFormat); + } + + /** + * Translates the user-facing SQL DDL options ({@link RocketMQGrpcConnectorOptions}) into a + * {@link Configuration} keyed by the programmatic/SDK options that the source and sink builders + * and their runtime read. + */ + private static Configuration buildConfiguration(ReadableConfig options) { + final Configuration configuration = new Configuration(); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.ENDPOINTS, + RocketMQGrpcOptions.ENDPOINTS); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.NAMESPACE, + RocketMQGrpcOptions.NAMESPACE); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.ACCESS_KEY, + RocketMQGrpcOptions.ACCESS_KEY); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.SECRET_KEY, + RocketMQGrpcOptions.SECRET_KEY); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.CREDENTIALS_RESOLVER_CLASS, + RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.TLS_ENABLED, + RocketMQGrpcOptions.TLS_ENABLED); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.REQUEST_TIMEOUT, + RocketMQGrpcOptions.REQUEST_TIMEOUT); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.TOPIC, + RocketMQGrpcSinkOptions.TOPIC); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.LITE_TOPIC, + RocketMQGrpcSinkOptions.LITE_TOPIC); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.CONSUMER_GROUP, + RocketMQGrpcSourceOptions.CONSUMER_GROUP); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.AWAIT_DURATION, + RocketMQGrpcSourceOptions.AWAIT_DURATION); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.INVISIBLE_DURATION, + RocketMQGrpcSourceOptions.INVISIBLE_DURATION); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.MAX_MESSAGE_NUM, + RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.FETCH_CONCURRENCY, + RocketMQGrpcSourceOptions.FETCH_CONCURRENCY); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.RENEWAL_POLICY_CLASS, + RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.RENEWAL_AHEAD_TIME, + RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME); + copyMapped( + options, + configuration, + RocketMQGrpcConnectorOptions.MAX_ATTEMPTS, + RocketMQGrpcSinkOptions.MAX_ATTEMPTS); + return configuration; + } + + private static void copyMapped( + ReadableConfig source, + Configuration target, + ConfigOption sourceOption, + ConfigOption targetOption) { + final T value = source.get(sourceOption); + if (value != null) { + target.set(targetOption, value); + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java new file mode 100644 index 0000000..af35c2e --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.sink.SinkV2Provider; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; + +import java.util.Objects; + +/** A {@link DynamicTableSink} for the RocketMQ gRPC connector (at-least-once). */ +@Internal +public class RocketMQGrpcDynamicTableSink implements DynamicTableSink { + + private final Configuration configuration; + private final String topic; + private final String liteTopic; + private final DataType physicalDataType; + private final EncodingFormat> encodingFormat; + + public RocketMQGrpcDynamicTableSink( + Configuration configuration, + String topic, + String liteTopic, + DataType physicalDataType, + EncodingFormat> encodingFormat) { + this.configuration = configuration; + this.topic = topic; + this.liteTopic = liteTopic; + this.physicalDataType = physicalDataType; + this.encodingFormat = encodingFormat; + } + + @Override + public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { + return ChangelogMode.insertOnly(); + } + + @Override + public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { + final SerializationSchema valueSerialization = + encodingFormat.createRuntimeEncoder(context, physicalDataType); + final RocketMQGrpcRowDataConverter converter = + RocketMQGrpcRowDataConverter.forSink(topic, liteTopic, valueSerialization); + + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setConfig(configuration) + .setSerializer(converter) + .build(); + + return SinkV2Provider.of(sink); + } + + @Override + public DynamicTableSink copy() { + return new RocketMQGrpcDynamicTableSink( + configuration, topic, liteTopic, physicalDataType, encodingFormat); + } + + @Override + public String asSummaryString() { + return "RocketMQGrpc"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final RocketMQGrpcDynamicTableSink that = (RocketMQGrpcDynamicTableSink) o; + return Objects.equals(configuration, that.configuration) + && Objects.equals(topic, that.topic) + && Objects.equals(liteTopic, that.liteTopic) + && Objects.equals(physicalDataType, that.physicalDataType) + && Objects.equals(encodingFormat, that.encodingFormat); + } + + @Override + public int hashCode() { + return Objects.hash(configuration, topic, liteTopic, physicalDataType, encodingFormat); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java new file mode 100644 index 0000000..5cb7f54 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.ProviderContext; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.source.DataStreamScanProvider; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.connector.source.ScanTableSource; +import org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.DataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** A {@link ScanTableSource} for the RocketMQ gRPC connector. */ +@Internal +public class RocketMQGrpcDynamicTableSource implements ScanTableSource, SupportsReadingMetadata { + + private final Configuration configuration; + private final String topic; + private final Boundedness boundedness; + private final DataType physicalDataType; + private final DecodingFormat> decodingFormat; + + /** The metadata keys applied by {@link #applyReadableMetadata(List, DataType)}. */ + private List metadataKeys; + + /** The produced data type including the appended metadata columns. */ + private DataType producedDataType; + + public RocketMQGrpcDynamicTableSource( + Configuration configuration, + String topic, + Boundedness boundedness, + DataType physicalDataType, + DecodingFormat> decodingFormat) { + this.configuration = configuration; + this.topic = topic; + this.boundedness = boundedness; + this.physicalDataType = physicalDataType; + this.decodingFormat = decodingFormat; + this.metadataKeys = Collections.emptyList(); + this.producedDataType = physicalDataType; + } + + @Override + public ChangelogMode getChangelogMode() { + return decodingFormat.getChangelogMode(); + } + + @Override + public Map listReadableMetadata() { + final Map metadataMap = new LinkedHashMap<>(); + Stream.of(RocketMQGrpcReadableMetadata.values()) + .forEach(metadata -> metadataMap.put(metadata.getKey(), metadata.getDataType())); + return metadataMap; + } + + @Override + public void applyReadableMetadata(List metadataKeys, DataType producedDataType) { + this.metadataKeys = metadataKeys; + this.producedDataType = producedDataType; + } + + @Override + public ScanRuntimeProvider getScanRuntimeProvider(ScanContext context) { + final DeserializationSchema valueDeserialization = + decodingFormat.createRuntimeDecoder(context, physicalDataType); + final TypeInformation producedType = + context.createTypeInformation(producedDataType); + final List metadataConverters = + metadataKeys.stream() + .map( + key -> + Stream.of(RocketMQGrpcReadableMetadata.values()) + .filter(metadata -> metadata.getKey().equals(key)) + .findFirst() + .orElseThrow(IllegalStateException::new) + .getConverter()) + .collect(Collectors.toList()); + final RocketMQGrpcRowDataConverter converter = + RocketMQGrpcRowDataConverter.forSource( + valueDeserialization, metadataConverters, producedType); + + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setConfig(configuration) + .setMainTopic(topic) + .setBoundedness(boundedness) + .setDeserializer(converter) + .build(); + + // The source produces AckableMessage; the SQL/Table path does not perform + // downstream acknowledgement, so the receipt handle is stripped here and only the RowData + // value is forwarded. + return new DataStreamScanProvider() { + @Override + public DataStream produceDataStream( + ProviderContext providerContext, StreamExecutionEnvironment execEnv) { + final DataStreamSource> sourceStream = + execEnv.fromSource( + source, WatermarkStrategy.noWatermarks(), "RocketMQGrpcSource"); + final SingleOutputStreamOperator valueStream = + sourceStream.map(AckableMessage::getValue).returns(producedType); + providerContext.generateUid("rocketmq-grpc-source").ifPresent(valueStream::uid); + return valueStream; + } + + @Override + public boolean isBounded() { + return Boundedness.BOUNDED == boundedness; + } + }; + } + + @Override + public DynamicTableSource copy() { + final RocketMQGrpcDynamicTableSource copy = + new RocketMQGrpcDynamicTableSource( + configuration, topic, boundedness, physicalDataType, decodingFormat); + copy.metadataKeys = new ArrayList<>(metadataKeys); + copy.producedDataType = producedDataType; + return copy; + } + + @Override + public String asSummaryString() { + return "RocketMQGrpc"; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final RocketMQGrpcDynamicTableSource that = (RocketMQGrpcDynamicTableSource) o; + return Objects.equals(configuration, that.configuration) + && Objects.equals(topic, that.topic) + && boundedness == that.boundedness + && Objects.equals(physicalDataType, that.physicalDataType) + && Objects.equals(decodingFormat, that.decodingFormat) + && Objects.equals(metadataKeys, that.metadataKeys) + && Objects.equals(producedDataType, that.producedDataType); + } + + @Override + public int hashCode() { + return Objects.hash( + configuration, + topic, + boundedness, + physicalDataType, + decodingFormat, + metadataKeys, + producedDataType); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java new file mode 100644 index 0000000..a2019c7 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericMapData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.DataType; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * The readable metadata of a RocketMQ gRPC message that a SQL table source can expose as {@code + * METADATA VIRTUAL} columns. + */ +@Internal +public enum RocketMQGrpcReadableMetadata { + MESSAGE_ID( + "message_id", + DataTypes.STRING().nullable(), + messageView -> StringData.fromString(messageView.getMessageId())), + + TOPIC( + "topic", + DataTypes.STRING().nullable(), + messageView -> StringData.fromString(messageView.getTopic())), + + TAG( + "tag", + DataTypes.STRING().nullable(), + messageView -> + messageView.getTag() == null + ? null + : StringData.fromString(messageView.getTag())), + + KEYS( + "keys", + DataTypes.ARRAY(DataTypes.STRING()).nullable(), + messageView -> + new GenericArrayData( + messageView.getKeys().stream().map(StringData::fromString).toArray())), + + DELIVERY_ATTEMPT( + "delivery_attempt", DataTypes.INT().nullable(), MessageView::getDeliveryAttempt), + + BORN_TIMESTAMP( + "born_timestamp", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).nullable(), + messageView -> TimestampData.fromEpochMillis(messageView.getEventTime())), + + PROPERTIES( + "properties", + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()).nullable(), + messageView -> { + final Map map = new HashMap<>(); + messageView + .getProperties() + .forEach( + (key, value) -> + map.put( + StringData.fromString(key), + StringData.fromString(value))); + return new GenericMapData(map); + }); + + private final String key; + private final DataType dataType; + private final MetadataConverter converter; + + RocketMQGrpcReadableMetadata(String key, DataType dataType, MetadataConverter converter) { + this.key = key; + this.dataType = dataType; + this.converter = converter; + } + + public String getKey() { + return key; + } + + public DataType getDataType() { + return dataType; + } + + public MetadataConverter getConverter() { + return converter; + } + + /** Converts a message attribute into the internal data structure of the metadata column. */ + @FunctionalInterface + public interface MetadataConverter extends Serializable { + @Nullable + Object read(MessageView messageView); + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java new file mode 100644 index 0000000..5ce50a4 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema; +import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.util.Collector; + +import org.apache.rocketmq.client.apis.message.Message; +import org.apache.rocketmq.client.apis.message.MessageBuilder; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Converts between RocketMQ gRPC {@link Message}/{@link MessageView} and Flink {@link RowData}. The + * value part of the record is (de)serialized with the format-provided value schema; the message + * body carries the encoded value. On the source side, requested metadata columns are appended after + * the physical columns produced by the value format. + */ +@Internal +public class RocketMQGrpcRowDataConverter + implements RocketMQGrpcDeserializationSchema, + RocketMQGrpcSerializationSchema { + + private static final long serialVersionUID = 1L; + + @Nullable private final String topic; + @Nullable private final String liteTopic; + @Nullable private final DeserializationSchema valueDeserialization; + @Nullable private final SerializationSchema valueSerialization; + private final List metadataConverters; + private final TypeInformation producedType; + + private transient MetadataAppendingCollector metadataAppendingCollector; + + private RocketMQGrpcRowDataConverter( + @Nullable String topic, + @Nullable String liteTopic, + @Nullable DeserializationSchema valueDeserialization, + @Nullable SerializationSchema valueSerialization, + List metadataConverters, + @Nullable TypeInformation producedType) { + this.topic = topic; + this.liteTopic = liteTopic; + this.valueDeserialization = valueDeserialization; + this.valueSerialization = valueSerialization; + this.metadataConverters = metadataConverters; + this.producedType = producedType; + } + + /** Create a converter used by the table source to decode a {@link MessageView}. */ + public static RocketMQGrpcRowDataConverter forSource( + DeserializationSchema valueDeserialization, + List metadataConverters, + TypeInformation producedType) { + return new RocketMQGrpcRowDataConverter( + null, + null, + checkNotNull(valueDeserialization), + null, + checkNotNull(metadataConverters), + checkNotNull(producedType)); + } + + /** Create a converter used by the table sink to encode a {@link RowData}. */ + public static RocketMQGrpcRowDataConverter forSink( + String topic, String liteTopic, SerializationSchema valueSerialization) { + return new RocketMQGrpcRowDataConverter( + checkNotNull(topic), + checkNotNull(liteTopic), + null, + checkNotNull(valueSerialization), + Collections.emptyList(), + null); + } + + @Override + public void open(DeserializationSchema.InitializationContext context) throws Exception { + checkNotNull(valueDeserialization).open(context); + this.metadataAppendingCollector = new MetadataAppendingCollector(metadataConverters); + } + + @Override + public void open(SerializationSchema.InitializationContext context) throws Exception { + checkNotNull(valueSerialization).open(context); + } + + @Override + public void deserialize(MessageView messageView, Collector out) throws IOException { + if (metadataConverters.isEmpty()) { + checkNotNull(valueDeserialization).deserialize(messageView.getBody(), out); + return; + } + metadataAppendingCollector.reset(messageView, out); + checkNotNull(valueDeserialization) + .deserialize(messageView.getBody(), metadataAppendingCollector); + } + + @Override + public Message serialize(RowData element, MessageBuilder messageBuilder, Long timestamp) { + final byte[] body = checkNotNull(valueSerialization).serialize(element); + return messageBuilder.setTopic(topic).setLiteTopic(liteTopic).setBody(body).build(); + } + + @Override + public TypeInformation getProducedType() { + return checkNotNull(producedType); + } + + /** + * Appends the requested metadata columns after the physical columns of each row produced by the + * value format. Like other connectors' metadata support, it relies on the format's runtime + * decoder producing {@link GenericRowData}. + */ + private static final class MetadataAppendingCollector implements Collector { + + private final List metadataConverters; + + private MessageView messageView; + private Collector out; + + private MetadataAppendingCollector( + List metadataConverters) { + this.metadataConverters = metadataConverters; + } + + private void reset(MessageView messageView, Collector out) { + this.messageView = messageView; + this.out = out; + } + + @Override + public void collect(RowData physicalRow) { + final GenericRowData physical = (GenericRowData) physicalRow; + final int physicalArity = physical.getArity(); + final GenericRowData produced = + new GenericRowData( + physical.getRowKind(), physicalArity + metadataConverters.size()); + for (int pos = 0; pos < physicalArity; pos++) { + produced.setField(pos, physical.getField(pos)); + } + for (int pos = 0; pos < metadataConverters.size(); pos++) { + produced.setField( + physicalArity + pos, metadataConverters.get(pos).read(messageView)); + } + out.collect(produced); + } + + @Override + public void close() {} + } +} diff --git a/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 0000000..7e42fa2 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.connector.rocketmq.grpc.table.RocketMQGrpcDynamicTableFactory diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java new file mode 100644 index 0000000..9ce6d06 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc; + +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.serialization.SimpleStringSchema; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQThrottleProcessFunction; +import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.test.util.AbstractTestBase; + +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientServiceProvider; +import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider; +import org.apache.rocketmq.client.apis.message.Message; +import org.apache.rocketmq.client.apis.producer.Producer; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for the RocketMQ gRPC connector. Requires a running RocketMQ 5.x instance with + * gRPC proxy enabled. Connection details are read from environment variables, so this test is + * {@link Disabled} by default and must be run manually after exporting {@code + * ROCKETMQ_GRPC_ENDPOINTS}, {@code ROCKETMQ_GRPC_ACCESS_KEY} and {@code ROCKETMQ_GRPC_SECRET_KEY}. + */ +@Disabled( + "Requires a running RocketMQ 5.x instance with gRPC proxy. Set the ROCKETMQ_GRPC_* " + + "environment variables and run manually.") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class RocketMQGrpcITCase extends AbstractTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcITCase.class); + + // -- Instance configuration (read from environment; never hard-code credentials) -- + private static final String ENDPOINTS = System.getenv("ROCKETMQ_GRPC_ENDPOINTS"); + private static final String ACCESS_KEY = System.getenv("ROCKETMQ_GRPC_ACCESS_KEY"); + private static final String SECRET_KEY = System.getenv("ROCKETMQ_GRPC_SECRET_KEY"); + + // -- Topics and groups -- + private static final String SINK_TOPIC = "flink-sink-1"; + private static final String SINK_LITE_TOPIC = "flink-sink-1-lite"; + private static final String SOURCE_TOPIC = "flink-source"; + private static final String E2E_SOURCE_TOPIC = "flink-source-1"; + private static final String E2E_SINK_TOPIC = "flink-sink-2"; + private static final String E2E_SINK_LITE_TOPIC = "flink-sink-2-lite"; + private static final String CONSUMER_GROUP = "GID-flink"; + + private static final int NUM_MESSAGES = 50; + private static final long COLLECT_TIMEOUT_SECONDS = 120; + + /** + * Each test run uses a unique prefix so that messages from previous runs can be distinguished + * from the current run's messages. + */ + private final String runId = UUID.randomUUID().toString().substring(0, 8); + + // ----------------------------------------------------------------------- + // Test 1: Sink writes messages + // ----------------------------------------------------------------------- + + @Test + @Order(1) + void testSinkWritesMessages() throws Exception { + final String prefix = "sink-" + runId + "-"; + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + final List messages = new ArrayList<>(NUM_MESSAGES); + for (int i = 0; i < NUM_MESSAGES; i++) { + messages.add(prefix + i); + } + + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setEndpoints(ENDPOINTS) + .setTopic(SINK_TOPIC) + .setLiteTopic(SINK_LITE_TOPIC) + .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY) + .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY) + .setValueOnlySerializer(new SimpleStringSchema()) + .build(); + + env.fromData(messages).sinkTo(sink); + env.execute("RocketMQ gRPC Sink Test"); + LOG.info("Successfully wrote {} messages to topic {}", NUM_MESSAGES, SINK_TOPIC); + } + + // ----------------------------------------------------------------------- + // Test 2: Source reads messages + // ----------------------------------------------------------------------- + + @Test + @Order(2) + void testSourceReadsMessages() throws Exception { + final String prefix = "src-" + runId + "-"; + + // Seed messages using the RocketMQ Producer API directly. + seedMessagesViaProducer(SOURCE_TOPIC, NUM_MESSAGES, prefix); + + // Read them back using the Flink source. + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setEndpoints(ENDPOINTS) + .setConsumerGroup(CONSUMER_GROUP) + .setMainTopic(SOURCE_TOPIC) + .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY) + .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY) + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); + + final DataStream stream = + env.fromSource(source, WatermarkStrategy.noWatermarks(), "RocketMQ gRPC Source") + .process( + new RocketMQThrottleProcessFunction( + ackConfiguration(), value -> Optional.empty())) + .returns(String.class); + stream.addSink(new CollectingSinkFunction()); + + // Collect messages, filtering by the current run's prefix. + final List results = + CollectingSinkFunction.startAndCollect(env, NUM_MESSAGES, prefix); + LOG.info("Collected {} messages (prefix={}) from source", results.size(), prefix); + + assertThat(results).hasSize(NUM_MESSAGES); + for (int i = 0; i < NUM_MESSAGES; i++) { + assertThat(results).contains(prefix + i); + } + } + + // ----------------------------------------------------------------------- + // Test 3: End-to-end pipeline (Source -> map -> Sink) + // ----------------------------------------------------------------------- + + @Test + @Order(3) + void testEndToEndPipeline() throws Exception { + final String prefix = "e2e-" + runId + "-"; + + // Seed messages into the e2e source topic. + seedMessagesViaProducer(E2E_SOURCE_TOPIC, NUM_MESSAGES, prefix); + + // Build pipeline: Source reads from flink-source-1 -> uppercase -> Sink to flink-sink-2. + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setEndpoints(ENDPOINTS) + .setConsumerGroup(CONSUMER_GROUP) + .setMainTopic(E2E_SOURCE_TOPIC) + .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY) + .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY) + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); + + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setEndpoints(ENDPOINTS) + .setTopic(E2E_SINK_TOPIC) + .setLiteTopic(E2E_SINK_LITE_TOPIC) + .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY) + .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY) + .setValueOnlySerializer(new SimpleStringSchema()) + .build(); + + final DataStream stream = + env.fromSource(source, WatermarkStrategy.noWatermarks(), "RocketMQ gRPC Source") + .process( + new RocketMQThrottleProcessFunction( + ackConfiguration(), value -> Optional.empty())) + .returns(String.class); + stream.map(String::toUpperCase).sinkTo(sink); + + // Run the pipeline in a background thread. + final Thread jobThread = + new Thread( + () -> { + try { + env.execute("E2E Pipeline"); + } catch (Exception e) { + LOG.info("E2E pipeline ended: {}", e.getMessage()); + } + }); + jobThread.setDaemon(true); + jobThread.start(); + + // Wait for the pipeline to process. + Thread.sleep(30_000); + + env.close(); + jobThread.interrupt(); + jobThread.join(10_000); + + // Verify by reading from the sink topic, filtering by the uppercased prefix. + final String upperPrefix = prefix.toUpperCase(); + final List results = + readMessagesViaConsumer(E2E_SINK_TOPIC, NUM_MESSAGES, upperPrefix); + LOG.info( + "Read {} messages from sink topic {} (prefix={})", + results.size(), + E2E_SINK_TOPIC, + upperPrefix); + + assertThat(results).hasSize(NUM_MESSAGES); + for (int i = 0; i < NUM_MESSAGES; i++) { + assertThat(results).contains(upperPrefix + i); + } + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + /** + * Build a client {@link Configuration} for the downstream ack operators; credentials come from + * the environment and never travel in the data stream. + */ + private Configuration ackConfiguration() { + final Configuration config = new Configuration(); + config.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS); + config.set(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY); + config.set(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY); + return config; + } + + /** Seed messages into the given topic using the RocketMQ gRPC Producer API directly. */ + private void seedMessagesViaProducer(String topic, int count, String prefix) throws Exception { + final ClientServiceProvider provider = ClientServiceProvider.loadService(); + final ClientConfiguration clientConfig = + ClientConfiguration.newBuilder() + .setEndpoints(ENDPOINTS) + .setCredentialProvider( + new StaticSessionCredentialsProvider(ACCESS_KEY, SECRET_KEY)) + .build(); + + try (Producer producer = + provider.newProducerBuilder() + .setClientConfiguration(clientConfig) + .setTopics(topic) + .build()) { + for (int i = 0; i < count; i++) { + final Message message = + provider.newMessageBuilder() + .setTopic(topic) + .setBody((prefix + i).getBytes(StandardCharsets.UTF_8)) + .build(); + producer.send(message); + } + } + LOG.info("Seeded {} messages into topic {} (prefix={})", count, topic, prefix); + + // Give the broker a moment to make messages available for consumers. + Thread.sleep(3000); + } + + /** + * Read messages from a topic using the RocketMQ gRPC SimpleConsumer API directly, filtering by + * the given prefix. Used to verify end-to-end pipeline output. + */ + private List readMessagesViaConsumer(String topic, int expectedCount, String prefix) + throws Exception { + final ClientServiceProvider provider = ClientServiceProvider.loadService(); + final ClientConfiguration clientConfig = + ClientConfiguration.newBuilder() + .setEndpoints(ENDPOINTS) + .setCredentialProvider( + new StaticSessionCredentialsProvider(ACCESS_KEY, SECRET_KEY)) + .build(); + + final List results = new ArrayList<>(); + final org.apache.rocketmq.client.apis.consumer.FilterExpression filterExpression = + new org.apache.rocketmq.client.apis.consumer.FilterExpression("*"); + final java.util.Map + subscriptions = Collections.singletonMap(topic, filterExpression); + + try (org.apache.rocketmq.client.apis.consumer.SimpleConsumer consumer = + provider.newSimpleConsumerBuilder() + .setClientConfiguration(clientConfig) + .setConsumerGroup(CONSUMER_GROUP) + .setAwaitDuration(Duration.ofSeconds(15)) + .setSubscriptionExpressions(subscriptions) + .build()) { + + final long deadline = System.currentTimeMillis() + COLLECT_TIMEOUT_SECONDS * 1000; + while (results.size() < expectedCount && System.currentTimeMillis() < deadline) { + final List messages = + consumer.receive(32, Duration.ofSeconds(15)); + for (org.apache.rocketmq.client.apis.message.MessageView msg : messages) { + final ByteBuffer buf = msg.getBody(); + final byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + final String body = new String(bytes, StandardCharsets.UTF_8); + consumer.ack(msg); + if (body.startsWith(prefix)) { + results.add(body); + } + } + } + } + return results; + } + + /** + * A collecting sink function that gathers emitted records into a thread-safe queue. Used to + * retrieve results from an unbounded source by cancelling the job once enough records matching + * the prefix are collected. + */ + @SuppressWarnings("deprecation") + private static class CollectingSinkFunction + extends org.apache.flink.streaming.api.functions.sink.RichSinkFunction { + + private static final long serialVersionUID = 1L; + + private static final LinkedBlockingQueue QUEUE = new LinkedBlockingQueue<>(); + + @Override + public void invoke(String value, Context context) { + QUEUE.add(value); + } + + /** + * Execute the given environment and collect at least {@code expectedCount} records matching + * the prefix. The job is cancelled once enough records are gathered or after a timeout. + */ + static List startAndCollect( + StreamExecutionEnvironment env, int expectedCount, String prefix) throws Exception { + QUEUE.clear(); + + final Thread jobThread = + new Thread( + () -> { + try { + env.execute("Collecting Job"); + } catch (Exception e) { + LOG.info("Collecting job ended: {}", e.getMessage()); + } + }); + jobThread.setDaemon(true); + jobThread.start(); + + // Collect messages matching the prefix until we have enough or timeout. + final List results = new ArrayList<>(); + final long deadline = System.currentTimeMillis() + COLLECT_TIMEOUT_SECONDS * 1000; + while (results.size() < expectedCount && System.currentTimeMillis() < deadline) { + final String item = QUEUE.poll(500, TimeUnit.MILLISECONDS); + if (item != null && item.startsWith(prefix)) { + results.add(item); + } + } + + env.close(); + jobThread.interrupt(); + jobThread.join(10_000); + + // Drain remaining matching items. + final List remaining = new ArrayList<>(); + QUEUE.drainTo(remaining); + for (String item : remaining) { + if (item.startsWith(prefix)) { + results.add(item); + } + } + return results; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java new file mode 100644 index 0000000..bea6673 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc; + +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.serialization.SimpleStringSchema; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQThrottleProcessFunction; +import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink; +import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicy; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource; +import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Manual end-to-end verification of the RocketMQ gRPC connector against a live cluster (public + * proxy SLB). Run with {@code mvn test-compile exec:java}. Phases: + * + *

    + *
  1. Sink: publish lite messages to the parent topic. + *
  2. Source + throttle: consume them; the throttle policy defers selected messages once by 10s + * (changeInvisibleDuration), verifying the throttle really delays re-delivery. + *
  3. Renewal: a slow pipeline keeps messages queued in the split reader beyond the renewal + * trigger point, verifying the invisible-duration renewal fires and prevents duplicates. + *
+ */ +public class RocketMQGrpcLiteE2EVerify { + + private static final String ENDPOINTS = + System.getProperty("rocketmq.endpoints", "47.98.110.140:8081"); + private static final String MAIN_TOPIC = "LiteTest"; + private static final String CONSUMER_GROUP = "LiteSimpleConsumerGroup"; + + private static final String RUN_ID = UUID.randomUUID().toString().substring(0, 8); + + // value -> arrival timestamps (ms) + private static final Map> ARRIVALS = new ConcurrentHashMap<>(); + // values the throttle policy has already deferred once + private static final Map THROTTLED_ONCE = new ConcurrentHashMap<>(); + // renewal invocations observed by the renewal policy + private static final AtomicInteger RENEWALS = new AtomicInteger(); + + public static void main(String[] args) { + // Run each phase in its own JVM: a finished local job cannot be cancelled cleanly, and a + // lingering same-group consumer would steal the next phase's messages. + final String phase = args.length > 0 ? args[0] : "phase1"; + System.out.println("=== RocketMQGrpcLiteE2EVerify " + phase + " runId=" + RUN_ID + " ==="); + try { + if ("phase2".equals(phase)) { + phase2Renewal(); + } else { + phase1SinkAndThrottle(); + } + System.out.println("=== " + phase + " PASSED ==="); + System.exit(0); + } catch (Throwable t) { + t.printStackTrace(); + System.exit(1); + } + } + + // ------------------------------------------------------------------ + // Phase 1: sink writes; source + throttle policy verification + // ------------------------------------------------------------------ + + private static void phase1SinkAndThrottle() throws Exception { + final String normalPrefix = "n-" + RUN_ID + "-"; + final String throttlePrefix = "thr-" + RUN_ID + "-"; + final List messages = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + messages.add(normalPrefix + i); + } + for (int i = 0; i < 5; i++) { + messages.add(throttlePrefix + i); + } + runSinkJob(messages, "flink-e2e-lite"); + System.out.println("[phase1] sink wrote " + messages.size() + " lite messages"); + + final StreamExecutionEnvironment env = localEnv(); + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setEndpoints(ENDPOINTS) + .setConsumerGroup(CONSUMER_GROUP) + .setMainTopic(MAIN_TOPIC) + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); + + final Configuration ackConfig = new Configuration(); + ackConfig.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS); + + final String tp = throttlePrefix; + final DataStream stream = + env.fromSource(source, WatermarkStrategy.noWatermarks(), "grpc-source") + .process( + new RocketMQThrottleProcessFunction( + ackConfig, + value -> { + if (value.startsWith(tp) + && THROTTLED_ONCE.putIfAbsent(value, true) + == null) { + return Optional.of(Duration.ofSeconds(10)); + } + return Optional.empty(); + })) + .returns(String.class); + stream.map(RocketMQGrpcLiteE2EVerify::record); + + ARRIVALS.clear(); + runJobUntil( + env, + 180, + () -> { + for (int i = 0; i < 10; i++) { + if (count(normalPrefix + i) < 1) { + return false; + } + } + for (int i = 0; i < 5; i++) { + if (count(tp + i) < 2) { + return false; + } + } + return true; + }); + + for (int i = 0; i < 5; i++) { + final List times = ARRIVALS.get(tp + i); + check(times != null && times.size() >= 2, "throttled message not redelivered: " + i); + final long gapMs = times.get(1) - times.get(0); + check(gapMs >= 8000, "throttle gap too small: " + gapMs + "ms for " + tp + i); + System.out.println("[phase1] " + tp + i + " redelivered after " + gapMs + " ms"); + } + System.out.println("[phase1] PASS: send/receive + throttle (defer via invisible duration)"); + } + + // ------------------------------------------------------------------ + // Phase 2: invisible-duration renewal under backpressure + // ------------------------------------------------------------------ + + private static void phase2Renewal() throws Exception { + final String prefix = "rnw-" + RUN_ID + "-"; + final int total = 60; + // Spread across lite topics: the broker paces event dispatch per lite topic when un-acked + // messages accumulate, which would starve the backpressure scenario otherwise. + final StreamExecutionEnvironment sinkEnv = localEnv(); + for (int t = 0; t < 20; t++) { + final List messages = new ArrayList<>(); + for (int i = t; i < total; i += 20) { + messages.add(prefix + i); + } + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setEndpoints(ENDPOINTS) + .setTopic(MAIN_TOPIC) + .setLiteTopic("flink-renew-lite-" + t) + .setValueOnlySerializer(new SimpleStringSchema()) + .build(); + sinkEnv.fromData(messages).sinkTo(sink); + } + sinkEnv.execute("lite-sink-renew"); + System.out.println("[phase2] sink wrote " + total + " lite messages"); + + final StreamExecutionEnvironment env = localEnv(); + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setEndpoints(ENDPOINTS) + .setConsumerGroup(CONSUMER_GROUP) + .setMainTopic(MAIN_TOPIC) + .setConfig( + RocketMQGrpcSourceOptions.INVISIBLE_DURATION, + Duration.ofSeconds(20)) + // Small batches so the downstream sleep keeps messages queued inside the + // split reader long enough to reach the renewal trigger point (20s - 18s). + .setConfig(RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM, 4) + .setFetchConcurrency(4) + .setRenewalPolicyClass(CountingRenewalPolicy.class.getName()) + .setRenewalAheadTime(Duration.ofSeconds(18)) + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); + + final Configuration ackConfig = new Configuration(); + ackConfig.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS); + + final DataStream stream = + env.fromSource(source, WatermarkStrategy.noWatermarks(), "grpc-source") + .process( + new RocketMQThrottleProcessFunction( + ackConfig, value -> Optional.empty())) + .returns(String.class); + stream.map( + value -> { + Thread.sleep(300); // backpressure so messages linger in the split reader + return record(value); + }); + + ARRIVALS.clear(); + RENEWALS.set(0); + runJobUntil( + env, + 240, + () -> { + for (int i = 0; i < total; i++) { + if (count(prefix + i) < 1) { + return false; + } + } + return true; + }); + + int duplicates = 0; + for (int i = 0; i < total; i++) { + if (count(prefix + i) > 1) { + duplicates++; + } + } + System.out.println( + "[phase2] renewals=" + RENEWALS.get() + ", duplicates=" + duplicates + "/" + total); + check(RENEWALS.get() > 0, "renewal policy never triggered"); + check(duplicates == 0, "unexpected duplicates: " + duplicates); + System.out.println("[phase2] PASS: invisible-duration renewal fired, no redelivery"); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + /** Renewal policy that counts invocations and extends by another 20s. */ + public static class CountingRenewalPolicy implements InvisibleDurationRenewalPolicy { + private static final long serialVersionUID = 1L; + + @Override + @Nullable + public Duration renew(MessageView messageView, int renewalCount) { + RENEWALS.incrementAndGet(); + return Duration.ofSeconds(20); + } + } + + private static void runSinkJob(List messages, String liteTopic) throws Exception { + final StreamExecutionEnvironment env = localEnv(); + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setEndpoints(ENDPOINTS) + .setTopic(MAIN_TOPIC) + .setLiteTopic(liteTopic) + .setValueOnlySerializer(new SimpleStringSchema()) + .build(); + env.fromData(messages).sinkTo(sink); + env.execute("lite-sink-" + liteTopic); + } + + private static StreamExecutionEnvironment localEnv() { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1); + env.setParallelism(1); + return env; + } + + private static String record(String value) { + ARRIVALS.computeIfAbsent(value, k -> new CopyOnWriteArrayList<>()) + .add(System.currentTimeMillis()); + return value; + } + + private static int count(String value) { + final List times = ARRIVALS.get(value); + return times == null ? 0 : times.size(); + } + + private static void runJobUntil( + StreamExecutionEnvironment env, int timeoutSeconds, Condition condition) + throws Exception { + final Thread jobThread = + new Thread( + () -> { + try { + env.execute("verify-job"); + } catch (Exception e) { + System.out.println("job ended: " + e.getMessage()); + } + }); + jobThread.setDaemon(true); + jobThread.start(); + + final long deadline = System.currentTimeMillis() + timeoutSeconds * 1000L; + boolean satisfied = false; + int ticks = 0; + while (System.currentTimeMillis() < deadline) { + if (condition.test()) { + satisfied = true; + break; + } + if (++ticks % 15 == 0) { + System.out.println( + "[progress] distinct=" + ARRIVALS.size() + " after " + ticks + "s"); + } + Thread.sleep(1000); + } + // Give in-flight acks a moment before tearing the job down. + Thread.sleep(3000); + env.close(); + jobThread.interrupt(); + jobThread.join(15000); + check(satisfied, "condition not satisfied within " + timeoutSeconds + "s"); + } + + private static void check(boolean condition, String message) { + if (!condition) { + throw new IllegalStateException("VERIFY FAILED: " + message); + } + } + + @FunctionalInterface + private interface Condition { + boolean test(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java new file mode 100644 index 0000000..c8b2204 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataOutputSerializer; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link AckableMessageTypeInfo.Serializer}. */ +class AckableMessageSerializerTest { + + private final AckableMessageTypeInfo.Serializer serializer = + new AckableMessageTypeInfo.Serializer<>(StringSerializer.INSTANCE); + + private static RocketMQReceiptHandle handle() { + return new RocketMQReceiptHandle( + "127.0.0.1:8080", "ns", "grp", "topic", "sub", "mid", "rh", 1); + } + + @Test + void roundTripWithValue() throws IOException { + final AckableMessage message = new AckableMessage<>("payload", handle()); + assertThat(roundTrip(message)).isEqualTo(message); + } + + @Test + void roundTripWithNullValue() throws IOException { + final AckableMessage message = new AckableMessage<>(null, handle()); + final AckableMessage deserialized = roundTrip(message); + assertThat(deserialized.getValue()).isNull(); + assertThat(deserialized).isEqualTo(message); + } + + @Test + void snapshotIsCompatibleWithSameValueSerializer() { + final TypeSerializerSnapshot> snapshot = + serializer.snapshotConfiguration(); + final AckableMessageTypeInfo.Serializer other = + new AckableMessageTypeInfo.Serializer<>(StringSerializer.INSTANCE); + final TypeSerializerSchemaCompatibility> compatibility = + other.snapshotConfiguration().resolveSchemaCompatibility(snapshot); + assertThat(compatibility.isCompatibleAsIs()).isTrue(); + } + + private AckableMessage roundTrip(AckableMessage message) throws IOException { + final DataOutputSerializer out = new DataOutputSerializer(64); + serializer.serialize(message, out); + final DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); + return serializer.deserialize(in); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java new file mode 100644 index 0000000..5c9dc5e --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for the reference counting behaviour of {@link RocketMQLiteAckClient#acquire}. */ +class RocketMQLiteAckClientTest { + + private static Configuration config(String endpoints) { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.ENDPOINTS, endpoints); + return configuration; + } + + @Test + void sameConfigurationSharesASingleReferenceCountedClient() { + final Configuration configuration = config("127.0.0.1:8080"); + assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero(); + + final RocketMQLiteAckClient first = RocketMQLiteAckClient.acquire(configuration); + final RocketMQLiteAckClient second = RocketMQLiteAckClient.acquire(configuration); + + // The same client is shared and the reference count is incremented per acquire. + assertThat(second).isSameAs(first); + assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isEqualTo(2); + + RocketMQLiteAckClient.release(configuration); + assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isEqualTo(1); + + RocketMQLiteAckClient.release(configuration); + assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero(); + } + + @Test + void differentConfigurationsGetDistinctClients() { + final Configuration a = config("host-a:8080"); + final Configuration b = config("host-b:8080"); + + final RocketMQLiteAckClient clientA = RocketMQLiteAckClient.acquire(a); + final RocketMQLiteAckClient clientB = RocketMQLiteAckClient.acquire(b); + try { + assertThat(clientA).isNotSameAs(clientB); + assertThat(RocketMQLiteAckClient.getReferenceCount(a)).isEqualTo(1); + assertThat(RocketMQLiteAckClient.getReferenceCount(b)).isEqualTo(1); + } finally { + RocketMQLiteAckClient.release(a); + RocketMQLiteAckClient.release(b); + } + + assertThat(RocketMQLiteAckClient.getReferenceCount(a)).isZero(); + assertThat(RocketMQLiteAckClient.getReferenceCount(b)).isZero(); + } + + @Test + void releaseWithoutAcquireIsANoOp() { + final Configuration configuration = config("no-acquire:8080"); + RocketMQLiteAckClient.release(configuration); + assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java new file mode 100644 index 0000000..7854705 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.rocketmq.client.apis.message.MessageId; +import org.apache.rocketmq.client.apis.message.MessageView; +import org.apache.rocketmq.client.java.message.MessageIdCodec; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.route.Endpoints; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RocketMQReceiptHandleCodec}. */ +class RocketMQReceiptHandleCodecTest { + + private static RocketMQReceiptHandle newHandle(String liteTopic) { + final String messageId = MessageIdCodec.getInstance().nextMessageId().toString(); + return new RocketMQReceiptHandle( + "127.0.0.1:8080", + "ns-1", + "GID-test", + "main-topic", + liteTopic, + messageId, + "receipt-handle-abc", + 2); + } + + @Test + void toAckableThenExtractRoundTripsAllFields() { + final RocketMQReceiptHandle original = newHandle("sub-topic-a"); + + final MessageView view = RocketMQReceiptHandleCodec.toAckable(original); + final RocketMQReceiptHandle roundTripped = + RocketMQReceiptHandleCodec.extract( + view, original.getNamespace(), original.getConsumerGroup()); + + assertThat(roundTripped).isEqualTo(original); + } + + @Test + void toAckableSupportsNullLiteTopic() { + final RocketMQReceiptHandle original = newHandle(null); + + final MessageView view = RocketMQReceiptHandleCodec.toAckable(original); + final RocketMQReceiptHandle roundTripped = + RocketMQReceiptHandleCodec.extract( + view, original.getNamespace(), original.getConsumerGroup()); + + assertThat(roundTripped.getLiteTopic()).isNull(); + assertThat(roundTripped).isEqualTo(original); + } + + @Test + void reconstructedViewCarriesParseableMessageIdAndEndpoints() { + final RocketMQReceiptHandle original = newHandle("sub-topic-a"); + + final MessageViewImpl view = + (MessageViewImpl) RocketMQReceiptHandleCodec.toAckable(original); + + // MessageId round-trips through the SDK codec. + final MessageId expectedId = MessageIdCodec.getInstance().decode(original.getMessageId()); + assertThat(view.getMessageId()).isEqualTo(expectedId); + + // Endpoints round-trip through the string form used by the handle. + final Endpoints endpoints = view.getEndpoints(); + assertThat(endpoints).isNotNull(); + assertThat(endpoints).isEqualTo(new Endpoints(original.getEndpoint())); + + assertThat(view.getTopic()).isEqualTo(original.getTopic()); + assertThat(view.getReceiptHandle()).isEqualTo(original.getReceiptHandle()); + assertThat(view.getDeliveryAttempt()).isEqualTo(original.getDeliveryAttempt()); + } + + @Test + void extractRejectsUnexpectedMessageViewType() { + final MessageView notImpl = + (MessageView) + Proxy.newProxyInstance( + MessageView.class.getClassLoader(), + new Class[] {MessageView.class}, + (proxy, method, args) -> { + throw new UnsupportedOperationException(); + }); + + assertThatThrownBy(() -> RocketMQReceiptHandleCodec.extract(notImpl, "ns", "grp")) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("internal message type"); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java new file mode 100644 index 0000000..4d7e56b --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataOutputSerializer; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RocketMQReceiptHandle.Serializer}. */ +class RocketMQReceiptHandleSerializerTest { + + private final RocketMQReceiptHandle.Serializer serializer = + RocketMQReceiptHandle.Serializer.INSTANCE; + + @Test + void roundTripWithLiteTopic() throws IOException { + final RocketMQReceiptHandle handle = + new RocketMQReceiptHandle( + "127.0.0.1:8080", "ns", "grp", "topic", "sub", "mid", "rh", 3); + + assertThat(roundTrip(handle)).isEqualTo(handle); + } + + @Test + void roundTripWithNullLiteTopic() throws IOException { + final RocketMQReceiptHandle handle = + new RocketMQReceiptHandle( + "127.0.0.1:8080", "", "grp", "topic", null, "mid", "rh", 0); + + final RocketMQReceiptHandle deserialized = roundTrip(handle); + assertThat(deserialized.getLiteTopic()).isNull(); + assertThat(deserialized).isEqualTo(handle); + } + + @Test + void snapshotIsCompatibleAsIs() { + final TypeSerializerSnapshot snapshot = + serializer.snapshotConfiguration(); + final TypeSerializerSchemaCompatibility compatibility = + serializer.snapshotConfiguration().resolveSchemaCompatibility(snapshot); + assertThat(compatibility.isCompatibleAsIs()).isTrue(); + } + + private RocketMQReceiptHandle roundTrip(RocketMQReceiptHandle handle) throws IOException { + final DataOutputSerializer out = new DataOutputSerializer(64); + serializer.serialize(handle, out); + final DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer()); + return serializer.deserialize(in); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java new file mode 100644 index 0000000..5469316 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.ack; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for the throttle branching and safeguards of {@link RocketMQThrottleProcessFunction}. */ +class RocketMQThrottleProcessFunctionTest { + + private static RocketMQReceiptHandle handle(String messageId, int deliveryAttempt) { + return new RocketMQReceiptHandle( + "127.0.0.1:8080", + "ns", + "grp", + "topic", + "sub", + messageId, + "rh-" + messageId, + deliveryAttempt); + } + + @Test + void emptyPolicyResultAcknowledges() throws Exception { + final RecordingThrottleFunction fn = + new RecordingThrottleFunction( + value -> Optional.empty(), 16, Duration.ofMinutes(30)); + final ListCollector out = new ListCollector(); + + fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out); + + assertThat(fn.acked).containsExactly("m1"); + assertThat(fn.changed).isEmpty(); + assertThat(out.values).containsExactly("v"); + } + + @Test + void policyDelayChangesInvisibleDuration() throws Exception { + final Duration delay = Duration.ofSeconds(10); + final RecordingThrottleFunction fn = + new RecordingThrottleFunction( + value -> Optional.of(delay), 16, Duration.ofMinutes(30)); + final ListCollector out = new ListCollector(); + + fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out); + + assertThat(fn.changed).containsEntry("m1", delay); + assertThat(fn.acked).isEmpty(); + // The value is always forwarded downstream. + assertThat(out.values).containsExactly("v"); + } + + @Test + void requestedDelayIsCappedToMaxInvisibleDuration() throws Exception { + final Duration cap = Duration.ofSeconds(30); + final RecordingThrottleFunction fn = + new RecordingThrottleFunction(value -> Optional.of(Duration.ofHours(1)), 16, cap); + final ListCollector out = new ListCollector(); + + fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out); + + assertThat(fn.changed).containsEntry("m1", cap); + } + + @Test + void deferSafeguardForcesAckOnceMaxDeliveryAttemptReached() throws Exception { + final int maxAttempt = 3; + final RecordingThrottleFunction fn = + new RecordingThrottleFunction( + value -> Optional.of(Duration.ofSeconds(10)), + maxAttempt, + Duration.ofMinutes(30)); + final ListCollector out = new ListCollector(); + + // Below the bound: still deferred. + fn.processElement(new AckableMessage<>("v", handle("m1", maxAttempt - 1)), null, out); + assertThat(fn.changed).containsKey("m1"); + assertThat(fn.acked).isEmpty(); + + // At the bound: the safeguard trips and the message is acknowledged instead. + fn.processElement(new AckableMessage<>("v", handle("m2", maxAttempt)), null, out); + assertThat(fn.acked).containsExactly("m2"); + } + + private static final class RecordingThrottleFunction + extends RocketMQThrottleProcessFunction { + + private static final long serialVersionUID = 1L; + + private final List acked = new ArrayList<>(); + private final java.util.Map changed = new java.util.LinkedHashMap<>(); + + RecordingThrottleFunction( + MessageThrottlePolicy policy, + int maxDeliveryAttempt, + Duration maxInvisible) { + super(new Configuration(), policy, maxDeliveryAttempt, maxInvisible); + } + + @Override + protected void ack(RocketMQReceiptHandle handle) { + acked.add(handle.getMessageId()); + } + + @Override + protected void changeInvisibleDuration( + RocketMQReceiptHandle handle, Duration invisibleDuration) { + changed.put(handle.getMessageId(), invisibleDuration); + } + } + + private static final class ListCollector implements Collector { + + private final List values = new ArrayList<>(); + + @Override + public void collect(String record) { + values.add(record); + } + + @Override + public void close() {} + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java new file mode 100644 index 0000000..375854f --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.common; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions; +import org.apache.flink.util.FlinkRuntimeException; + +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientConfigurationBuilder; +import org.apache.rocketmq.client.apis.SessionCredentialsProvider; +import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link CredentialsResolvers}. */ +class CredentialsResolversTest { + + private static final SessionCredentialsProvider RESOLVED_PROVIDER = + new StaticSessionCredentialsProvider("resolved-ak", "resolved-sk"); + + @Test + void createFromConfigurationReturnsNullWithoutResolverClass() { + assertThat(CredentialsResolvers.createFromConfiguration(new Configuration())).isNull(); + } + + @Test + void createFromConfigurationInstantiatesAndConfiguresTheResolver() { + final Configuration configuration = new Configuration(); + configuration.set( + RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, RecordingResolver.class.getName()); + + final CredentialsResolver resolver = + CredentialsResolvers.createFromConfiguration(configuration); + + assertThat(resolver).isInstanceOf(RecordingResolver.class); + assertThat(((RecordingResolver) resolver).configured).isSameAs(configuration); + } + + @Test + void createFromConfigurationFailsForNonResolverClass() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, String.class.getName()); + + assertThatThrownBy(() -> CredentialsResolvers.createFromConfiguration(configuration)) + .isInstanceOf(FlinkRuntimeException.class) + .hasMessageContaining(String.class.getName()); + } + + @Test + void createFromConfigurationFailsForMissingClass() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, "does.not.Exist"); + + assertThatThrownBy(() -> CredentialsResolvers.createFromConfiguration(configuration)) + .isInstanceOf(FlinkRuntimeException.class) + .hasMessageContaining("does.not.Exist"); + } + + @Test + void resolverTakesPrecedenceOverStaticKeys() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak"); + configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk"); + final RecordingResolver resolver = new RecordingResolver(); + + final ClientConfigurationBuilder builder = newBuilder(); + CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, resolver); + + assertThat(resolver.resolvedEndpoint).isEqualTo("host-a:8080"); + assertThat(builder.build().getCredentialsProvider()).contains(RESOLVED_PROVIDER); + } + + @Test + void staticKeysAreUsedWithoutAResolver() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak"); + configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk"); + + final ClientConfigurationBuilder builder = newBuilder(); + CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, null); + + assertThat(builder.build().getCredentialsProvider()) + .containsInstanceOf(StaticSessionCredentialsProvider.class); + } + + @Test + void noCredentialsAreSetWhenTheResolverReturnsNull() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak"); + configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk"); + + final ClientConfigurationBuilder builder = newBuilder(); + CredentialsResolvers.applyCredentials( + builder, "host-a:8080", configuration, endpoint -> null); + + // The resolver explicitly resolved "no credentials"; the static keys must not be used. + assertThat(builder.build().getCredentialsProvider()).isEmpty(); + } + + @Test + void noCredentialsAreSetWithoutResolverAndWithIncompleteStaticKeys() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak"); + + final ClientConfigurationBuilder builder = newBuilder(); + CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, null); + + assertThat(builder.build().getCredentialsProvider()).isEmpty(); + } + + private static ClientConfigurationBuilder newBuilder() { + return ClientConfiguration.newBuilder().setEndpoints("host-a:8080"); + } + + /** A resolver that records its interactions for assertions. */ + public static class RecordingResolver implements CredentialsResolver { + + @Nullable private Configuration configured; + @Nullable private String resolvedEndpoint; + + @Override + public void configure(Configuration configuration) { + this.configured = configuration; + } + + @Override + public SessionCredentialsProvider resolve(String endpoint) { + this.resolvedEndpoint = endpoint; + return RESOLVED_PROVIDER; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java new file mode 100644 index 0000000..2d559b5 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.sink; + +import org.apache.flink.api.common.serialization.SimpleStringSchema; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RocketMQGrpcSinkBuilder}. */ +class RocketMQGrpcSinkBuilderTest { + + @Test + void buildSucceedsWithValueOnlySerializerTest() { + final RocketMQGrpcSink sink = + RocketMQGrpcSink.builder() + .setEndpoints("127.0.0.1:8080") + .setTopic("topic") + .setLiteTopic("lite-topic") + .setValueOnlySerializer(new SimpleStringSchema()) + .build(); + assertThat(sink).isNotNull(); + } + + @Test + void buildFailsWithoutEndpointsTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSink.builder() + .setTopic("topic") + .setLiteTopic("lite-topic") + .setValueOnlySerializer(new SimpleStringSchema()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("endpoints"); + } + + @Test + void buildFailsWithoutSerializerTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSink.builder() + .setEndpoints("127.0.0.1:8080") + .setTopic("topic") + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("serializer"); + } + + @Test + void valueOnlySerializerRequiresTopicTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSink.builder() + .setEndpoints("127.0.0.1:8080") + .setValueOnlySerializer(new SimpleStringSchema())) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("topic"); + } + + @Test + void valueOnlySerializerRequiresLiteTopicTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSink.builder() + .setEndpoints("127.0.0.1:8080") + .setTopic("topic") + .setValueOnlySerializer(new SimpleStringSchema())) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("lite topic"); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java new file mode 100644 index 0000000..93225f8 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.util.FlinkRuntimeException; + +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Unit tests for {@link InvisibleDurationRenewalPolicies}. */ +class InvisibleDurationRenewalPoliciesTest { + + @Test + void returnsNullWhenNoPolicyConfiguredTest() { + assertThat(InvisibleDurationRenewalPolicies.createFromConfiguration(new Configuration())) + .isNull(); + } + + @Test + void instantiatesAndConfiguresPolicyTest() { + final Configuration configuration = new Configuration(); + configuration.set( + RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, + RecordingRenewalPolicy.class.getName()); + final InvisibleDurationRenewalPolicy policy = + InvisibleDurationRenewalPolicies.createFromConfiguration(configuration); + assertThat(policy).isInstanceOf(RecordingRenewalPolicy.class); + assertThat(((RecordingRenewalPolicy) policy).configured).isTrue(); + } + + @Test + void failsForUnknownClassTest() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, "does.not.Exist"); + assertThatThrownBy( + () -> + InvisibleDurationRenewalPolicies.createFromConfiguration( + configuration)) + .isInstanceOf(FlinkRuntimeException.class); + } + + @Test + void failsForNonPolicyClassTest() { + final Configuration configuration = new Configuration(); + configuration.set(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, String.class.getName()); + assertThatThrownBy( + () -> + InvisibleDurationRenewalPolicies.createFromConfiguration( + configuration)) + .isInstanceOf(FlinkRuntimeException.class); + } + + /** A renewal policy that records whether it was configured. */ + public static class RecordingRenewalPolicy implements InvisibleDurationRenewalPolicy { + + private static final long serialVersionUID = 1L; + + boolean configured; + + @Override + public void configure(Configuration configuration) { + this.configured = true; + } + + @Nullable + @Override + public Duration renew(MessageView messageView, int renewalCount) { + return null; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java new file mode 100644 index 0000000..21a8c6a --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.api.common.serialization.SimpleStringSchema; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RocketMQGrpcSourceBuilder}. */ +class RocketMQGrpcSourceBuilderTest { + + @Test + void buildSucceedsWithAllRequiredOptionsTest() { + final RocketMQGrpcSource source = + RocketMQGrpcSource.builder() + .setEndpoints("127.0.0.1:8080") + .setConsumerGroup("group") + .setMainTopic("topic") + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build(); + assertThat(source).isNotNull(); + } + + @Test + void buildFailsWithoutEndpointsTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSource.builder() + .setConsumerGroup("group") + .setMainTopic("topic") + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("endpoints"); + } + + @Test + void buildFailsWithoutConsumerGroupTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSource.builder() + .setEndpoints("127.0.0.1:8080") + .setMainTopic("topic") + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("consumer group"); + } + + @Test + void buildFailsWithoutMainTopicTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSource.builder() + .setEndpoints("127.0.0.1:8080") + .setConsumerGroup("group") + .setValueOnlyDeserializer(new SimpleStringSchema()) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("main topic"); + } + + @Test + void buildFailsWithoutDeserializerTest() { + assertThatThrownBy( + () -> + RocketMQGrpcSource.builder() + .setEndpoints("127.0.0.1:8080") + .setConsumerGroup("group") + .setMainTopic("topic") + .build()) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("deserializer"); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java new file mode 100644 index 0000000..39917d9 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumState; +import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumStateSerializer; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RocketMQGrpcSourceEnumStateSerializer}. */ +class RocketMQGrpcSourceEnumStateSerializerTest { + + @Test + void serializeAndDeserializeRoundTripTest() throws Exception { + final RocketMQGrpcSourceEnumStateSerializer serializer = + new RocketMQGrpcSourceEnumStateSerializer(); + + final byte[] serialized = serializer.serialize(new RocketMQGrpcSourceEnumState()); + final RocketMQGrpcSourceEnumState deserialized = + serializer.deserialize(serializer.getVersion(), serialized); + + assertThat(deserialized).isNotNull(); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java new file mode 100644 index 0000000..3a3d1f5 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source; + +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit; +import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitSerializer; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link RocketMQGrpcSourceSplitSerializer}. */ +class RocketMQGrpcSourceSplitSerializerTest { + + @Test + void serializeAndDeserializeRoundTripTest() throws Exception { + final RocketMQGrpcSourceSplitSerializer serializer = + new RocketMQGrpcSourceSplitSerializer(); + final RocketMQGrpcSourceSplit split = new RocketMQGrpcSourceSplit(); + + final byte[] serialized = serializer.serialize(split); + final RocketMQGrpcSourceSplit deserialized = + serializer.deserialize(serializer.getVersion(), serialized); + + assertThat(deserialized).isEqualTo(split); + assertThat(deserialized.splitId()).isEqualTo(RocketMQGrpcSourceSplit.SPLIT_ID); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java new file mode 100644 index 0000000..29bee9a --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.source.reader; + +import org.apache.rocketmq.client.apis.message.MessageId; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.FutureTask; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Unit tests for the renewal/emit bookkeeping of {@link MessageViewImpl}. */ +class MessageViewImplTest { + + @Test + void markEmittedCancelsPendingRenewalTest() { + final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView()); + final FutureTask future = new FutureTask<>(() -> null); + view.setRenewalFuture(future); + + assertThat(view.isEmitted()).isFalse(); + view.markEmitted(); + + assertThat(view.isEmitted()).isTrue(); + assertThat(future.isCancelled()).isTrue(); + } + + @Test + void renewalFutureSetAfterEmitIsCancelledTest() { + final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView()); + view.markEmitted(); + + final FutureTask future = new FutureTask<>(() -> null); + view.setRenewalFuture(future); + + assertThat(future.isCancelled()).isTrue(); + } + + @Test + void copiesBodyAndExposesAttributesTest() { + final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView()); + + assertThat(view.getBody()).isEqualTo("body".getBytes(StandardCharsets.UTF_8)); + assertThat(view.getTopic()).isEqualTo("topic"); + assertThat(view.getTag()).isNull(); + assertThat(view.getDeliveryAttempt()).isEqualTo(2); + assertThat(view.getEventTime()).isEqualTo(1234L); + } + + /** A minimal SDK message view stub. */ + private static class TestingSdkMessageView + implements org.apache.rocketmq.client.apis.message.MessageView { + + @Override + public MessageId getMessageId() { + return null; + } + + @Override + public String getTopic() { + return "topic"; + } + + @Override + public ByteBuffer getBody() { + return ByteBuffer.wrap("body".getBytes(StandardCharsets.UTF_8)); + } + + @Override + public Map getProperties() { + return Collections.emptyMap(); + } + + @Override + public Optional getTag() { + return Optional.empty(); + } + + @Override + public Collection getKeys() { + return Collections.emptyList(); + } + + @Override + public Optional getMessageGroup() { + return Optional.empty(); + } + + @Override + public Optional getLiteTopic() { + return Optional.empty(); + } + + @Override + public Optional getDeliveryTimestamp() { + return Optional.empty(); + } + + @Override + public Optional getPriority() { + return Optional.empty(); + } + + @Override + public String getBornHost() { + return "localhost"; + } + + @Override + public long getBornTimestamp() { + return 1234L; + } + + @Override + public int getDeliveryAttempt() { + return 2; + } + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java new file mode 100644 index 0000000..67aa3d4 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.utils.FactoryMocks; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RocketMQGrpcDynamicTableFactory}. */ +class RocketMQGrpcDynamicTableFactoryTest { + + private static final ResolvedSchema SCHEMA = + ResolvedSchema.of(Column.physical("f0", DataTypes.STRING())); + + private static Map baseOptions() { + final Map options = new HashMap<>(); + options.put("connector", RocketMQGrpcDynamicTableFactory.IDENTIFIER); + options.put("endpoints", "127.0.0.1:8080"); + options.put("topic", "test-topic"); + options.put("sink.lite-topic", "test-lite-topic"); + options.put("format", "test-format"); + options.put("test-format.delimiter", ","); + return options; + } + + @Test + void createTableSourceTest() { + final Map options = baseOptions(); + options.put("source.consumer-group", "test-group"); + + final DynamicTableSource source = FactoryMocks.createTableSource(SCHEMA, options); + assertThat(source).isInstanceOf(RocketMQGrpcDynamicTableSource.class); + } + + @Test + void createTableSinkTest() { + final DynamicTableSink sink = FactoryMocks.createTableSink(SCHEMA, baseOptions()); + assertThat(sink).isInstanceOf(RocketMQGrpcDynamicTableSink.class); + } + + @Test + void createTableSourceFailsWithoutConsumerGroupTest() { + assertThatThrownBy(() -> FactoryMocks.createTableSource(SCHEMA, baseOptions())) + .isInstanceOf(ValidationException.class) + .hasStackTraceContaining("consumer-group"); + } + + @Test + void createTableSinkFailsWithoutLiteTopicTest() { + final Map options = baseOptions(); + options.remove("sink.lite-topic"); + assertThatThrownBy(() -> FactoryMocks.createTableSink(SCHEMA, options)) + .hasStackTraceContaining("lite-topic"); + } +} diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java new file mode 100644 index 0000000..135adb8 --- /dev/null +++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.rocketmq.grpc.table; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.util.Collector; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Unit tests for the metadata handling of {@link RocketMQGrpcRowDataConverter}. */ +class RocketMQGrpcRowDataConverterTest { + + @Test + void appendsRequestedMetadataColumnsTest() throws Exception { + final List converters = + Arrays.asList( + RocketMQGrpcReadableMetadata.TAG.getConverter(), + RocketMQGrpcReadableMetadata.DELIVERY_ATTEMPT.getConverter(), + RocketMQGrpcReadableMetadata.BORN_TIMESTAMP.getConverter()); + final RocketMQGrpcRowDataConverter converter = + RocketMQGrpcRowDataConverter.forSource( + new BodyAsStringDeserializationSchema(), + converters, + TypeInformation.of(RowData.class)); + converter.open((DeserializationSchema.InitializationContext) null); + + final List output = new ArrayList<>(); + converter.deserialize(new TestingMessageView(), new ListCollector(output)); + + assertThat(output).hasSize(1); + final GenericRowData row = (GenericRowData) output.get(0); + assertThat(row.getArity()).isEqualTo(4); + assertThat(row.getField(0)).isEqualTo(StringData.fromString("body")); + assertThat(row.getField(1)).isEqualTo(StringData.fromString("tagA")); + assertThat(row.getField(2)).isEqualTo(7); + assertThat(row.getField(3)).isEqualTo(TimestampData.fromEpochMillis(1234L)); + } + + @Test + void forwardsPhysicalRowWithoutMetadataTest() throws Exception { + final RocketMQGrpcRowDataConverter converter = + RocketMQGrpcRowDataConverter.forSource( + new BodyAsStringDeserializationSchema(), + Collections.emptyList(), + TypeInformation.of(RowData.class)); + converter.open((DeserializationSchema.InitializationContext) null); + + final List output = new ArrayList<>(); + converter.deserialize(new TestingMessageView(), new ListCollector(output)); + + assertThat(output).hasSize(1); + final GenericRowData row = (GenericRowData) output.get(0); + assertThat(row.getArity()).isEqualTo(1); + assertThat(row.getField(0)).isEqualTo(StringData.fromString("body")); + } + + /** Decodes the message body into a single-field row containing the body as a string. */ + private static class BodyAsStringDeserializationSchema + implements DeserializationSchema { + + private static final long serialVersionUID = 1L; + + @Override + public RowData deserialize(byte[] message) { + return GenericRowData.of( + StringData.fromString(new String(message, StandardCharsets.UTF_8))); + } + + @Override + public boolean isEndOfStream(RowData nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(RowData.class); + } + } + + private static class ListCollector implements Collector { + + private final List output; + + private ListCollector(List output) { + this.output = output; + } + + @Override + public void collect(RowData record) { + output.add(record); + } + + @Override + public void close() {} + } + + /** A connector-level message view with fixed attributes. */ + private static class TestingMessageView implements MessageView { + + @Override + public String getMessageId() { + return "MSG-1"; + } + + @Override + public String getTopic() { + return "topic"; + } + + @Override + public String getTag() { + return "tagA"; + } + + @Override + public Collection getKeys() { + return Arrays.asList("k1", "k2"); + } + + @Override + public byte[] getBody() { + return "body".getBytes(StandardCharsets.UTF_8); + } + + @Override + public int getDeliveryAttempt() { + return 7; + } + + @Override + public long getEventTime() { + return 1234L; + } + + @Override + public Map getProperties() { + return Collections.singletonMap("p1", "v1"); + } + } +} diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java index 39ada10..197d6c2 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java @@ -97,8 +97,10 @@ public RocketMQConfigValidatorBuilder requiredOption(ConfigOption option) { } public RocketMQConfigValidator build() { - List>> conflict = Collections.unmodifiableList(new ArrayList<>(conflictOptions)); - Set> required = Collections.unmodifiableSet(new HashSet<>(requiredOptions)); + List>> conflict = + Collections.unmodifiableList(new ArrayList<>(conflictOptions)); + Set> required = + Collections.unmodifiableSet(new HashSet<>(requiredOptions)); return new RocketMQConfigValidator(conflict, required); } diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java index f2562e5..3c89a27 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java @@ -21,13 +21,13 @@ import org.apache.flink.api.connector.sink2.Committer; import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; import org.apache.flink.connector.rocketmq.sink.committer.RocketMQCommitter; import org.apache.flink.connector.rocketmq.sink.committer.SendCommittable; import org.apache.flink.connector.rocketmq.sink.committer.SendCommittableSerializer; import org.apache.flink.connector.rocketmq.sink.writer.RocketMQWriter; import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema; import org.apache.flink.core.io.SimpleVersionedSerializer; +import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; @PublicEvolving public class RocketMQSink implements TwoPhaseCommittingSink { diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java index 5895612..21e194c 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java @@ -26,10 +26,10 @@ import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigBuilder; import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigValidator; import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions; -import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema; import org.apache.flink.connector.rocketmq.source.RocketMQSource; import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions; +import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; import org.apache.commons.lang3.StringUtils; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java index 61e7263..a220e50 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java @@ -97,7 +97,9 @@ public class RocketMQSinkOptions extends RocketMQOptions { ConfigOptions.key("rocketmq.sink.tag.dynamic.column").stringType().noDefaultValue(); public static final ConfigOption OPTIONAL_WRITE_DYNAMIC_TAG_COLUMN_WRITE_INCLUDED = - ConfigOptions.key("rocketmq.sink.tag.dynamic.write.included").booleanType().defaultValue(true); + ConfigOptions.key("rocketmq.sink.tag.dynamic.write.included") + .booleanType() + .defaultValue(true); public static final ConfigOption OPTIONAL_WRITE_KEY_COLUMNS = ConfigOptions.key("rocketmq.sink.key.columns").stringType().noDefaultValue(); diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java index a5f2a6f..ae1b4e6 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java @@ -22,7 +22,6 @@ import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.DeliveryGuarantee; -import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; import org.apache.flink.connector.rocketmq.sink.InnerProducer; import org.apache.flink.connector.rocketmq.sink.InnerProducerImpl; import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions; @@ -30,6 +29,7 @@ import org.apache.flink.connector.rocketmq.sink.writer.context.RocketMQSinkContext; import org.apache.flink.connector.rocketmq.sink.writer.context.RocketMQSinkContextImpl; import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema; +import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; import org.apache.flink.util.FlinkRuntimeException; import org.apache.rocketmq.client.producer.SendResult; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java index c1c6907..8c79e0f 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java @@ -74,10 +74,8 @@ public InnerConsumerImpl(Configuration configuration) { String accessKey = configuration.getString(RocketMQSourceOptions.OPTIONAL_ACCESS_KEY); String secretKey = configuration.getString(RocketMQSourceOptions.OPTIONAL_SECRET_KEY); - boolean enableTrace = - configuration.getBoolean(RocketMQSourceOptions.ENABLE_MESSAGE_TRACE); - String traceTopic = - configuration.getString(RocketMQSourceOptions.CUSTOMIZED_TRACE_TOPIC); + boolean enableTrace = configuration.getBoolean(RocketMQSourceOptions.ENABLE_MESSAGE_TRACE); + String traceTopic = configuration.getString(RocketMQSourceOptions.CUSTOMIZED_TRACE_TOPIC); // Note: sync pull thread num may not enough if (!StringUtils.isNullOrWhitespaceOnly(accessKey) @@ -496,17 +494,13 @@ public Map committedOffsets(Collection message @Override public Map minOffsets(Collection messageQueues) { return fetchOffsets( - messageQueues, - mq -> innerConsumer.seekMinOffset(mq), - "fetch min offset"); + messageQueues, mq -> innerConsumer.seekMinOffset(mq), "fetch min offset"); } @Override public Map maxOffsets(Collection messageQueues) { return fetchOffsets( - messageQueues, - mq -> innerConsumer.seekMaxOffset(mq), - "fetch max offset"); + messageQueues, mq -> innerConsumer.seekMaxOffset(mq), "fetch max offset"); } @Override diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java index 7f26d87..3430a04 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java @@ -48,7 +48,6 @@ import org.apache.flink.metrics.MetricGroup; import org.apache.flink.util.UserCodeClassLoader; - import java.util.function.Supplier; /** The Source implementation of RocketMQ. */ diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java index e14667f..4683184 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java @@ -22,8 +22,8 @@ import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigValidator; import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; import org.apache.flink.connector.rocketmq.source.enumerator.allocate.AllocateStrategyFactory; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; /** Includes config options of RocketMQ connector type. */ public class RocketMQSourceOptions extends RocketMQOptions { diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java index 68ceff2..c914f96 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java @@ -19,9 +19,9 @@ package org.apache.flink.connector.rocketmq.source.enumerator.offset; import org.apache.flink.annotation.PublicEvolving; -import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy; import org.apache.flink.connector.rocketmq.source.RocketMQSource; import org.apache.flink.connector.rocketmq.source.split.RocketMQSourceSplit; +import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageQueue; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java index e04b771..a74b831 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java @@ -18,6 +18,7 @@ package org.apache.flink.connector.rocketmq.source.reader; import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.connector.base.source.reader.SourceReaderBase; import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager; @@ -25,7 +26,6 @@ import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherTask; import org.apache.flink.connector.base.source.reader.splitreader.SplitReader; import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; -import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.rocketmq.source.split.RocketMQSourceSplit; import org.apache.rocketmq.common.message.MessageQueue; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java index ac9ef7d..65ac1f4 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java @@ -85,8 +85,7 @@ private static BytesMessage toBytesMessage(MessageView message) { bytesMessage.setProperties(message.getProperties()); } bytesMessage.setProperty("__topic__", message.getTopic()); - bytesMessage.setProperty( - "__store_timestamp__", String.valueOf(message.getIngestionTime())); + bytesMessage.setProperty("__store_timestamp__", String.valueOf(message.getIngestionTime())); bytesMessage.setProperty("__born_timestamp__", String.valueOf(message.getEventTime())); bytesMessage.setProperty("__queue_id__", String.valueOf(message.getQueueId())); bytesMessage.setProperty("__queue_offset__", String.valueOf(message.getQueueOffset())); diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java index d95c212..bc3ef5c 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java @@ -17,14 +17,14 @@ package org.apache.flink.streaming.connectors.rocketmq; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; -import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils; import org.apache.flink.metrics.Meter; import org.apache.flink.runtime.state.FunctionInitializationContext; import org.apache.flink.runtime.state.FunctionSnapshotContext; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector; +import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils; import org.apache.flink.util.StringUtils; import org.apache.commons.lang.Validate; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java index c660335..42dc415 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java @@ -25,12 +25,6 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.ResultTypeQueryable; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy; -import org.apache.flink.streaming.connectors.rocketmq.common.config.StartupMode; -import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema; -import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils; -import org.apache.flink.streaming.connectors.rocketmq.common.util.RetryUtil; -import org.apache.flink.streaming.connectors.rocketmq.common.util.RocketMQUtils; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Meter; import org.apache.flink.metrics.MeterView; @@ -41,6 +35,12 @@ import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy; +import org.apache.flink.streaming.connectors.rocketmq.common.config.StartupMode; +import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema; +import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils; +import org.apache.flink.streaming.connectors.rocketmq.common.util.RetryUtil; +import org.apache.flink.streaming.connectors.rocketmq.common.util.RocketMQUtils; import org.apache.flink.util.Preconditions; import org.apache.commons.collections.CollectionUtils; @@ -70,7 +70,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -268,8 +267,7 @@ private void awaitTermination() throws InterruptedException { } } - private void consumeMessages( - MessageQueue mq, SourceContext context, int pullBatchSize) { + private void consumeMessages(MessageQueue mq, SourceContext context, int pullBatchSize) { RetryUtil.call( () -> { pollAndEmitLoop(mq, context, pullBatchSize); @@ -279,8 +277,7 @@ private void consumeMessages( runningChecker); } - private void pollAndEmitLoop( - MessageQueue mq, SourceContext context, int pullBatchSize) { + private void pollAndEmitLoop(MessageQueue mq, SourceContext context, int pullBatchSize) { while (runningChecker.isRunning()) { try { pollAndEmit(mq, context, pullBatchSize); @@ -313,8 +310,7 @@ private void pollAndEmit(MessageQueue mq, SourceContext context, int pullBa } if (!found) { - RetryUtil.waitForMs( - RocketMQConfig.DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND); + RetryUtil.waitForMs(RocketMQConfig.DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND); } } @@ -322,9 +318,7 @@ private void emitMessages(List messages, SourceContext context) long fetchTime = System.currentTimeMillis(); for (MessageExt msg : messages) { byte[] key = - msg.getKeys() != null - ? msg.getKeys().getBytes(StandardCharsets.UTF_8) - : null; + msg.getKeys() != null ? msg.getKeys().getBytes(StandardCharsets.UTF_8) : null; byte[] value = msg.getBody(); OUT data = schema.deserializeKeyAndValue(key, value); diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java index bb05b67..1a4bb04 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java @@ -21,9 +21,9 @@ import org.apache.flink.api.java.typeutils.RowTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions; -import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions; import org.apache.flink.connector.rocketmq.sink.RocketMQSink; import org.apache.flink.connector.rocketmq.sink.RocketMQSinkBuilder; +import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.ChangelogMode; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java index 4807c4d..c5754c7 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java @@ -20,9 +20,9 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions; import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.descriptors.DescriptorProperties; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java index 5cc0755..3a3418d 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java @@ -26,8 +26,8 @@ /** * A {@link RocketMQSerializationSchema} adapter that bridges the Table/SQL {@link RowData} to - * RocketMQ {@link Message} conversion. Delegates to {@link RocketMQRowDataConverter} for the - * actual conversion logic. + * RocketMQ {@link Message} conversion. Delegates to {@link RocketMQRowDataConverter} for the actual + * conversion logic. */ public class RocketMQRowDataSerializationSchema implements RocketMQSerializationSchema { diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java index 138d79c..cc8ee35 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java @@ -16,8 +16,8 @@ import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink; import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink; import org.apache.flink.table.data.RowData; import org.apache.rocketmq.common.message.Message; diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java index 32afa6e..6d12961 100644 --- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java +++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java @@ -19,10 +19,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction; -import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema; -import org.apache.flink.streaming.connectors.rocketmq.common.serialization.RowKeyValueDeserializationSchema; import org.apache.flink.connector.rocketmq.source.RocketMQSource; import org.apache.flink.connector.rocketmq.source.RocketMQSourceBuilder; import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions; @@ -32,6 +28,10 @@ import org.apache.flink.connector.rocketmq.source.reader.deserializer.RocketMQDeserializationSchema; import org.apache.flink.connector.rocketmq.source.reader.deserializer.RocketMQRowDeserializationSchema; import org.apache.flink.connector.rocketmq.source.reader.deserializer.RowDeserializationSchema.MetadataConverter; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction; +import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema; +import org.apache.flink.streaming.connectors.rocketmq.common.serialization.RowKeyValueDeserializationSchema; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.ChangelogMode; diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java index e8f2c1f..1611b02 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java @@ -23,15 +23,9 @@ import org.apache.flink.table.catalog.CatalogPartition; import org.apache.flink.table.catalog.CatalogPartitionSpec; import org.apache.flink.table.catalog.ObjectPath; -import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException; -import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException; import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; -import org.apache.flink.table.catalog.exceptions.FunctionAlreadyExistException; import org.apache.flink.table.catalog.exceptions.FunctionNotExistException; -import org.apache.flink.table.catalog.exceptions.PartitionAlreadyExistsException; import org.apache.flink.table.catalog.exceptions.PartitionNotExistException; -import org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException; -import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException; import org.apache.flink.table.catalog.stats.CatalogColumnStatistics; @@ -215,8 +209,7 @@ public void testCreateTable() { @Test public void testDropTable() { assertThrows( - UnsupportedOperationException.class, - () -> rocketMQCatalog.dropTable(null, false)); + UnsupportedOperationException.class, () -> rocketMQCatalog.dropTable(null, false)); } @Test @@ -239,8 +232,7 @@ public void testGetFunction() { @Test public void testFunctionExists() { assertThrows( - UnsupportedOperationException.class, - () -> rocketMQCatalog.functionExists(null)); + UnsupportedOperationException.class, () -> rocketMQCatalog.functionExists(null)); } @Test @@ -274,8 +266,7 @@ public void testAlterDatabase() { @Test public void testListViews() { assertThrows( - UnsupportedOperationException.class, - () -> rocketMQCatalog.listViews("default")); + UnsupportedOperationException.class, () -> rocketMQCatalog.listViews("default")); } @Test diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java index 27c4f3a..0b973e6 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java @@ -43,9 +43,7 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.flink.connector.rocketmq.example.ConnectorConfig.ACCESS_KEY; import static org.apache.flink.connector.rocketmq.example.ConnectorConfig.CONSUMER_GROUP; @@ -122,8 +120,8 @@ public void sourceToSinkPipelineTest() throws Exception { // ---- Step 1: Start Flink Source → Sink pipeline ---- private Thread startFlinkPipeline() throws Exception { - System.out.println("[Step 1] Starting Flink pipeline: " - + SOURCE_TOPIC_1 + " → " + SINK_TOPIC_1); + System.out.println( + "[Step 1] Starting Flink pipeline: " + SOURCE_TOPIC_1 + " → " + SINK_TOPIC_1); StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1); @@ -138,8 +136,7 @@ private Thread startFlinkPipeline() throws Exception { .setMinOffsets(OffsetsSelector.latest()) .setConfig(RocketMQOptions.OPTIONAL_ACCESS_KEY, ACCESS_KEY) .setConfig(RocketMQOptions.OPTIONAL_SECRET_KEY, SECRET_KEY) - .setConfig( - RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD) + .setConfig(RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD) .setDeserializer(new StringBodyDeserializationSchema()) .build(); @@ -150,8 +147,7 @@ private Thread startFlinkPipeline() throws Exception { .setGroupId(PRODUCER_GROUP) .setConfig(RocketMQOptions.OPTIONAL_ACCESS_KEY, ACCESS_KEY) .setConfig(RocketMQOptions.OPTIONAL_SECRET_KEY, SECRET_KEY) - .setConfig( - RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD) + .setConfig(RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD) .setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) .setSerializer( (element, context, timestamp) -> @@ -215,8 +211,7 @@ private void produceMessages() throws Exception { producer.shutdown(); } - System.out.printf( - "[Step 2] Done: %d/%d messages sent%n%n", successCount, MESSAGE_COUNT); + System.out.printf("[Step 2] Done: %d/%d messages sent%n%n", successCount, MESSAGE_COUNT); if (successCount < MESSAGE_COUNT) { throw new RuntimeException( "Failed to produce all messages: " + successCount + "/" + MESSAGE_COUNT); @@ -265,7 +260,9 @@ private void verifySinkMessages() throws Exception { receivedMessages.size(), MESSAGE_COUNT); assertTrue( - "Expected at least " + MESSAGE_COUNT + " messages in sink, got " + "Expected at least " + + MESSAGE_COUNT + + " messages in sink, got " + receivedMessages.size(), receivedMessages.size() >= MESSAGE_COUNT); } diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java index 9d78fe6..f00336b 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java @@ -128,7 +128,8 @@ public void sqlSourceToSinkTest() throws Exception { } private void produceMessages() throws Exception { - System.out.println("[Step 2] Producing " + MESSAGE_COUNT + " messages to " + SOURCE_TOPIC_2); + System.out.println( + "[Step 2] Producing " + MESSAGE_COUNT + " messages to " + SOURCE_TOPIC_2); DefaultMQProducer producer = new DefaultMQProducer(PRODUCER_GROUP, getAclRpcHook(), true, null); producer.setNamesrvAddr(ENDPOINTS); diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java index 7bbd015..266ec75 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java @@ -54,8 +54,8 @@ *
  • Record the current max offsets of the transaction topic *
  • Run a bounded Flink job writing messages with EXACTLY_ONCE; checkpointing triggers the * two-phase commit (prepareCommit → Committer.commit → endTransaction COMMIT) - *
  • Consume the topic from the recorded offsets and verify every message became visible, - * which only happens after a successful transaction commit + *
  • Consume the topic from the recorded offsets and verify every message became visible, which + * only happens after a successful transaction commit * * *

    Skipped automatically when environment variables are not set. The topic must be a diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java index 7de3db7..84a216e 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java @@ -29,7 +29,8 @@ /** Tests for verifying that configuration parameters are applied to the RocketMQ producer. */ class InnerProducerImplTest { - private TransactionMQProducer getProducerViaReflection(InnerProducerImpl impl) throws Exception { + private TransactionMQProducer getProducerViaReflection(InnerProducerImpl impl) + throws Exception { Field field = InnerProducerImpl.class.getDeclaredField("producer"); field.setAccessible(true); return (TransactionMQProducer) field.get(impl); diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java index 1e60937..5f6c7a9 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java @@ -18,10 +18,10 @@ package org.apache.flink.streaming.connectors.rocketmq; +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema; import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleKeyValueDeserializationSchema; import org.apache.flink.streaming.connectors.rocketmq.common.util.TestUtils; -import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.PullResult; diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java index c65fa1e..684680a 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java @@ -19,18 +19,18 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink; -import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction; -import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleTupleDeserializationSchema; -import org.apache.flink.streaming.connectors.rocketmq.function.SinkMapFunction; -import org.apache.flink.streaming.connectors.rocketmq.function.SourceMapFunction; import org.apache.flink.runtime.state.memory.MemoryStateBackend; import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink; +import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction; +import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleTupleDeserializationSchema; +import org.apache.flink.streaming.connectors.rocketmq.function.SinkMapFunction; +import org.apache.flink.streaming.connectors.rocketmq.function.SourceMapFunction; import org.apache.rocketmq.client.AccessChannel; diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java index c9db6f1..f0ce69a 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java @@ -35,7 +35,6 @@ import java.util.Map; import static org.apache.flink.table.api.DataTypes.BIGINT; -import static org.apache.flink.table.api.DataTypes.STRING; import static org.assertj.core.api.Assertions.assertThat; /** diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java index bcf163b..13a5c7b 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java @@ -66,8 +66,7 @@ void tagFilterShouldBePreservedOnCopy() { @Test void sqlFilterShouldBePreservedOnCopy() { DescriptorProperties props = new DescriptorProperties(); - TableSchema schema = - TableSchema.builder().field("id", DataTypes.BIGINT()).build(); + TableSchema schema = TableSchema.builder().field("id", DataTypes.BIGINT()).build(); RocketMQScanTableSource source = new RocketMQScanTableSource( @@ -97,8 +96,7 @@ void sqlFilterShouldBePreservedOnCopy() { void wildcardTagShouldMatchAllMessages() { // "*" is the default tag and should not filter any messages DescriptorProperties props = new DescriptorProperties(); - TableSchema schema = - TableSchema.builder().field("id", DataTypes.BIGINT()).build(); + TableSchema schema = TableSchema.builder().field("id", DataTypes.BIGINT()).build(); RocketMQScanTableSource source = new RocketMQScanTableSource( diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java index 0000f9f..0fa158b 100644 --- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java +++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java @@ -17,7 +17,6 @@ package org.apache.flink.streaming.connectors.rocketmq.table; -import org.apache.flink.connector.rocketmq.source.RocketMQSource; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.connector.source.ScanTableSource; diff --git a/flink-sql-connector-rocketmq-grpc/pom.xml b/flink-sql-connector-rocketmq-grpc/pom.xml new file mode 100644 index 0000000..963169a --- /dev/null +++ b/flink-sql-connector-rocketmq-grpc/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connector-rocketmq-parent + 1.0.0-SNAPSHOT + + + flink-sql-connector-rocketmq-grpc + Flink : Connectors : RocketMQ gRPC : SQL + jar + + + true + + + + + org.apache.flink + flink-connector-rocketmq-grpc + ${project.version} + + + + org.apache.flink + flink-connector-base + provided + + + + + org.apache.flink + flink-test-utils + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.apache.flink + flink-table-common + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.flink:flink-connector-rocketmq-grpc + org.apache.rocketmq:rocketmq-client-java + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + org.apache.rocketmq:* + + LICENSE + NOTICE + + + + + + org.apache.rocketmq + org.apache.flink.rocketmq.grpc.shaded.org.apache.rocketmq + + + + + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze-deps + + analyze + + verify + + + org.apache.flink:flink-connector-rocketmq-grpc + + + + + + + + diff --git a/pom.xml b/pom.xml index c92cf4f..7570a25 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,9 @@ under the License. flink-connector-rocketmq + flink-connector-rocketmq-grpc flink-sql-connector-rocketmq + flink-sql-connector-rocketmq-grpc