From 1f8b55bac7af5816ff42295b51822967e6069439 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 1 Sep 2026 19:59:34 +0800 Subject: [PATCH 1/3] [kafka] Establish compatibility framework Introduce request dispatch, ApiVersions, metadata discovery, and topic lifecycle operations as the reviewable foundation for subsequent Produce support. Co-Authored-By: Codex AI-Model: gpt-5.6-sol AI-Contributed/Feature: 2698/2698 AI-Contributed/UT: 1160/1160 --- .../apache/fluss/config/ConfigOptions.java | 16 + fluss-kafka/pom.xml | 16 +- .../fluss/kafka/KafkaChannelInitializer.java | 5 +- .../fluss/kafka/KafkaCommandDecoder.java | 45 ++- .../fluss/kafka/KafkaProtocolPlugin.java | 14 +- .../org/apache/fluss/kafka/KafkaRequest.java | 18 + .../fluss/kafka/KafkaRequestContext.java | 96 +++++ .../fluss/kafka/KafkaRequestHandler.java | 291 ++++---------- .../kafka/api/admin/CreateTopicsHandler.java | 218 ++++++++++ .../kafka/api/admin/DeleteTopicsHandler.java | 109 +++++ .../kafka/api/metadata/MetadataHandler.java | 198 ++++++++++ .../api/versions/ApiVersionsHandler.java | 78 ++++ .../admin/GatewayKafkaTopicAdminBackend.java | 268 +++++++++++++ .../backend/admin/KafkaTopicAdminBackend.java | 172 ++++++++ .../metadata/GatewayKafkaMetadataBackend.java | 281 +++++++++++++ .../metadata/KafkaClusterMetadata.java | 210 ++++++++++ .../metadata/KafkaMetadataBackend.java | 30 ++ .../backend/metadata/KafkaMetadataQuery.java | 102 +++++ .../kafka/dispatcher/KafkaApiHandler.java | 37 ++ .../kafka/dispatcher/KafkaApiRegistry.java | 80 ++++ .../fluss/kafka/dispatcher/KafkaApiSpec.java | 82 ++++ .../dispatcher/KafkaRequestDispatcher.java | 108 +++++ .../fluss/kafka/error/KafkaErrorMapper.java | 45 +++ .../fluss/kafka/format/KafkaDataFormat.java | 61 +++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 65 +++ .../fluss/kafka/KafkaCommandDecoderTest.java | 118 ++++++ .../apache/fluss/kafka/KafkaConfigsTest.java | 10 + .../fluss/kafka/KafkaMetadataHandlerTest.java | 372 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 189 +++++++-- .../kafka/KafkaTopicAdminHandlerTest.java | 301 ++++++++++++++ .../dispatcher/KafkaApiRegistryTest.java | 127 ++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 43 ++ .../rpc/gateway/AdminGatewayProvider.java | 28 ++ .../fluss/rpc/netty/server/NettyServer.java | 13 +- .../fluss/server/tablet/TabletService.java | 12 +- 35 files changed, 3602 insertions(+), 256 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminGatewayProvider.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 3fdb77970af..8c3a727bb3f 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2645,6 +2645,22 @@ public class ConfigOptions { .withDescription( "The database for fluss kafka. The default database is `kafka`."); + public static final ConfigOption KAFKA_DEFAULT_KEY_FORMAT = + key("kafka.default.key.format") + .stringType() + .defaultValue("raw") + .withDescription( + "The default format for Kafka record keys when a CreateTopics request does not specify fluss.key.format. " + + "Supported formats are raw and string."); + + public static final ConfigOption KAFKA_DEFAULT_VALUE_FORMAT = + key("kafka.default.value.format") + .stringType() + .defaultValue("raw") + .withDescription( + "The default format for Kafka record values when a CreateTopics request does not specify fluss.value.format. " + + "Supported formats are raw and string."); + public static final ConfigOption KAFKA_CONNECTION_MAX_IDLE_TIME = key("kafka.connection.max-idle-time") .durationType() diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index 124001ca7b5..042a720cb02 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,11 +64,25 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils + + org.apache.fluss + fluss-client + ${project.version} + test + + org.apache.fluss fluss-common @@ -92,4 +106,4 @@ test - \ No newline at end of file + diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java index 5e7551a9af7..29bdc745ca9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java @@ -31,17 +31,20 @@ public class KafkaChannelInitializer extends NettyChannelInitializer { private final RequestChannel[] requestChannels; + private final String listenerName; private final int maxRequestSize; private final LengthFieldPrepender prepender = new LengthFieldPrepender(4); private final boolean preferHeap; public KafkaChannelInitializer( RequestChannel[] requestChannels, + String listenerName, long maxIdleTimeSeconds, int maxRequestSize, boolean preferHeap) { super(maxIdleTimeSeconds); this.requestChannels = requestChannels; + this.listenerName = listenerName; this.maxRequestSize = maxRequestSize; this.preferHeap = preferHeap; } @@ -53,6 +56,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(prepender); addFrameDecoder(ch, maxRequestSize, 4, preferHeap); ch.pipeline().addLast("flowController", new FlowControlHandler()); - ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels)); + ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels, listenerName)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java index 43a0533b2d3..1dcf1cca90a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java @@ -27,6 +27,7 @@ import org.apache.fluss.utils.MathUtils; import org.apache.kafka.common.errors.LeaderNotAvailableException; +import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -55,6 +56,7 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { private final RequestChannel[] requestChannels; private final int numChannels; + private final String listenerName; // Need to use a Queue to store the inflight responses, because Kafka clients require the // responses to be sent in order. @@ -65,18 +67,18 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { protected volatile ChannelHandlerContext ctx; protected SocketAddress remoteAddress; - public KafkaCommandDecoder(RequestChannel[] requestChannels) { + public KafkaCommandDecoder(RequestChannel[] requestChannels, String listenerName) { super(false); this.requestChannels = requestChannels; this.numChannels = requestChannels.length; + this.listenerName = listenerName; } @Override public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception { CompletableFuture future = new CompletableFuture<>(); - boolean needRelease = false; try { - KafkaRequest request = parseRequest(ctx, future, buffer); + KafkaRequest request = parseRequest(ctx, future, buffer, listenerName); inflightResponses.addLast(request); future.whenCompleteAsync((r, t) -> sendResponse(ctx), ctx.executor()); int channelIndex = @@ -86,16 +88,15 @@ public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Excep if (!isActive.get()) { LOG.warn("Received a request on an inactive channel: {}", remoteAddress); request.fail(new LeaderNotAvailableException("Channel is inactive")); - needRelease = true; } } catch (Throwable t) { - needRelease = true; LOG.error("Error handling request", t); future.completeExceptionally(t); } finally { - if (needRelease) { - ReferenceCountUtil.release(buffer); - } + // KafkaRequest retains the buffer because Kafka record sets can reference its memory + // asynchronously. Release the decoder's ownership on every path; the request releases + // its retained reference after response handling or cancellation. + ReferenceCountUtil.release(buffer); } } @@ -184,19 +185,39 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E } private static KafkaRequest parseRequest( - ChannelHandlerContext ctx, CompletableFuture future, ByteBuf buffer) { + ChannelHandlerContext ctx, + CompletableFuture future, + ByteBuf buffer, + String listenerName) { ByteBuffer nioBuffer = buffer.nioBuffer(); RequestHeader header = RequestHeader.parse(nioBuffer); if (isUnsupportedApiVersionRequest(header)) { ApiVersionsRequest request = - new ApiVersionsRequest.Builder(header.apiVersion()).build(); + new ApiVersionsRequest( + new ApiVersionsRequestData(), + API_VERSIONS.oldestVersion(), + header.apiVersion()); return new KafkaRequest( - API_VERSIONS, header.apiVersion(), header, request, buffer, ctx, future); + API_VERSIONS, + header.apiVersion(), + header, + request, + listenerName, + buffer, + ctx, + future); } RequestAndSize request = AbstractRequest.parseRequest(header.apiKey(), header.apiVersion(), nioBuffer); return new KafkaRequest( - header.apiKey(), header.apiVersion(), header, request.request, buffer, ctx, future); + header.apiKey(), + header.apiVersion(), + header, + request.request, + listenerName, + buffer, + ctx, + future); } private static boolean isUnsupportedApiVersionRequest(RequestHeader header) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index d92ba5e68fc..65d2f8d7af6 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,7 +19,9 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.AdminGatewayProvider; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -53,6 +55,7 @@ public ChannelHandler createChannelHandler( RequestChannel[] requestChannels, String listenerName) { return new KafkaChannelInitializer( requestChannels, + listenerName, conf.get(ConfigOptions.KAFKA_CONNECTION_MAX_IDLE_TIME).getSeconds(), (int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(), conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST)); @@ -66,6 +69,15 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + if (service instanceof AdminGatewayProvider) { + return new KafkaRequestHandler( + service, + gateway, + ((AdminGatewayProvider) service).getAdminGateway(), + conf.get(ConfigOptions.KAFKA_DATABASE), + KafkaDataFormat.parse(conf.get(ConfigOptions.KAFKA_DEFAULT_KEY_FORMAT)), + KafkaDataFormat.parse(conf.get(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT))); + } + return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java index 25e409a7455..20d8bf01898 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java @@ -46,6 +46,7 @@ public class KafkaRequest implements RpcRequest { private final long requestId = ID_GENERATOR.getAndIncrement(); private final RequestHeader header; private final AbstractRequest request; + private final String listenerName; private final ByteBuf buffer; private final ChannelHandlerContext ctx; private final long startTimeMs; @@ -60,10 +61,23 @@ protected KafkaRequest( ByteBuf buffer, ChannelHandlerContext ctx, CompletableFuture future) { + this(apiKey, apiVersion, header, request, "UNKNOWN", buffer, ctx, future); + } + + protected KafkaRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest request, + String listenerName, + ByteBuf buffer, + ChannelHandlerContext ctx, + CompletableFuture future) { this.apiKey = apiKey; this.apiVersion = apiVersion; this.header = header; this.request = request; + this.listenerName = listenerName; this.buffer = buffer.retain(); this.ctx = ctx; this.startTimeMs = System.currentTimeMillis(); @@ -100,6 +114,10 @@ public T request() { return (T) request; } + public String listenerName() { + return listenerName; + } + public ChannelHandlerContext ctx() { return ctx; } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java new file mode 100644 index 00000000000..e75a20babcc --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.shaded.netty4.io.netty.channel.Channel; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.net.SocketAddress; + +/** Immutable wire-level context made available to Kafka API handlers. */ +@Internal +public final class KafkaRequestContext { + + private final int correlationId; + private final String clientId; + private final ApiKeys apiKey; + private final short apiVersion; + private final String listenerName; + private final SocketAddress localAddress; + private final SocketAddress remoteAddress; + private final long receivedTimeMs; + + private KafkaRequestContext(KafkaRequest request) { + this.correlationId = request.header().correlationId(); + this.clientId = request.header().clientId(); + this.apiKey = request.apiKey(); + this.apiVersion = request.apiVersion(); + this.listenerName = request.listenerName(); + Channel channel = request.ctx().channel(); + this.localAddress = channel == null ? null : channel.localAddress(); + this.remoteAddress = channel == null ? null : channel.remoteAddress(); + this.receivedTimeMs = request.startTimeMs(); + } + + /** Creates a context from a network request. */ + public static KafkaRequestContext fromRequest(KafkaRequest request) { + return new KafkaRequestContext(request); + } + + /** Returns the request correlation ID. */ + public int correlationId() { + return correlationId; + } + + /** Returns the client ID, or {@code null} when the request did not provide one. */ + public String clientId() { + return clientId; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the Kafka request version. */ + public short apiVersion() { + return apiVersion; + } + + /** Returns the listener that accepted the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the local socket address. */ + public SocketAddress localAddress() { + return localAddress; + } + + /** Returns the remote socket address. */ + public SocketAddress remoteAddress() { + return remoteAddress; + } + + /** Returns the wall-clock time at which the request was received. */ + public long receivedTimeMs() { + return receivedTimeMs; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..bbfa4fb30bc 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,87 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.api.admin.CreateTopicsHandler; +import org.apache.fluss.kafka.api.admin.DeleteTopicsHandler; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.admin.GatewayKafkaTopicAdminBackend; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.AdminGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; +import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; + + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler( + RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(kafkaDatabase); + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.register( + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + /** Creates a Kafka request handler including topic lifecycle capabilities. */ + public KafkaRequestHandler( + RpcGatewayService service, + TabletServerGateway gateway, + AdminGateway adminGateway, + String kafkaDatabase) { + this( + service, + gateway, + adminGateway, + kafkaDatabase, + KafkaDataFormat.RAW, + KafkaDataFormat.RAW); + } - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** + * Creates a Kafka request handler including topic lifecycle and default format capabilities. + */ + public KafkaRequestHandler( + RpcGatewayService service, + TabletServerGateway gateway, + AdminGateway adminGateway, + String kafkaDatabase, + KafkaDataFormat defaultKeyFormat, + KafkaDataFormat defaultValueFormat) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(adminGateway); + checkNotNull(kafkaDatabase); + checkNotNull(defaultKeyFormat); + checkNotNull(defaultValueFormat); + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.register( + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase), true)); + GatewayKafkaTopicAdminBackend topicAdminBackend = + new GatewayKafkaTopicAdminBackend(service, adminGateway, kafkaDatabase); + registry.register( + new CreateTopicsHandler(topicAdminBackend, defaultKeyFormat, defaultValueFormat)); + registry.register(new DeleteTopicsHandler(topicAdminBackend)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +107,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); - } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java new file mode 100644 index 00000000000..095420e72ec --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.admin; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend.CreateTopic; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.format.KafkaDataFormat; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic; +import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicConfig; +import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka CreateTopics by creating fixed-schema Arrow log tables in Fluss. */ +@Internal +public final class CreateTopicsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.CREATE_TOPICS, + ApiKeys.CREATE_TOPICS.oldestVersion(), + ApiKeys.CREATE_TOPICS.latestVersion(), + true); + + private final KafkaTopicAdminBackend backend; + private final KafkaDataFormat defaultKeyFormat; + private final KafkaDataFormat defaultValueFormat; + + /** Creates a CreateTopics handler. */ + public CreateTopicsHandler(KafkaTopicAdminBackend backend) { + this(backend, KafkaDataFormat.RAW, KafkaDataFormat.RAW); + } + + /** Creates a CreateTopics handler with formats used when a request omits format configs. */ + public CreateTopicsHandler( + KafkaTopicAdminBackend backend, + KafkaDataFormat defaultKeyFormat, + KafkaDataFormat defaultValueFormat) { + this.backend = checkNotNull(backend); + this.defaultKeyFormat = checkNotNull(defaultKeyFormat); + this.defaultValueFormat = checkNotNull(defaultValueFormat); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, CreateTopicsRequest request) { + List validTopics = new ArrayList<>(); + Map localResults = new LinkedHashMap<>(); + for (CreatableTopic topic : request.data().topics()) { + TopicResult invalid = validate(topic); + if (invalid == null) { + try { + FormatConfig formats = parseFormats(topic); + validTopics.add( + new CreateTopic( + topic.name(), + topic.numPartitions(), + topic.replicationFactor(), + formats.keyFormat, + formats.valueFormat)); + } catch (IllegalArgumentException e) { + localResults.put( + topic.name(), invalid(topic, Errors.INVALID_CONFIG, e.getMessage())); + } + } else { + localResults.put(topic.name(), invalid); + } + } + + return backend.createTopics( + validTopics, + request.data().validateOnly(), + context.listenerName(), + clientAddress(context.remoteAddress())) + .thenApply(results -> toResponse(request, localResults, results)); + } + + private static @Nullable TopicResult validate(CreatableTopic topic) { + if (!Topic.isValid(topic.name())) { + return invalid(topic, Errors.INVALID_TOPIC_EXCEPTION, "Invalid Kafka topic name."); + } + if (topic.numPartitions() <= 0) { + return invalid( + topic, + Errors.INVALID_PARTITIONS, + "A positive partition count is required for a Fluss topic table."); + } + if (topic.replicationFactor() == 0 || topic.replicationFactor() < -1) { + return invalid(topic, Errors.INVALID_REPLICATION_FACTOR, "Invalid replication factor."); + } + if (!topic.assignments().isEmpty()) { + return invalid( + topic, + Errors.INVALID_REPLICA_ASSIGNMENT, + "Explicit Kafka replica assignments are not supported by Fluss."); + } + return null; + } + + private FormatConfig parseFormats(CreatableTopic topic) { + KafkaDataFormat keyFormat = defaultKeyFormat; + KafkaDataFormat valueFormat = defaultValueFormat; + Map configs = new LinkedHashMap<>(); + for (CreatableTopicConfig config : topic.configs()) { + if (configs.containsKey(config.name())) { + throw new IllegalArgumentException( + "Duplicate Kafka topic config '" + config.name() + "'."); + } + configs.put(config.name(), config.value()); + } + for (Map.Entry config : configs.entrySet()) { + if (KafkaDataFormat.KEY_FORMAT_CONFIG.equals(config.getKey())) { + keyFormat = KafkaDataFormat.parse(config.getValue()); + } else if (KafkaDataFormat.VALUE_FORMAT_CONFIG.equals(config.getKey())) { + valueFormat = KafkaDataFormat.parse(config.getValue()); + } else { + throw new IllegalArgumentException( + "Unsupported Kafka topic config '" + config.getKey() + "'."); + } + } + return new FormatConfig(keyFormat, valueFormat); + } + + private static TopicResult invalid(CreatableTopic topic, Errors error, String message) { + return new TopicResult( + topic.name(), + Uuid.ZERO_UUID, + error, + message, + topic.numPartitions(), + topic.replicationFactor()); + } + + private static CreateTopicsResponse toResponse( + CreateTopicsRequest request, + Map localResults, + List backendResults) { + Map results = new LinkedHashMap<>(localResults); + for (TopicResult result : backendResults) { + results.put(result.name(), result); + } + CreateTopicsResponseData response = new CreateTopicsResponseData().setThrottleTimeMs(0); + for (CreatableTopic topic : request.data().topics()) { + TopicResult result = results.get(topic.name()); + response.topics() + .add( + new CreateTopicsResponseData.CreatableTopicResult() + .setName(result.name()) + .setTopicId(result.topicId()) + .setErrorCode(result.error().code()) + .setErrorMessage(result.errorMessage()) + .setNumPartitions(result.numPartitions()) + .setReplicationFactor(result.replicationFactor())); + } + return new CreateTopicsResponse(response); + } + + private static @Nullable InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } + + private static final class FormatConfig { + private final KafkaDataFormat keyFormat; + private final KafkaDataFormat valueFormat; + + private FormatConfig(KafkaDataFormat keyFormat, KafkaDataFormat valueFormat) { + this.keyFormat = keyFormat; + this.valueFormat = valueFormat; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java new file mode 100644 index 00000000000..6a2bf1d6eae --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.admin; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend.DeleteTopic; +import org.apache.fluss.kafka.backend.admin.KafkaTopicAdminBackend.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.DeleteTopicsRequestData.DeleteTopicState; +import org.apache.kafka.common.message.DeleteTopicsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.DeleteTopicsRequest; +import org.apache.kafka.common.requests.DeleteTopicsResponse; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka DeleteTopics by deleting the corresponding Fluss tables. */ +@Internal +public final class DeleteTopicsHandler implements KafkaApiHandler { + + private static final short TOPIC_ID_VERSION = 6; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.DELETE_TOPICS, + ApiKeys.DELETE_TOPICS.oldestVersion(), + ApiKeys.DELETE_TOPICS.latestVersion(), + true); + + private final KafkaTopicAdminBackend backend; + + /** Creates a DeleteTopics handler. */ + public DeleteTopicsHandler(KafkaTopicAdminBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, DeleteTopicsRequest request) { + List topics = new ArrayList<>(); + if (request.version() < TOPIC_ID_VERSION) { + for (String topicName : request.data().topicNames()) { + topics.add(new DeleteTopic(topicName, Uuid.ZERO_UUID)); + } + } else { + for (DeleteTopicState topic : request.data().topics()) { + topics.add(new DeleteTopic(topic.name(), topic.topicId())); + } + } + return backend.deleteTopics( + topics, context.listenerName(), clientAddress(context.remoteAddress())) + .thenApply(DeleteTopicsHandler::toResponse); + } + + private static DeleteTopicsResponse toResponse(List results) { + DeleteTopicsResponseData response = new DeleteTopicsResponseData().setThrottleTimeMs(0); + for (TopicResult result : results) { + response.responses() + .add( + new DeleteTopicsResponseData.DeletableTopicResult() + .setName(result.name()) + .setTopicId(result.topicId()) + .setErrorCode(result.error().code()) + .setErrorMessage(result.errorMessage())); + } + return new DeleteTopicsResponse(response); + } + + private static @Nullable InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..bcfc2655d9c --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + private final boolean controllerAvailable; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend) { + this(backend, false); + } + + /** Creates a Metadata handler and optionally exposes a Kafka-reachable controller. */ + public MetadataHandler(KafkaMetadataBackend backend, boolean controllerAvailable) { + this.backend = checkNotNull(backend); + this.controllerAvailable = controllerAvailable; + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() != null && !Topic.isValid(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics), + controllerAvailable); + }); + } + + private static MetadataResponse toResponse( + short version, KafkaClusterMetadata metadata, boolean controllerAvailable) { + int controllerId = + controllerAvailable && !metadata.brokers().isEmpty() + ? metadata.brokers().get(0).id() + : MetadataResponse.NO_CONTROLLER_ID; + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(controllerId) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + .setTopicId(topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..c38d7bc6cb2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java new file mode 100644 index 00000000000..6fe0321835a --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.admin; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.messages.CreateTableRequest; +import org.apache.fluss.rpc.messages.DropTableRequest; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Uses the TabletServer's existing coordinator gateway for Kafka topic administration. */ +@Internal +public final class GatewayKafkaTopicAdminBackend implements KafkaTopicAdminBackend { + + private final RpcGatewayService service; + private final AdminGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + + /** Creates a topic backend backed by the Fluss coordinator admin gateway. */ + public GatewayKafkaTopicAdminBackend( + RpcGatewayService service, AdminGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture> createTopics( + List topics, + boolean validateOnly, + String listenerName, + @Nullable InetAddress clientAddress) { + List> futures = new ArrayList<>(); + for (CreateTopic topic : topics) { + futures.add(createTopic(topic, validateOnly, listenerName, clientAddress)); + } + return collect(futures); + } + + @Override + public CompletableFuture> deleteTopics( + List topics, String listenerName, @Nullable InetAddress clientAddress) { + List> futures = new ArrayList<>(); + for (DeleteTopic topic : topics) { + futures.add(deleteTopic(topic, listenerName, clientAddress)); + } + return collect(futures); + } + + private CompletableFuture createTopic( + CreateTopic topic, + boolean validateOnly, + String listenerName, + @Nullable InetAddress clientAddress) { + TableDescriptor descriptor = createDescriptor(topic); + if (validateOnly) { + return CompletableFuture.completedFuture(success(topic, Uuid.ZERO_UUID)); + } + + CreateTableRequest request = new CreateTableRequest(); + request.setTableJson(descriptor.toJsonBytes()) + .setIgnoreIfExists(false) + .setTablePath() + .setDatabaseName(databaseName) + .setTableName(topic.name()); + setCurrentSession(listenerName, clientAddress); + return gateway.createTable(request) + .thenCompose(ignored -> getCreatedTopic(topic, listenerName, clientAddress)) + .exceptionally(failure -> failed(topic.name(), failure)); + } + + private CompletableFuture getCreatedTopic( + CreateTopic topic, String listenerName, @Nullable InetAddress clientAddress) { + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath().setDatabaseName(databaseName).setTableName(topic.name()); + setCurrentSession(listenerName, clientAddress); + return gateway.getTableInfo(request) + .thenApply( + response -> success(topic, topicMapper.toTopicId(response.getTableId()))); + } + + private CompletableFuture deleteTopic( + DeleteTopic topic, String listenerName, @Nullable InetAddress clientAddress) { + if (topic.name() == null) { + return CompletableFuture.completedFuture( + new TopicResult( + null, + topic.topicId(), + Errors.UNKNOWN_TOPIC_ID, + "Deleting a Fluss table by Kafka topic id is not supported.", + -1, + (short) -1)); + } + DropTableRequest request = new DropTableRequest(); + request.setIgnoreIfNotExists(false) + .setTablePath() + .setDatabaseName(databaseName) + .setTableName(topic.name()); + setCurrentSession(listenerName, clientAddress); + return gateway.dropTable(request) + .thenApply( + ignored -> + new TopicResult( + topic.name(), + topic.topicId(), + Errors.NONE, + null, + -1, + (short) -1)) + .exceptionally(failure -> failed(topic.name(), topic.topicId(), failure)); + } + + private static TableDescriptor createDescriptor(CreateTopic topic) { + TableDescriptor.Builder builder = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("record_key", dataType(topic.keyFormat())) + .column("payload", dataType(topic.valueFormat())) + .column( + "event_time", + DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "headers", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "name", + DataTypes.STRING() + .copy(false)), + DataTypes.FIELD( + "value", + DataTypes.BYTES())))) + .build()) + .distributedBy(topic.numPartitions()) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, topic.keyFormat().value()) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, topic.valueFormat().value()); + if (topic.replicationFactor() > 0) { + builder.property( + ConfigOptions.TABLE_REPLICATION_FACTOR, (int) topic.replicationFactor()); + } + return builder.build(); + } + + private static DataType dataType(KafkaDataFormat format) { + return format == KafkaDataFormat.RAW ? DataTypes.BYTES() : DataTypes.STRING(); + } + + private static TopicResult success(CreateTopic topic, Uuid topicId) { + return new TopicResult( + topic.name(), + topicId, + Errors.NONE, + null, + topic.numPartitions(), + topic.replicationFactor()); + } + + private static TopicResult failed(String topicName, Throwable failure) { + return failed(topicName, Uuid.ZERO_UUID, failure); + } + + private static TopicResult failed(String topicName, Uuid topicId, Throwable failure) { + Throwable cause = unwrap(failure); + return new TopicResult( + topicName, topicId, toKafkaError(cause), cause.getMessage(), -1, (short) -1); + } + + private static Errors toKafkaError(Throwable failure) { + org.apache.fluss.rpc.protocol.Errors error = + org.apache.fluss.rpc.protocol.Errors.forException(failure); + switch (error) { + case TABLE_ALREADY_EXIST: + return Errors.TOPIC_ALREADY_EXISTS; + case TABLE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case INVALID_TABLE_EXCEPTION: + return Errors.INVALID_REQUEST; + case INVALID_REPLICATION_FACTOR: + return Errors.INVALID_REPLICATION_FACTOR; + case BUCKET_MAX_NUM_EXCEPTION: + return Errors.INVALID_PARTITIONS; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case DELETION_DISABLED_EXCEPTION: + return Errors.TOPIC_DELETION_DISABLED; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_COORDINATOR_LEADER_EXCEPTION: + return Errors.NOT_CONTROLLER; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(String listenerName, @Nullable InetAddress clientAddress) { + service.setCurrentSession( + new Session( + (short) 0, listenerName, false, clientAddress, FlussPrincipal.ANONYMOUS)); + } + + private static CompletableFuture> collect( + List> futures) { + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return results; + }); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java new file mode 100644 index 00000000000..653d03367c4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.admin; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** Backend contract for mapping Kafka topic lifecycle operations to Fluss tables. */ +@Internal +public interface KafkaTopicAdminBackend { + + /** Creates or validates the requested topics. */ + CompletableFuture> createTopics( + List topics, + boolean validateOnly, + String listenerName, + @Nullable InetAddress clientAddress); + + /** Deletes the requested topics. */ + CompletableFuture> deleteTopics( + List topics, String listenerName, @Nullable InetAddress clientAddress); + + /** A validated request to create one topic. */ + final class CreateTopic { + private final String name; + private final int numPartitions; + private final short replicationFactor; + private final KafkaDataFormat keyFormat; + private final KafkaDataFormat valueFormat; + + /** Creates a topic specification. */ + public CreateTopic( + String name, + int numPartitions, + short replicationFactor, + KafkaDataFormat keyFormat, + KafkaDataFormat valueFormat) { + this.name = name; + this.numPartitions = numPartitions; + this.replicationFactor = replicationFactor; + this.keyFormat = keyFormat; + this.valueFormat = valueFormat; + } + + /** Returns the Kafka topic name. */ + public String name() { + return name; + } + + /** Returns the requested partition count. */ + public int numPartitions() { + return numPartitions; + } + + /** Returns the requested replication factor, or {@code -1} for the Fluss default. */ + public short replicationFactor() { + return replicationFactor; + } + + /** Returns the interpretation of Kafka record keys. */ + public KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the interpretation of Kafka record values. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + } + + /** A request to delete one topic by name or Kafka topic id. */ + final class DeleteTopic { + private final @Nullable String name; + private final Uuid topicId; + + /** Creates a topic deletion reference. */ + public DeleteTopic(@Nullable String name, Uuid topicId) { + this.name = name; + this.topicId = topicId; + } + + /** Returns the topic name, if supplied. */ + public @Nullable String name() { + return name; + } + + /** Returns the Kafka topic id, or {@link Uuid#ZERO_UUID}. */ + public Uuid topicId() { + return topicId; + } + } + + /** Result of one topic lifecycle operation. */ + final class TopicResult { + private final @Nullable String name; + private final Uuid topicId; + private final Errors error; + private final @Nullable String errorMessage; + private final int numPartitions; + private final short replicationFactor; + + /** Creates a topic operation result. */ + public TopicResult( + @Nullable String name, + Uuid topicId, + Errors error, + @Nullable String errorMessage, + int numPartitions, + short replicationFactor) { + this.name = name; + this.topicId = topicId; + this.error = error; + this.errorMessage = errorMessage; + this.numPartitions = numPartitions; + this.replicationFactor = replicationFactor; + } + + /** Returns the topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the Kafka topic id, if known. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the optional error detail. */ + public @Nullable String errorMessage() { + return errorMessage; + } + + /** Returns the created partition count. */ + public int numPartitions() { + return numPartitions; + } + + /** Returns the created replication factor. */ + public short replicationFactor() { + return replicationFactor; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..087eb6a8d46 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.Uuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + + /** Creates a metadata backend backed by the local TabletServer gateway. */ + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenCompose( + response -> + requestFlussMetadata( + query, + new LinkedHashSet<>(response.getTableNamesList()))); + } + + Set topicNames = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + topicNames.add(topic.topicName()); + } + } + return requestFlussMetadata(query, topicNames); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames) { + return requestFlussMetadata(query, topicNames, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (String topicName : topicNames) { + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(databaseName) + .setTableName(topicName))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTopicNames(query) + .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenApply( + response -> { + Set currentNames = + new LinkedHashSet<>(response.getTableNamesList()); + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedNames = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedNames.add(topic.topicName()); + } + } + currentNames.retainAll(requestedNames); + } + return currentNames; + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByName = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + if (!databaseName.equals(table.getTablePath().getDatabaseName())) { + continue; + } + Topic topic = toKafkaTopic(table, aliveBrokerIds); + topicsByName.put(topic.name(), topic); + topicsById.put(topic.topicId(), topic); + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + topics.addAll(topicsByName.values()); + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByName.get(reference.topicName()); + if (topic != null && matches(reference, topic)) { + topics.add(topic); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (aliveBrokerIds.contains(replicaId)) { + isr.add(replicaId); + } else { + offlineReplicas.add(replicaId); + } + } + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.NONE, + partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private static boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null || reference.topicName().equals(topic.name())) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java new file mode 100644 index 00000000000..36995b8e27f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +/** Handles one Kafka API without blocking the request processor thread. */ +@Internal +public interface KafkaApiHandler { + + /** Returns the capability implemented by this handler. */ + KafkaApiSpec apiSpec(); + + /** Handles a parsed request asynchronously. */ + CompletableFuture handle(KafkaRequestContext context, R request); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java new file mode 100644 index 00000000000..b4a1dfd8ea3 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Registry and single source of truth for Kafka APIs exposed by one server. */ +@Internal +public final class KafkaApiRegistry { + + private final Map> handlers = new HashMap<>(); + private boolean frozen; + + /** Creates an empty API registry. */ + public KafkaApiRegistry() {} + + /** Registers a handler. Registrations are rejected after {@link #freeze()} is called. */ + public void register(KafkaApiHandler handler) { + checkNotNull(handler); + checkState(!frozen, "Kafka API registry is already frozen."); + ApiKeys apiKey = handler.apiSpec().apiKey(); + checkArgument(!handlers.containsKey(apiKey), "Kafka API %s is already registered.", apiKey); + handlers.put(apiKey, handler); + } + + /** Prevents further registrations. */ + public void freeze() { + frozen = true; + } + + /** Returns a routable handler, or {@code null} when the API is not exposed by this server. */ + public KafkaApiHandler lookup(ApiKeys apiKey) { + KafkaApiHandler handler = handlers.get(apiKey); + if (handler == null || !handler.apiSpec().advertised()) { + return null; + } + return handler; + } + + /** Returns the sorted API specifications advertised by this server. */ + public List advertisedApiSpecs() { + List specs = new ArrayList<>(); + for (KafkaApiHandler handler : handlers.values()) { + KafkaApiSpec spec = handler.apiSpec(); + if (spec.advertised()) { + specs.add(spec); + } + } + Collections.sort(specs, Comparator.comparingInt(spec -> spec.apiKey().id)); + return Collections.unmodifiableList(specs); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java new file mode 100644 index 00000000000..50d6a7ebab2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Describes the versions actually supported by a Kafka API handler. */ +@Internal +public final class KafkaApiSpec { + + private final ApiKeys apiKey; + private final short minVersion; + private final short maxVersion; + private final boolean advertised; + + /** Creates an API specification. */ + public KafkaApiSpec(ApiKeys apiKey, short minVersion, short maxVersion, boolean advertised) { + this.apiKey = checkNotNull(apiKey); + checkArgument(minVersion >= 0, "Minimum version must not be negative."); + checkArgument( + minVersion <= maxVersion, + "Minimum version %s must not exceed maximum version %s.", + minVersion, + maxVersion); + checkArgument( + minVersion >= apiKey.oldestVersion() && maxVersion <= apiKey.latestVersion(), + "Version range [%s, %s] is outside the Kafka library range [%s, %s] for %s.", + minVersion, + maxVersion, + apiKey.oldestVersion(), + apiKey.latestVersion(), + apiKey); + this.minVersion = minVersion; + this.maxVersion = maxVersion; + this.advertised = advertised; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the oldest supported request version. */ + public short minVersion() { + return minVersion; + } + + /** Returns the newest supported request version. */ + public short maxVersion() { + return maxVersion; + } + + /** Returns whether this API is allowed to be routed and advertised. */ + public boolean advertised() { + return advertised; + } + + /** Returns whether the supplied request version is supported. */ + public boolean supportsVersion(short version) { + return version >= minVersion && version <= maxVersion; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java new file mode 100644 index 00000000000..efdfe33dd66 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequest; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.error.KafkaErrorMapper; + +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Validates and dispatches parsed Kafka requests to independently registered API handlers. */ +@Internal +public final class KafkaRequestDispatcher { + + private final KafkaApiRegistry registry; + private final KafkaErrorMapper errorMapper; + + /** Creates a dispatcher backed by the supplied registry and error mapper. */ + public KafkaRequestDispatcher(KafkaApiRegistry registry, KafkaErrorMapper errorMapper) { + this.registry = checkNotNull(registry); + this.errorMapper = checkNotNull(errorMapper); + } + + /** Dispatches a request and always completes with a Kafka protocol response. */ + public CompletableFuture dispatch(KafkaRequest request) { + AbstractRequest abstractRequest = request.request(); + KafkaApiHandler handler = registry.lookup(request.apiKey()); + if (handler == null) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + "Kafka API " + request.apiKey() + " is not supported by this server.")); + } + + KafkaApiSpec spec = handler.apiSpec(); + if (!spec.supportsVersion(request.apiVersion())) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + String.format( + "Version %s is not supported for %s. Supported versions are [%s, %s].", + request.apiVersion(), + request.apiKey(), + spec.minVersion(), + spec.maxVersion()))); + } + + CompletableFuture responseFuture; + try { + responseFuture = + invoke(handler, KafkaRequestContext.fromRequest(request), abstractRequest); + if (responseFuture == null) { + throw new NullPointerException("Kafka API handler returned a null future."); + } + } catch (Throwable t) { + return completedErrorResponse(abstractRequest, t); + } + + CompletableFuture result = new CompletableFuture<>(); + responseFuture.whenComplete( + (response, failure) -> { + if (failure == null && response != null) { + result.complete(response); + } else { + Throwable responseFailure = + failure == null + ? new NullPointerException( + "Kafka API handler returned a null response.") + : failure; + result.complete(errorMapper.toResponse(abstractRequest, responseFailure)); + } + }); + return result; + } + + @SuppressWarnings("unchecked") + private static CompletableFuture invoke( + KafkaApiHandler handler, KafkaRequestContext context, AbstractRequest request) { + return ((KafkaApiHandler) handler).handle(context, request); + } + + private CompletableFuture completedErrorResponse( + AbstractRequest request, Throwable failure) { + return CompletableFuture.completedFuture(errorMapper.toResponse(request, failure)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java new file mode 100644 index 00000000000..4396566396d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.error; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + +/** Maps failures from the compatibility layer to version-aware Kafka responses. */ +@Internal +public final class KafkaErrorMapper { + + /** Converts a failure to the error response defined by the parsed Kafka request. */ + public AbstractResponse toResponse(AbstractRequest request, Throwable failure) { + return request.getErrorResponse(unwrap(failure)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..0581abdfe36 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Kafka topic config and Fluss custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "fluss.key.format"; + + /** Kafka topic config and Fluss custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "fluss.value.format"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a topic config value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted topic config value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..6b75bdfd0d1 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + return TablePath.of(databaseName, topicName); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java new file mode 100644 index 00000000000..a2b8a0dc60b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests response ordering and ownership in {@link KafkaCommandDecoder}. */ +public class KafkaCommandDecoderTest { + + @Test + public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() { + RequestChannel requestChannel = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requestChannel}, "KAFKA")); + short produceVersion = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest produceRequest = + new ProduceRequest( + new ProduceRequestData().setAcks((short) 0).setTimeoutMs(1000), + produceVersion); + RequestHeader produceHeader = + new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1); + ByteBuf produceBuffer = serialize(produceHeader, produceRequest); + + short apiVersionsVersion = ApiKeys.API_VERSIONS.latestVersion(); + ApiVersionsRequest apiVersionsRequest = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData(), + apiVersionsVersion, + apiVersionsVersion) + .build(); + RequestHeader apiVersionsHeader = + new RequestHeader(ApiKeys.API_VERSIONS, apiVersionsVersion, "client", 2); + ByteBuf apiVersionsBuffer = serialize(apiVersionsHeader, apiVersionsRequest); + + try { + channel.writeInbound(produceBuffer); + channel.writeInbound(apiVersionsBuffer); + KafkaRequest first = (KafkaRequest) requestChannel.pollRequest(1000); + KafkaRequest second = (KafkaRequest) requestChannel.pollRequest(1000); + assertThat(first).isNotNull(); + assertThat(second).isNotNull(); + + second.complete(new ApiVersionsResponse(new ApiVersionsResponseData())); + channel.runPendingTasks(); + Object blockedResponse = channel.readOutbound(); + assertThat(blockedResponse).isNull(); + + first.complete(new ProduceResponse(new ProduceResponseData())); + channel.runPendingTasks(); + + ByteBuf response = channel.readOutbound(); + try { + assertThat(response).isNotNull(); + ResponseHeader responseHeader = + ResponseHeader.parse( + response.nioBuffer(), + apiVersionsHeader.toResponseHeader().headerVersion()); + assertThat(responseHeader.correlationId()).isEqualTo(2); + Object additionalResponse = channel.readOutbound(); + assertThat(additionalResponse).isNull(); + } finally { + if (response != null) { + response.release(); + } + } + + assertThat(produceBuffer.refCnt()).isZero(); + assertThat(apiVersionsBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + + private static ByteBuf serialize(RequestHeader header, AbstractRequest request) { + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + return Unpooled.wrappedBuffer(serialized); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 7f502494f2e..1c52120fae4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -37,12 +37,18 @@ public void testFromMap() throws Exception { map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); map.put(ConfigOptions.KAFKA_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_KEY_FORMAT.key(), "string"); + map.put(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT.key(), "string"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("fluss"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_KEY_FORMAT)) + .isEqualTo("string"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT)) + .isEqualTo("string"); } @Test @@ -52,5 +58,9 @@ public void testFromDefault() throws Exception { assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("kafka"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_KEY_FORMAT)) + .isEqualTo("raw"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT)) + .isEqualTo("raw"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..f7e7dd9bd47 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList("topic"))), + version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + } + } + + @Test + public void testAllTopicsAndMissingAndInvalidTopic() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse allTopics = + handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); + assertThat(allTopics.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactlyInAnyOrder("other", "topic"); + + MetadataRequest requestedTopics = + new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + .build((short) 9); + MetadataResponse errors = handle(service, requestedTopics, (short) 9); + assertThat(errors.errors()) + .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testTopicIdentityAcrossDeleteAndRecreate() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + + service.removeTable("topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(deleted.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + + service.putTable("topic", 223L); + Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); + MetadataResponse recreatedByName = + handle( + service, + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11), + (short) 11); + assertThat(recreatedByName.data().topics().find("topic").topicId()) + .isEqualTo(recreatedTopicId) + .isNotEqualTo(TOPIC_ID); + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(topicName) + .setTopicId(Uuid.ZERO_UUID))), + (short) 11); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + KafkaRequest request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Map tables = new LinkedHashMap<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + + private TestingMetadataGatewayService() { + tables.put("topic", 123L); + tables.put("other", 124L); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + assertThat(request.getDatabaseName()).isEqualTo("kafka"); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + Long tableId = tables.get(tablePath.getTableName()); + if (tableId != null) { + topics.add( + tableMetadata( + tablePath.getTableName(), + tableId, + !"topic".equals(tablePath.getTableName()) + || topicLeaderAvailable)); + } + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + tables.put(topic, tableId); + } + + private void removeTable(String topic) { + tables.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, long tableId, boolean leaderAvailable) { + return new PbTableMetadata() + .setTablePath(new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}))); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 24e4ce8a6ce..1f3032c219c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -18,22 +18,33 @@ package org.apache.fluss.kafka; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.gateway.AdminGateway; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.mock; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -54,7 +65,7 @@ public void testKafkaApiVersionsNotSupported() { ByteBufAllocator.DEFAULT.buffer(), ctx, new CompletableFuture<>()); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ByteBuf responseBuffer = request.responseBuffer(); ApiVersionsResponse response = @@ -66,53 +77,171 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testAdminCapabilitiesAreAdvertisedWhenCoordinatorGatewayIsAvailable() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + KafkaRequestHandler handler = + new KafkaRequestHandler(service, service, mock(AdminGateway.class), "kafka"); + short version = ApiKeys.API_VERSIONS.latestVersion(); + ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder().build(version); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + requestBody, + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + + handler.processRequest(request); + + ApiVersionsResponse response = parseApiVersionsResponse(request); + assertSuccessfulResponseDefaults(response); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion()), + tuple( + ApiKeys.CREATE_TOPICS.id, + ApiKeys.CREATE_TOPICS.oldestVersion(), + ApiKeys.CREATE_TOPICS.latestVersion()), + tuple( + ApiKeys.DELETE_TOPICS.id, + ApiKeys.DELETE_TOPICS.oldestVersion(), + ApiKeys.DELETE_TOPICS.latestVersion())); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ByteBufAllocator.DEFAULT.buffer(), + ctx, + new CompletableFuture<>()); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + ByteBuf responseBuffer = request.responseBuffer(); + return (ApiVersionsResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + + @Test + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = new KafkaRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, + requestBody, ByteBufAllocator.DEFAULT.buffer(), - ctx, + new TestingChannelHandlerContext(), new CompletableFuture<>()); - handler.handleApiVersionsRequest(request); + + handler.processRequest(request); ByteBuf responseBuffer = request.responseBuffer(); ApiVersionsResponse response = (ApiVersionsResponse) AbstractResponse.parseResponse( responseBuffer.nioBuffer(), request.header()); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + new KafkaRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + + handler.processRequest(request); + + ByteBuf responseBuffer = request.responseBuffer(); + CreateTopicsResponse response = + (CreateTopicsResponse) + AbstractResponse.parseResponse( + responseBuffer.nioBuffer(), request.header()); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(service, service, "kafka"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java new file mode 100644 index 00000000000..c32b8cd0298 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.TableAlreadyExistException; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.messages.CreateTableRequest; +import org.apache.fluss.rpc.messages.CreateTableResponse; +import org.apache.fluss.rpc.messages.DropTableRequest; +import org.apache.fluss.rpc.messages.DropTableResponse; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.message.CreateTopicsResponseData; +import org.apache.kafka.common.message.DeleteTopicsRequestData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; +import org.apache.kafka.common.requests.DeleteTopicsRequest; +import org.apache.kafka.common.requests.DeleteTopicsResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests the Kafka topic lifecycle mapping to Fluss tables. */ +public class KafkaTopicAdminHandlerTest { + + @Test + public void testCreateTopicCreatesArrowTable() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + when(adminGateway.createTable(any(CreateTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new CreateTableResponse())); + when(adminGateway.getTableInfo(any(GetTableInfoRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + new GetTableInfoResponse().setTableId(123L))); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequest requestBody = createTopicsRequest(version); + KafkaRequest request = kafkaRequest(ApiKeys.CREATE_TOPICS, requestBody, version); + + new KafkaRequestHandler(service, service, adminGateway, "kafka").processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + CreateTopicsResponseData.CreatableTopicResult result = + response.data().topics().find("topic"); + assertThat(result.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(result.topicId()).isNotEqualTo(Uuid.ZERO_UUID); + ArgumentCaptor captor = + ArgumentCaptor.forClass(CreateTableRequest.class); + verify(adminGateway).createTable(captor.capture()); + CreateTableRequest flussRequest = captor.getValue(); + assertThat(flussRequest.getTablePath().getDatabaseName()).isEqualTo("kafka"); + assertThat(flussRequest.getTablePath().getTableName()).isEqualTo("topic"); + TableDescriptor descriptor = TableDescriptor.fromJsonBytes(flussRequest.getTableJson()); + assertThat(descriptor.getSchema().getColumnNames()) + .containsExactly("record_key", "payload", "event_time", "headers"); + assertThat(descriptor.getSchema().getRowType().getTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(1)).isEqualTo(DataTypes.BYTES()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(2)) + .isEqualTo(DataTypes.TIMESTAMP_LTZ(3).copy(false)); + assertThat(descriptor.getSchema().getRowType().getTypeAt(3)) + .isEqualTo( + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING().copy(false)), + DataTypes.FIELD("value", DataTypes.BYTES())))); + assertThat(descriptor.getTableDistribution().get().getBucketCount().get()).isEqualTo(3); + assertThat(descriptor.getProperties()) + .containsEntry(ConfigOptions.TABLE_LOG_FORMAT.key(), LogFormat.ARROW.toString()) + .containsEntry(ConfigOptions.TABLE_REPLICATION_FACTOR.key(), "2"); + assertThat(descriptor.getCustomProperties()) + .containsEntry(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .containsEntry(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw"); + } + + @Test + public void testCreateTopicSupportsIndependentStringFormats() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + when(adminGateway.createTable(any(CreateTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new CreateTableResponse())); + when(adminGateway.getTableInfo(any(GetTableInfoRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + new GetTableInfoResponse().setTableId(123L))); + Map configs = new LinkedHashMap<>(); + configs.put(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"); + configs.put(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw"); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest request = + kafkaRequest(ApiKeys.CREATE_TOPICS, createTopicsRequest(version, configs), version); + + new KafkaRequestHandler(service, service, adminGateway, "kafka").processRequest(request); + + assertThat(((CreateTopicsResponse) parseResponse(request)).errorCounts()) + .containsOnlyKeys(Errors.NONE); + ArgumentCaptor captor = + ArgumentCaptor.forClass(CreateTableRequest.class); + verify(adminGateway).createTable(captor.capture()); + TableDescriptor descriptor = + TableDescriptor.fromJsonBytes(captor.getValue().getTableJson()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(0)).isEqualTo(DataTypes.STRING()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(1)).isEqualTo(DataTypes.BYTES()); + assertThat(descriptor.getCustomProperties()).containsAllEntriesOf(configs); + } + + @Test + public void testCreateTopicUsesConfiguredDefaultFormats() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + when(adminGateway.createTable(any(CreateTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new CreateTableResponse())); + when(adminGateway.getTableInfo(any(GetTableInfoRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + new GetTableInfoResponse().setTableId(123L))); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest request = + kafkaRequest(ApiKeys.CREATE_TOPICS, createTopicsRequest(version), version); + + new KafkaRequestHandler( + service, + service, + adminGateway, + "kafka", + KafkaDataFormat.STRING, + KafkaDataFormat.STRING) + .processRequest(request); + + assertThat(((CreateTopicsResponse) parseResponse(request)).errorCounts()) + .containsOnlyKeys(Errors.NONE); + ArgumentCaptor captor = + ArgumentCaptor.forClass(CreateTableRequest.class); + verify(adminGateway).createTable(captor.capture()); + TableDescriptor descriptor = + TableDescriptor.fromJsonBytes(captor.getValue().getTableJson()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(0)).isEqualTo(DataTypes.STRING()); + assertThat(descriptor.getSchema().getRowType().getTypeAt(1)).isEqualTo(DataTypes.STRING()); + assertThat(descriptor.getCustomProperties()) + .containsEntry(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .containsEntry(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + } + + @Test + public void testCreateTopicRejectsInvalidFormat() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest request = + kafkaRequest( + ApiKeys.CREATE_TOPICS, + createTopicsRequest( + version, + Collections.singletonMap( + KafkaDataFormat.VALUE_FORMAT_CONFIG, "json")), + version); + + new KafkaRequestHandler(service, service, adminGateway, "kafka").processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_CONFIG, 1); + assertThat(response.data().topics().find("topic").errorMessage()) + .contains("Expected raw or string"); + verify(adminGateway, never()).createTable(any(CreateTableRequest.class)); + } + + @Test + public void testCreateTopicMapsAlreadyExists() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + CompletableFuture failure = new CompletableFuture<>(); + failure.completeExceptionally(new TableAlreadyExistException("already exists")); + when(adminGateway.createTable(any(CreateTableRequest.class))).thenReturn(failure); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest request = + kafkaRequest(ApiKeys.CREATE_TOPICS, createTopicsRequest(version), version); + + new KafkaRequestHandler(service, service, adminGateway, "kafka").processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.TOPIC_ALREADY_EXISTS, 1); + } + + @Test + public void testDeleteTopicDropsTable() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + when(adminGateway.dropTable(any(DropTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new DropTableResponse())); + short version = ApiKeys.DELETE_TOPICS.latestVersion(); + DeleteTopicsRequestData data = + new DeleteTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + Collections.singletonList( + new DeleteTopicsRequestData.DeleteTopicState() + .setName("topic") + .setTopicId(Uuid.ZERO_UUID))); + DeleteTopicsRequest requestBody = new DeleteTopicsRequest.Builder(data).build(version); + KafkaRequest request = kafkaRequest(ApiKeys.DELETE_TOPICS, requestBody, version); + + new KafkaRequestHandler(service, service, adminGateway, "kafka").processRequest(request); + + DeleteTopicsResponse response = (DeleteTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + ArgumentCaptor captor = ArgumentCaptor.forClass(DropTableRequest.class); + verify(adminGateway).dropTable(captor.capture()); + assertThat(captor.getValue().getTablePath().getDatabaseName()).isEqualTo("kafka"); + assertThat(captor.getValue().getTablePath().getTableName()).isEqualTo("topic"); + } + + private static CreateTopicsRequest createTopicsRequest(short version) { + return createTopicsRequest(version, Collections.emptyMap()); + } + + private static CreateTopicsRequest createTopicsRequest( + short version, Map configs) { + CreateTopicsRequestData.CreatableTopic topic = + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(3) + .setReplicationFactor((short) 2); + for (Map.Entry config : configs.entrySet()) { + topic.configs() + .add( + new CreateTopicsRequestData.CreatableTopicConfig() + .setName(config.getKey()) + .setValue(config.getValue())); + } + CreateTopicsRequestData data = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList(topic).iterator())); + return new CreateTopicsRequest.Builder(data).build(version); + } + + private static KafkaRequest kafkaRequest( + ApiKeys apiKey, AbstractRequest requestBody, short version) { + return new KafkaRequest( + apiKey, + version, + new RequestHeader(apiKey, version, "client-id", 1), + requestBody, + "KAFKA", + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } + + private static AbstractResponse parseResponse(KafkaRequest request) { + ByteBuf responseBuffer = request.responseBuffer(); + try { + return AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java new file mode 100644 index 00000000000..84a889c16bd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaApiRegistry}. */ +public class KafkaApiRegistryTest { + + @Test + public void testRejectDuplicateRegistrationAndRegistrationAfterFreeze() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("already registered"); + + registry.freeze(); + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already frozen"); + } + + @Test + public void testOnlyAdvertiseEnabledHandlers() { + KafkaApiRegistry registry = brokerRegistry(); + registry.register(new TestingApiVersionsHandler(true)); + assertThat(registry.advertisedApiSpecs()).hasSize(1); + + KafkaApiRegistry hiddenRegistry = brokerRegistry(); + hiddenRegistry.register(new TestingApiVersionsHandler(false)); + assertThat(hiddenRegistry.advertisedApiSpecs()).isEmpty(); + assertThat(hiddenRegistry.lookup(ApiKeys.API_VERSIONS)).isNull(); + } + + @Test + public void testAdvertisedSpecIsSameSpecUsedForRouting() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + registry.freeze(); + + KafkaApiSpec advertisedSpec = registry.advertisedApiSpecs().get(0); + KafkaApiHandler routedHandler = registry.lookup(ApiKeys.API_VERSIONS); + + assertThat(routedHandler).isSameAs(handler); + assertThat(routedHandler.apiSpec()).isSameAs(advertisedSpec); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + assertThat(advertisedSpec.supportsVersion(version)).isTrue(); + } + assertThat( + advertisedSpec.supportsVersion( + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1))) + .isFalse(); + } + + @Test + public void testRejectInvalidVersionRange() { + assertThatThrownBy(() -> new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 1, (short) 0, true)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1), + true)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static KafkaApiRegistry brokerRegistry() { + return new KafkaApiRegistry(); + } + + private static final class TestingApiVersionsHandler + implements KafkaApiHandler { + + private final KafkaApiSpec spec; + + private TestingApiVersionsHandler(boolean advertised) { + this.spec = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + advertised); + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..a0a2036d4d5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.kafka.common.Uuid; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminGatewayProvider.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminGatewayProvider.java new file mode 100644 index 00000000000..901060cfa1b --- /dev/null +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminGatewayProvider.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.gateway; + +import org.apache.fluss.annotation.Internal; + +/** Provides the admin gateway used by a server service for delegated metadata mutations. */ +@Internal +public interface AdminGatewayProvider { + + /** Returns the admin gateway available to the server service. */ + AdminGateway getAdminGateway(); +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index 03d798fb371..df2256985c2 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -232,8 +232,17 @@ private static List loadProtocols( NetworkProtocolPlugin kafkaPlugin = loadProtocolPlugin(NetworkProtocolPlugin.KAFKA_PROTOCOL_NAME); kafkaPlugin.setup(conf); - listeners.removeAll(kafkaPlugin.listenerNames()); - protocolPlugins.add(kafkaPlugin); + List kafkaListenerNames = kafkaPlugin.listenerNames(); + boolean hasKafkaEndpoint = + endpoints.stream() + .anyMatch( + endpoint -> + kafkaListenerNames.contains( + endpoint.getListenerName())); + if (hasKafkaEndpoint) { + listeners.removeAll(kafkaListenerNames); + protocolPlugins.add(kafkaPlugin); + } } // Add the Fluss protocol plugin in the end to allow other protocol diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..4230637bdab 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -37,6 +37,7 @@ import org.apache.fluss.rpc.entity.LookupResultForBucket; import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; +import org.apache.fluss.rpc.gateway.AdminGatewayProvider; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.FetchLogRequest; @@ -160,7 +161,8 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toPutKvDataForBuckets; /** An RPC Gateway service for tablet server. */ -public final class TabletService extends RpcServiceBase implements TabletServerGateway { +public final class TabletService extends RpcServiceBase + implements TabletServerGateway, AdminGatewayProvider { private final String serviceName; private final ReplicaManager replicaManager; @@ -209,6 +211,14 @@ public String name() { return serviceName; } + /** + * Returns the coordinator admin gateway used by this tablet service for internal forwarding. + */ + @Override + public CoordinatorGateway getAdminGateway() { + return coordinatorGateway; + } + @Override public void shutdown() {} From 12e1897a80de8292c2540e2825964925c694ccdb Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 1 Sep 2026 20:03:51 +0800 Subject: [PATCH 2/3] [kafka] Add basic Produce compatibility Support raw and string Produce requests, validate acks=all semantics, and wake delayed follower fetches after successful appends. Co-Authored-By: Codex AI-Model: gpt-5.6-sol AI-Contributed/Feature: 1204/1204 AI-Contributed/UT: 2066/2066 --- fluss-dist/src/main/resources/server.yaml | 4 + .../fluss/kafka/KafkaRequestHandler.java | 17 + .../kafka/api/produce/ProduceHandler.java | 191 +++++++ .../produce/GatewayKafkaProduceBackend.java | 251 +++++++++ .../backend/produce/KafkaProduceBackend.java | 29 ++ .../backend/produce/KafkaProduceCommand.java | 197 +++++++ .../backend/produce/KafkaProduceResult.java | 111 ++++ .../transcode/ArrowKafkaRecordTranscoder.java | 198 +++++++ .../KafkaRecordEncodingException.java | 30 ++ .../transcode/KafkaRecordTranscoder.java | 32 ++ .../fluss/kafka/KafkaAcksAllITCase.java | 292 +++++++++++ .../kafka/KafkaFlussRoundTripITCase.java | 225 ++++++++ .../kafka/KafkaMetadataFailoverITCase.java | 324 ++++++++++++ .../fluss/kafka/KafkaProduceHandlerTest.java | 486 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 2 + .../fluss/kafka/KafkaRequestITCase.java | 402 ++++++++++++++- .../apache/fluss/rpc/RpcGatewayService.java | 7 + .../rpc/netty/server/FlussRequestHandler.java | 2 + .../netty/server/FlussRequestHandlerTest.java | 133 +++++ .../apache/fluss/server/replica/Replica.java | 9 +- .../fluss/server/replica/ReplicaManager.java | 27 + .../server/replica/delay/ActionQueue.java | 34 ++ .../replica/delay/DelayedActionQueue.java | 60 +++ .../fluss/server/tablet/TabletService.java | 5 + .../fluss/server/replica/AdjustIsrITCase.java | 5 +- .../fluss/server/replica/ReplicaTest.java | 21 + .../replica/delay/DelayedActionQueueTest.java | 77 +++ .../replica/delay/DelayedFetchLogTest.java | 99 +++- 28 files changed, 3250 insertions(+), 20 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaAcksAllITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaFlussRoundTripITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataFailoverITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java create mode 100644 fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java diff --git a/fluss-dist/src/main/resources/server.yaml b/fluss-dist/src/main/resources/server.yaml index 24352ceb2aa..aa735d91180 100644 --- a/fluss-dist/src/main/resources/server.yaml +++ b/fluss-dist/src/main/resources/server.yaml @@ -39,6 +39,10 @@ default.bucket.number: 1 # factor is specified for the table. default.replication.factor: 1 +# The minimum number of in-sync replicas required for writes configured with acks=all (-1). +# A replicated production deployment commonly sets this to 2 or higher. The default is 1. +log.replica.min-in-sync-replicas-number: 1 + # The local data directory to be used for Fluss to storing kv and log data. data.dir: /tmp/fluss-data diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index bbfa4fb30bc..43cc67b74c7 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -20,13 +20,16 @@ import org.apache.fluss.kafka.api.admin.CreateTopicsHandler; import org.apache.fluss.kafka.api.admin.DeleteTopicsHandler; import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.admin.GatewayKafkaTopicAdminBackend; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.AdminGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; @@ -51,6 +54,13 @@ public KafkaRequestHandler( registry.register( new MetadataHandler( new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); + registry.register( + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + gateway, + kafkaDatabase, + new ArrowKafkaRecordTranscoder()))); registry.freeze(); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -91,6 +101,13 @@ public KafkaRequestHandler( registry.register( new MetadataHandler( new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase), true)); + registry.register( + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + gateway, + kafkaDatabase, + new ArrowKafkaRecordTranscoder()))); GatewayKafkaTopicAdminBackend topicAdminBackend = new GatewayKafkaTopicAdminBackend(service, adminGateway, kafkaDatabase); registry.register( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..2749bd721a7 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, ApiKeys.PRODUCE.latestVersion(), true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } + topics.add(new TopicWrite(topic.name(), partitions)); + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.write(command).thenApply(ProduceHandler::toResponse); + } + + private static void validateRequest(ProduceRequest request) { + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + List copied = new ArrayList<>(); + for (RecordBatch batch : records.batches()) { + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + for (org.apache.kafka.common.record.Record record : batch) { + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + return copied; + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..151c6564077 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.protocol.Errors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaRecordTranscoder transcoder; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + String databaseName, + KafkaRecordTranscoder transcoder) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.transcoder = checkNotNull(transcoder); + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath().setDatabaseName(databaseName).setTableName(topic.topicName()); + return gateway.getTableInfo(request) + .thenCompose(response -> produceTopic(command, topic, toTableInfo(topic, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + List retainedRecords = new ArrayList<>(); + try { + for (PartitionWrite partition : topic.partitions()) { + BytesView records = transcoder.transcode(partition.records(), tableInfo); + retainedRecords.add(records); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } + } catch (Exception e) { + CompletableFuture failure = new CompletableFuture<>(); + failure.completeExceptionally(e); + return failure; + } + + setCurrentSession(command); + return gateway.produceLog(request) + .thenApply( + response -> { + // Keep the native buffers reachable until the asynchronous append has + // completed. + retainedRecords.size(); + return toTopicResult(topic, response); + }); + } + + private TableInfo toTableInfo(TopicWrite topic, GetTableInfoResponse response) { + return TableInfo.of( + TablePath.of(databaseName, topic.topicName()), + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime()); + } + + private static TopicResult toTopicResult(TopicWrite topic, ProduceLogResponse response) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode()) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forException(cause)); + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult( + partition.partitionId(), kafkaError, -1L, cause.getMessage())); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..722f3d3d245 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order. */ + public List records() { + return records; + } + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + private final long timestamp; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** Creates a copied Kafka record. */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the Kafka record timestamp. */ + public long timestamp() { + return timestamp; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..218c566bde4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..a94eda00ad3 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Transcodes Kafka records into the fixed-schema Arrow log table used for Kafka topics. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + + /** Column containing the nullable Kafka record key. */ + public static final String KEY_COLUMN = "record_key"; + + /** Column containing the nullable Kafka record value. */ + public static final String VALUE_COLUMN = "payload"; + + /** Column containing the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN = "event_time"; + + /** Column containing the Kafka record headers. */ + public static final String HEADERS_COLUMN = "headers"; + + private static final String[] COLUMN_NAMES = { + KEY_COLUMN, VALUE_COLUMN, TIMESTAMP_COLUMN, HEADERS_COLUMN + }; + private static final int INITIAL_PAGE_SIZE = 4096; + + @Override + public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + validateTable(tableInfo); + KafkaDataFormat keyFormat = dataFormat(tableInfo, KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat valueFormat = dataFormat(tableInfo, KafkaDataFormat.VALUE_FORMAT_CONFIG); + RowType rowType = tableInfo.getRowType(); + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + rowType, + tableInfo.getTableConfig().getArrowCompressionInfo()); + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + for (Record record : records) { + builder.append( + ChangeType.APPEND_ONLY, + GenericRow.of( + transcodeBytes(record.key(), keyFormat, KEY_COLUMN), + transcodeBytes(record.value(), valueFormat, VALUE_COLUMN), + TimestampLtz.fromEpochMillis(record.timestamp()), + toHeaders(record.headers()))); + } + return builder.build(); + } + } + + private static void validateTable(TableInfo tableInfo) { + checkArgument(!tableInfo.hasPrimaryKey(), "Kafka topic table must be a log table."); + checkArgument(!tableInfo.isPartitioned(), "Partitioned Fluss tables are not supported."); + checkArgument( + tableInfo.getTableConfig().getLogFormat() == LogFormat.ARROW, + "Kafka topic table must use the Arrow log format."); + RowType rowType = tableInfo.getRowType(); + KafkaDataFormat keyFormat = dataFormat(tableInfo, KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat valueFormat = dataFormat(tableInfo, KafkaDataFormat.VALUE_FORMAT_CONFIG); + checkArgument( + rowType.getFieldNames().equals(Arrays.asList(COLUMN_NAMES)), + "Kafka topic table columns must be %s.", + Arrays.toString(COLUMN_NAMES)); + checkDataType(rowType, 0, KEY_COLUMN, keyFormat); + checkDataType(rowType, 1, VALUE_COLUMN, valueFormat); + checkArgument( + rowType.getTypeAt(2) instanceof LocalZonedTimestampType + && !rowType.getTypeAt(2).isNullable() + && ((LocalZonedTimestampType) rowType.getTypeAt(2)).getPrecision() == 3, + "Kafka event_time column must be TIMESTAMP_LTZ(3) NOT NULL."); + checkHeadersType(rowType); + } + + private static GenericArray toHeaders(List headers) { + Object[] rows = new Object[headers.size()]; + for (int i = 0; i < headers.size(); i++) { + RecordHeader header = headers.get(i); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + } + return new GenericArray(rows); + } + + private static KafkaDataFormat dataFormat(TableInfo tableInfo, String configKey) { + String value = tableInfo.getCustomProperties().toMap().get(configKey); + return value == null ? KafkaDataFormat.RAW : KafkaDataFormat.parse(value); + } + + private static void checkDataType( + RowType rowType, int position, String columnName, KafkaDataFormat format) { + boolean validType = + format == KafkaDataFormat.RAW + ? rowType.getTypeAt(position) instanceof BytesType + : rowType.getTypeAt(position) instanceof StringType; + checkArgument( + validType && rowType.getTypeAt(position).isNullable(), + "Kafka %s column must be nullable %s for format %s.", + columnName, + format == KafkaDataFormat.RAW ? "BYTES" : "STRING", + format.value()); + } + + private static @Nullable Object transcodeBytes( + @Nullable byte[] bytes, KafkaDataFormat format, String columnName) { + if (bytes == null || format == KafkaDataFormat.RAW) { + return bytes; + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException( + "Kafka " + columnName + " is not valid UTF-8 for string format.", e); + } + } + + private static void checkHeadersType(RowType rowType) { + checkArgument( + rowType.getTypeAt(3) instanceof ArrayType && rowType.getTypeAt(3).isNullable(), + "Kafka headers column must be nullable ARRAY>."); + ArrayType headersType = (ArrayType) rowType.getTypeAt(3); + checkArgument( + headersType.getElementType() instanceof RowType, + "Kafka headers elements must be ROW."); + RowType headerType = (RowType) headersType.getElementType(); + checkArgument( + headerType.getFieldNames().equals(Arrays.asList("name", "value")) + && headerType.getTypeAt(0) instanceof StringType + && !headerType.getTypeAt(0).isNullable() + && headerType.getTypeAt(1) instanceof BytesType + && headerType.getTypeAt(1).isNullable(), + "Kafka headers elements must be ROW."); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..f206c25bbef --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..eeff2f3d14f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Transcodes records according to the target Fluss table schema and log format. */ + BytesView transcode(List records, TableInfo tableInfo) throws Exception; +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaAcksAllITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaAcksAllITCase.java new file mode 100644 index 00000000000..f0da87d5650 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaAcksAllITCase.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.CreateTopicsRequestData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Certifies cluster-level minimum ISR semantics through the Kafka Produce path. */ +public class KafkaAcksAllITCase { + + private static final String DATABASE = "kafka"; + private static final String REPLICATED_TOPIC = "acks-all-replicated"; + private static final String UNDER_REPLICATED_TOPIC = "acks-all-under-replicated"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final AtomicInteger CORRELATION_ID = new AtomicInteger(); + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + private Connection connection; + private org.apache.fluss.client.admin.Admin flussAdmin; + private String bootstrapServers; + + @BeforeEach + public void setup() throws Exception { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER_EXTENSION.getClientConfig()); + flussAdmin = connection.getAdmin(); + flussAdmin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + bootstrapServers = + FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(",")); + } + + @AfterEach + public void teardown() throws Exception { + if (flussAdmin != null) { + flussAdmin.close(); + } + if (connection != null) { + connection.close(); + } + } + + @Test + public void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + try { + CreateTopicsResponse createResponse = createTopics(); + assertThat(createResponse.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(createResponse.data().topics().find(REPLICATED_TOPIC).replicationFactor()) + .isEqualTo((short) 3); + assertThat( + createResponse + .data() + .topics() + .find(UNDER_REPLICATED_TOPIC) + .replicationFactor()) + .isEqualTo((short) 1); + + RecordMetadata replicated = send(REPLICATED_TOPIC, "replicated-with-min-isr-two"); + assertThat(replicated.offset()).isZero(); + + assertThatThrownBy( + () -> send(UNDER_REPLICATED_TOPIC, "must-not-append-with-min-isr-two")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class) + .hasMessageContaining("minimum ISR 2"); + + flussAdmin + .alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER + .key(), + "1", + AlterConfigOpType.SET))) + .get(1, TimeUnit.MINUTES); + + retry( + Duration.ofMinutes(1), + () -> + assertThat(FLUSS_CLUSTER_EXTENSION.getTabletServers()) + .allSatisfy( + tabletServer -> + assertThat( + tabletServer + .getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(1))); + + assertThat(send(UNDER_REPLICATED_TOPIC, "accepted-after-reconfigure").offset()) + .isZero(); + + assertSingleFlussValue( + UNDER_REPLICATED_TOPIC, + "accepted-after-reconfigure".getBytes(StandardCharsets.UTF_8)); + } finally { + dropTablesIgnoringErrors(); + flussAdmin + .alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER + .key(), + "2", + AlterConfigOpType.SET))) + .get(1, TimeUnit.MINUTES); + } + } + + private CreateTopicsResponse createTopics() throws Exception { + return sendCreateTopicsRequest(); + } + + private CreateTopicsResponse sendCreateTopicsRequest() throws Exception { + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + List topics = + Arrays.asList( + new CreateTopicsRequestData.CreatableTopic() + .setName(REPLICATED_TOPIC) + .setNumPartitions(1) + .setReplicationFactor((short) 3), + new CreateTopicsRequestData.CreatableTopic() + .setName(UNDER_REPLICATED_TOPIC) + .setNumPartitions(1) + .setReplicationFactor((short) 1)); + CreateTopicsRequest request = + new CreateTopicsRequest.Builder( + new CreateTopicsRequestData() + .setTimeoutMs(30000) + .setTopics( + new CreateTopicsRequestData + .CreatableTopicCollection( + topics.iterator()))) + .build(version); + ServerNode node = FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA").get(0); + RequestHeader header = + new RequestHeader( + ApiKeys.CREATE_TOPICS, + version, + "acks-all-test", + CORRELATION_ID.incrementAndGet()); + return sendRequest(node, header, request); + } + + private static CreateTopicsResponse sendRequest( + ServerNode node, RequestHeader header, CreateTopicsRequest request) throws Exception { + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + try (Socket socket = new Socket(node.host(), node.port()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + socket.setSoTimeout(10000); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + + int responseSize = input.readInt(); + byte[] responseBytes = new byte[responseSize]; + input.readFully(responseBytes); + return (CreateTopicsResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } + + private void dropTablesIgnoringErrors() { + try { + flussAdmin.dropTable(TablePath.of(DATABASE, REPLICATED_TOPIC), true).get(); + flussAdmin.dropTable(TablePath.of(DATABASE, UNDER_REPLICATED_TOPIC), true).get(); + } catch (Exception ignored) { + // Preserve the primary test failure when cleanup cannot complete. + } + } + + private RecordMetadata send(String topic, String value) throws Exception { + try (KafkaProducer producer = new KafkaProducer<>(producerConfig())) { + return producer.send( + new ProducerRecord<>( + topic, KEY, value.getBytes(StandardCharsets.UTF_8))) + .get(30, TimeUnit.SECONDS); + } + } + + private void assertSingleFlussValue(String topic, byte[] expectedValue) throws Exception { + TablePath tablePath = TablePath.of(DATABASE, topic); + try (Table table = connection.getTable(tablePath); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + ScanRecords records = scanner.poll(Duration.ofSeconds(10)); + assertThat(records).hasSize(1); + ScanRecord record = records.iterator().next(); + assertThat(record.getRow().getBytes(0)).containsExactly(KEY); + assertThat(record.getRow().getBytes(1)).containsExactly(expectedValue); + } + } + + private Map producerConfig() { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, "all"); + config.put(ProducerConfig.RETRIES_CONFIG, 0); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 10000); + return config; + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaFlussRoundTripITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaFlussRoundTripITCase.java new file mode 100644 index 00000000000..dc269a829c6 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaFlussRoundTripITCase.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies that records written through Kafka can be consumed through the native Fluss client. */ +public class KafkaFlussRoundTripITCase { + + private static final String DATABASE = "kafka"; + private static final String TOPIC = "round-trip-topic"; + private static final String STRING_TOPIC = "round-trip-string-topic"; + private static final long TIMESTAMP = 123456789L; + private static final byte[] KEY = "kafka-key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "kafka-value".getBytes(StandardCharsets.UTF_8); + private static final String HEADER_KEY = "source"; + private static final byte[] HEADER_VALUE = "kafka".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + private Connection connection; + private org.apache.fluss.client.admin.Admin flussAdmin; + private String kafkaBootstrapServer; + + @BeforeEach + public void setup() throws Exception { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER_EXTENSION.getClientConfig()); + flussAdmin = connection.getAdmin(); + flussAdmin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + ServerNode kafkaNode = FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA").get(0); + kafkaBootstrapServer = kafkaNode.host() + ":" + kafkaNode.port(); + } + + @AfterEach + public void teardown() throws Exception { + if (flussAdmin != null) { + flussAdmin.close(); + } + if (connection != null) { + connection.close(); + } + } + + @Test + public void testKafkaWriteCanBeConsumedByFluss() throws Exception { + testRoundTrip(TOPIC, Collections.emptyMap(), false); + } + + @Test + public void testKafkaStringFormatsCanBeConsumedByFluss() throws Exception { + Map configs = new HashMap<>(); + configs.put(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"); + configs.put(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + testRoundTrip(STRING_TOPIC, configs, true); + } + + private void testRoundTrip(String topic, Map topicConfigs, boolean stringFormat) + throws Exception { + Map adminConfig = new HashMap<>(); + adminConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBootstrapServer); + adminConfig.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 60000); + try (Admin kafkaAdmin = Admin.create(adminConfig)) { + NewTopic newTopic = new NewTopic(topic, 1, (short) 1); + newTopic.configs(topicConfigs); + kafkaAdmin.createTopics(Collections.singleton(newTopic)).all().get(); + + writeKafkaRecord(topic); + assertFlussRecord(topic, stringFormat); + assertProjectedFlussRecord(topic, stringFormat); + + kafkaAdmin.deleteTopics(Collections.singleton(topic)).all().get(); + } + } + + private void writeKafkaRecord(String topic) throws Exception { + Map producerConfig = new HashMap<>(); + producerConfig.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBootstrapServer); + producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerConfig.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerConfig.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + producerConfig.put(ProducerConfig.ACKS_CONFIG, "1"); + RecordHeaders headers = + new RecordHeaders( + Collections.singletonList(new RecordHeader(HEADER_KEY, HEADER_VALUE))); + ProducerRecord record = + new ProducerRecord<>(topic, 0, TIMESTAMP, KEY, VALUE, headers); + try (KafkaProducer producer = new KafkaProducer<>(producerConfig)) { + producer.send(record).get(); + } + } + + private void assertFlussRecord(String topic, boolean stringFormat) throws Exception { + TablePath tablePath = TablePath.of(DATABASE, topic); + try (Table table = connection.getTable(tablePath); + LogScanner scanner = table.newScan().createLogScanner()) { + assertThat(table.getTableInfo().getTableConfig().getLogFormat()) + .isEqualTo(LogFormat.ARROW); + scanner.subscribeFromBeginning(0); + for (int attempt = 0; attempt < 30; attempt++) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertRow(record.getRow(), stringFormat); + return; + } + } + } + throw new AssertionError("Kafka record was not visible through the Fluss LogScanner."); + } + + private void assertProjectedFlussRecord(String topic, boolean stringFormat) throws Exception { + TablePath tablePath = TablePath.of(DATABASE, topic); + try (Table table = connection.getTable(tablePath); + LogScanner scanner = + table.newScan().project(new int[] {0, 1, 2}).createLogScanner()) { + scanner.subscribeFromBeginning(0); + for (int attempt = 0; attempt < 30; attempt++) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertProjectedRow(record.getRow(), stringFormat); + return; + } + } + } + throw new AssertionError( + "Kafka record was not visible through the projected Fluss LogScanner."); + } + + private static void assertRow(InternalRow row, boolean stringFormat) { + if (stringFormat) { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(KEY, StandardCharsets.UTF_8)); + assertThat(row.getString(1).toString()) + .isEqualTo(new String(VALUE, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(0)).containsExactly(KEY); + assertThat(row.getBytes(1)).containsExactly(VALUE); + } + assertThat(row.getTimestampLtz(2, 3).getEpochMillisecond()).isEqualTo(TIMESTAMP); + + InternalArray headers = row.getArray(3); + assertThat(headers.size()).isEqualTo(1); + InternalRow header = headers.getRow(0, 2); + assertThat(header.getString(0).toString()).isEqualTo(HEADER_KEY); + assertThat(header.getBytes(1)).containsExactly(HEADER_VALUE); + } + + private static void assertProjectedRow(InternalRow row, boolean stringFormat) { + if (stringFormat) { + assertThat(row.getString(0).toString()) + .isEqualTo(new String(KEY, StandardCharsets.UTF_8)); + assertThat(row.getString(1).toString()) + .isEqualTo(new String(VALUE, StandardCharsets.UTF_8)); + } else { + assertThat(row.getBytes(0)).containsExactly(KEY); + assertThat(row.getBytes(1)).containsExactly(VALUE); + } + assertThat(row.getTimestampLtz(2, 3).getEpochMillisecond()).isEqualTo(TIMESTAMP); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataFailoverITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataFailoverITCase.java new file mode 100644 index 00000000000..8358b9b7a5a --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataFailoverITCase.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.StringSerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Three-node integration tests for Kafka Metadata routing and topic identity. */ +public class KafkaMetadataFailoverITCase { + + private static final String DATABASE = "kafka"; + private static final short METADATA_VERSION = 11; + private static final AtomicInteger CORRELATION_ID = new AtomicInteger(); + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + private Connection connection; + private org.apache.fluss.client.admin.Admin flussAdmin; + private String bootstrapServers; + + @BeforeEach + public void setup() throws Exception { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER_EXTENSION.getClientConfig()); + flussAdmin = connection.getAdmin(); + flussAdmin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + bootstrapServers = + FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(",")); + } + + @AfterEach + public void teardown() throws Exception { + if (flussAdmin != null) { + flussAdmin.close(); + } + if (connection != null) { + connection.close(); + } + } + + @Test + public void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { + String topic = "metadata-lifecycle-topic"; + try (Admin admin = Admin.create(adminConfig())) { + deleteIgnoringErrors(admin, topic); + createTopic(admin, topic); + + MetadataResponse initial = waitForTopic(topic); + MetadataResponseTopic initialTopic = initial.data().topics().find(topic); + Uuid initialTopicId = initialTopic.topicId(); + assertThat(initialTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertKafkaListenerEndpoints(initial); + + admin.deleteTopics(Collections.singleton(topic)).all().get(1, TimeUnit.MINUTES); + retry( + Duration.ofMinutes(1), + () -> + assertThat(fetchTopicMetadata(topic).errors()) + .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); + + createTopic(admin, topic); + MetadataResponse recreated = waitForTopic(topic); + assertThat(recreated.data().topics().find(topic).topicId()) + .isNotEqualTo(Uuid.ZERO_UUID) + .isNotEqualTo(initialTopicId); + } finally { + try (Admin cleanupAdmin = Admin.create(adminConfig())) { + deleteIgnoringErrors(cleanupAdmin, topic); + } + } + } + + @Test + public void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + String topic = "metadata-leader-failover-topic"; + int stoppedLeader = -1; + try (Admin admin = Admin.create(adminConfig())) { + deleteIgnoringErrors(admin, topic); + createTopic(admin, topic); + + MetadataResponse initial = waitForTopic(topic); + MetadataResponsePartition initialPartition = + initial.data().topics().find(topic).partitions().get(0); + stoppedLeader = initialPartition.leaderId(); + int initialEpoch = initialPartition.leaderEpoch(); + + try (KafkaProducer producer = new KafkaProducer<>(producerConfig())) { + long firstOffset = + producer.send(new ProducerRecord<>(topic, 0, "key-1", "before-failover")) + .get(1, TimeUnit.MINUTES) + .offset(); + + FLUSS_CLUSTER_EXTENSION.stopTabletServer(stoppedLeader); + AtomicReference newPartition = new AtomicReference<>(); + int previousLeader = stoppedLeader; + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchTopicMetadata(topic); + MetadataResponseTopic responseTopic = + response.data().topics().find(topic); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = responseTopic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(previousLeader); + assertThat(partition.leaderEpoch()).isGreaterThan(initialEpoch); + newPartition.set(partition); + }); + + long secondOffset = + producer.send(new ProducerRecord<>(topic, 0, "key-2", "after-failover")) + .get(1, TimeUnit.MINUTES) + .offset(); + // acks=1 does not guarantee that the first leader replicated its acknowledged + // record before failover, so offset reuse is possible after data loss. The test + // certifies Metadata refresh and continued routing, not all-replica durability. + assertThat(secondOffset).isNotNegative(); + assertThat(firstOffset).isNotNegative(); + assertThat(newPartition.get()).isNotNull(); + } + } finally { + if (stoppedLeader >= 0 + && FLUSS_CLUSTER_EXTENSION.getTabletServerById(stoppedLeader) == null) { + FLUSS_CLUSTER_EXTENSION.startTabletServer(stoppedLeader); + FLUSS_CLUSTER_EXTENSION.assertHasTabletServerNumber(3); + } + try (Admin cleanupAdmin = Admin.create(adminConfig())) { + deleteIgnoringErrors(cleanupAdmin, topic); + } + } + } + + private MetadataResponse waitForTopic(String topic) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchTopicMetadata(topic); + MetadataResponseTopic responseTopic = response.data().topics().find(topic); + assertThat(responseTopic).isNotNull(); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(responseTopic.partitions()).hasSize(1); + assertThat(responseTopic.partitions().get(0).leaderId()).isNotEqualTo(-1); + result.set(response); + }); + return result.get(); + } + + private void assertKafkaListenerEndpoints(MetadataResponse response) { + List kafkaNodes = FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA"); + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + kafkaNodes.stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private MetadataResponse fetchTopicMetadata(String topic) throws Exception { + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + Exception lastFailure = null; + for (ServerNode node : FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA")) { + try { + return sendMetadataRequest(node, request); + } catch (Exception e) { + lastFailure = e; + } + } + throw new IllegalStateException("No Kafka listener returned Metadata.", lastFailure); + } + + private static MetadataResponse sendMetadataRequest(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader( + ApiKeys.METADATA, + METADATA_VERSION, + "metadata-failover-test", + CORRELATION_ID.incrementAndGet()); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + try (Socket socket = new Socket(node.host(), node.port()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + + int responseSize = input.readInt(); + byte[] responseBytes = new byte[responseSize]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } + + private Map adminConfig() { + Map config = new HashMap<>(); + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 60000); + return config; + } + + private Map producerConfig() { + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, "1"); + config.put(ProducerConfig.RETRIES_CONFIG, 20); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 60000); + config.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000); + return config; + } + + private static void createTopic(Admin admin, String topic) throws Exception { + admin.createTopics(Collections.singleton(new NewTopic(topic, 1, (short) 3))) + .all() + .get(1, TimeUnit.MINUTES); + } + + private static void deleteIgnoringErrors(Admin admin, String topic) { + try { + admin.deleteTopics(Collections.singleton(topic)).all().get(1, TimeUnit.MINUTES); + } catch (Exception ignored) { + // The topic may not exist before or after a failed test. + } + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java new file mode 100644 index 00000000000..2556430142d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java @@ -0,0 +1,486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.server.utils.ServerRpcMessageUtils; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol and native-record tests for the minimal Kafka Produce implementation. */ +public class KafkaProduceHandlerTest { + + private static final int SCHEMA_ID = 1; + private static final long TABLE_ID = 123L; + private static final long TIMESTAMP = 123456L; + + @Test + public void testProduceTranscodesAndWritesKafkaRecord() throws Exception { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + short version = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest requestBody = produceRequest(version, (short) 1); + KafkaRequest request = kafkaRequest(requestBody, version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponse response = parseResponse(request); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + ProduceResponseData.PartitionProduceResponse partitionResponse = + response.data().responses().find("topic").partitionResponses().get(0); + assertThat(partitionResponse.baseOffset()).isEqualTo(42L); + assertThat(service.lastProduceRequest.getTableId()).isEqualTo(TABLE_ID); + assertThat(service.lastProduceRequest.getAcks()).isEqualTo(1); + assertThat(service.lastProduceRequest.getTimeoutMs()).isEqualTo(1000); + + MemoryLogRecords records = + ServerRpcMessageUtils.getProduceLogData(service.lastProduceRequest) + .values() + .iterator() + .next(); + LogRecordBatch batch = records.batches().iterator().next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) SCHEMA_ID); + assertThat(batch.getRecordCount()).isEqualTo(1); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + service.schema.getRowType(), + SCHEMA_ID, + new TestingSchemaGetter( + new SchemaInfo(service.schema, SCHEMA_ID))); + CloseableIterator iterator = batch.records(context)) { + LogRecord record = iterator.next(); + InternalRow row = record.getRow(); + assertThat(row.getBytes(0)).isEqualTo(bytes("key")); + assertThat(row.getBytes(1)).isEqualTo(bytes("value")); + assertThat(row.getTimestampLtz(2, 3).getEpochMillisecond()).isEqualTo(TIMESTAMP); + InternalArray headers = row.getArray(3); + assertThat(headers.size()).isEqualTo(1); + InternalRow header = headers.getRow(0, 2); + assertThat(header.getString(0).toString()).isEqualTo("header"); + assertThat(header.getBytes(1)).isEqualTo(bytes("header-value")); + assertThat(iterator.hasNext()).isFalse(); + } + } + + @Test + public void testAcksZeroCompletesWithoutChangingWriteSemantics() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) 0), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + assertThat(request.future()).isCompleted(); + assertThat(service.lastProduceRequest.getAcks()).isZero(); + } + + @Test + public void testAcksAllAndTimeoutAreForwardedUnchanged() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) -1, 4321), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + assertThat(parseResponse(request).errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(service.lastProduceRequest.getAcks()).isEqualTo(-1); + assertThat(service.lastProduceRequest.getTimeoutMs()).isEqualTo(4321); + } + + @Test + public void testInvalidAcksReturnsInvalidRequiredAcksWithoutCallingBackend() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) 2), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponseData.PartitionProduceResponse partition = + parseResponse(request).data().responses().find("topic").partitionResponses().get(0); + assertThat(Errors.forCode(partition.errorCode())).isEqualTo(Errors.INVALID_REQUIRED_ACKS); + assertThat(partition.baseOffset()).isEqualTo(-1L); + assertThat(service.lastProduceRequest).isNull(); + } + + @Test + public void testProduceTranscodesStringKeyAndValue() throws Exception { + TestingProduceGatewayService service = + new TestingProduceGatewayService(KafkaDataFormat.STRING, KafkaDataFormat.STRING); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) 1), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + assertThat(parseResponse(request).errorCounts()).containsOnlyKeys(Errors.NONE); + MemoryLogRecords records = + ServerRpcMessageUtils.getProduceLogData(service.lastProduceRequest) + .values() + .iterator() + .next(); + LogRecordBatch batch = records.batches().iterator().next(); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + service.schema.getRowType(), + SCHEMA_ID, + new TestingSchemaGetter( + new SchemaInfo(service.schema, SCHEMA_ID))); + CloseableIterator iterator = batch.records(context)) { + InternalRow row = iterator.next().getRow(); + assertThat(row.getString(0).toString()).isEqualTo("key"); + assertThat(row.getString(1).toString()).isEqualTo("value"); + } + } + + @Test + public void testProduceRejectsInvalidUtf8ForStringFormat() { + TestingProduceGatewayService service = + new TestingProduceGatewayService(KafkaDataFormat.STRING, KafkaDataFormat.RAW); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = + kafkaRequest( + produceRequest( + version, (short) 1, new byte[] {(byte) 0xc3, 0x28}, bytes("value")), + version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponse response = parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.CORRUPT_MESSAGE, 1); + assertThat(service.lastProduceRequest).isNull(); + } + + @Test + public void testMapsAcksAllOutcomesToPartitionResponses() { + assertAcksAllOutcome(org.apache.fluss.rpc.protocol.Errors.NONE, Errors.NONE, 42L); + assertAcksAllOutcome( + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + Errors.NOT_ENOUGH_REPLICAS, + -1L); + assertAcksAllOutcome( + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + -1L); + assertAcksAllOutcome( + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + Errors.REQUEST_TIMED_OUT, + -1L); + assertAcksAllOutcome( + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER, + Errors.NOT_LEADER_OR_FOLLOWER, + -1L); + } + + @Test + public void testMapsMixedPartitionResultsIndependently() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + service.produceResponse = + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(42L), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode( + org.apache.fluss.rpc.protocol.Errors + .NOT_ENOUGH_REPLICAS_EXCEPTION + .code()))); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(twoPartitionProduceRequest(version), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponse response = parseResponse(request); + ProduceResponseData.PartitionProduceResponse successful = + response.data().responses().find("topic").partitionResponses().get(0); + ProduceResponseData.PartitionProduceResponse failed = + response.data().responses().find("topic").partitionResponses().get(1); + assertThat(Errors.forCode(successful.errorCode())).isEqualTo(Errors.NONE); + assertThat(successful.baseOffset()).isEqualTo(42L); + assertThat(Errors.forCode(failed.errorCode())).isEqualTo(Errors.NOT_ENOUGH_REPLICAS); + assertThat(failed.baseOffset()).isEqualTo(-1L); + } + + @Test + public void testMissingBucketResponseFailsOnlyThatPartition() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + service.produceResponse = + new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(42L))); + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(twoPartitionProduceRequest(version), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponse response = parseResponse(request); + ProduceResponseData.PartitionProduceResponse successful = + response.data().responses().find("topic").partitionResponses().get(0); + ProduceResponseData.PartitionProduceResponse missing = + response.data().responses().find("topic").partitionResponses().get(1); + assertThat(Errors.forCode(successful.errorCode())).isEqualTo(Errors.NONE); + assertThat(successful.baseOffset()).isEqualTo(42L); + assertThat(Errors.forCode(missing.errorCode())).isEqualTo(Errors.UNKNOWN_SERVER_ERROR); + assertThat(missing.baseOffset()).isEqualTo(-1L); + assertThat(missing.errorMessage()).contains("omitted this bucket"); + } + + @Test + public void testEveryAdvertisedProduceVersion() { + for (short version = 3; version <= ApiKeys.PRODUCE.latestVersion(); version++) { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) 1), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + assertThat(parseResponse(request).errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + private static ProduceRequest produceRequest(short version, short acks) { + return produceRequest(version, acks, 1000); + } + + private static ProduceRequest produceRequest(short version, short acks, int timeoutMs) { + return produceRequest(version, acks, timeoutMs, bytes("key"), bytes("value")); + } + + private static ProduceRequest produceRequest( + short version, short acks, byte[] key, byte[] value) { + return produceRequest(version, acks, 1000, key, value); + } + + private static ProduceRequest produceRequest( + short version, short acks, int timeoutMs, byte[] key, byte[] value) { + Header[] headers = {new RecordHeader("header", bytes("header-value"))}; + MemoryRecords records = + MemoryRecords.withRecords( + org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2, + TIMESTAMP, + Compression.NONE, + new SimpleRecord(TIMESTAMP, key, value, headers)); + TopicProduceData topic = + new TopicProduceData() + .setName("topic") + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords(records))); + ProduceRequestData data = + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(timeoutMs) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())); + return new ProduceRequest(data, version); + } + + private static ProduceRequest twoPartitionProduceRequest(short version) { + TopicProduceData topic = + new TopicProduceData() + .setName("topic") + .setPartitionData( + Arrays.asList( + new PartitionProduceData() + .setIndex(0) + .setRecords(memoryRecords()), + new PartitionProduceData() + .setIndex(1) + .setRecords(memoryRecords()))); + ProduceRequestData data = + new ProduceRequestData() + .setAcks((short) -1) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())); + return new ProduceRequest(data, version); + } + + private static MemoryRecords memoryRecords() { + return MemoryRecords.withRecords( + org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2, + TIMESTAMP, + Compression.NONE, + new SimpleRecord(TIMESTAMP, bytes("key"), bytes("value"))); + } + + private static void assertAcksAllOutcome( + org.apache.fluss.rpc.protocol.Errors flussError, + Errors expectedKafkaError, + long expectedBaseOffset) { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + service.produceError = flussError; + short version = ApiKeys.PRODUCE.latestVersion(); + KafkaRequest request = kafkaRequest(produceRequest(version, (short) -1, 4321), version); + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + ProduceResponseData.PartitionProduceResponse partition = + parseResponse(request).data().responses().find("topic").partitionResponses().get(0); + assertThat(Errors.forCode(partition.errorCode())).isEqualTo(expectedKafkaError); + assertThat(partition.baseOffset()).isEqualTo(expectedBaseOffset); + assertThat(service.lastProduceRequest.getAcks()).isEqualTo(-1); + assertThat(service.lastProduceRequest.getTimeoutMs()).isEqualTo(4321); + } + + private static KafkaRequest kafkaRequest(ProduceRequest requestBody, short version) { + return new KafkaRequest( + ApiKeys.PRODUCE, + version, + new RequestHeader(ApiKeys.PRODUCE, version, "client-id", 1), + requestBody, + "KAFKA", + org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } + + private static ProduceResponse parseResponse(KafkaRequest request) { + org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf responseBuffer = + request.responseBuffer(); + try { + return (ProduceResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + } + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static final class TestingProduceGatewayService extends TestingTabletGatewayService { + private final Schema schema; + private final TableDescriptor tableDescriptor; + private ProduceLogRequest lastProduceRequest; + private ProduceLogResponse produceResponse; + private org.apache.fluss.rpc.protocol.Errors produceError = + org.apache.fluss.rpc.protocol.Errors.NONE; + + private TestingProduceGatewayService() { + this(KafkaDataFormat.RAW, KafkaDataFormat.RAW); + } + + private TestingProduceGatewayService( + KafkaDataFormat keyFormat, KafkaDataFormat valueFormat) { + schema = + Schema.newBuilder() + .column("record_key", dataType(keyFormat)) + .column("payload", dataType(valueFormat)) + .column("event_time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "headers", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "name", DataTypes.STRING().copy(false)), + DataTypes.FIELD("value", DataTypes.BYTES())))) + .build(); + tableDescriptor = + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, keyFormat.value()) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat.value()) + .build(); + } + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + return CompletableFuture.completedFuture( + new GetTableInfoResponse() + .setTableId(TABLE_ID) + .setSchemaId(SCHEMA_ID) + .setTableJson(tableDescriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L)); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + lastProduceRequest = request; + if (produceResponse != null) { + return CompletableFuture.completedFuture(produceResponse); + } + PbProduceLogRespForBucket bucket = + new PbProduceLogRespForBucket().setBucketId(0).setBaseOffset(42L); + if (produceError != org.apache.fluss.rpc.protocol.Errors.NONE) { + bucket.clearBaseOffset().setErrorCode(produceError.code()); + } + return CompletableFuture.completedFuture( + new ProduceLogResponse().addAllBucketsResps(Collections.singletonList(bucket))); + } + + private static DataType dataType(KafkaDataFormat format) { + return format == KafkaDataFormat.RAW ? DataTypes.BYTES() : DataTypes.STRING(); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 1f3032c219c..e99f6a82213 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -111,6 +111,7 @@ public void testAdminCapabilitiesAreAdvertisedWhenCoordinatorGatewayIsAvailable( assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, ApiKeys.PRODUCE.latestVersion()), tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), tuple( ApiKeys.API_VERSIONS.id, @@ -164,6 +165,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, ApiKeys.PRODUCE.latestVersion()), tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), tuple( ApiKeys.API_VERSIONS.id, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestITCase.java index 6656a87f125..c0730c0b6ff 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestITCase.java @@ -20,40 +20,102 @@ import org.apache.fluss.cluster.Endpoint; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metrics.groups.MetricGroup; import org.apache.fluss.metrics.util.NOPMetricsGroup; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.gateway.AdminGatewayProvider; +import org.apache.fluss.rpc.messages.CreateTableRequest; +import org.apache.fluss.rpc.messages.CreateTableResponse; +import org.apache.fluss.rpc.messages.DropTableRequest; +import org.apache.fluss.rpc.messages.DropTableResponse; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.netty.server.NettyServer; import org.apache.fluss.rpc.netty.server.RequestsMetrics; +import org.apache.fluss.types.DataTypes; import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.clients.producer.internals.ProducerMetrics; import org.apache.kafka.clients.producer.internals.Sender; import org.apache.kafka.common.Node; +import org.apache.kafka.common.compress.Compression; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; import org.apache.kafka.common.metrics.KafkaMetricsContext; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Integration test for Kafka request handling. */ public class KafkaRequestITCase { @@ -61,6 +123,7 @@ public class KafkaRequestITCase { private NettyServer nettyServer; private NetworkClient client; private Node node; + private TestingKafkaGatewayService gatewayService; @BeforeEach public void setup() throws Exception { @@ -69,13 +132,14 @@ public void setup() throws Exception { conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); conf.set(ConfigOptions.KAFKA_ENABLED, true); nettyServer = startNettyServer(); - client = createNetworkClient(); Endpoint endpoint = nettyServer.getBindEndpoints().stream() .filter(e -> e.getListenerName().equals("KAFKA")) .findFirst() .get(); node = new Node(0, endpoint.getHost(), endpoint.getPort()); + client = createNetworkClient(node); + gatewayService.kafkaPort = endpoint.getPort(); } @AfterEach @@ -88,44 +152,246 @@ public void cleanup() throws Exception { @Test public void testApiVersionsRequest() { // initiate the connection - client.ready(node, 100); + client.ready(node, Time.SYSTEM.milliseconds()); // handle the connection, send the ApiVersionsRequest - client.poll(0, 1); - - // check that the ApiVersionsRequest has been initiated - assertThat(client.hasInFlightRequests(node.idString())).isTrue(); + client.poll(0, Time.SYSTEM.milliseconds()); retry( Duration.ofMinutes(1), () -> { // handle completed receives - client.poll(0, 100); + client.poll(0, Time.SYSTEM.milliseconds()); // the ApiVersionsRequest is gone assertThat(client.hasInFlightRequests(node.idString())).isFalse(); // various assertions - assertThat(client.isReady(node, 100)).isTrue(); + assertThat(client.isReady(node, Time.SYSTEM.milliseconds())).isTrue(); }); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testApiVersionsWireVersions(short version) throws Exception { + ApiVersionsRequestData requestData = new ApiVersionsRequestData(); + if (version >= 3) { + requestData + .setClientSoftwareName("fluss-compatibility-test") + .setClientSoftwareVersion("1.0"); + requestData.unknownTaggedFields().add(new RawTaggedField(100, new byte[] {1, 2, 3})); + } + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, version, "test-client", 40 + version); + ApiVersionsRequest request = + new ApiVersionsRequest.Builder(requestData, version, version).build(version); + + ByteBuffer responseBuffer = sendRequest(header, request); + ApiVersionsResponse response = + (ApiVersionsResponse) AbstractResponse.parseResponse(responseBuffer, header); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, ApiKeys.PRODUCE.latestVersion()), + tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion()), + tuple( + ApiKeys.CREATE_TOPICS.id, + ApiKeys.CREATE_TOPICS.oldestVersion(), + ApiKeys.CREATE_TOPICS.latestVersion()), + tuple( + ApiKeys.DELETE_TOPICS.id, + ApiKeys.DELETE_TOPICS.oldestVersion(), + ApiKeys.DELETE_TOPICS.latestVersion())); + } + + @Test + public void testFutureApiVersionsWireVersion() throws Exception { + short futureVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, futureVersion, "test-client", 49); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), + header.headerVersion(), + new ApiVersionsRequestData(), + ApiKeys.API_VERSIONS.oldestVersion()); + + ByteBuffer responseBuffer = sendSerializedRequest(serialized); + ResponseHeader responseHeader = + ResponseHeader.parse(responseBuffer, header.toResponseHeader().headerVersion()); + ApiVersionsResponse response = + ApiVersionsResponse.parse(responseBuffer, ApiKeys.API_VERSIONS.oldestVersion()); + + assertThat(responseHeader.correlationId()).isEqualTo(header.correlationId()); + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNSUPPORTED_VERSION, 1)); + } + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) + public void testMetadataWireVersions(short version) throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, version, "test-client", 42 + version); + MetadataRequestData metadataRequestData = new MetadataRequestData(); + metadataRequestData.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataRequest request = new MetadataRequest(metadataRequestData, version); + if (version >= 9) { + request.data().unknownTaggedFields().add(new RawTaggedField(100, new byte[] {1, 2, 3})); + } + + MetadataResponse response = + (MetadataResponse) + AbstractResponse.parseResponse(sendRequest(header, request), header); + + assertThat(response.brokers()).hasSize(1); + Node responseBroker = response.brokers().iterator().next(); + assertThat(responseBroker.host()).isEqualTo("localhost"); + assertThat(responseBroker.port()).isEqualTo(node.port()); + assertThat(response.errors()).isEmpty(); + assertThat(response.throttleTimeMs()).isZero(); + } + + private ByteBuffer sendRequest(RequestHeader header, AbstractRequest request) throws Exception { + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + return sendSerializedRequest(serialized); + } + + private ByteBuffer sendSerializedRequest(ByteBuffer serialized) throws Exception { + try (Socket socket = new Socket(node.host(), node.port()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + + int responseSize = input.readInt(); + byte[] responseBytes = new byte[responseSize]; + input.readFully(responseBytes); + return ByteBuffer.wrap(responseBytes); + } + } + + @Test + public void testProduceRequest() throws Exception { + short version = ApiKeys.PRODUCE.latestVersion(); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, version, "test-client", 43); + MemoryRecords records = + MemoryRecords.withRecords( + org.apache.kafka.common.record.RecordBatch.MAGIC_VALUE_V2, + 123L, + Compression.NONE, + new SimpleRecord(123L, new byte[] {1}, new byte[] {2})); + TopicProduceData topic = + new TopicProduceData() + .setName("topic") + .setPartitionData( + Collections.singletonList( + new PartitionProduceData() + .setIndex(0) + .setRecords(records))); + ProduceRequest request = + new ProduceRequest( + new ProduceRequestData() + .setAcks((short) 1) + .setTimeoutMs(1000) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())), + version); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + + try (Socket socket = new Socket(node.host(), node.port()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + + int responseSize = input.readInt(); + byte[] responseBytes = new byte[responseSize]; + input.readFully(responseBytes); + ProduceResponse response = + (ProduceResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + } + } + + @Test + public void testStandardClientCreateProduceDeleteLifecycle() throws Exception { + Map config = new HashMap<>(); + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 60000); + try (Admin admin = Admin.create(config)) { + admin.createTopics(Collections.singleton(new NewTopic("topic", 1, (short) 1))) + .all() + .get(); + + Map producerConfig = new HashMap<>(); + producerConfig.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + producerConfig.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + producerConfig.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + producerConfig.put(ProducerConfig.ACKS_CONFIG, "1"); + try (KafkaProducer producer = new KafkaProducer<>(producerConfig)) { + assertThat(producer.send(new ProducerRecord<>("topic", "key", "value")).get()) + .isNotNull(); + } + + admin.deleteTopics(Collections.singleton("topic")).all().get(); + } + verify(gatewayService.adminGateway).createTable(any(CreateTableRequest.class)); + assertThat(gatewayService.produced).isTrue(); + verify(gatewayService.adminGateway).dropTable(any(DropTableRequest.class)); + } + private NettyServer startNettyServer() throws Exception { MetricGroup metricGroup = NOPMetricsGroup.newInstance(); + gatewayService = new TestingKafkaGatewayService(); NettyServer server = new NettyServer( conf, Arrays.asList( new Endpoint("localhost", 0, "INTERNAL"), new Endpoint("localhost", 0, "KAFKA")), - new TestingTabletGatewayService(), + gatewayService, metricGroup, RequestsMetrics.createCoordinatorServerRequestMetrics(metricGroup)); server.start(); return server; } - private NetworkClient createNetworkClient() throws Exception { + private NetworkClient createNetworkClient(Node bootstrapNode) throws Exception { Map config = new HashMap<>(); config.put("key.serializer", StringSerializer.class.getName()); config.put("value.serializer", StringSerializer.class.getName()); @@ -145,6 +411,9 @@ private NetworkClient createNetworkClient() throws Exception { metadataExpireMs, new LogContext(), new ClusterResourceListeners()); + metadata.bootstrap( + Collections.singletonList( + new InetSocketAddress(bootstrapNode.host(), bootstrapNode.port()))); ProducerMetrics metricsRegistry = new ProducerMetrics(metrics); Sensor sensor = Sender.throttleTimeSensor(metricsRegistry.senderMetrics); return ClientUtils.createNetworkClient( @@ -159,4 +428,117 @@ private NetworkClient createNetworkClient() throws Exception { sensor, null); } + + private static final class TestingKafkaGatewayService extends TestingTabletGatewayService + implements AdminGatewayProvider { + + private final AdminGateway adminGateway = mock(AdminGateway.class); + private volatile int kafkaPort; + private volatile boolean produced; + + private final TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("record_key", DataTypes.BYTES()) + .column("payload", DataTypes.BYTES()) + .column( + "event_time", + DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "headers", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "name", + DataTypes.STRING() + .copy(false)), + DataTypes.FIELD( + "value", + DataTypes.BYTES())))) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_LOG_FORMAT, LogFormat.ARROW) + .build(); + + private TestingKafkaGatewayService() { + when(adminGateway.createTable(any(CreateTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new CreateTableResponse())); + when(adminGateway.dropTable(any(DropTableRequest.class))) + .thenReturn(CompletableFuture.completedFuture(new DropTableResponse())); + when(adminGateway.getTableInfo(any(GetTableInfoRequest.class))) + .thenReturn( + CompletableFuture.completedFuture( + new GetTableInfoResponse() + .setTableId(123L) + .setSchemaId(1) + .setTableJson(tableDescriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L))); + } + + @Override + public AdminGateway getAdminGateway() { + return adminGateway; + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + return CompletableFuture.completedFuture(new ListTablesResponse()); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + org.apache.fluss.rpc.messages.MetadataResponse response = + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Collections.singletonList( + new PbServerNode() + .setNodeId(0) + .setHost("localhost") + .setPort(kafkaPort))); + for (PbTablePath tablePath : request.getTablePathsList()) { + if ("topic".equals(tablePath.getTableName())) { + response.addAllTableMetadatas( + Collections.singletonList( + new PbTableMetadata() + .setTablePath(tablePath) + .setTableId(123L) + .addAllBucketMetadatas( + Collections.singletonList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(0) + .setLeaderEpoch(1) + .setReplicaIds( + new int[] {0}))))); + } + } + return CompletableFuture.completedFuture(response); + } + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + return CompletableFuture.completedFuture( + new GetTableInfoResponse() + .setTableId(123L) + .setSchemaId(1) + .setTableJson(tableDescriptor.toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L)); + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + produced = true; + return CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setBaseOffset(42L)))); + } + } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 8f2365add70..d1738a54ada 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -85,4 +85,11 @@ public String currentListenerName() { /** Shutdown the gateway service, release any resources. */ public abstract void shutdown(); + + /** + * Tries to complete actions that were deferred while handling the current request. + * + *

Services without deferred actions do not need to override this method. + */ + public void tryCompleteActions() {} } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java index 2ff5e0691bb..747a0f9b796 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java @@ -96,6 +96,8 @@ public void processRequest(FlussRequest request) { } catch (Throwable t) { LOG.debug("Error while executing RPC {}", api, t); request.fail(stripException(t, InvocationTargetException.class)); + } finally { + service.tryCompleteActions(); } } } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java new file mode 100644 index 00000000000..e1eff680fc8 --- /dev/null +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.netty.server; + +import org.apache.fluss.rpc.TestingGatewayService; +import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; +import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.ApiManager; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; + +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link FlussRequestHandler}. */ +class FlussRequestHandlerTest { + + @Test + void testCompletesActionsAfterSuccessfulInvocation() { + CompletableFuture serviceResponse = + CompletableFuture.completedFuture(new ApiVersionsResponse()); + TestingActionGatewayService service = new TestingActionGatewayService(serviceResponse); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThat(request.getResponseFuture()).isCompletedWithValue(serviceResponse.join()); + } + + @Test + void testCompletesActionsAfterSynchronousInvocationFailure() { + IllegalStateException expected = new IllegalStateException("expected test failure"); + TestingActionGatewayService service = new TestingActionGatewayService(expected); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThatThrownBy(request.getResponseFuture()::join) + .isInstanceOf(CompletionException.class) + .hasCause(expected); + } + + @Test + void testCompletesActionsBeforeAsynchronousResponseFinishes() { + CompletableFuture serviceResponse = new CompletableFuture<>(); + TestingActionGatewayService service = new TestingActionGatewayService(serviceResponse); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThat(request.getResponseFuture()).isNotDone(); + + ApiVersionsResponse response = new ApiVersionsResponse(); + serviceResponse.complete(response); + assertThat(request.getResponseFuture()).isCompletedWithValue(response); + assertThat(service.completedActions()).isOne(); + } + + private static FlussRequest createApiVersionsRequest() { + return new FlussRequest( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.highestSupportedVersion, + 1, + ApiManager.forApiKey(ApiKeys.API_VERSIONS.id), + new ApiVersionsRequest(), + Unpooled.EMPTY_BUFFER, + "FLUSS", + false, + FlussPrincipal.ANONYMOUS, + InetAddress.getLoopbackAddress(), + new CompletableFuture()); + } + + private static final class TestingActionGatewayService extends TestingGatewayService { + private final CompletableFuture response; + private final RuntimeException synchronousFailure; + private final AtomicInteger completedActions = new AtomicInteger(); + + private TestingActionGatewayService(CompletableFuture response) { + this.response = response; + this.synchronousFailure = null; + } + + private TestingActionGatewayService(RuntimeException synchronousFailure) { + this.response = null; + this.synchronousFailure = synchronousFailure; + } + + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + if (synchronousFailure != null) { + throw synchronousFailure; + } + return response; + } + + @Override + public void tryCompleteActions() { + completedActions.incrementAndGet(); + } + + private int completedActions() { + return completedActions.get(); + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index ee28567756f..7cb4db9d7e4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -2416,12 +2416,13 @@ private boolean isFollowerOutOfSync( private void validateInSyncReplicaSize(int requiredAcks) { int inSyncSize = isrState.isr().size(); - if (inSyncSize < minInSyncReplicasSupplier.getAsInt() && requiredAcks == -1) { + int minInSyncReplicas = minInSyncReplicasSupplier.getAsInt(); + if (inSyncSize < minInSyncReplicas && requiredAcks == -1) { throw new NotEnoughReplicasException( String.format( - "The size of the current ISR %s is insufficient to satisfy " - + "the required acks %s for table bucket %s.", - isrState.isr(), requiredAcks, tableBucket)); + "The current ISR %s has %d replicas, below the configured minimum ISR " + + "%d required for acks=all on table bucket %s.", + isrState.isr(), inSyncSize, minInSyncReplicas, tableBucket)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..f2a1e5db115 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -108,6 +108,8 @@ import org.apache.fluss.server.metrics.group.BucketMetricGroup; import org.apache.fluss.server.metrics.group.TableMetricGroup; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.server.replica.delay.ActionQueue; +import org.apache.fluss.server.replica.delay.DelayedActionQueue; import org.apache.fluss.server.replica.delay.DelayedFetchLog; import org.apache.fluss.server.replica.delay.DelayedFetchLog.FetchBucketStatus; import org.apache.fluss.server.replica.delay.DelayedOperationManager; @@ -212,6 +214,8 @@ public class ReplicaManager implements ServerReconfigurable { */ private final DelayedOperationManager delayedFetchLogManager; + private final ActionQueue actionQueue; + private final ReplicaFetcherManager replicaFetcherManager; // The manager used to manager the replica alter, especially the isr expand and shrink. private final AdjustIsrManager adjustIsrManager; @@ -339,6 +343,7 @@ public ReplicaManager( "delay fetch log", serverId, conf.getInt(ConfigOptions.LOG_REPLICA_FETCH_OPERATION_PURGE_NUMBER)); + this.actionQueue = new DelayedActionQueue(); this.internalListenerName = conf.get(ConfigOptions.INTERNAL_LISTENER_NAME); this.replicaFetcherManager = @@ -683,6 +688,10 @@ public void appendRecordsToLog( appendToLocalLog(entriesPerBucket, requiredAcks, userContext); LOG.debug("Append records to local log in {} ms", System.currentTimeMillis() - startTime); + // Queue fetch completion before registering a delayed write. The request handler drains + // the actions only after both steps have returned. + enqueueDelayedFetchCompletions(appendResult); + // maybe do delay write operation. maybeAddDelayedWrite( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); @@ -2152,6 +2161,24 @@ private boolean isNonCriticalFetchError(Errors error) { || error == Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION; } + private void enqueueDelayedFetchCompletions( + Map appendResults) { + appendResults.forEach( + (tableBucket, appendResult) -> { + if (appendResult.succeeded()) { + actionQueue.add( + () -> + delayedFetchLogManager.checkAndComplete( + new DelayedTableBucketKey(tableBucket))); + } + }); + } + + /** Tries to complete actions deferred by log appends. */ + public void tryCompleteActions() { + actionQueue.tryCompleteActions(); + } + private void completeDelayedOperations(TableBucket tableBucket) { DelayedTableBucketKey delayedTableBucketKey = new DelayedTableBucketKey(tableBucket); delayedWriteManager.checkAndComplete(delayedTableBucketKey); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java new file mode 100644 index 00000000000..997295f6718 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; + +/** + * A queue for collecting actions that must run after the current request invocation releases its + * write-path locks. + */ +@Internal +public interface ActionQueue { + + /** Adds an action to the queue. */ + void add(Runnable action); + + /** Tries to execute pending actions without waiting for actions added concurrently. */ + void tryCompleteActions(); +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java new file mode 100644 index 00000000000..a5070de3a55 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Thread-safe {@link ActionQueue} backed by a concurrent queue. + * + *

Each drain bounds its work using the queue's weakly consistent size at the start. Actions + * added while draining may remain available for a later drain. A failing action is logged and does + * not prevent the remaining bounded set from running. + */ +@Internal +public class DelayedActionQueue implements ActionQueue { + private static final Logger LOG = LoggerFactory.getLogger(DelayedActionQueue.class); + + private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + + @Override + public void add(Runnable action) { + queue.add(action); + } + + @Override + public void tryCompleteActions() { + int actionsToComplete = queue.size(); + for (int completed = 0; completed < actionsToComplete; completed++) { + Runnable action = queue.poll(); + if (action == null) { + return; + } + try { + action.run(); + } catch (Exception e) { + LOG.error("Failed to complete delayed action.", e); + } + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index 4230637bdab..b0993e51e6d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -222,6 +222,11 @@ public CoordinatorGateway getAdminGateway() { @Override public void shutdown() {} + @Override + public void tryCompleteActions() { + replicaManager.tryCompleteActions(); + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java index 846a1f5f56c..74b164b0cca 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java @@ -222,8 +222,9 @@ void testIsrSetSizeLessThanMinInSynReplicasNumber() throws Exception { assertThat(respForBucket.getErrorMessage()) .contains( String.format( - "The size of the current ISR [%s] is insufficient to satisfy the " - + "required acks -1 for table bucket TableBucket{tableId=%s, bucket=0}.", + "The current ISR [%s] has 1 replicas, below the configured minimum " + + "ISR 2 required for acks=all on table bucket " + + "TableBucket{tableId=%s, bucket=0}.", leader, tableId)); // check again leader highWatermark not increase because the isr set < min_isr assertThat( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index b1e98b9d30f..de8b498123a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -38,6 +38,7 @@ import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.record.ProjectionPushdownCache; +import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.kv.KvFlushScheduler; @@ -268,6 +269,26 @@ void testPhysicalStorageLocalLogSizeIsScopedPerBucket() throws Exception { .containsEntry("bucket", "2"); } + @Test + void testAcksAllUsesUpdatedMinIsrWhileWaitingForAcknowledgement() throws Exception { + conf.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 1); + Replica logReplica = + makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, new TableBucket(DATA1_TABLE_ID, 1)); + makeLogReplicaAsLeader(logReplica); + + logReplica.appendRecordsToLeader(genMemoryLogRecordsByObject(DATA1), 0); + long requiredOffset = logReplica.getLocalLogEndOffset(); + logReplica.getLogTablet().updateHighWatermark(requiredOffset); + + assertThat(logReplica.checkEnoughReplicasReachOffset(requiredOffset)) + .isEqualTo(Tuple2.of(true, Errors.NONE)); + + conf.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + + assertThat(logReplica.checkEnoughReplicasReachOffset(requiredOffset)) + .isEqualTo(Tuple2.of(true, Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION)); + } + @Test void testAppendRecordsWithOutOfOrderBatchSequence() throws Exception { Replica logReplica = diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java new file mode 100644 index 00000000000..27bc0fa70ad --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link DelayedActionQueue}. */ +class DelayedActionQueueTest { + + @Test + void testActionsExecuteExactlyOnce() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + AtomicInteger executions = new AtomicInteger(); + actionQueue.add(executions::incrementAndGet); + actionQueue.add(executions::incrementAndGet); + + actionQueue.tryCompleteActions(); + actionQueue.tryCompleteActions(); + + assertThat(executions).hasValue(2); + } + + @Test + void testActionFailureDoesNotPreventLaterActions() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + AtomicInteger executions = new AtomicInteger(); + actionQueue.add( + () -> { + throw new RuntimeException("expected test failure"); + }); + actionQueue.add(executions::incrementAndGet); + + actionQueue.tryCompleteActions(); + actionQueue.tryCompleteActions(); + + assertThat(executions).hasValue(1); + } + + @Test + void testDrainUsesPendingActionSnapshot() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + List executions = new ArrayList<>(); + actionQueue.add( + () -> { + executions.add(1); + actionQueue.add(() -> executions.add(2)); + }); + + actionQueue.tryCompleteActions(); + assertThat(executions).containsExactly(1); + + actionQueue.tryCompleteActions(); + assertThat(executions).isEqualTo(Arrays.asList(1, 2)); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java index 73d620842dc..87df0b844d2 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java @@ -32,6 +32,7 @@ import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -83,7 +84,7 @@ void testCompleteDelayedFetchLog() throws Exception { assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(1); assertThat(delayedFetchLogManager.watched()).isEqualTo(1); - // write data. + // Appending data enqueues completion, but does not run it under the append call. assertThat(delayedResponse.isDone()).isFalse(); CompletableFuture> future = new CompletableFuture<>(); replicaManager.appendRecordsToLog( @@ -93,10 +94,10 @@ void testCompleteDelayedFetchLog() throws Exception { null, future::complete); assertThat(future.get()).containsOnly(new ProduceLogResultForBucket(tb, 0, 10L)); + assertThat(delayedResponse.isDone()).isFalse(); - // check and complete manually - numComplete = delayedFetchLogManager.checkAndComplete(delayedTableBucketKey); - assertThat(numComplete).isEqualTo(1); + replicaManager.tryCompleteActions(); + assertThat(delayedResponse.isDone()).isTrue(); assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(0); assertThat(delayedFetchLogManager.watched()).isEqualTo(0); @@ -107,6 +108,72 @@ void testCompleteDelayedFetchLog() throws Exception { assertLogRecordsEquals(DATA1_ROW_TYPE, resultForBucket.records(), DATA1); } + @Test + void testSuccessfulBucketCompletesWhenAnotherBucketAppendFails() throws Exception { + TableBucket successfulBucket = new TableBucket(DATA1_TABLE_ID, 1); + TableBucket failedBucket = new TableBucket(DATA1_TABLE_ID, 2); + makeLogTableAsLeader(successfulBucket.getBucket()); + CompletableFuture> delayedResponse = + watchDelayedFetch(successfulBucket); + + Map entries = new HashMap<>(); + entries.put(successfulBucket, genMemoryLogRecordsByObject(DATA1)); + entries.put(failedBucket, genMemoryLogRecordsByObject(DATA1)); + CompletableFuture> produceResponse = + new CompletableFuture<>(); + + replicaManager.appendRecordsToLog(20000, 1, entries, null, produceResponse::complete); + + List produceResults = produceResponse.get(); + assertThat(produceResults).hasSize(2); + assertThat(produceResults) + .filteredOn(result -> result.getTableBucket().equals(successfulBucket)) + .hasSize(1) + .allSatisfy(result -> assertThat(result.succeeded()).isTrue()); + assertThat(produceResults) + .filteredOn(result -> result.getTableBucket().equals(failedBucket)) + .hasSize(1) + .allSatisfy(result -> assertThat(result.failed()).isTrue()); + assertThat(delayedResponse).isNotDone(); + + replicaManager.tryCompleteActions(); + + assertThat(delayedResponse).isDone(); + assertThat(replicaManager.getDelayedFetchLogManager().numDelayed()).isZero(); + } + + @Test + void testDrainCompletesDelayedFetchesForMultipleBuckets() throws Exception { + TableBucket firstBucket = new TableBucket(DATA1_TABLE_ID, 1); + TableBucket secondBucket = new TableBucket(DATA1_TABLE_ID, 2); + makeLogTableAsLeader(firstBucket.getBucket()); + makeLogTableAsLeader(secondBucket.getBucket()); + CompletableFuture> firstResponse = + watchDelayedFetch(firstBucket); + CompletableFuture> secondResponse = + watchDelayedFetch(secondBucket); + + Map entries = new HashMap<>(); + entries.put(firstBucket, genMemoryLogRecordsByObject(DATA1)); + entries.put(secondBucket, genMemoryLogRecordsByObject(DATA1)); + CompletableFuture> produceResponse = + new CompletableFuture<>(); + + replicaManager.appendRecordsToLog(20000, 1, entries, null, produceResponse::complete); + + assertThat(produceResponse.get()) + .hasSize(2) + .allSatisfy(result -> assertThat(result.succeeded()).isTrue()); + assertThat(firstResponse).isNotDone(); + assertThat(secondResponse).isNotDone(); + + replicaManager.tryCompleteActions(); + + assertThat(firstResponse).isDone(); + assertThat(secondResponse).isDone(); + assertThat(replicaManager.getDelayedFetchLogManager().numDelayed()).isZero(); + } + @Test void testDelayFetchLogTimeout() { TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); @@ -165,4 +232,28 @@ private DelayedFetchLog createDelayedFetchLogRequest( TestingMetricGroups.TABLET_SERVER_METRICS, null); } + + private CompletableFuture> watchDelayedFetch( + TableBucket tableBucket) { + FetchLogResultForBucket previousResult = + new FetchLogResultForBucket(tableBucket, MemoryLogRecords.EMPTY, 0L); + CompletableFuture> response = + new CompletableFuture<>(); + DelayedFetchLog delayedFetchLog = + createDelayedFetchLogRequest( + tableBucket, + 1, + Duration.ofMinutes(3).toMillis(), + new FetchBucketStatus( + new FetchReqInfo(150001L, 0L, Integer.MAX_VALUE), + new LogOffsetMetadata(0L, 0L, 0), + previousResult), + response::complete); + replicaManager + .getDelayedFetchLogManager() + .tryCompleteElseWatch( + delayedFetchLog, + Collections.singletonList(new DelayedTableBucketKey(tableBucket))); + return response; + } } From 263282d0d4368b7471fe558b0996d4968b8280bf Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 1 Sep 2026 20:14:01 +0800 Subject: [PATCH 3/3] [kafka] Add SASL/PLAIN authentication Authenticate Kafka protocol connections with SASL/PLAIN and propagate the authenticated principal through request dispatch and authorization. Co-Authored-By: Codex AI-Model: gpt-5.6-sol AI-Contributed/Feature: 1503/1503 AI-Contributed/UT: 1514/1514 --- .../fluss/config/ConfigurationUtils.java | 1 + .../SaslServerAuthenticator.java | 31 +- .../security/auth/sasl/jaas/LoginManager.java | 14 +- .../auth/sasl/jaas/SaslServerFactory.java | 8 + .../plain/PlainSaslServerConfigManager.java | 206 +++++++++++++ .../fluss/config/ConfigurationTest.java | 4 + .../auth/sasl/jaas/LoginManagerTest.java | 11 +- .../PlainSaslServerConfigManagerTest.java | 176 +++++++++++ .../fluss/kafka/KafkaChannelInitializer.java | 29 +- .../fluss/kafka/KafkaCommandDecoder.java | 91 +++++- .../fluss/kafka/KafkaProtocolPlugin.java | 103 ++++++- .../org/apache/fluss/kafka/KafkaRequest.java | 72 ++++- .../fluss/kafka/KafkaRequestContext.java | 19 ++ .../fluss/kafka/KafkaRequestHandler.java | 51 +++- .../kafka/api/admin/CreateTopicsHandler.java | 3 +- .../kafka/api/admin/DeleteTopicsHandler.java | 5 +- .../kafka/api/metadata/MetadataHandler.java | 3 +- .../kafka/api/produce/ProduceHandler.java | 3 +- .../api/sasl/SaslAuthenticateHandler.java | 105 +++++++ .../kafka/api/sasl/SaslHandshakeHandler.java | 122 ++++++++ .../api/versions/ApiVersionsHandler.java | 14 + .../admin/GatewayKafkaTopicAdminBackend.java | 79 ++++- .../backend/admin/KafkaTopicAdminBackend.java | 20 ++ .../metadata/GatewayKafkaMetadataBackend.java | 3 +- .../backend/metadata/KafkaMetadataQuery.java | 18 ++ .../produce/GatewayKafkaProduceBackend.java | 3 +- .../backend/produce/KafkaProduceCommand.java | 19 ++ .../dispatcher/KafkaRequestDispatcher.java | 9 +- .../kafka/security/KafkaSaslConnection.java | 263 +++++++++++++++++ .../fluss/kafka/KafkaCommandDecoderTest.java | 213 ++++++++++++++ .../apache/fluss/kafka/KafkaConfigsTest.java | 55 ++++ .../fluss/kafka/KafkaMetadataHandlerTest.java | 32 ++ .../fluss/kafka/KafkaProduceHandlerTest.java | 35 +++ .../fluss/kafka/KafkaRequestHandlerTest.java | 71 +++++ .../KafkaSaslPlainAuthenticationITCase.java | 233 +++++++++++++++ .../kafka/KafkaTopicAdminHandlerTest.java | 146 +++++++++- .../kafka/api/sasl/SaslHandlersTest.java | 255 ++++++++++++++++ .../security/KafkaSaslConnectionTest.java | 274 ++++++++++++++++++ .../rpc/gateway/AdminOperationAuthorizer.java | 32 ++ .../rpc/netty/server/FlussProtocolPlugin.java | 153 +--------- .../rpc/netty/server/NettyServerHandler.java | 13 +- .../rpc/TestingTabletGatewayService.java | 9 +- .../fluss/server/tablet/TabletService.java | 11 +- 43 files changed, 2805 insertions(+), 212 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManager.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManagerTest.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslAuthenticateHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslHandshakeHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/security/KafkaSaslConnection.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaSaslPlainAuthenticationITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/api/sasl/SaslHandlersTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/security/KafkaSaslConnectionTest.java create mode 100644 fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminOperationAuthorizer.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigurationUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigurationUtils.java index 1d08102565c..6a780fa0f43 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigurationUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigurationUtils.java @@ -51,6 +51,7 @@ public class ConfigurationUtils { "token", "basic-auth", "jaas.config", + "security.sasl.plain.credentials", "http-headers", "private.key", "private-key", diff --git a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/authenticator/SaslServerAuthenticator.java b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/authenticator/SaslServerAuthenticator.java index 48231932e9c..8225010c60f 100644 --- a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/authenticator/SaslServerAuthenticator.java +++ b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/authenticator/SaslServerAuthenticator.java @@ -46,6 +46,7 @@ public class SaslServerAuthenticator implements ServerAuthenticator { private static final String SERVER_AUTHENTICATOR_PREFIX = "security.sasl."; private final List enabledMechanisms; private SaslServer saslServer; + private LoginManager loginManager; private final Map configs; public SaslServerAuthenticator(Configuration configuration) { @@ -60,6 +61,7 @@ public SaslServerAuthenticator(Configuration configuration) { @Override public void initialize(AuthenticateContext context) { + close(); String mechanism = context.protocol(); String listenerName = context.listenerName(); String address = context.ipAddress(); @@ -102,16 +104,22 @@ public void initialize(AuthenticateContext context) { JaasContext jaasContext = JaasContext.loadServerContext(listenerName, dynamicJaasConfig); + LoginManager acquiredLoginManager = null; try { - LoginManager loginManager = LoginManager.acquireLoginManager(jaasContext); - saslServer = + acquiredLoginManager = LoginManager.acquireLoginManager(jaasContext); + SaslServer newSaslServer = createSaslServer( mechanism, address, configs, - loginManager, + acquiredLoginManager, jaasContext.configurationEntries()); + loginManager = acquiredLoginManager; + saslServer = newSaslServer; } catch (Exception e) { + if (acquiredLoginManager != null) { + acquiredLoginManager.release(); + } throw new RuntimeException(e); } } @@ -150,4 +158,21 @@ public boolean isCompleted() { public FlussPrincipal createPrincipal() { return new FlussPrincipal(saslServer.getAuthorizationID(), "User"); } + + @Override + public void close() { + if (saslServer != null) { + try { + saslServer.dispose(); + } catch (SaslException e) { + LOG.debug("Failed to dispose SASL server.", e); + } finally { + saslServer = null; + } + } + if (loginManager != null) { + loginManager.release(); + loginManager = null; + } + } } diff --git a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/LoginManager.java b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/LoginManager.java index a425130f543..0e5722a6c80 100644 --- a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/LoginManager.java +++ b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/LoginManager.java @@ -44,6 +44,7 @@ public class LoginManager { private int refCount; private final String loginKey; + private final boolean dynamic; /** * A global cache of LoginManager instances keyed by static JAAS configuration names (e.g., @@ -68,11 +69,13 @@ public class LoginManager { * @throws LoginException if the login operation fails due to invalid credentials, missing * modules, or misconfigured JAAS settings */ - private LoginManager(JaasContext jaasContext, String loginKey) throws LoginException { + private LoginManager(JaasContext jaasContext, String loginKey, boolean dynamic) + throws LoginException { this.login = new DefaultLogin(); login.configure(jaasContext.name(), jaasContext.getConfiguration()); login.login(); this.loginKey = loginKey; + this.dynamic = dynamic; } public Subject subject() { @@ -90,14 +93,14 @@ public static LoginManager acquireLoginManager(JaasContext jaasContext) throws L if (jaasConfigValue != null) { loginManager = DYNAMIC_INSTANCES.get(jaasConfigValue); if (loginManager == null) { - loginManager = new LoginManager(jaasContext, jaasConfigValue); + loginManager = new LoginManager(jaasContext, jaasConfigValue, true); DYNAMIC_INSTANCES.put(jaasConfigValue, loginManager); } } else { String jaasContextName = jaasContext.name(); loginManager = STATIC_INSTANCES.get(jaasContextName); if (loginManager == null) { - loginManager = new LoginManager(jaasContext, jaasContextName); + loginManager = new LoginManager(jaasContext, jaasContextName, false); STATIC_INSTANCES.put(jaasContextName, loginManager); } } @@ -117,6 +120,11 @@ public void release() { if (refCount == 0) { throw new IllegalStateException("release() called on disposed " + this); } else if (refCount == 1) { + if (dynamic) { + DYNAMIC_INSTANCES.remove(loginKey, this); + } else { + STATIC_INSTANCES.remove(loginKey, this); + } login.close(); } --refCount; diff --git a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/SaslServerFactory.java b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/SaslServerFactory.java index 87fe2ab351f..f938ede74d5 100644 --- a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/SaslServerFactory.java +++ b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/jaas/SaslServerFactory.java @@ -17,6 +17,7 @@ package org.apache.fluss.security.auth.sasl.jaas; +import org.apache.fluss.security.auth.sasl.plain.PlainSaslServer; import org.apache.fluss.security.auth.sasl.plain.PlainServerCallbackHandler; import org.slf4j.Logger; @@ -61,6 +62,13 @@ public static SaslServer createSaslServer( } callbackHandler.configure(mechanism, configurationEntries); + // Construct Fluss's PLAIN server directly. Kafka clients register a provider with the + // same JVM provider name as Fluss's PLAIN provider. Delegating this server-side path + // to Sasl.createSaslServer would therefore make the selected callback type depend on + // class-loading order when both implementations share a process. + if (PlainSaslServer.PLAIN_MECHANISM.equals(mechanism)) { + return new PlainSaslServer(callbackHandler); + } SaslServer saslServer = Subject.doAs( loginManager.subject(), diff --git a/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManager.java b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManager.java new file mode 100644 index 00000000000..8abff33317f --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManager.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.security.auth.sasl.plain; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.ServerReconfigurable; +import org.apache.fluss.exception.ConfigException; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** + * Manages the effective server configuration for SASL/PLAIN authentication. + * + *

{@link ConfigOptions#SERVER_SASL_CREDENTIALS} is a convenient credential map, while the SASL + * implementation consumes {@link ConfigOptions#SERVER_SASL_PLAIN_JAAS_CONFIG}. This manager + * validates the credential map and converts it to a JAAS configuration. Credentials from the map + * are merged with credentials in the initial JAAS configuration and take precedence when the same + * username is present in both sources. + * + *

The managed {@link Configuration} has a stable identity so authenticator suppliers that + * capture it during server startup see later credential updates. Callers must treat the returned + * configuration as read-only and perform updates through this manager. + */ +@Internal +public final class PlainSaslServerConfigManager implements ServerReconfigurable { + + private static final String PLAIN_CREDENTIALS_CONFIG = + ConfigOptions.SERVER_SASL_CREDENTIALS.key(); + + /** Pattern to match {@code user_=""} entries in a JAAS config. */ + private static final Pattern JAAS_USER_PATTERN = Pattern.compile("user_(\\w+)=\"([^\"]*)\""); + + /** Usernames become JAAS option keys, so only word characters are accepted. */ + private static final Pattern VALID_USERNAME_PATTERN = Pattern.compile("\\w+"); + + /** Characters that would break the credential-map syntax or generated JAAS statement. */ + private static final Pattern INVALID_PASSWORD_PATTERN = + Pattern.compile("[,:\"\\\\;]|[\\x00-\\x1F\\x7F]"); + + private final Map initialPlainCredentialsFromJaasConfig; + + private final Configuration configuration; + + // Access is guarded by synchronized validate/reconfigure calls. + private Map currentPlainCredentials; + + /** + * Creates a manager from the initial server configuration. + * + * @param configuration initial server configuration + * @throws ConfigException if the configured credential map is invalid + */ + public PlainSaslServerConfigManager(Configuration configuration) throws ConfigException { + checkNotNull(configuration, "configuration must not be null"); + this.configuration = new Configuration(configuration); + this.initialPlainCredentialsFromJaasConfig = parseCredentialsFromJaasConfig(configuration); + validate(configuration); + reconfigure(configuration); + } + + /** + * Returns the managed configuration containing the effective generated JAAS configuration. + * + *

The returned object has a stable identity and must be treated as read-only by callers. + * + * @return the managed effective configuration + */ + public Configuration getConfiguration() { + return configuration; + } + + @Override + public synchronized void validate(Configuration newConfiguration) throws ConfigException { + Map newCredentials = readPlainCredentials(newConfiguration); + if (Objects.equals(newCredentials, currentPlainCredentials)) { + return; + } + + if (newCredentials != null && !newCredentials.isEmpty()) { + int index = 0; + for (Map.Entry credential : newCredentials.entrySet()) { + validateUsername(credential.getKey()); + validatePassword(index, credential.getKey(), credential.getValue()); + index++; + } + } + + // Build the value during validation so reconfigure cannot fail after validation succeeds. + generateMergedJaasConfig(newCredentials); + } + + @Override + public synchronized void reconfigure(Configuration newConfiguration) throws ConfigException { + // DynamicServerConfig may continue to reconfigure other components after a validation + // failure when it is applying a best-effort update. Defensively validate here as well so + // malformed credentials can never be rendered into an effective JAAS statement. + validate(newConfiguration); + Map newCredentials = readPlainCredentials(newConfiguration); + if (Objects.equals(newCredentials, currentPlainCredentials)) { + return; + } + + configuration.setString( + ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG, + generateMergedJaasConfig(newCredentials)); + currentPlainCredentials = copyCredentials(newCredentials); + } + + private static Map readPlainCredentials(Configuration configuration) + throws ConfigException { + try { + return copyCredentials(configuration.get(ConfigOptions.SERVER_SASL_CREDENTIALS)); + } catch (IllegalArgumentException | IllegalStateException e) { + throw new ConfigException( + String.format( + "Failed to parse %s: %s", PLAIN_CREDENTIALS_CONFIG, e.getMessage()), + e); + } + } + + private static Map copyCredentials(Map credentials) { + return credentials == null ? null : new LinkedHashMap<>(credentials); + } + + private static void validateUsername(String username) throws ConfigException { + if (username == null || !VALID_USERNAME_PATTERN.matcher(username).matches()) { + throw new ConfigException( + String.format( + "%s: username '%s' contains invalid characters. " + + "Only letters, digits, and underscores are allowed.", + PLAIN_CREDENTIALS_CONFIG, username)); + } + } + + private static void validatePassword(int index, String username, String password) + throws ConfigException { + if (password == null || password.isEmpty()) { + throw new ConfigException( + String.format( + "%s[%d]: password for user '%s' must not be empty.", + PLAIN_CREDENTIALS_CONFIG, index, username)); + } + if (INVALID_PASSWORD_PATTERN.matcher(password).find()) { + throw new ConfigException( + String.format( + "%s[%d]: password for user '%s' contains invalid characters. " + + "Commas, colons, quotes, semicolons, backslashes, and control characters are not allowed.", + PLAIN_CREDENTIALS_CONFIG, index, username)); + } + } + + private String generateMergedJaasConfig(Map newCredentials) { + Map mergedCredentials = + new LinkedHashMap<>(initialPlainCredentialsFromJaasConfig); + if (newCredentials != null) { + mergedCredentials.putAll(newCredentials); + } + + StringBuilder jaasConfig = + new StringBuilder(PlainLoginModule.class.getName()).append(" required"); + for (Map.Entry entry : mergedCredentials.entrySet()) { + jaasConfig + .append(" user_") + .append(entry.getKey()) + .append("=\"") + .append(entry.getValue()) + .append('"'); + } + return jaasConfig.append(';').toString(); + } + + private static Map parseCredentialsFromJaasConfig(Configuration configuration) { + Map credentials = new LinkedHashMap<>(); + String existingJaas = configuration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG); + if (existingJaas != null) { + Matcher matcher = JAAS_USER_PATTERN.matcher(existingJaas); + while (matcher.find()) { + credentials.put(matcher.group(1), matcher.group(2)); + } + } + return credentials; + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/config/ConfigurationTest.java b/fluss-common/src/test/java/org/apache/fluss/config/ConfigurationTest.java index d0809994850..a165b5fbc5b 100644 --- a/fluss-common/src/test/java/org/apache/fluss/config/ConfigurationTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/config/ConfigurationTest.java @@ -512,6 +512,10 @@ void testHideSensitiveValue() { .isEqualTo(Password.HIDDEN_CONTENT); assertThat(ConfigurationUtils.hideSensitiveValue("client.security.sasl.password", "pwd")) .isEqualTo(Password.HIDDEN_CONTENT); + assertThat( + ConfigurationUtils.hideSensitiveValue( + ConfigOptions.SERVER_SASL_CREDENTIALS.key(), "admin:admin-secret")) + .isEqualTo(Password.HIDDEN_CONTENT); assertThat(ConfigurationUtils.hideSensitiveValue("plain.key", new Password("pwd"))) .isEqualTo(Password.HIDDEN_CONTENT); assertThat(ConfigurationUtils.hideSensitiveValue("plain.key", "value")).isEqualTo("value"); diff --git a/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/jaas/LoginManagerTest.java b/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/jaas/LoginManagerTest.java index 39da239001a..953ca4399a0 100644 --- a/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/jaas/LoginManagerTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/jaas/LoginManagerTest.java @@ -96,13 +96,12 @@ private void verifyLoginManagerRelease( assertThat(LoginManager.acquireLoginManager(jaasContext)).isEqualTo(loginManager); } - // Release all references and verify that new LoginManager is created on next acquire - for (int i = 0; i < 2; i++) { - // release all references + // Release all references and verify that a new LoginManager is created on next acquire. + for (int i = 0; i < acquireCount; i++) { loginManager.release(); - LoginManager newLoginManager = LoginManager.acquireLoginManager(jaasContext); - assertThat(newLoginManager).isEqualTo(loginManager); - newLoginManager.release(); } + LoginManager newLoginManager = LoginManager.acquireLoginManager(jaasContext); + assertThat(newLoginManager).isNotSameAs(loginManager); + newLoginManager.release(); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManagerTest.java b/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManagerTest.java new file mode 100644 index 00000000000..782ee56d72a --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/security/auth/sasl/plain/PlainSaslServerConfigManagerTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.security.auth.sasl.plain; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; + +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PlainSaslServerConfigManagerTest { + + @Test + void testInitialCredentialMapIsMergedWithJaasConfig() { + Configuration initialConfiguration = new Configuration(); + initialConfiguration.setString( + ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG, + PlainLoginModule.class.getName() + + " required user_admin=\"old-secret\" user_alice=\"alice-secret\";"); + initialConfiguration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, + credentials("admin", "new-secret", "bob", "bob-secret")); + + PlainSaslServerConfigManager manager = + new PlainSaslServerConfigManager(initialConfiguration); + + assertThat( + manager.getConfiguration() + .getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .isEqualTo( + PlainLoginModule.class.getName() + + " required user_admin=\"new-secret\"" + + " user_alice=\"alice-secret\"" + + " user_bob=\"bob-secret\";"); + assertThat(initialConfiguration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .contains("user_admin=\"old-secret\"") + .doesNotContain("user_bob"); + } + + @Test + void testReconfigureUpdatesStableManagedConfiguration() { + Configuration initialConfiguration = new Configuration(); + initialConfiguration.setString( + ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG, + PlainLoginModule.class.getName() + " required user_admin=\"admin-secret\";"); + PlainSaslServerConfigManager manager = + new PlainSaslServerConfigManager(initialConfiguration); + Configuration managedConfiguration = manager.getConfiguration(); + + Configuration addBobConfiguration = new Configuration(); + addBobConfiguration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("bob", "bob-secret")); + manager.validate(addBobConfiguration); + manager.reconfigure(addBobConfiguration); + + assertThat(manager.getConfiguration()).isSameAs(managedConfiguration); + assertThat(managedConfiguration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .isEqualTo( + PlainLoginModule.class.getName() + + " required user_admin=\"admin-secret\"" + + " user_bob=\"bob-secret\";"); + + Configuration removeBobConfiguration = new Configuration(); + manager.validate(removeBobConfiguration); + manager.reconfigure(removeBobConfiguration); + + assertThat(manager.getConfiguration()).isSameAs(managedConfiguration); + assertThat(managedConfiguration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .isEqualTo( + PlainLoginModule.class.getName() + + " required user_admin=\"admin-secret\";"); + } + + @Test + void testValidationDoesNotApplyCredentials() { + PlainSaslServerConfigManager manager = + new PlainSaslServerConfigManager(new Configuration()); + Configuration managedConfiguration = manager.getConfiguration(); + Configuration newConfiguration = new Configuration(); + newConfiguration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("alice", "alice-secret")); + + manager.validate(newConfiguration); + + assertThat(managedConfiguration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .isNull(); + } + + @Test + void testInitialConfigurationRejectsInvalidCredentials() { + Configuration invalidUsername = new Configuration(); + invalidUsername.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("user-name", "secret")); + assertThatThrownBy(() -> new PlainSaslServerConfigManager(invalidUsername)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("username 'user-name' contains invalid characters"); + + Configuration invalidPassword = new Configuration(); + invalidPassword.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("user", "pass;word")); + assertThatThrownBy(() -> new PlainSaslServerConfigManager(invalidPassword)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("password for user 'user' contains invalid characters"); + + Configuration emptyPassword = new Configuration(); + emptyPassword.set(ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("user", "")); + assertThatThrownBy(() -> new PlainSaslServerConfigManager(emptyPassword)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("password for user 'user' must not be empty"); + } + + @Test + void testRejectsMalformedCredentialMapString() { + PlainSaslServerConfigManager manager = + new PlainSaslServerConfigManager(new Configuration()); + Configuration malformedConfiguration = new Configuration(); + malformedConfiguration.setString( + ConfigOptions.SERVER_SASL_CREDENTIALS.key(), "bob:pass,word"); + + assertThatThrownBy(() -> manager.validate(malformedConfiguration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("Failed to parse security.sasl.plain.credentials") + .hasMessageNotContaining("pass,word"); + } + + @Test + void testReconfigureRejectsInvalidCredentialsWithoutMutatingManagedConfiguration() { + Configuration initialConfiguration = new Configuration(); + initialConfiguration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("admin", "admin-secret")); + PlainSaslServerConfigManager manager = + new PlainSaslServerConfigManager(initialConfiguration); + String effectiveJaas = + manager.getConfiguration().getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG); + Configuration invalidConfiguration = new Configuration(); + invalidConfiguration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, credentials("bob", "pass;word")); + + assertThatThrownBy(() -> manager.reconfigure(invalidConfiguration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("password for user 'bob' contains invalid characters"); + assertThat( + manager.getConfiguration() + .getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG)) + .isEqualTo(effectiveJaas); + } + + private static Map credentials(String... values) { + Map credentials = new LinkedHashMap<>(); + for (int i = 0; i < values.length; i += 2) { + credentials.put(values[i], values[i + 1]); + } + return credentials; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java index 29bdc745ca9..7285d20cb2a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java @@ -19,11 +19,16 @@ import org.apache.fluss.rpc.netty.NettyChannelInitializer; import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.security.auth.ServerAuthenticator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelInitializer; import org.apache.fluss.shaded.netty4.io.netty.channel.socket.SocketChannel; import org.apache.fluss.shaded.netty4.io.netty.handler.codec.LengthFieldPrepender; import org.apache.fluss.shaded.netty4.io.netty.handler.flow.FlowControlHandler; +import javax.annotation.Nullable; + +import java.util.function.Supplier; + /** * A {@link ChannelInitializer} for initializing {@link SocketChannel} instances that will be used * by the server to handle the Kafka requests for the client. @@ -33,29 +38,51 @@ public class KafkaChannelInitializer extends NettyChannelInitializer { private final RequestChannel[] requestChannels; private final String listenerName; private final int maxRequestSize; + private final @Nullable Supplier authenticatorSupplier; private final LengthFieldPrepender prepender = new LengthFieldPrepender(4); private final boolean preferHeap; + /** Creates a PLAINTEXT channel initializer. */ public KafkaChannelInitializer( RequestChannel[] requestChannels, String listenerName, long maxIdleTimeSeconds, int maxRequestSize, boolean preferHeap) { + this(requestChannels, listenerName, maxIdleTimeSeconds, maxRequestSize, preferHeap, null); + } + + /** Creates a channel initializer with an optional per-connection authenticator supplier. */ + public KafkaChannelInitializer( + RequestChannel[] requestChannels, + String listenerName, + long maxIdleTimeSeconds, + int maxRequestSize, + boolean preferHeap, + @Nullable Supplier authenticatorSupplier) { super(maxIdleTimeSeconds); this.requestChannels = requestChannels; this.listenerName = listenerName; this.maxRequestSize = maxRequestSize; this.preferHeap = preferHeap; + this.authenticatorSupplier = authenticatorSupplier; } @Override protected void initChannel(SocketChannel ch) throws Exception { super.initChannel(ch); + // NettyLogger dumps full buffers at TRACE. A SASL/PLAIN frame contains the clear-text + // credential token, so authenticated listeners must never install the payload logger. + if (authenticatorSupplier != null && ch.pipeline().get("loggingHandler") != null) { + ch.pipeline().remove("loggingHandler"); + } addIdleStateHandler(ch); ch.pipeline().addLast(prepender); addFrameDecoder(ch, maxRequestSize, 4, preferHeap); ch.pipeline().addLast("flowController", new FlowControlHandler()); - ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels, listenerName)); + ch.pipeline() + .addLast( + new KafkaCommandDecoder( + requestChannels, listenerName, authenticatorSupplier)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java index 1dcf1cca90a..1a5cf57911e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java @@ -17,8 +17,11 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.security.KafkaSaslConnection; import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.security.auth.ServerAuthenticator; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelFuture; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; import org.apache.fluss.shaded.netty4.io.netty.channel.SimpleChannelInboundHandler; import org.apache.fluss.shaded.netty4.io.netty.handler.timeout.IdleState; @@ -38,11 +41,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.net.SocketAddress; import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import static org.apache.kafka.common.protocol.ApiKeys.API_VERSIONS; import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE; @@ -57,6 +63,7 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { private final RequestChannel[] requestChannels; private final int numChannels; private final String listenerName; + private final KafkaSaslConnection saslConnection; // Need to use a Queue to store the inflight responses, because Kafka clients require the // responses to be sent in order. @@ -67,22 +74,51 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { protected volatile ChannelHandlerContext ctx; protected SocketAddress remoteAddress; + /** Creates a decoder for a PLAINTEXT Kafka connection. */ public KafkaCommandDecoder(RequestChannel[] requestChannels, String listenerName) { + this(requestChannels, listenerName, null); + } + + /** Creates a decoder that requires SASL when an authenticator supplier is provided. */ + public KafkaCommandDecoder( + RequestChannel[] requestChannels, + String listenerName, + @Nullable Supplier authenticatorSupplier) { super(false); this.requestChannels = requestChannels; this.numChannels = requestChannels.length; this.listenerName = listenerName; + this.saslConnection = + authenticatorSupplier == null + ? KafkaSaslConnection.plaintext() + : KafkaSaslConnection.sasl(authenticatorSupplier); } @Override public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception { CompletableFuture future = new CompletableFuture<>(); try { - KafkaRequest request = parseRequest(ctx, future, buffer, listenerName); + ByteBuffer nioBuffer = buffer.nioBuffer(); + RequestHeader header = RequestHeader.parse(nioBuffer); + if (!saslConnection.isRequestAllowed(header.apiKey())) { + LOG.warn( + "Rejecting Kafka API {} before authentication completes on listener {}", + header.apiKey(), + listenerName); + close(); + return; + } + KafkaRequest request = + parseRequest( + ctx, future, buffer, listenerName, saslConnection, header, nioBuffer); inflightResponses.addLast(request); future.whenCompleteAsync((r, t) -> sendResponse(ctx), ctx.executor()); int channelIndex = MathUtils.murmurHash(ctx.channel().id().asLongText().hashCode()) % numChannels; + // The worker and the ordered-response queue own independent references. This lets a + // disconnect release response-side ownership without invalidating a Produce request + // that is still waiting in the shared RequestChannel. + request.retainBufferForProcessor(); requestChannels[channelIndex].putRequest(request); if (!isActive.get()) { @@ -91,7 +127,7 @@ public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Excep } } catch (Throwable t) { LOG.error("Error handling request", t); - future.completeExceptionally(t); + close(); } finally { // KafkaRequest retains the buffer because Kafka record sets can reference its memory // asynchronously. Release the decoder's ownership on every path; the request releases @@ -112,8 +148,9 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { - super.channelInactive(ctx); LOG.info("Connection closed from {}", ctx.channel().remoteAddress()); + deactivate(); + super.channelInactive(ctx); // TODO Channel metrics } @@ -146,20 +183,30 @@ private void sendResponse(ChannelHandlerContext ctx) { } } - if (!isDone) { - break; - } - if (cancelled) { inflightResponses.pollFirst(); request.releaseBuffer(); continue; } + if (!isDone) { + break; + } + inflightResponses.pollFirst(); if (isActive.get()) { ByteBuf buffer = request.responseBuffer(); - ctx.writeAndFlush(buffer); + ChannelFuture responseFuture = ctx.writeAndFlush(buffer); + if (request.shouldCloseConnectionAfterResponse()) { + isActive.set(false); + saslConnection.close(); + responseFuture.addListener( + ignored -> { + releasePendingRequests(); + ctx.close(); + }); + break; + } } else { request.releaseBuffer(); } @@ -167,14 +214,27 @@ private void sendResponse(ChannelHandlerContext ctx) { } protected void close() { - isActive.set(false); - ctx.close(); + deactivate(); + if (ctx != null) { + ctx.close(); + } LOG.warn( "Close channel {} with {} pending requests.", remoteAddress, inflightResponses.size()); - for (KafkaRequest request : inflightResponses) { + } + + private void deactivate() { + isActive.set(false); + saslConnection.close(); + releasePendingRequests(); + } + + private void releasePendingRequests() { + KafkaRequest request; + while ((request = inflightResponses.pollFirst()) != null) { request.cancel(); + request.releaseBuffer(); } } @@ -188,9 +248,10 @@ private static KafkaRequest parseRequest( ChannelHandlerContext ctx, CompletableFuture future, ByteBuf buffer, - String listenerName) { - ByteBuffer nioBuffer = buffer.nioBuffer(); - RequestHeader header = RequestHeader.parse(nioBuffer); + String listenerName, + KafkaSaslConnection saslConnection, + RequestHeader header, + ByteBuffer nioBuffer) { if (isUnsupportedApiVersionRequest(header)) { ApiVersionsRequest request = new ApiVersionsRequest( @@ -203,6 +264,7 @@ private static KafkaRequest parseRequest( header, request, listenerName, + saslConnection, buffer, ctx, future); @@ -215,6 +277,7 @@ private static KafkaRequest parseRequest( header, request.request, listenerName, + saslConnection, buffer, ctx, future); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 65d2f8d7af6..6ff4af738a2 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -19,21 +19,41 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.ServerReconfigurable; +import org.apache.fluss.exception.ConfigException; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.AdminGatewayProvider; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; +import org.apache.fluss.security.auth.AuthenticationFactory; +import org.apache.fluss.security.auth.ServerAuthenticator; +import org.apache.fluss.security.auth.sasl.plain.PlainSaslServerConfigManager; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** The Kafka protocol plugin. */ -public class KafkaProtocolPlugin implements NetworkProtocolPlugin { +public class KafkaProtocolPlugin implements NetworkProtocolPlugin, ServerReconfigurable { + + private static final String SASL_AUTH_PROTOCOL = "sasl"; + private static final String PLAINTEXT_AUTH_PROTOCOL = "plaintext"; private Configuration conf; + private PlainSaslServerConfigManager plainSaslServerConfigManager; + private Map> authenticatorSuppliers = + Collections.emptyMap(); + private Set saslListenerNames = Collections.emptySet(); @Override public String name() { @@ -42,7 +62,12 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = conf; + validateKafkaAuthenticationConfiguration(conf); + this.saslListenerNames = saslListenerNames(conf); + this.plainSaslServerConfigManager = new PlainSaslServerConfigManager(conf); + this.conf = plainSaslServerConfigManager.getConfiguration(); + this.authenticatorSuppliers = + AuthenticationFactory.loadServerAuthenticatorSuppliers(this.conf); } @Override @@ -53,12 +78,21 @@ public List listenerNames() { @Override public ChannelHandler createChannelHandler( RequestChannel[] requestChannels, String listenerName) { + Supplier authenticatorSupplier = null; + if (saslListenerNames.contains(listenerName)) { + authenticatorSupplier = + checkNotNull( + authenticatorSuppliers.get(listenerName), + "No SASL server authenticator is configured for Kafka listener %s.", + listenerName); + } return new KafkaChannelInitializer( requestChannels, listenerName, conf.get(ConfigOptions.KAFKA_CONNECTION_MAX_IDLE_TIME).getSeconds(), (int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(), - conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST)); + conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST), + authenticatorSupplier); } @Override @@ -70,14 +104,77 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { } TabletServerGateway gateway = (TabletServerGateway) service; if (service instanceof AdminGatewayProvider) { + if (!(service instanceof AdminOperationAuthorizer)) { + throw new IllegalArgumentException( + "Kafka topic administration requires the TabletServer service to authorize external admin operations before internal forwarding."); + } return new KafkaRequestHandler( service, gateway, ((AdminGatewayProvider) service).getAdminGateway(), + (AdminOperationAuthorizer) service, conf.get(ConfigOptions.KAFKA_DATABASE), KafkaDataFormat.parse(conf.get(ConfigOptions.KAFKA_DEFAULT_KEY_FORMAT)), KafkaDataFormat.parse(conf.get(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT))); } return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } + + @Override + public void validate(Configuration newConfig) throws ConfigException { + validateKafkaAuthenticationConfiguration(newConfig); + plainSaslServerConfigManager.validate(newConfig); + } + + @Override + public void reconfigure(Configuration newConfig) throws ConfigException { + plainSaslServerConfigManager.reconfigure(newConfig); + } + + private static void validateKafkaAuthenticationConfiguration(Configuration configuration) { + Map protocolMap = + configuration.get(ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP); + List kafkaListeners = configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES); + boolean saslEnabled = false; + for (String listenerName : kafkaListeners) { + String protocol = protocolMap.get(listenerName); + if (protocol == null) { + continue; + } + if (PLAINTEXT_AUTH_PROTOCOL.equalsIgnoreCase(protocol)) { + continue; + } + if (!SASL_AUTH_PROTOCOL.equalsIgnoreCase(protocol)) { + throw new ConfigException( + String.format( + "Kafka listener '%s' supports only PLAINTEXT or SASL authentication, but '%s' is configured.", + listenerName, protocol)); + } + saslEnabled = true; + } + if (!saslEnabled) { + return; + } + + List mechanisms = + configuration.get(ConfigOptions.SERVER_SASL_ENABLED_MECHANISMS_CONFIG); + if (mechanisms == null + || !mechanisms.stream() + .anyMatch(mechanism -> "PLAIN".equalsIgnoreCase(mechanism))) { + throw new ConfigException( + "Kafka SASL listeners require PLAIN in security.sasl.enabled.mechanisms."); + } + } + + private static Set saslListenerNames(Configuration configuration) { + Map protocolMap = + configuration.get(ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP); + Set listenerNames = new HashSet<>(); + for (String listenerName : configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) { + if (SASL_AUTH_PROTOCOL.equalsIgnoreCase(protocolMap.get(listenerName))) { + listenerNames.add(listenerName); + } + } + return Collections.unmodifiableSet(listenerNames); + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java index 20d8bf01898..b32e7afa13c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java @@ -17,8 +17,10 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.security.KafkaSaslConnection; import org.apache.fluss.rpc.netty.server.RpcRequest; import org.apache.fluss.rpc.protocol.RequestType; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; import org.apache.fluss.shaded.netty4.io.netty.util.ReferenceCountUtil; @@ -47,12 +49,16 @@ public class KafkaRequest implements RpcRequest { private final RequestHeader header; private final AbstractRequest request; private final String listenerName; + private final KafkaSaslConnection saslConnection; + private final FlussPrincipal principal; private final ByteBuf buffer; private final ChannelHandlerContext ctx; private final long startTimeMs; private final CompletableFuture future; private volatile boolean cancelled = false; + private volatile boolean closeConnectionAfterResponse; + /** Creates an anonymous request with an unknown listener name. */ protected KafkaRequest( ApiKeys apiKey, short apiVersion, @@ -61,15 +67,48 @@ protected KafkaRequest( ByteBuf buffer, ChannelHandlerContext ctx, CompletableFuture future) { - this(apiKey, apiVersion, header, request, "UNKNOWN", buffer, ctx, future); - } - + this( + apiKey, + apiVersion, + header, + request, + "UNKNOWN", + KafkaSaslConnection.plaintext(), + buffer, + ctx, + future); + } + + /** Creates an anonymous request for the supplied listener. */ + protected KafkaRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest request, + String listenerName, + ByteBuf buffer, + ChannelHandlerContext ctx, + CompletableFuture future) { + this( + apiKey, + apiVersion, + header, + request, + listenerName, + KafkaSaslConnection.plaintext(), + buffer, + ctx, + future); + } + + /** Creates a request that snapshots identity from the supplied connection security state. */ protected KafkaRequest( ApiKeys apiKey, short apiVersion, RequestHeader header, AbstractRequest request, String listenerName, + KafkaSaslConnection saslConnection, ByteBuf buffer, ChannelHandlerContext ctx, CompletableFuture future) { @@ -78,6 +117,8 @@ protected KafkaRequest( this.header = header; this.request = request; this.listenerName = listenerName; + this.saslConnection = saslConnection; + this.principal = saslConnection.principal(); this.buffer = buffer.retain(); this.ctx = ctx; this.startTimeMs = System.currentTimeMillis(); @@ -94,6 +135,11 @@ public void releaseBuffer() { ReferenceCountUtil.safeRelease(buffer); } + /** Retains the request buffer for ownership by the RequestProcessor queue. */ + void retainBufferForProcessor() { + buffer.retain(); + } + public ApiKeys apiKey() { return apiKey; } @@ -118,6 +164,16 @@ public String listenerName() { return listenerName; } + /** Returns the connection-level SASL state associated with this request. */ + public KafkaSaslConnection saslConnection() { + return saslConnection; + } + + /** Returns the principal captured when this request was parsed. */ + public FlussPrincipal principal() { + return principal; + } + public ChannelHandlerContext ctx() { return ctx; } @@ -146,6 +202,16 @@ public boolean cancelled() { return cancelled; } + /** Marks this request so the channel closes only after its response has been flushed. */ + public void closeConnectionAfterResponse() { + closeConnectionAfterResponse = true; + } + + /** Returns whether the channel must close after this request's response is flushed. */ + public boolean shouldCloseConnectionAfterResponse() { + return closeConnectionAfterResponse; + } + public ByteBuf responseBuffer() { try { AbstractResponse response = future.join(); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java index e75a20babcc..69cae0b1e5d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java @@ -18,6 +18,8 @@ package org.apache.fluss.kafka; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.security.KafkaSaslConnection; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.fluss.shaded.netty4.io.netty.channel.Channel; import org.apache.kafka.common.protocol.ApiKeys; @@ -36,8 +38,10 @@ public final class KafkaRequestContext { private final SocketAddress localAddress; private final SocketAddress remoteAddress; private final long receivedTimeMs; + private final KafkaRequest request; private KafkaRequestContext(KafkaRequest request) { + this.request = request; this.correlationId = request.header().correlationId(); this.clientId = request.header().clientId(); this.apiKey = request.apiKey(); @@ -93,4 +97,19 @@ public SocketAddress remoteAddress() { public long receivedTimeMs() { return receivedTimeMs; } + + /** Returns the authenticated principal captured when this request was received. */ + public FlussPrincipal principal() { + return request.principal(); + } + + /** Returns this network connection's SASL state machine. */ + public KafkaSaslConnection saslConnection() { + return request.saslConnection(); + } + + /** Closes the connection after this request's response has been flushed. */ + public void closeConnectionAfterResponse() { + request.closeConnectionAfterResponse(); + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 43cc67b74c7..47f6a77a11c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -21,6 +21,8 @@ import org.apache.fluss.kafka.api.admin.DeleteTopicsHandler; import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.api.sasl.SaslAuthenticateHandler; +import org.apache.fluss.kafka.api.sasl.SaslHandshakeHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.admin.GatewayKafkaTopicAdminBackend; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; @@ -32,6 +34,7 @@ import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; @@ -51,6 +54,8 @@ public KafkaRequestHandler( checkNotNull(kafkaDatabase); KafkaApiRegistry registry = new KafkaApiRegistry(); registry.register(new ApiVersionsHandler(registry)); + registry.register(new SaslHandshakeHandler()); + registry.register(new SaslAuthenticateHandler()); registry.register( new MetadataHandler( new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); @@ -71,10 +76,21 @@ public KafkaRequestHandler( TabletServerGateway gateway, AdminGateway adminGateway, String kafkaDatabase) { + this(service, gateway, adminGateway, adminOperationAuthorizer(service), kafkaDatabase); + } + + /** Creates a Kafka request handler with explicit authorization for topic lifecycle requests. */ + public KafkaRequestHandler( + RpcGatewayService service, + TabletServerGateway gateway, + AdminGateway adminGateway, + AdminOperationAuthorizer adminOperationAuthorizer, + String kafkaDatabase) { this( service, gateway, adminGateway, + adminOperationAuthorizer, kafkaDatabase, KafkaDataFormat.RAW, KafkaDataFormat.RAW); @@ -90,14 +106,38 @@ public KafkaRequestHandler( String kafkaDatabase, KafkaDataFormat defaultKeyFormat, KafkaDataFormat defaultValueFormat) { + this( + service, + gateway, + adminGateway, + adminOperationAuthorizer(service), + kafkaDatabase, + defaultKeyFormat, + defaultValueFormat); + } + + /** + * Creates a Kafka request handler with explicit authorization and default format capabilities. + */ + public KafkaRequestHandler( + RpcGatewayService service, + TabletServerGateway gateway, + AdminGateway adminGateway, + AdminOperationAuthorizer adminOperationAuthorizer, + String kafkaDatabase, + KafkaDataFormat defaultKeyFormat, + KafkaDataFormat defaultValueFormat) { checkNotNull(service); checkNotNull(gateway); checkNotNull(adminGateway); + checkNotNull(adminOperationAuthorizer); checkNotNull(kafkaDatabase); checkNotNull(defaultKeyFormat); checkNotNull(defaultValueFormat); KafkaApiRegistry registry = new KafkaApiRegistry(); registry.register(new ApiVersionsHandler(registry)); + registry.register(new SaslHandshakeHandler()); + registry.register(new SaslAuthenticateHandler()); registry.register( new MetadataHandler( new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase), true)); @@ -109,7 +149,8 @@ public KafkaRequestHandler( kafkaDatabase, new ArrowKafkaRecordTranscoder()))); GatewayKafkaTopicAdminBackend topicAdminBackend = - new GatewayKafkaTopicAdminBackend(service, adminGateway, kafkaDatabase); + new GatewayKafkaTopicAdminBackend( + service, adminGateway, adminOperationAuthorizer, kafkaDatabase); registry.register( new CreateTopicsHandler(topicAdminBackend, defaultKeyFormat, defaultValueFormat)); registry.register(new DeleteTopicsHandler(topicAdminBackend)); @@ -117,6 +158,14 @@ public KafkaRequestHandler( this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } + private static AdminOperationAuthorizer adminOperationAuthorizer(RpcGatewayService service) { + if (!(service instanceof AdminOperationAuthorizer)) { + throw new IllegalArgumentException( + "Kafka topic administration requires an AdminOperationAuthorizer."); + } + return (AdminOperationAuthorizer) service; + } + @Override public RequestType requestType() { return RequestType.KAFKA; diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java index 095420e72ec..f811d84efaf 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/CreateTopicsHandler.java @@ -115,7 +115,8 @@ public CompletableFuture handle( validTopics, request.data().validateOnly(), context.listenerName(), - clientAddress(context.remoteAddress())) + clientAddress(context.remoteAddress()), + context.principal()) .thenApply(results -> toResponse(request, localResults, results)); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java index 6a2bf1d6eae..2f1ae5a2cff 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/admin/DeleteTopicsHandler.java @@ -82,7 +82,10 @@ public CompletableFuture handle( } } return backend.deleteTopics( - topics, context.listenerName(), clientAddress(context.remoteAddress())) + topics, + context.listenerName(), + clientAddress(context.remoteAddress()), + context.principal()) .thenApply(DeleteTopicsHandler::toResponse); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index bcfc2655d9c..794655fe19d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -107,7 +107,8 @@ public CompletableFuture handle( request.isAllTopics(), validTopics, context.listenerName(), - clientAddress(context.remoteAddress())); + clientAddress(context.remoteAddress()), + context.principal()); return backend.getMetadata(query) .thenApply( metadata -> { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index 2749bd721a7..3e6f22da0f5 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -101,7 +101,8 @@ public CompletableFuture handle( request.timeout(), topics, context.listenerName(), - clientAddress(context.remoteAddress())); + clientAddress(context.remoteAddress()), + context.principal()); return backend.write(command).thenApply(ProduceHandler::toResponse); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslAuthenticateHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslAuthenticateHandler.java new file mode 100644 index 00000000000..361ad506f95 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslAuthenticateHandler.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.sasl; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.security.KafkaSaslConnection; + +import org.apache.kafka.common.message.SaslAuthenticateResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.SaslAuthenticateRequest; +import org.apache.kafka.common.requests.SaslAuthenticateResponse; + +import java.util.concurrent.CompletableFuture; + +/** Implements Kafka-framed SASL/PLAIN token authentication. */ +@Internal +public final class SaslAuthenticateHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.SASL_AUTHENTICATE, (short) 0, (short) 2, true); + private static final String AUTHENTICATION_FAILURE_MESSAGE = + "Authentication failed due to invalid credentials with SASL mechanism PLAIN."; + private static final String ILLEGAL_STATE_MESSAGE = + "SASL authentication is not active on this connection."; + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, SaslAuthenticateRequest request) { + return handle(context.saslConnection(), request, context::closeConnectionAfterResponse); + } + + CompletableFuture handle( + KafkaSaslConnection connection, SaslAuthenticateRequest request) { + return handle(connection, request, () -> {}); + } + + CompletableFuture handle( + KafkaSaslConnection connection, + SaslAuthenticateRequest request, + Runnable closeConnectionAfterResponse) { + if (!connection.authenticationEnabled() || !connection.isAuthenticating()) { + return failure( + connection, + closeConnectionAfterResponse, + Errors.ILLEGAL_SASL_STATE, + ILLEGAL_STATE_MESSAGE); + } + + try { + byte[] challenge = connection.authenticate(request.data().authBytes()); + return CompletableFuture.completedFuture(response(Errors.NONE, null, challenge)); + } catch (RuntimeException e) { + return failure( + connection, + closeConnectionAfterResponse, + Errors.SASL_AUTHENTICATION_FAILED, + AUTHENTICATION_FAILURE_MESSAGE); + } + } + + private static CompletableFuture failure( + KafkaSaslConnection connection, + Runnable closeConnectionAfterResponse, + Errors error, + String errorMessage) { + connection.failAuthentication(); + closeConnectionAfterResponse.run(); + return CompletableFuture.completedFuture(response(error, errorMessage, new byte[0])); + } + + private static SaslAuthenticateResponse response( + Errors error, String errorMessage, byte[] authBytes) { + return new SaslAuthenticateResponse( + new SaslAuthenticateResponseData() + .setErrorCode(error.code()) + .setErrorMessage(errorMessage) + .setAuthBytes(authBytes) + .setSessionLifetimeMs(0L)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslHandshakeHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslHandshakeHandler.java new file mode 100644 index 00000000000..ceed6c58612 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/sasl/SaslHandshakeHandler.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.sasl; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.security.KafkaSaslConnection; + +import org.apache.kafka.common.message.SaslHandshakeResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.SaslHandshakeRequest; +import org.apache.kafka.common.requests.SaslHandshakeResponse; + +import java.net.SocketAddress; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +/** Implements the Kafka SASL handshake for the PLAIN mechanism. */ +@Internal +public final class SaslHandshakeHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.SASL_HANDSHAKE, (short) 1, (short) 1, true); + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, SaslHandshakeRequest request) { + return handle( + context.saslConnection(), + context.listenerName(), + context.remoteAddress(), + request, + context::closeConnectionAfterResponse); + } + + CompletableFuture handle( + KafkaSaslConnection connection, + String listenerName, + SocketAddress remoteAddress, + SaslHandshakeRequest request) { + return handle(connection, listenerName, remoteAddress, request, () -> {}); + } + + CompletableFuture handle( + KafkaSaslConnection connection, + String listenerName, + SocketAddress remoteAddress, + SaslHandshakeRequest request, + Runnable closeConnectionAfterResponse) { + if (!connection.authenticationEnabled() || !connection.isAwaitingHandshake()) { + return failure( + connection, + closeConnectionAfterResponse, + Errors.ILLEGAL_SASL_STATE, + Collections.singletonList(KafkaSaslConnection.PLAIN_MECHANISM)); + } + + String mechanism = request.data().mechanism(); + if (!KafkaSaslConnection.PLAIN_MECHANISM.equals(mechanism)) { + return failure( + connection, + closeConnectionAfterResponse, + Errors.UNSUPPORTED_SASL_MECHANISM, + Collections.singletonList(KafkaSaslConnection.PLAIN_MECHANISM)); + } + + try { + connection.beginAuthentication(mechanism, listenerName, remoteAddress); + return CompletableFuture.completedFuture( + response( + Errors.NONE, + Collections.singletonList(KafkaSaslConnection.PLAIN_MECHANISM))); + } catch (RuntimeException e) { + return failure( + connection, + closeConnectionAfterResponse, + Errors.SASL_AUTHENTICATION_FAILED, + Collections.singletonList(KafkaSaslConnection.PLAIN_MECHANISM)); + } + } + + private static CompletableFuture failure( + KafkaSaslConnection connection, + Runnable closeConnectionAfterResponse, + Errors error, + java.util.List mechanisms) { + connection.failAuthentication(); + closeConnectionAfterResponse.run(); + return CompletableFuture.completedFuture(response(error, mechanisms)); + } + + private static SaslHandshakeResponse response(Errors error, java.util.List mechanisms) { + return new SaslHandshakeResponse( + new SaslHandshakeResponseData() + .setErrorCode(error.code()) + .setMechanisms(mechanisms)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index c38d7bc6cb2..1125a88c66b 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -60,12 +60,22 @@ public KafkaApiSpec apiSpec() { @Override public CompletableFuture handle( KafkaRequestContext context, ApiVersionsRequest request) { + if (context.saslConnection().isAuthenticating()) { + context.closeConnectionAfterResponse(); + return CompletableFuture.completedFuture( + new ApiVersionsResponse( + new ApiVersionsResponseData() + .setErrorCode(Errors.ILLEGAL_SASL_STATE.code()))); + } if (!request.isValid()) { return CompletableFuture.completedFuture( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } ApiVersionsResponseData data = new ApiVersionsResponseData(); for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + if (isSaslApi(spec.apiKey()) && !context.saslConnection().authenticationEnabled()) { + continue; + } data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() @@ -75,4 +85,8 @@ public CompletableFuture handle( } return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); } + + private static boolean isSaslApi(ApiKeys apiKey) { + return apiKey == ApiKeys.SASL_HANDSHAKE || apiKey == ApiKeys.SASL_AUTHENTICATE; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java index 6fe0321835a..e45a5c2a3cd 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/GatewayKafkaTopicAdminBackend.java @@ -26,11 +26,14 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.messages.CreateTableRequest; import org.apache.fluss.rpc.messages.DropTableRequest; import org.apache.fluss.rpc.messages.GetTableInfoRequest; import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.acl.OperationType; +import org.apache.fluss.security.acl.Resource; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -53,14 +56,19 @@ public final class GatewayKafkaTopicAdminBackend implements KafkaTopicAdminBacke private final RpcGatewayService service; private final AdminGateway gateway; + private final AdminOperationAuthorizer adminOperationAuthorizer; private final String databaseName; private final KafkaTopicMapper topicMapper; /** Creates a topic backend backed by the Fluss coordinator admin gateway. */ public GatewayKafkaTopicAdminBackend( - RpcGatewayService service, AdminGateway gateway, String databaseName) { + RpcGatewayService service, + AdminGateway gateway, + AdminOperationAuthorizer adminOperationAuthorizer, + String databaseName) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); + this.adminOperationAuthorizer = checkNotNull(adminOperationAuthorizer); this.databaseName = checkNotNull(databaseName); this.topicMapper = new KafkaTopicMapper(databaseName); } @@ -71,9 +79,20 @@ public CompletableFuture> createTopics( boolean validateOnly, String listenerName, @Nullable InetAddress clientAddress) { + return createTopics( + topics, validateOnly, listenerName, clientAddress, FlussPrincipal.ANONYMOUS); + } + + @Override + public CompletableFuture> createTopics( + List topics, + boolean validateOnly, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { List> futures = new ArrayList<>(); for (CreateTopic topic : topics) { - futures.add(createTopic(topic, validateOnly, listenerName, clientAddress)); + futures.add(createTopic(topic, validateOnly, listenerName, clientAddress, principal)); } return collect(futures); } @@ -81,9 +100,18 @@ public CompletableFuture> createTopics( @Override public CompletableFuture> deleteTopics( List topics, String listenerName, @Nullable InetAddress clientAddress) { + return deleteTopics(topics, listenerName, clientAddress, FlussPrincipal.ANONYMOUS); + } + + @Override + public CompletableFuture> deleteTopics( + List topics, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { List> futures = new ArrayList<>(); for (DeleteTopic topic : topics) { - futures.add(deleteTopic(topic, listenerName, clientAddress)); + futures.add(deleteTopic(topic, listenerName, clientAddress, principal)); } return collect(futures); } @@ -92,8 +120,16 @@ private CompletableFuture createTopic( CreateTopic topic, boolean validateOnly, String listenerName, - @Nullable InetAddress clientAddress) { + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { TableDescriptor descriptor = createDescriptor(topic); + Session session = clientSession(listenerName, clientAddress, principal); + try { + adminOperationAuthorizer.authorize( + session, OperationType.CREATE, Resource.database(databaseName)); + } catch (RuntimeException failure) { + return CompletableFuture.completedFuture(failed(topic.name(), failure)); + } if (validateOnly) { return CompletableFuture.completedFuture(success(topic, Uuid.ZERO_UUID)); } @@ -104,24 +140,26 @@ private CompletableFuture createTopic( .setTablePath() .setDatabaseName(databaseName) .setTableName(topic.name()); - setCurrentSession(listenerName, clientAddress); + setCurrentSession(session); return gateway.createTable(request) - .thenCompose(ignored -> getCreatedTopic(topic, listenerName, clientAddress)) + .thenCompose(ignored -> getCreatedTopic(topic, session)) .exceptionally(failure -> failed(topic.name(), failure)); } - private CompletableFuture getCreatedTopic( - CreateTopic topic, String listenerName, @Nullable InetAddress clientAddress) { + private CompletableFuture getCreatedTopic(CreateTopic topic, Session session) { GetTableInfoRequest request = new GetTableInfoRequest(); request.setTablePath().setDatabaseName(databaseName).setTableName(topic.name()); - setCurrentSession(listenerName, clientAddress); + setCurrentSession(session); return gateway.getTableInfo(request) .thenApply( response -> success(topic, topicMapper.toTopicId(response.getTableId()))); } private CompletableFuture deleteTopic( - DeleteTopic topic, String listenerName, @Nullable InetAddress clientAddress) { + DeleteTopic topic, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { if (topic.name() == null) { return CompletableFuture.completedFuture( new TopicResult( @@ -132,12 +170,20 @@ private CompletableFuture deleteTopic( -1, (short) -1)); } + Session session = clientSession(listenerName, clientAddress, principal); + try { + adminOperationAuthorizer.authorize( + session, OperationType.DROP, Resource.table(databaseName, topic.name())); + } catch (RuntimeException failure) { + return CompletableFuture.completedFuture( + failed(topic.name(), topic.topicId(), failure)); + } DropTableRequest request = new DropTableRequest(); request.setIgnoreIfNotExists(false) .setTablePath() .setDatabaseName(databaseName) .setTableName(topic.name()); - setCurrentSession(listenerName, clientAddress); + setCurrentSession(session); return gateway.dropTable(request) .thenApply( ignored -> @@ -238,10 +284,13 @@ private static Errors toKafkaError(Throwable failure) { } } - private void setCurrentSession(String listenerName, @Nullable InetAddress clientAddress) { - service.setCurrentSession( - new Session( - (short) 0, listenerName, false, clientAddress, FlussPrincipal.ANONYMOUS)); + private static Session clientSession( + String listenerName, @Nullable InetAddress clientAddress, FlussPrincipal principal) { + return new Session((short) 0, listenerName, false, clientAddress, checkNotNull(principal)); + } + + private void setCurrentSession(Session session) { + service.setCurrentSession(session); } private static CompletableFuture> collect( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java index 653d03367c4..7be4510bff4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/admin/KafkaTopicAdminBackend.java @@ -19,6 +19,7 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.protocol.Errors; @@ -40,10 +41,29 @@ CompletableFuture> createTopics( String listenerName, @Nullable InetAddress clientAddress); + /** Creates or validates topics on behalf of the authenticated Kafka principal. */ + default CompletableFuture> createTopics( + List topics, + boolean validateOnly, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { + return createTopics(topics, validateOnly, listenerName, clientAddress); + } + /** Deletes the requested topics. */ CompletableFuture> deleteTopics( List topics, String listenerName, @Nullable InetAddress clientAddress); + /** Deletes topics on behalf of the authenticated Kafka principal. */ + default CompletableFuture> deleteTopics( + List topics, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { + return deleteTopics(topics, listenerName, clientAddress); + } + /** A validated request to create one topic. */ final class CreateTopic { private final String name; diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 087eb6a8d46..35c48b8a2a6 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -34,7 +34,6 @@ import org.apache.fluss.rpc.messages.PbTableMetadata; import org.apache.fluss.rpc.messages.PbTablePath; import org.apache.fluss.rpc.netty.server.Session; -import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.kafka.common.Uuid; import org.slf4j.Logger; @@ -259,7 +258,7 @@ private void setCurrentSession(KafkaMetadataQuery query) { query.listenerName(), false, query.clientAddress(), - FlussPrincipal.ANONYMOUS)); + query.principal())); } private static boolean containsTopicId(List topics) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java index 01e21fa4440..4594e5a9e26 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.metadata; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.kafka.common.Uuid; @@ -38,6 +39,7 @@ public final class KafkaMetadataQuery { private final List topics; private final String listenerName; private final @Nullable InetAddress clientAddress; + private final FlussPrincipal principal; /** Creates a metadata query. */ public KafkaMetadataQuery( @@ -45,10 +47,21 @@ public KafkaMetadataQuery( List topics, String listenerName, @Nullable InetAddress clientAddress) { + this(allTopics, topics, listenerName, clientAddress, FlussPrincipal.ANONYMOUS); + } + + /** Creates a metadata query for the authenticated Kafka principal. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { this.allTopics = allTopics; this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); this.listenerName = checkNotNull(listenerName); this.clientAddress = clientAddress; + this.principal = checkNotNull(principal); } /** Returns whether all Kafka topics should be returned. */ @@ -71,6 +84,11 @@ public String listenerName() { return clientAddress; } + /** Returns the authenticated Kafka principal. */ + public FlussPrincipal principal() { + return principal; + } + /** Kafka topic name and ID supplied by a Metadata request. */ @Internal public static final class TopicReference { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java index 151c6564077..7b2a80a000d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -36,7 +36,6 @@ import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.ProduceLogResponse; import org.apache.fluss.rpc.netty.server.Session; -import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.kafka.common.protocol.Errors; @@ -238,7 +237,7 @@ private void setCurrentSession(KafkaProduceCommand command) { command.listenerName(), false, command.clientAddress(), - FlussPrincipal.ANONYMOUS)); + command.principal())); } private static Throwable unwrap(Throwable failure) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 722f3d3d245..c07274204de 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.produce; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.security.acl.FlussPrincipal; import javax.annotation.Nullable; @@ -37,6 +38,7 @@ public final class KafkaProduceCommand { private final List topics; private final String listenerName; private final @Nullable InetAddress clientAddress; + private final FlussPrincipal principal; /** Creates a Kafka write command. */ public KafkaProduceCommand( @@ -45,11 +47,23 @@ public KafkaProduceCommand( List topics, String listenerName, @Nullable InetAddress clientAddress) { + this(acks, timeoutMs, topics, listenerName, clientAddress, FlussPrincipal.ANONYMOUS); + } + + /** Creates a Kafka write command for the authenticated Kafka principal. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress, + FlussPrincipal principal) { this.acks = acks; this.timeoutMs = timeoutMs; this.topics = immutableCopy(topics); this.listenerName = checkNotNull(listenerName); this.clientAddress = clientAddress; + this.principal = checkNotNull(principal); } /** Returns Kafka required acknowledgements. */ @@ -77,6 +91,11 @@ public String listenerName() { return clientAddress; } + /** Returns the authenticated Kafka principal. */ + public FlussPrincipal principal() { + return principal; + } + private static List immutableCopy(List values) { return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java index efdfe33dd66..921964b085f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java @@ -23,6 +23,7 @@ import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -55,7 +56,8 @@ public CompletableFuture dispatch(KafkaRequest request) { } KafkaApiSpec spec = handler.apiSpec(); - if (!spec.supportsVersion(request.apiVersion())) { + if (!spec.supportsVersion(request.apiVersion()) + && !shouldDispatchBeforeVersionValidation(request)) { return completedErrorResponse( abstractRequest, new UnsupportedVersionException( @@ -95,6 +97,11 @@ public CompletableFuture dispatch(KafkaRequest request) { return result; } + private static boolean shouldDispatchBeforeVersionValidation(KafkaRequest request) { + return request.apiKey() == ApiKeys.API_VERSIONS + && request.saslConnection().isAuthenticating(); + } + @SuppressWarnings("unchecked") private static CompletableFuture invoke( KafkaApiHandler handler, KafkaRequestContext context, AbstractRequest request) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/security/KafkaSaslConnection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/security/KafkaSaslConnection.java new file mode 100644 index 00000000000..223610f8da5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/security/KafkaSaslConnection.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.security; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.AuthenticationException; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.auth.ServerAuthenticator; + +import org.apache.kafka.common.protocol.ApiKeys; + +import javax.annotation.Nullable; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.function.Supplier; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Per-connection authentication state for a Kafka protocol channel. */ +@Internal +public final class KafkaSaslConnection implements AutoCloseable { + + /** The only Kafka SASL mechanism supported by the initial implementation. */ + public static final String PLAIN_MECHANISM = "PLAIN"; + + private enum State { + AUTHENTICATION_REQUIRED, + AUTHENTICATING, + READY, + FAILED, + CLOSED + } + + @Nullable private final Supplier authenticatorSupplier; + + private volatile State state; + private volatile FlussPrincipal principal; + @Nullable private ServerAuthenticator authenticator; + + private KafkaSaslConnection( + @Nullable Supplier authenticatorSupplier, State initialState) { + this.authenticatorSupplier = authenticatorSupplier; + this.state = initialState; + this.principal = FlussPrincipal.ANONYMOUS; + } + + /** Creates an unauthenticated PLAINTEXT connection that is immediately ready. */ + public static KafkaSaslConnection plaintext() { + return new KafkaSaslConnection(null, State.READY); + } + + /** Creates a SASL connection that must authenticate before normal requests are accepted. */ + public static KafkaSaslConnection sasl(Supplier authenticatorSupplier) { + return new KafkaSaslConnection( + checkNotNull(authenticatorSupplier), State.AUTHENTICATION_REQUIRED); + } + + /** Returns whether SASL authentication is enabled for this connection. */ + public boolean authenticationEnabled() { + return authenticatorSupplier != null; + } + + /** Returns whether this connection is waiting for a SASL handshake. */ + public boolean isAwaitingHandshake() { + return state == State.AUTHENTICATION_REQUIRED; + } + + /** Returns whether this connection is exchanging SASL authentication tokens. */ + public boolean isAuthenticating() { + return state == State.AUTHENTICATING; + } + + /** Returns whether this connection is ready to serve normal Kafka requests. */ + public boolean isReady() { + return state == State.READY; + } + + /** + * Returns whether the request is allowed in the current connection state. + * + *

Both SASL APIs are always routed while the connection is active so that their handlers can + * return Kafka's precise illegal-state error. ApiVersions is additionally routed before and + * during authentication so its handler can return Kafka's state-specific response. Normal + * business APIs are accepted only after authentication succeeds. + */ + public boolean isRequestAllowed(ApiKeys apiKey) { + checkNotNull(apiKey); + State currentState = state; + if (currentState == State.FAILED || currentState == State.CLOSED) { + return false; + } + if (apiKey == ApiKeys.SASL_HANDSHAKE || apiKey == ApiKeys.SASL_AUTHENTICATE) { + return true; + } + if (currentState == State.READY) { + return true; + } + if (currentState == State.AUTHENTICATION_REQUIRED || currentState == State.AUTHENTICATING) { + return apiKey == ApiKeys.API_VERSIONS; + } + return false; + } + + /** Returns the authenticated principal, or the anonymous principal before authentication. */ + public FlussPrincipal principal() { + return principal; + } + + /** Returns whether the network channel must close after its pending response is flushed. */ + public boolean shouldClose() { + State currentState = state; + return currentState == State.FAILED || currentState == State.CLOSED; + } + + /** + * Starts a PLAIN authentication exchange for this connection. + * + * @param mechanism mechanism selected by the Kafka client + * @param listenerName listener on which the client connected + * @param remoteAddress remote client address + */ + public synchronized void beginAuthentication( + String mechanism, String listenerName, @Nullable SocketAddress remoteAddress) { + if (state != State.AUTHENTICATION_REQUIRED) { + throw new IllegalStateException("SASL handshake is not allowed in the current state."); + } + if (!PLAIN_MECHANISM.equals(mechanism)) { + throw new AuthenticationException("Unsupported SASL mechanism."); + } + + ServerAuthenticator newAuthenticator = null; + try { + newAuthenticator = checkNotNull(authenticatorSupplier).get(); + newAuthenticator.initialize( + new DefaultAuthenticateContext( + listenerName, clientIpAddress(remoteAddress), mechanism)); + authenticator = newAuthenticator; + state = State.AUTHENTICATING; + } catch (AuthenticationException e) { + closeAuthenticator(newAuthenticator); + failAuthentication(); + throw e; + } catch (RuntimeException e) { + closeAuthenticator(newAuthenticator); + failAuthentication(); + throw new AuthenticationException("Failed to initialize SASL authentication.", e); + } + } + + /** + * Evaluates one client SASL token and advances the connection to ready when authentication + * completes. + */ + public synchronized byte[] authenticate(byte[] token) { + if (state != State.AUTHENTICATING || authenticator == null) { + throw new IllegalStateException( + "SASL authentication is not allowed in the current state."); + } + + try { + byte[] challenge = authenticator.evaluateResponse(checkNotNull(token)); + if (authenticator.isCompleted()) { + principal = checkNotNull(authenticator.createPrincipal()); + state = State.READY; + closeAuthenticator(authenticator); + authenticator = null; + } + return challenge == null ? new byte[0] : challenge; + } catch (AuthenticationException e) { + failAuthentication(); + throw e; + } catch (RuntimeException e) { + failAuthentication(); + throw new AuthenticationException("SASL authentication failed.", e); + } + } + + /** Marks authentication as failed and releases its authenticator. */ + public synchronized void failAuthentication() { + if (state == State.CLOSED) { + return; + } + closeAuthenticator(authenticator); + authenticator = null; + principal = FlussPrincipal.ANONYMOUS; + state = State.FAILED; + } + + /** Releases the per-connection authenticator. */ + @Override + public synchronized void close() { + closeAuthenticator(authenticator); + authenticator = null; + state = State.CLOSED; + } + + private static String clientIpAddress(@Nullable SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + InetSocketAddress inetAddress = (InetSocketAddress) remoteAddress; + if (inetAddress.getAddress() != null) { + return inetAddress.getAddress().getHostAddress(); + } + return inetAddress.getHostString(); + } + return remoteAddress == null ? "UNKNOWN" : remoteAddress.toString(); + } + + private static void closeAuthenticator(@Nullable ServerAuthenticator authenticator) { + if (authenticator == null) { + return; + } + try { + authenticator.close(); + } catch (Exception ignored) { + // The connection is already closing or failed; there is no recovery action here. + } + } + + private static final class DefaultAuthenticateContext + implements ServerAuthenticator.AuthenticateContext { + + private final String listenerName; + private final String ipAddress; + private final String protocol; + + private DefaultAuthenticateContext(String listenerName, String ipAddress, String protocol) { + this.listenerName = checkNotNull(listenerName); + this.ipAddress = checkNotNull(ipAddress); + this.protocol = checkNotNull(protocol); + } + + @Override + public String ipAddress() { + return ipAddress; + } + + @Override + public String listenerName() { + return listenerName; + } + + @Override + public String protocol() { + return protocol; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index a2b8a0dc60b..11632f2173a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,17 +17,25 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.security.auth.ServerAuthenticator; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; +import org.apache.fluss.shaded.netty4.io.netty.handler.codec.LengthFieldPrepender; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.message.SaslHandshakeRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.ProduceRequest; @@ -35,15 +43,178 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.requests.SaslHandshakeRequest; +import org.apache.kafka.common.requests.SaslHandshakeResponse; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; /** Tests response ordering and ownership in {@link KafkaCommandDecoder}. */ public class KafkaCommandDecoderTest { + @Test + public void testApiVersionsDuringAuthenticationIsFlushedBeforeChannelCloses() throws Exception { + RequestChannel requestChannel = new RequestChannel(100); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + KafkaRequestHandler requestHandler = new KafkaRequestHandler(service, service, "kafka"); + EmbeddedChannel channel = + new EmbeddedChannel( + new LengthFieldPrepender(4), + new KafkaCommandDecoder( + new RequestChannel[] {requestChannel}, + "KAFKA", + () -> mock(ServerAuthenticator.class))); + + short handshakeVersion = 1; + RequestHeader handshakeHeader = + new RequestHeader(ApiKeys.SASL_HANDSHAKE, handshakeVersion, "client", 16); + SaslHandshakeRequest handshakeRequest = + new SaslHandshakeRequest( + new SaslHandshakeRequestData().setMechanism("PLAIN"), handshakeVersion); + ByteBuf handshakeBuffer = serialize(handshakeHeader, handshakeRequest); + + short unsupportedVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + RequestHeader apiVersionsHeader = + new RequestHeader(ApiKeys.API_VERSIONS, unsupportedVersion, "client", 17); + ByteBuf apiVersionsBuffer = serializeHeaderOnly(apiVersionsHeader); + + try { + channel.writeInbound(handshakeBuffer); + processNextRequest(requestChannel, requestHandler); + channel.runPendingTasks(); + + ByteBuf handshakeResponseLength = channel.readOutbound(); + ByteBuf handshakeResponseBuffer = channel.readOutbound(); + try { + assertThat(handshakeResponseLength).isNotNull(); + assertThat(handshakeResponseBuffer).isNotNull(); + assertThat(handshakeResponseLength.readInt()) + .isEqualTo(handshakeResponseBuffer.readableBytes()); + SaslHandshakeResponse handshakeResponse = + (SaslHandshakeResponse) + AbstractResponse.parseResponse( + handshakeResponseBuffer.nioBuffer(), handshakeHeader); + assertThat(handshakeResponse.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + } finally { + if (handshakeResponseLength != null) { + handshakeResponseLength.release(); + } + if (handshakeResponseBuffer != null) { + handshakeResponseBuffer.release(); + } + } + assertThat(channel.isActive()).isTrue(); + + channel.writeInbound(apiVersionsBuffer); + processNextRequest(requestChannel, requestHandler); + channel.runPendingTasks(); + + ByteBuf apiVersionsResponseLength = channel.readOutbound(); + ByteBuf apiVersionsResponseBuffer = channel.readOutbound(); + try { + assertThat(apiVersionsResponseLength).isNotNull(); + assertThat(apiVersionsResponseBuffer).isNotNull(); + assertThat(apiVersionsResponseLength.readInt()) + .isEqualTo(apiVersionsResponseBuffer.readableBytes()); + ByteBuffer responsePayload = apiVersionsResponseBuffer.nioBuffer(); + ResponseHeader responseHeader = + ResponseHeader.parse( + responsePayload, + apiVersionsHeader.toResponseHeader().headerVersion()); + ApiVersionsResponse apiVersionsResponse = + ApiVersionsResponse.parse( + responsePayload, ApiKeys.API_VERSIONS.oldestVersion()); + + assertThat(responseHeader.correlationId()).isEqualTo(17); + assertThat(apiVersionsResponse.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.ILLEGAL_SASL_STATE, 1)); + assertThat(apiVersionsResponse.data().apiKeys()).isEmpty(); + } finally { + if (apiVersionsResponseLength != null) { + apiVersionsResponseLength.release(); + } + if (apiVersionsResponseBuffer != null) { + apiVersionsResponseBuffer.release(); + } + } + assertThat(channel.isActive()).isFalse(); + Object additionalResponse = channel.readOutbound(); + assertThat(additionalResponse).isNull(); + assertThat(handshakeBuffer.refCnt()).isZero(); + assertThat(apiVersionsBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + + @Test + public void testUnauthenticatedProduceIsRejectedBeforeBodyParsing() { + RequestChannel requestChannel = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder( + new RequestChannel[] {requestChannel}, + "KAFKA", + () -> { + throw new AssertionError( + "Authenticator must not be created for a Produce request."); + })); + short produceVersion = ApiKeys.PRODUCE.latestVersion(); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1); + ByteBuf headerOnlyBuffer = serializeHeader(header); + + try { + channel.writeInbound(headerOnlyBuffer); + channel.runPendingTasks(); + + assertThat(requestChannel.requestsCount()).isZero(); + assertThat(channel.isActive()).isFalse(); + assertThat(headerOnlyBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + + @Test + public void testDisconnectDoesNotReleaseQueuedRequestBuffer() { + RequestChannel requestChannel = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requestChannel}, "KAFKA")); + short produceVersion = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest produceRequest = + new ProduceRequest( + new ProduceRequestData().setAcks((short) 1).setTimeoutMs(1000), + produceVersion); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1); + ByteBuf requestBuffer = serialize(header, produceRequest); + + try { + channel.writeInbound(requestBuffer); + KafkaRequest queuedRequest = (KafkaRequest) requestChannel.pollRequest(1000); + assertThat(queuedRequest).isNotNull(); + assertThat(requestBuffer.refCnt()).isEqualTo(2); + + channel.close(); + channel.runPendingTasks(); + + // The response-queue reference is released on disconnect, while the independent + // RequestProcessor ownership remains valid until the worker finishes the request. + assertThat(requestBuffer.refCnt()).isOne(); + assertThat(queuedRequest.request().acks()).isEqualTo((short) 1); + queuedRequest.releaseBuffer(); + assertThat(requestBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + @Test public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() { RequestChannel requestChannel = new RequestChannel(100); @@ -78,6 +249,12 @@ public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() { assertThat(first).isNotNull(); assertThat(second).isNotNull(); + // Polling the requests above stands in for RequestProcessor. In production its finally + // block releases the processor-owned reference after dispatching each request, while + // the ordered-response queue keeps its independent reference until completion. + first.releaseBuffer(); + second.releaseBuffer(); + second.complete(new ApiVersionsResponse(new ApiVersionsResponseData())); channel.runPendingTasks(); Object blockedResponse = channel.readOutbound(); @@ -115,4 +292,40 @@ private static ByteBuf serialize(RequestHeader header, AbstractRequest request) header.data(), header.headerVersion(), request.data(), request.version()); return Unpooled.wrappedBuffer(serialized); } + + private static ByteBuf serializeHeaderOnly(RequestHeader header) { + ObjectSerializationCache cache = new ObjectSerializationCache(); + int headerSize = header.data().size(cache, header.headerVersion()); + ByteBuffer serialized = ByteBuffer.allocate(headerSize); + header.data().write(new ByteBufferAccessor(serialized), cache, header.headerVersion()); + serialized.flip(); + return Unpooled.wrappedBuffer(serialized); + } + + private static void processNextRequest( + RequestChannel requestChannel, KafkaRequestHandler requestHandler) throws Exception { + KafkaRequest request = (KafkaRequest) requestChannel.pollRequest(1000); + assertThat(request).isNotNull(); + try { + requestHandler.processRequest(request); + } finally { + request.releaseBuffer(); + } + } + + private static ByteBuf serializeHeader(RequestHeader header) { + ProduceRequest emptyProduceRequest = + new ProduceRequest( + new ProduceRequestData().setAcks((short) 1).setTimeoutMs(1000), + header.apiVersion()); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), + header.headerVersion(), + emptyProduceRequest.data(), + emptyProduceRequest.version()); + int headerSize = header.data().size(new ObjectSerializationCache(), header.headerVersion()); + serialized.limit(headerSize); + return Unpooled.wrappedBuffer(serialized); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 1c52120fae4..2cbfdd6d5ab 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.ConfigException; import org.junit.jupiter.api.Test; @@ -28,6 +29,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for Kafka configuration. */ public class KafkaConfigsTest { @@ -63,4 +65,57 @@ public void testFromDefault() throws Exception { assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_VALUE_FORMAT)) .isEqualTo("raw"); } + + @Test + public void testKafkaSaslPlainConfiguration() { + Configuration configuration = new Configuration(); + configuration.set( + ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP, + Collections.singletonMap("KAFKA", "sasl")); + configuration.set( + ConfigOptions.SERVER_SASL_ENABLED_MECHANISMS_CONFIG, + Collections.singletonList("PLAIN")); + configuration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, + Collections.singletonMap("writer", "writer-secret")); + + new KafkaProtocolPlugin().setup(configuration); + } + + @Test + public void testKafkaSaslRequiresPlainMechanism() { + Configuration configuration = new Configuration(); + configuration.set( + ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP, + Collections.singletonMap("KAFKA", "sasl")); + configuration.set( + ConfigOptions.SERVER_SASL_ENABLED_MECHANISMS_CONFIG, + Collections.singletonList("SCRAM-SHA-256")); + + assertThatThrownBy(() -> new KafkaProtocolPlugin().setup(configuration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("require PLAIN"); + } + + @Test + public void testKafkaListenerRejectsNonSaslAuthenticationPlugin() { + Configuration configuration = new Configuration(); + configuration.set( + ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP, + Collections.singletonMap("KAFKA", "custom")); + + assertThatThrownBy(() -> new KafkaProtocolPlugin().setup(configuration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("supports only PLAINTEXT or SASL authentication"); + } + + @Test + public void testKafkaListenerAcceptsExplicitPlaintextProtocol() { + Configuration configuration = new Configuration(); + configuration.set( + ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP, + Collections.singletonMap("KAFKA", "PLAINTEXT")); + + new KafkaProtocolPlugin().setup(configuration); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index f7e7dd9bd47..412fbc2c3fc 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -25,6 +25,7 @@ import org.apache.fluss.rpc.messages.PbServerNode; import org.apache.fluss.rpc.messages.PbTableMetadata; import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; @@ -58,6 +59,35 @@ public class KafkaMetadataHandlerTest { private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + @Test + public void testAuthenticatedPrincipalPropagatesToMetadataGatewaySession() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + FlussPrincipal principal = new FlussPrincipal("kafka-user", "User"); + MetadataRequest requestBody = namedTopicRequest("topic"); + short version = requestBody.version(); + KafkaRequest request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()) { + @Override + public FlussPrincipal principal() { + return principal; + } + }; + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + responseBuffer.release(); + + assertThat(service.lastPrincipal).isEqualTo(principal); + } + @Test public void testNamedTopicForEverySupportedVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); @@ -281,6 +311,7 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private final Map tables = new LinkedHashMap<>(); private String lastListenerName; + private FlussPrincipal lastPrincipal; private boolean topicLeaderAvailable = true; private boolean failMetadata; private boolean failNextMetadataAsMissing; @@ -301,6 +332,7 @@ public CompletableFuture listTables(ListTablesRequest reques public CompletableFuture metadata( org.apache.fluss.rpc.messages.MetadataRequest request) { lastListenerName = currentListenerName(); + lastPrincipal = currentSession().getPrincipal(); if (failMetadata) { CompletableFuture failure = new CompletableFuture<>(); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java index 2556430142d..4fdb5422938 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceHandlerTest.java @@ -36,6 +36,7 @@ import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; import org.apache.fluss.rpc.messages.ProduceLogRequest; import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.security.acl.FlussPrincipal; import org.apache.fluss.server.utils.ServerRpcMessageUtils; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -72,6 +73,36 @@ public class KafkaProduceHandlerTest { private static final long TABLE_ID = 123L; private static final long TIMESTAMP = 123456L; + @Test + public void testAuthenticatedPrincipalPropagatesToProduceGatewaySessions() { + TestingProduceGatewayService service = new TestingProduceGatewayService(); + FlussPrincipal principal = new FlussPrincipal("kafka-user", "User"); + short version = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest requestBody = produceRequest(version, (short) 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + version, + new RequestHeader(ApiKeys.PRODUCE, version, "client-id", 1), + requestBody, + "KAFKA", + org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator.DEFAULT + .buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()) { + @Override + public FlussPrincipal principal() { + return principal; + } + }; + + new KafkaRequestHandler(service, service, "kafka").processRequest(request); + + assertThat(parseResponse(request).errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(service.getTableInfoPrincipal).isEqualTo(principal); + assertThat(service.producePrincipal).isEqualTo(principal); + } + @Test public void testProduceTranscodesAndWritesKafkaRecord() throws Exception { TestingProduceGatewayService service = new TestingProduceGatewayService(); @@ -420,6 +451,8 @@ private static final class TestingProduceGatewayService extends TestingTabletGat private final TableDescriptor tableDescriptor; private ProduceLogRequest lastProduceRequest; private ProduceLogResponse produceResponse; + private FlussPrincipal getTableInfoPrincipal; + private FlussPrincipal producePrincipal; private org.apache.fluss.rpc.protocol.Errors produceError = org.apache.fluss.rpc.protocol.Errors.NONE; @@ -455,6 +488,7 @@ private TestingProduceGatewayService( @Override public CompletableFuture getTableInfo(GetTableInfoRequest request) { + getTableInfoPrincipal = currentSession().getPrincipal(); return CompletableFuture.completedFuture( new GetTableInfoResponse() .setTableId(TABLE_ID) @@ -466,6 +500,7 @@ public CompletableFuture getTableInfo(GetTableInfoRequest @Override public CompletableFuture produceLog(ProduceLogRequest request) { + producePrincipal = currentSession().getPrincipal(); lastProduceRequest = request; if (produceResponse != null) { return CompletableFuture.completedFuture(produceResponse); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index e99f6a82213..6f19a9e2aba 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,8 +17,10 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.security.KafkaSaslConnection; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.security.auth.ServerAuthenticator; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; @@ -127,6 +129,75 @@ public void testAdminCapabilitiesAreAdvertisedWhenCoordinatorGatewayIsAvailable( ApiKeys.DELETE_TOPICS.latestVersion())); } + @Test + public void testSaslCapabilitiesAreAdvertisedOnlyForSaslConnection() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.API_VERSIONS.latestVersion(); + ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder().build(version); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + requestBody, + "KAFKA", + KafkaSaslConnection.sasl(() -> mock(ServerAuthenticator.class)), + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + + handler.processRequest(request); + + ApiVersionsResponse response = parseApiVersionsResponse(request); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .contains( + tuple(ApiKeys.SASL_HANDSHAKE.id, (short) 1, (short) 1), + tuple(ApiKeys.SASL_AUTHENTICATE.id, (short) 0, (short) 2)); + } + + @Test + public void testApiVersionsDuringSaslAuthenticationReturnsIllegalState() { + assertApiVersionsDuringSaslAuthenticationReturnsIllegalState( + ApiKeys.API_VERSIONS.oldestVersion()); + } + + @Test + public void testUnsupportedApiVersionsDuringSaslAuthenticationReturnsIllegalState() { + assertApiVersionsDuringSaslAuthenticationReturnsIllegalState( + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1)); + } + + private static void assertApiVersionsDuringSaslAuthenticationReturnsIllegalState( + short requestVersion) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short parsedVersion = ApiKeys.API_VERSIONS.oldestVersion(); + KafkaSaslConnection connection = + KafkaSaslConnection.sasl(() -> mock(ServerAuthenticator.class)); + connection.beginAuthentication("PLAIN", "KAFKA", null); + ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder().build(parsedVersion); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + requestVersion, + new RequestHeader(ApiKeys.API_VERSIONS, requestVersion, "client-id", 17), + requestBody, + "KAFKA", + connection, + ByteBufAllocator.DEFAULT.buffer(), + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + + handler.processRequest(request); + + ApiVersionsResponse response = parseApiVersionsResponse(request); + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.ILLEGAL_SASL_STATE, 1)); + assertThat(response.data().apiKeys()).isEmpty(); + assertThat(request.shouldCloseConnectionAfterResponse()).isTrue(); + assertThat(request.header().correlationId()).isEqualTo(17); + } + private static ApiVersionsResponse requestApiVersions( KafkaRequestHandler handler, short version) { ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaSaslPlainAuthenticationITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaSaslPlainAuthenticationITCase.java new file mode 100644 index 00000000000..c3ee170db33 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaSaslPlainAuthenticationITCase.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.security.acl.AccessControlEntry; +import org.apache.fluss.security.acl.AclBinding; +import org.apache.fluss.security.acl.AclBindingFilter; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.acl.OperationType; +import org.apache.fluss.security.acl.PermissionType; +import org.apache.fluss.security.acl.Resource; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.errors.SaslAuthenticationException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Integration test for Kafka SASL_PLAINTEXT with the PLAIN mechanism. */ +public class KafkaSaslPlainAuthenticationITCase { + + private static final String DATABASE = "kafka"; + private static final String TOPIC = "sasl-plain-topic"; + private static final String USERNAME = "writer"; + private static final String PASSWORD = "writer-secret"; + private static final byte[] KEY = "authenticated-key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "authenticated-value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + private Connection connection; + private org.apache.fluss.client.admin.Admin flussAdmin; + private String bootstrapServers; + + @BeforeEach + public void setup() throws Exception { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER_EXTENSION.getClientConfig()); + flussAdmin = connection.getAdmin(); + flussAdmin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + bootstrapServers = + FLUSS_CLUSTER_EXTENSION.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(",")); + } + + @AfterEach + public void teardown() throws Exception { + if (flussAdmin != null) { + try { + flussAdmin.dropTable(TablePath.of(DATABASE, TOPIC), true).get(); + } catch (Exception ignored) { + // Preserve the primary test failure when cleanup cannot complete. + } + try { + flussAdmin.dropAcls(Collections.singletonList(AclBindingFilter.ANY)).all().get(); + } catch (Exception ignored) { + // Preserve the primary test failure when cleanup cannot complete. + } + flussAdmin.close(); + } + if (connection != null) { + connection.close(); + } + } + + @Test + public void testAuthenticatedAdminAndProducerLifecycle() throws Exception { + grantWriterDatabaseAccess(); + Map clientConfig = kafkaClientConfig(USERNAME, PASSWORD); + try (Admin admin = Admin.create(clientConfig)) { + admin.createTopics(Collections.singleton(new NewTopic(TOPIC, 1, (short) 1))) + .all() + .get(30, TimeUnit.SECONDS); + + Map producerConfig = new HashMap<>(clientConfig); + producerConfig.put( + ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerConfig.put( + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + producerConfig.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + producerConfig.put(ProducerConfig.ACKS_CONFIG, "1"); + try (KafkaProducer producer = new KafkaProducer<>(producerConfig)) { + assertThat(producer.send(new ProducerRecord<>(TOPIC, KEY, VALUE)).get()) + .isNotNull(); + } + + assertFlussRecord(); + admin.deleteTopics(Collections.singleton(TOPIC)).all().get(30, TimeUnit.SECONDS); + } + } + + @Test + public void testAuthenticatedUserWithoutAdminAclCannotCreateTopic() throws Exception { + try (Admin admin = Admin.create(kafkaClientConfig(USERNAME, PASSWORD))) { + assertThatThrownBy( + () -> + admin.createTopics( + Collections.singleton( + new NewTopic(TOPIC, 1, (short) 1))) + .all() + .get(30, TimeUnit.SECONDS)) + .hasRootCauseInstanceOf(TopicAuthorizationException.class); + } + assertThat(flussAdmin.tableExists(TablePath.of(DATABASE, TOPIC)).get()).isFalse(); + } + + @Test + public void testWrongPasswordIsRejectedAsAuthenticationFailure() { + try (Admin admin = Admin.create(kafkaClientConfig(USERNAME, "wrong-password"))) { + assertThatThrownBy(() -> admin.describeCluster().nodes().get(30, TimeUnit.SECONDS)) + .hasRootCauseInstanceOf(SaslAuthenticationException.class); + } + } + + private void assertFlussRecord() throws Exception { + try (Table table = connection.getTable(TablePath.of(DATABASE, TOPIC)); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + for (int attempt = 0; attempt < 30; attempt++) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + assertThat(record.getRow().getBytes(0)).containsExactly(KEY); + assertThat(record.getRow().getBytes(1)).containsExactly(VALUE); + return; + } + } + } + throw new AssertionError("Authenticated Kafka record was not visible through Fluss."); + } + + private void grantWriterDatabaseAccess() throws Exception { + AclBinding aclBinding = + new AclBinding( + Resource.database(DATABASE), + new AccessControlEntry( + new FlussPrincipal(USERNAME, "User"), + AccessControlEntry.WILD_CARD_HOST, + OperationType.ALL, + PermissionType.ALLOW)); + flussAdmin.createAcls(Collections.singletonList(aclBinding)).all().get(); + FLUSS_CLUSTER_EXTENSION.waitUntilAuthenticationSync( + Collections.singletonList(aclBinding), true); + } + + private Map kafkaClientConfig(String username, String password) { + Map config = new HashMap<>(); + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + config.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, 10000); + config.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put( + CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_PLAINTEXT.name); + config.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + config.put( + SaslConfigs.SASL_JAAS_CONFIG, + String.format( + "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";", + username, password)); + return config; + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 1); + config.set( + ConfigOptions.SERVER_SECURITY_PROTOCOL_MAP, + Collections.singletonMap("KAFKA", "sasl")); + config.set( + ConfigOptions.SERVER_SASL_ENABLED_MECHANISMS_CONFIG, + Collections.singletonList("PLAIN")); + config.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, + Collections.singletonMap(USERNAME, PASSWORD)); + config.set(ConfigOptions.AUTHORIZER_ENABLED, true); + config.set(ConfigOptions.SUPER_USERS, "User:ANONYMOUS"); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java index c32b8cd0298..05f7f101b8f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaTopicAdminHandlerTest.java @@ -18,18 +18,23 @@ package org.apache.fluss.kafka; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.exception.AuthorizationException; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.gateway.AdminGateway; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.messages.CreateTableRequest; import org.apache.fluss.rpc.messages.CreateTableResponse; import org.apache.fluss.rpc.messages.DropTableRequest; import org.apache.fluss.rpc.messages.DropTableResponse; import org.apache.fluss.rpc.messages.GetTableInfoRequest; import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.acl.OperationType; +import org.apache.fluss.security.acl.Resource; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; @@ -50,13 +55,17 @@ import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -65,6 +74,119 @@ /** Tests the Kafka topic lifecycle mapping to Fluss tables. */ public class KafkaTopicAdminHandlerTest { + @Test + public void testAuthenticatedPrincipalPropagatesToTopicAdminGatewaySessions() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + AdminOperationAuthorizer adminOperationAuthorizer = mock(AdminOperationAuthorizer.class); + FlussPrincipal principal = new FlussPrincipal("kafka-user", "User"); + List observedPrincipals = new ArrayList<>(); + when(adminGateway.createTable(any(CreateTableRequest.class))) + .thenAnswer( + ignored -> { + observedPrincipals.add(service.currentSession().getPrincipal()); + return CompletableFuture.completedFuture(new CreateTableResponse()); + }); + when(adminGateway.getTableInfo(any(GetTableInfoRequest.class))) + .thenAnswer( + ignored -> { + observedPrincipals.add(service.currentSession().getPrincipal()); + return CompletableFuture.completedFuture( + new GetTableInfoResponse().setTableId(123L)); + }); + when(adminGateway.dropTable(any(DropTableRequest.class))) + .thenAnswer( + ignored -> { + observedPrincipals.add(service.currentSession().getPrincipal()); + return CompletableFuture.completedFuture(new DropTableResponse()); + }); + + short createVersion = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest createRequest = + kafkaRequest( + ApiKeys.CREATE_TOPICS, + createTopicsRequest(createVersion), + createVersion, + principal); + new KafkaRequestHandler(service, service, adminGateway, adminOperationAuthorizer, "kafka") + .processRequest(createRequest); + assertThat(((CreateTopicsResponse) parseResponse(createRequest)).errorCounts()) + .containsOnlyKeys(Errors.NONE); + + short deleteVersion = ApiKeys.DELETE_TOPICS.latestVersion(); + DeleteTopicsRequest deleteRequestBody = + new DeleteTopicsRequest.Builder( + new DeleteTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + Collections.singletonList( + new DeleteTopicsRequestData + .DeleteTopicState() + .setName("topic") + .setTopicId(Uuid.ZERO_UUID)))) + .build(deleteVersion); + KafkaRequest deleteRequest = + kafkaRequest(ApiKeys.DELETE_TOPICS, deleteRequestBody, deleteVersion, principal); + new KafkaRequestHandler(service, service, adminGateway, adminOperationAuthorizer, "kafka") + .processRequest(deleteRequest); + assertThat(((DeleteTopicsResponse) parseResponse(deleteRequest)).errorCounts()) + .containsOnlyKeys(Errors.NONE); + + assertThat(observedPrincipals).containsExactly(principal, principal, principal); + verify(adminOperationAuthorizer) + .authorize( + org.mockito.ArgumentMatchers.argThat( + session -> session.getPrincipal().equals(principal)), + eq(OperationType.CREATE), + eq(Resource.database("kafka"))); + verify(adminOperationAuthorizer) + .authorize( + org.mockito.ArgumentMatchers.argThat( + session -> session.getPrincipal().equals(principal)), + eq(OperationType.DROP), + eq(Resource.table("kafka", "topic"))); + } + + @Test + public void testUnauthorizedTopicAdminRequestsDoNotReachCoordinatorGateway() { + TestingTabletGatewayService service = new TestingTabletGatewayService(); + AdminGateway adminGateway = mock(AdminGateway.class); + AdminOperationAuthorizer adminOperationAuthorizer = mock(AdminOperationAuthorizer.class); + doThrow(new AuthorizationException("denied")) + .when(adminOperationAuthorizer) + .authorize(any(), any(), any()); + KafkaRequestHandler handler = + new KafkaRequestHandler( + service, service, adminGateway, adminOperationAuthorizer, "kafka"); + + short createVersion = ApiKeys.CREATE_TOPICS.latestVersion(); + KafkaRequest createRequest = + kafkaRequest( + ApiKeys.CREATE_TOPICS, + createTopicsRequest(createVersion), + createVersion, + new FlussPrincipal("denied-user", "User")); + handler.processRequest(createRequest); + assertThat(((CreateTopicsResponse) parseResponse(createRequest)).errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.TOPIC_AUTHORIZATION_FAILED, 1)); + + short deleteVersion = ApiKeys.DELETE_TOPICS.latestVersion(); + KafkaRequest deleteRequest = + kafkaRequest( + ApiKeys.DELETE_TOPICS, + deleteTopicsRequest(deleteVersion), + deleteVersion, + new FlussPrincipal("denied-user", "User")); + handler.processRequest(deleteRequest); + assertThat(((DeleteTopicsResponse) parseResponse(deleteRequest)).errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.TOPIC_AUTHORIZATION_FAILED, 1)); + + verify(adminGateway, never()).createTable(any(CreateTableRequest.class)); + verify(adminGateway, never()).dropTable(any(DropTableRequest.class)); + } + @Test public void testCreateTopicCreatesArrowTable() { TestingTabletGatewayService service = new TestingTabletGatewayService(); @@ -277,8 +399,25 @@ private static CreateTopicsRequest createTopicsRequest( return new CreateTopicsRequest.Builder(data).build(version); } + private static DeleteTopicsRequest deleteTopicsRequest(short version) { + DeleteTopicsRequestData data = + new DeleteTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + Collections.singletonList( + new DeleteTopicsRequestData.DeleteTopicState() + .setName("topic") + .setTopicId(Uuid.ZERO_UUID))); + return new DeleteTopicsRequest.Builder(data).build(version); + } + private static KafkaRequest kafkaRequest( ApiKeys apiKey, AbstractRequest requestBody, short version) { + return kafkaRequest(apiKey, requestBody, version, FlussPrincipal.ANONYMOUS); + } + + private static KafkaRequest kafkaRequest( + ApiKeys apiKey, AbstractRequest requestBody, short version, FlussPrincipal principal) { return new KafkaRequest( apiKey, version, @@ -287,7 +426,12 @@ private static KafkaRequest kafkaRequest( "KAFKA", ByteBufAllocator.DEFAULT.buffer(), new TestingChannelHandlerContext(), - new CompletableFuture<>()); + new CompletableFuture<>()) { + @Override + public FlussPrincipal principal() { + return principal; + } + }; } private static AbstractResponse parseResponse(KafkaRequest request) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/sasl/SaslHandlersTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/sasl/SaslHandlersTest.java new file mode 100644 index 00000000000..e57ba0694de --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/sasl/SaslHandlersTest.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.sasl; + +import org.apache.fluss.exception.AuthenticationException; +import org.apache.fluss.kafka.security.KafkaSaslConnection; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.auth.ServerAuthenticator; + +import org.apache.kafka.common.message.SaslAuthenticateRequestData; +import org.apache.kafka.common.message.SaslHandshakeRequestData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.SaslAuthenticateRequest; +import org.apache.kafka.common.requests.SaslAuthenticateResponse; +import org.apache.kafka.common.requests.SaslHandshakeRequest; +import org.apache.kafka.common.requests.SaslHandshakeResponse; +import org.junit.jupiter.api.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol response tests for the Kafka SASL handlers. */ +public class SaslHandlersTest { + + @Test + public void testHandshakeSupportsOnlyV1AndPlain() { + SaslHandshakeHandler handler = new SaslHandshakeHandler(); + assertThat(handler.apiSpec().minVersion()).isEqualTo((short) 1); + assertThat(handler.apiSpec().maxVersion()).isEqualTo((short) 1); + + KafkaSaslConnection connection = KafkaSaslConnection.sasl(TestingServerAuthenticator::new); + SaslHandshakeResponse response = + handler.handle( + connection, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092), + handshake("PLAIN")) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())).isEqualTo(Errors.NONE); + assertThat(response.data().mechanisms()).containsExactly("PLAIN"); + assertThat(connection.isAuthenticating()).isTrue(); + } + + @Test + public void testUnsupportedHandshakeMechanismClosesAfterErrorResponse() { + SaslHandshakeHandler handler = new SaslHandshakeHandler(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(TestingServerAuthenticator::new); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslHandshakeResponse response = + handler.handle( + connection, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092), + handshake("SCRAM-SHA-256"), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.UNSUPPORTED_SASL_MECHANISM); + assertThat(response.data().mechanisms()).containsExactly("PLAIN"); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + @Test + public void testHandshakeOnPlaintextConnectionReturnsIllegalState() { + SaslHandshakeHandler handler = new SaslHandshakeHandler(); + KafkaSaslConnection connection = KafkaSaslConnection.plaintext(); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslHandshakeResponse response = + handler.handle( + connection, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092), + handshake("PLAIN"), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.ILLEGAL_SASL_STATE); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + @Test + public void testRepeatedHandshakeReturnsIllegalState() { + SaslHandshakeHandler handler = new SaslHandshakeHandler(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(TestingServerAuthenticator::new); + InetSocketAddress remoteAddress = new InetSocketAddress("127.0.0.1", 9092); + handler.handle(connection, "KAFKA", remoteAddress, handshake("PLAIN")).join(); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslHandshakeResponse response = + handler.handle( + connection, + "KAFKA", + remoteAddress, + handshake("PLAIN"), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.ILLEGAL_SASL_STATE); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + @Test + public void testAuthenticateSupportsV0ThroughV2AndReturnsPrincipal() { + SaslAuthenticateHandler handler = new SaslAuthenticateHandler(); + assertThat(handler.apiSpec().minVersion()).isZero(); + assertThat(handler.apiSpec().maxVersion()).isEqualTo((short) 2); + + KafkaSaslConnection connection = authenticatedHandshakeConnection(); + SaslAuthenticateResponse response = + handler.handle(connection, authenticate("valid-token", (short) 2)).join(); + + assertThat(Errors.forCode(response.data().errorCode())).isEqualTo(Errors.NONE); + assertThat(response.data().errorMessage()).isNull(); + assertThat(response.data().authBytes()).isEmpty(); + assertThat(response.data().sessionLifetimeMs()).isZero(); + assertThat(connection.isReady()).isTrue(); + assertThat(connection.principal()).isEqualTo(new FlussPrincipal("alice", "User")); + } + + @Test + public void testBadCredentialsReturnSafeFailureAndCloseAfterResponse() { + SaslAuthenticateHandler handler = new SaslAuthenticateHandler(); + KafkaSaslConnection connection = authenticatedHandshakeConnection(); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslAuthenticateResponse response = + handler.handle( + connection, + authenticate("secret-value", (short) 1), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.SASL_AUTHENTICATION_FAILED); + assertThat(response.data().errorMessage()) + .doesNotContain("secret-value") + .contains("invalid credentials"); + assertThat(response.data().authBytes()).isEmpty(); + assertThat(response.data().sessionLifetimeMs()).isZero(); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + @Test + public void testAuthenticateBeforeHandshakeReturnsIllegalState() { + SaslAuthenticateHandler handler = new SaslAuthenticateHandler(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(TestingServerAuthenticator::new); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslAuthenticateResponse response = + handler.handle( + connection, + authenticate("valid-token", (short) 0), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.ILLEGAL_SASL_STATE); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + @Test + public void testAuthenticateAfterCompletionReturnsIllegalState() { + SaslAuthenticateHandler handler = new SaslAuthenticateHandler(); + KafkaSaslConnection connection = authenticatedHandshakeConnection(); + handler.handle(connection, authenticate("valid-token", (short) 2)).join(); + AtomicBoolean closeAfterResponse = new AtomicBoolean(); + + SaslAuthenticateResponse response = + handler.handle( + connection, + authenticate("valid-token", (short) 2), + () -> closeAfterResponse.set(true)) + .join(); + + assertThat(Errors.forCode(response.data().errorCode())) + .isEqualTo(Errors.ILLEGAL_SASL_STATE); + assertThat(connection.shouldClose()).isTrue(); + assertThat(closeAfterResponse).isTrue(); + } + + private static KafkaSaslConnection authenticatedHandshakeConnection() { + KafkaSaslConnection connection = KafkaSaslConnection.sasl(TestingServerAuthenticator::new); + connection.beginAuthentication("PLAIN", "KAFKA", new InetSocketAddress("127.0.0.1", 9092)); + return connection; + } + + private static SaslHandshakeRequest handshake(String mechanism) { + return new SaslHandshakeRequest( + new SaslHandshakeRequestData().setMechanism(mechanism), (short) 1); + } + + private static SaslAuthenticateRequest authenticate(String token, short version) { + return new SaslAuthenticateRequest( + new SaslAuthenticateRequestData() + .setAuthBytes(token.getBytes(StandardCharsets.UTF_8)), + version); + } + + private static final class TestingServerAuthenticator implements ServerAuthenticator { + private boolean completed; + + @Override + public String protocol() { + return "sasl"; + } + + @Override + public byte[] evaluateResponse(byte[] token) { + if (!java.util.Arrays.equals(token, "valid-token".getBytes(StandardCharsets.UTF_8))) { + throw new AuthenticationException("Rejected token contents"); + } + completed = true; + return new byte[0]; + } + + @Override + public boolean isCompleted() { + return completed; + } + + @Override + public FlussPrincipal createPrincipal() { + return new FlussPrincipal("alice", "User"); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/security/KafkaSaslConnectionTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/security/KafkaSaslConnectionTest.java new file mode 100644 index 00000000000..80ed986f9fb --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/security/KafkaSaslConnectionTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.security; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.AuthenticationException; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.security.auth.ServerAuthenticator; +import org.apache.fluss.security.auth.sasl.authenticator.SaslServerAuthenticator; +import org.apache.fluss.security.auth.sasl.plain.PlainSaslServerConfigManager; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.junit.jupiter.api.Test; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for the per-connection Kafka SASL state machine. */ +public class KafkaSaslConnectionTest { + + @Test + public void testPlaintextConnectionIsImmediatelyReady() { + KafkaSaslConnection connection = KafkaSaslConnection.plaintext(); + + assertThat(connection.authenticationEnabled()).isFalse(); + assertThat(connection.isReady()).isTrue(); + assertThat(connection.principal()).isEqualTo(FlussPrincipal.ANONYMOUS); + assertThat(connection.isRequestAllowed(ApiKeys.METADATA)).isTrue(); + assertThat(connection.shouldClose()).isFalse(); + } + + @Test + public void testSuccessfulAuthenticationTransitionsToReady() { + TestingServerAuthenticator authenticator = new TestingServerAuthenticator(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(() -> authenticator); + + assertThat(connection.authenticationEnabled()).isTrue(); + assertThat(connection.isAwaitingHandshake()).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.API_VERSIONS)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.SASL_HANDSHAKE)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.SASL_AUTHENTICATE)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.METADATA)).isFalse(); + + connection.beginAuthentication( + KafkaSaslConnection.PLAIN_MECHANISM, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092)); + + assertThat(connection.isAuthenticating()).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.API_VERSIONS)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.SASL_AUTHENTICATE)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.SASL_HANDSHAKE)).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.METADATA)).isFalse(); + assertThat(authenticator.listenerName).isEqualTo("KAFKA"); + assertThat(authenticator.ipAddress).isEqualTo("127.0.0.1"); + assertThat(authenticator.mechanism).isEqualTo(KafkaSaslConnection.PLAIN_MECHANISM); + + byte[] challenge = connection.authenticate(bytes("valid-token")); + + assertThat(challenge).isEmpty(); + assertThat(connection.isReady()).isTrue(); + assertThat(connection.principal()).isEqualTo(new FlussPrincipal("alice", "User")); + assertThat(connection.isRequestAllowed(ApiKeys.PRODUCE)).isTrue(); + assertThat(connection.shouldClose()).isFalse(); + assertThat(authenticator.closed).isTrue(); + } + + @Test + public void testFlussPlainAuthenticatorAcceptsKafkaPlainToken() { + Configuration configuration = new Configuration(); + configuration.set( + ConfigOptions.SERVER_SASL_ENABLED_MECHANISMS_CONFIG, + Collections.singletonList(KafkaSaslConnection.PLAIN_MECHANISM)); + configuration.set( + ConfigOptions.SERVER_SASL_CREDENTIALS, + Collections.singletonMap("writer", "writer-secret")); + PlainSaslServerConfigManager configManager = + new PlainSaslServerConfigManager(configuration); + KafkaSaslConnection connection = + KafkaSaslConnection.sasl( + () -> new SaslServerAuthenticator(configManager.getConfiguration())); + + connection.beginAuthentication( + KafkaSaslConnection.PLAIN_MECHANISM, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092)); + connection.authenticate(bytes("\u0000writer\u0000writer-secret")); + + assertThat(connection.isReady()).isTrue(); + assertThat(connection.principal()).isEqualTo(new FlussPrincipal("writer", "User")); + } + + @Test + public void testAuthenticationFailureRequiresConnectionClose() { + TestingServerAuthenticator authenticator = new TestingServerAuthenticator(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(() -> authenticator); + connection.beginAuthentication( + KafkaSaslConnection.PLAIN_MECHANISM, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092)); + + assertThatThrownBy(() -> connection.authenticate(bytes("bad-token"))) + .isInstanceOf(AuthenticationException.class); + + assertThat(connection.shouldClose()).isTrue(); + assertThat(connection.isReady()).isFalse(); + assertThat(connection.principal()).isEqualTo(FlussPrincipal.ANONYMOUS); + assertThat(authenticator.closed).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.API_VERSIONS)).isFalse(); + } + + @Test + public void testCloseReleasesAuthenticator() { + TestingServerAuthenticator authenticator = new TestingServerAuthenticator(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(() -> authenticator); + connection.beginAuthentication( + KafkaSaslConnection.PLAIN_MECHANISM, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092)); + + connection.close(); + + assertThat(authenticator.closed).isTrue(); + assertThat(connection.shouldClose()).isTrue(); + assertThat(connection.isRequestAllowed(ApiKeys.SASL_AUTHENTICATE)).isFalse(); + } + + @Test + public void testCloseWaitsForInProgressAuthentication() throws Exception { + BlockingServerAuthenticator authenticator = new BlockingServerAuthenticator(); + KafkaSaslConnection connection = KafkaSaslConnection.sasl(() -> authenticator); + connection.beginAuthentication( + KafkaSaslConnection.PLAIN_MECHANISM, + "KAFKA", + new InetSocketAddress("127.0.0.1", 9092)); + CompletableFuture authentication = + CompletableFuture.supplyAsync(() -> connection.authenticate(bytes("token"))); + + assertThat(authenticator.entered.await(10, TimeUnit.SECONDS)).isTrue(); + CountDownLatch closeStarted = new CountDownLatch(1); + CompletableFuture closeFuture = + CompletableFuture.runAsync( + () -> { + closeStarted.countDown(); + connection.close(); + }); + assertThat(closeStarted.await(10, TimeUnit.SECONDS)).isTrue(); + try { + assertThatThrownBy(() -> closeFuture.get(100, TimeUnit.MILLISECONDS)) + .isInstanceOf(TimeoutException.class); + } finally { + authenticator.proceed.countDown(); + } + + assertThat(authentication.get(10, TimeUnit.SECONDS)).isEmpty(); + closeFuture.get(10, TimeUnit.SECONDS); + assertThat(connection.shouldClose()).isTrue(); + assertThat(connection.isReady()).isFalse(); + assertThat(authenticator.closeCount).hasValue(1); + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static final class TestingServerAuthenticator implements ServerAuthenticator { + private String listenerName; + private String ipAddress; + private String mechanism; + private boolean completed; + private boolean closed; + + @Override + public String protocol() { + return "sasl"; + } + + @Override + public void initialize(AuthenticateContext context) { + listenerName = context.listenerName(); + ipAddress = context.ipAddress(); + mechanism = context.protocol(); + } + + @Override + public byte[] evaluateResponse(byte[] token) { + if (!java.util.Arrays.equals(token, bytes("valid-token"))) { + throw new AuthenticationException("Invalid credentials"); + } + completed = true; + return new byte[0]; + } + + @Override + public boolean isCompleted() { + return completed; + } + + @Override + public FlussPrincipal createPrincipal() { + return new FlussPrincipal("alice", "User"); + } + + @Override + public void close() { + closed = true; + } + } + + private static final class BlockingServerAuthenticator implements ServerAuthenticator { + private final CountDownLatch entered = new CountDownLatch(1); + private final CountDownLatch proceed = new CountDownLatch(1); + private final AtomicInteger closeCount = new AtomicInteger(); + private boolean completed; + + @Override + public String protocol() { + return "sasl"; + } + + @Override + public byte[] evaluateResponse(byte[] token) { + entered.countDown(); + try { + proceed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AuthenticationException("Interrupted while testing authentication.", e); + } + completed = true; + return new byte[0]; + } + + @Override + public boolean isCompleted() { + return completed; + } + + @Override + public FlussPrincipal createPrincipal() { + return new FlussPrincipal("alice", "User"); + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminOperationAuthorizer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminOperationAuthorizer.java new file mode 100644 index 00000000000..8c549943a42 --- /dev/null +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/gateway/AdminOperationAuthorizer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.gateway; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.OperationType; +import org.apache.fluss.security.acl.Resource; + +/** Authorizes an administrative operation before it is forwarded through an internal RPC. */ +@Internal +@FunctionalInterface +public interface AdminOperationAuthorizer { + + /** Authorizes an administrative operation using the original external client session. */ + void authorize(Session session, OperationType operationType, Resource resource); +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussProtocolPlugin.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussProtocolPlugin.java index 25ae1894148..9cc032092f9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussProtocolPlugin.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussProtocolPlugin.java @@ -27,49 +27,19 @@ import org.apache.fluss.rpc.protocol.NetworkProtocolPlugin; import org.apache.fluss.security.auth.AuthenticationFactory; import org.apache.fluss.security.auth.PlainTextAuthenticationPlugin; +import org.apache.fluss.security.auth.sasl.plain.PlainSaslServerConfigManager; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandler; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** Build-in protocol plugin for Fluss. */ public class FlussProtocolPlugin implements NetworkProtocolPlugin, ServerReconfigurable { - private static final String PLAIN_CREDENTIALS_CONFIG = - ConfigOptions.SERVER_SASL_CREDENTIALS.key(); - - /** Pattern to match {@code user_=""} entries in JAAS config strings. */ - private static final Pattern JAAS_USER_PATTERN = Pattern.compile("user_(\\w+)=\"([^\"]*)\""); - - /** - * Valid username pattern. Only letters, digits, and underscores are allowed because the - * username is used as part of the JAAS option key {@code user_}. - */ - private static final Pattern VALID_USERNAME_PATTERN = Pattern.compile("\\w+"); - - /** - * Characters forbidden in passwords. These would break the map format or the generated JAAS - * config string: comma (entry separator), colon (key-value separator), double-quote (JAAS value - * delimiter), semicolon (JAAS statement terminator), backslash (escape char), and control - * characters. - */ - private static final Pattern INVALID_PASSWORD_PATTERN = - Pattern.compile("[,:\"\\\\;]|[\\x00-\\x1F\\x7F]"); - private final ApiManager apiManager; private final List listeners; private final RequestsMetrics requestsMetrics; - private Configuration conf; - /** Initial credentials from `security.sasl.plain.jaas.config`. */ - private Map initialPlainCredentialsFromJaasConfig; - - /** Current config `security.sasl.plain.credentials`. */ - private Map currentPlainCredentials; + private PlainSaslServerConfigManager plainSaslServerConfigManager; public FlussProtocolPlugin( ServerType serverType, List listeners, RequestsMetrics requestsMetrics) { @@ -85,9 +55,7 @@ public String name() { @Override public void setup(Configuration conf) { - this.conf = new Configuration(conf); - this.initialPlainCredentialsFromJaasConfig = parseCredentialsFromJaasConfig(conf); - enrichWithJaasConfig(conf); + this.plainSaslServerConfigManager = new PlainSaslServerConfigManager(conf); } @Override @@ -98,6 +66,7 @@ public List listenerNames() { @Override public ChannelHandler createChannelHandler( RequestChannel[] requestChannels, String listenerName) { + Configuration conf = plainSaslServerConfigManager.getConfiguration(); return new ServerChannelInitializer( requestChannels, apiManager, @@ -107,7 +76,7 @@ public ChannelHandler createChannelHandler( conf.get(ConfigOptions.NETTY_CONNECTION_MAX_IDLE_TIME).getSeconds(), (int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(), Optional.ofNullable( - AuthenticationFactory.loadServerAuthenticatorSuppliers(this.conf) + AuthenticationFactory.loadServerAuthenticatorSuppliers(conf) .get(listenerName)) .orElse(PlainTextAuthenticationPlugin.PlainTextServerAuthenticator::new)); } @@ -121,119 +90,11 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { @Override public void validate(Configuration newConfig) throws ConfigException { - Map newCredentials = readPlainCredentials(newConfig); - if (Objects.equals(newCredentials, currentPlainCredentials)) { - return; - } - if (newCredentials != null && !newCredentials.isEmpty()) { - int index = 0; - for (Map.Entry credential : newCredentials.entrySet()) { - validateUsername(credential.getKey()); - validatePassword(index, credential.getKey(), credential.getValue()); - index++; - } - } - - // Generate the merged JAAS config value to ensure it is valid. - generateMergedJaasConfig(newCredentials); + plainSaslServerConfigManager.validate(newConfig); } @Override public void reconfigure(Configuration newConfig) throws ConfigException { - enrichWithJaasConfig(newConfig); - } - - /** - * Enriches the plugin's configuration with a generated JAAS config string by merging: - * - *

    - *
  1. Existing credentials parsed from the current {@code security.sasl.plain.jaas.config} - *
  2. New credentials from the {@code security.sasl.plain.credentials} map in {@code - * newConfig} - *
- * - *

New credentials take priority when a username exists in both sources. If the credentials - * map is not present in {@code newConfig}, the configuration is returned unchanged. - */ - private void enrichWithJaasConfig(Configuration newConfig) throws ConfigException { - Map newCredentials = readPlainCredentials(newConfig); - if (Objects.equals(newCredentials, currentPlainCredentials)) { - return; - } - - conf.setString( - ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG, - generateMergedJaasConfig(newCredentials)); - currentPlainCredentials = newCredentials; - } - - private static Map readPlainCredentials(Configuration config) - throws ConfigException { - try { - return config.get(ConfigOptions.SERVER_SASL_CREDENTIALS); - } catch (IllegalArgumentException | IllegalStateException e) { - throw new ConfigException( - String.format( - "Failed to parse %s: %s", PLAIN_CREDENTIALS_CONFIG, e.getMessage()), - e); - } - } - - private static void validateUsername(String username) throws ConfigException { - if (!VALID_USERNAME_PATTERN.matcher(username).matches()) { - throw new ConfigException( - String.format( - "%s: username '%s' contains invalid characters. " - + "Only letters, digits, and underscores are allowed.", - PLAIN_CREDENTIALS_CONFIG, username)); - } - } - - private static void validatePassword(int index, String username, String password) - throws ConfigException { - if (password == null || INVALID_PASSWORD_PATTERN.matcher(password).find()) { - throw new ConfigException( - String.format( - "%s[%d]: password for user '%s' contains invalid characters. " - + "Commas, colons, quotes, semicolons, backslashes, and control characters are not allowed.", - PLAIN_CREDENTIALS_CONFIG, index, username)); - } - } - - /** - * Generates the merged JAAS config string by combining existing credentials from the current - * {@code security.sasl.plain.jaas.config} with the given new credentials map. New credentials - * take priority on username conflict. - * - * @param newCredentials map of username to password from SERVER_SASL_CREDENTIALS - * @return the generated JAAS config string - */ - private String generateMergedJaasConfig(Map newCredentials) { - Map mergedCredentials = - new LinkedHashMap<>(initialPlainCredentialsFromJaasConfig); - if (newCredentials != null) { - mergedCredentials.putAll(newCredentials); - } - - StringBuilder sb = - new StringBuilder( - "org.apache.fluss.security.auth.sasl.plain.PlainLoginModule required"); - for (Map.Entry entry : mergedCredentials.entrySet()) { - sb.append(String.format(" user_%s=\"%s\"", entry.getKey(), entry.getValue())); - } - sb.append(";"); - return sb.toString(); - } - - private static Map parseCredentialsFromJaasConfig(Configuration configuration) { - Map credentials = new LinkedHashMap<>(); - String existingJaas = configuration.getString(ConfigOptions.SERVER_SASL_PLAIN_JAAS_CONFIG); - if (existingJaas != null) { - Matcher matcher = JAAS_USER_PATTERN.matcher(existingJaas); - while (matcher.find()) { - credentials.put(matcher.group(1), matcher.group(2)); - } - } - return credentials; + plainSaslServerConfigManager.reconfigure(newConfig); } } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java index b04722d5c01..154c7c3e5b9 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServerHandler.java @@ -184,11 +184,14 @@ public void channelActive(ChannelHandlerContext ctx) throws Exception { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { - super.channelInactive(ctx); - - // Unregister this channel from its RequestChannel. The RequestChannel will clean up both - // the association and any paused state. - requestChannel.unregisterChannel(ctx.channel()); + try { + IOUtils.closeQuietly(authenticator); + // Unregister this channel from its RequestChannel. The RequestChannel will clean up + // both the association and any paused state. + requestChannel.unregisterChannel(ctx.channel()); + } finally { + super.channelInactive(ctx); + } } @Override diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java index f465bb4a69a..d5e5053cfb2 100644 --- a/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/TestingTabletGatewayService.java @@ -18,6 +18,7 @@ package org.apache.fluss.rpc; import org.apache.fluss.cluster.ServerType; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.DatabaseExistsRequest; import org.apache.fluss.rpc.messages.DatabaseExistsResponse; @@ -83,12 +84,18 @@ import org.apache.fluss.rpc.messages.TableExistsResponse; import org.apache.fluss.rpc.messages.UpdateMetadataRequest; import org.apache.fluss.rpc.messages.UpdateMetadataResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.OperationType; +import org.apache.fluss.security.acl.Resource; import java.util.concurrent.CompletableFuture; /** A testing implementation of the {@link TabletServerGateway} interface. */ public class TestingTabletGatewayService extends TestingGatewayService - implements TabletServerGateway { + implements TabletServerGateway, AdminOperationAuthorizer { + + @Override + public void authorize(Session session, OperationType operationType, Resource resource) {} @Override public ServerType providerType() { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index b0993e51e6d..32e13eef624 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -38,6 +38,7 @@ import org.apache.fluss.rpc.entity.PrefixLookupResultForBucket; import org.apache.fluss.rpc.entity.ResultForBucket; import org.apache.fluss.rpc.gateway.AdminGatewayProvider; +import org.apache.fluss.rpc.gateway.AdminOperationAuthorizer; import org.apache.fluss.rpc.gateway.CoordinatorGateway; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.FetchLogRequest; @@ -77,6 +78,7 @@ import org.apache.fluss.rpc.messages.StopReplicaResponse; import org.apache.fluss.rpc.messages.UpdateMetadataRequest; import org.apache.fluss.rpc.messages.UpdateMetadataResponse; +import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.rpc.protocol.ApiError; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.FetchLogReadPreference; @@ -162,7 +164,7 @@ /** An RPC Gateway service for tablet server. */ public final class TabletService extends RpcServiceBase - implements TabletServerGateway, AdminGatewayProvider { + implements TabletServerGateway, AdminGatewayProvider, AdminOperationAuthorizer { private final String serviceName; private final ReplicaManager replicaManager; @@ -219,6 +221,13 @@ public CoordinatorGateway getAdminGateway() { return coordinatorGateway; } + @Override + public void authorize(Session session, OperationType operationType, Resource resource) { + if (authorizer != null) { + authorizer.authorize(session, operationType, resource); + } + } + @Override public void shutdown() {}