[docker] Add kafka relate case (#24180)

Add kafka relate case

docker/thirdparties/docker-compose/kafka/kafka.env

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DOCKER_ZOOKEEPER_EXTERNAL_PORT=12181
+DOCKER_KAFKA_EXTERNAL_PORT=19193

docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+    doris--zookeeper:
+        image: wurstmeister/zookeeper
+        restart: always
+        container_name: doris--zookeeper
+        ports:
+            - ${DOCKER_ZOOKEEPER_EXTERNAL_PORT}:2181
+    doris--kafka:
+        image: wurstmeister/kafka 
+        restart: always
+        container_name: doris--kafka
+        depends_on:
+            - doris--zookeeper
+        ports:
+            - ${DOCKER_KAFKA_EXTERNAL_PORT}:19193
+        environment:
+            KAFKA_ZOOKEEPER_CONNECT: doris--zookeeper:2181/kafka
+            KAFKA_LISTENERS: PLAINTEXT://:19193
+            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19193
+            KAFKA_BROKER_ID: 1
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+            

docker/thirdparties/docker-compose/kafka/scripts/test.csv

@@ -0,0 +1 @@
+1,2,3

docker/thirdparties/run-thirdparties-docker.sh

@@ -37,7 +37,7 @@ Usage: $0 <options>
      --stop             stop the specified components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino,kafka
   "
     exit 1
 }
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino"
+    COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka"
         fi
     fi
 fi
@@ -130,6 +130,7 @@ RUN_ES=0
 RUN_ICEBERG=0
 RUN_HUDI=0
 RUN_TRINO=0
+RUN_KAFKA=0
 
 for element in "${COMPONENTS_ARR[@]}"; do
     if [[ "${element}"x == "mysql"x ]]; then
@@ -146,6 +147,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
         RUN_ES=1
     elif [[ "${element}"x == "hive"x ]]; then
         RUN_HIVE=1
+    elif [[ "${element}"x == "kafka"x ]]; then
+        RUN_KAFKA=1
     elif [[ "${element}"x == "iceberg"x ]]; then
         RUN_ICEBERG=1
     elif [[ "${element}"x == "hudi"x ]]; then
@@ -235,6 +238,26 @@ if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
     fi
 fi
 
+if [[ "${RUN_KAFKA}" -eq 1 ]]; then
+    # kafka
+    KAFKA_CONTAINER_ID="${CONTAINER_UID}kafka"
+    eth0_num=$(ifconfig -a|grep flags=|grep -n ^eth0|awk -F ':' '{print $1}')
+    IP_HOST=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"|tail -n +${eth0_num}|head -n 1)
+    cp "${ROOT}"/docker-compose/kafka/kafka.yaml.tpl "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sed -i "s/localhost/${IP_HOST}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env down
+    if [[ "${STOP}" -ne 1 ]]; then
+        sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env up --build --remove-orphans -d
+        sleep 30s
+        while IFS= read -r line
+            do
+                docker exec -i ${KAFKA_CONTAINER_ID} bash -c "echo "$line" | /opt/kafka/bin/kafka-console-producer.sh --broker-list '${IP_HOST}:19193' --topic test"
+                sleep 1
+            done < ""${ROOT}"/docker-compose/kafka/scripts/test.csv"
+    fi
+fi
+
 if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # hive
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"