From 3d6d40db333e96963ca606559d080651403a6769 Mon Sep 17 00:00:00 2001
From: XuJianxu <xjxlzlpzl@163.com>
Date: Mon, 11 Sep 2023 19:41:21 +0800
Subject: [PATCH] [docker] Add kafka relate case (#24180)

Add kafka relate case
---
 .../docker-compose/kafka/kafka.env            | 20 +++++++++
 .../docker-compose/kafka/kafka.yaml.tpl       | 42 +++++++++++++++++++
 .../docker-compose/kafka/scripts/test.csv     |  1 +
 .../thirdparties/run-thirdparties-docker.sh   | 29 +++++++++++--
 4 files changed, 89 insertions(+), 3 deletions(-)
 create mode 100644 docker/thirdparties/docker-compose/kafka/kafka.env
 create mode 100644 docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl
 create mode 100644 docker/thirdparties/docker-compose/kafka/scripts/test.csv

diff --git a/docker/thirdparties/docker-compose/kafka/kafka.env b/docker/thirdparties/docker-compose/kafka/kafka.env
new file mode 100644
index 0000000000..6c430af182
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/kafka.env
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DOCKER_ZOOKEEPER_EXTERNAL_PORT=12181
+DOCKER_KAFKA_EXTERNAL_PORT=19193
diff --git a/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl b/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl
new file mode 100644
index 0000000000..15c61d8d76
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3"
+
+services:
+    doris--zookeeper:
+        image: wurstmeister/zookeeper
+        restart: always
+        container_name: doris--zookeeper
+        ports:
+            - ${DOCKER_ZOOKEEPER_EXTERNAL_PORT}:2181
+    doris--kafka:
+        image: wurstmeister/kafka 
+        restart: always
+        container_name: doris--kafka
+        depends_on:
+            - doris--zookeeper
+        ports:
+            - ${DOCKER_KAFKA_EXTERNAL_PORT}:19193
+        environment:
+            KAFKA_ZOOKEEPER_CONNECT: doris--zookeeper:2181/kafka
+            KAFKA_LISTENERS: PLAINTEXT://:19193
+            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19193
+            KAFKA_BROKER_ID: 1
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+            
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kafka/scripts/test.csv b/docker/thirdparties/docker-compose/kafka/scripts/test.csv
new file mode 100644
index 0000000000..3fde4e2023
--- /dev/null
+++ b/docker/thirdparties/docker-compose/kafka/scripts/test.csv
@@ -0,0 +1 @@
+1,2,3
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index ebdb1209d0..b9a82b4218 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -37,7 +37,7 @@ Usage: $0 <options>
      --stop             stop the specified components
 
   All valid components:
-    mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino
+    mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino,kafka
   "
     exit 1
 }
@@ -60,7 +60,7 @@ STOP=0
 
 if [[ "$#" == 1 ]]; then
     # default
-    COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino"
+    COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka"
 else
     while true; do
         case "$1" in
@@ -92,7 +92,7 @@ else
     done
     if [[ "${COMPONENTS}"x == ""x ]]; then
         if [[ "${STOP}" -eq 1 ]]; then
-            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino"
+            COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka"
         fi
     fi
 fi
@@ -130,6 +130,7 @@ RUN_ES=0
 RUN_ICEBERG=0
 RUN_HUDI=0
 RUN_TRINO=0
+RUN_KAFKA=0
 
 for element in "${COMPONENTS_ARR[@]}"; do
     if [[ "${element}"x == "mysql"x ]]; then
@@ -146,6 +147,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
         RUN_ES=1
     elif [[ "${element}"x == "hive"x ]]; then
         RUN_HIVE=1
+    elif [[ "${element}"x == "kafka"x ]]; then
+        RUN_KAFKA=1
     elif [[ "${element}"x == "iceberg"x ]]; then
         RUN_ICEBERG=1
     elif [[ "${element}"x == "hudi"x ]]; then
@@ -235,6 +238,26 @@ if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
     fi
 fi
 
+if [[ "${RUN_KAFKA}" -eq 1 ]]; then
+    # kafka
+    KAFKA_CONTAINER_ID="${CONTAINER_UID}kafka"
+    eth0_num=$(ifconfig -a|grep flags=|grep -n ^eth0|awk -F ':' '{print $1}')
+    IP_HOST=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"|tail -n +${eth0_num}|head -n 1)
+    cp "${ROOT}"/docker-compose/kafka/kafka.yaml.tpl "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sed -i "s/localhost/${IP_HOST}/g" "${ROOT}"/docker-compose/kafka/kafka.yaml
+    sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env down
+    if [[ "${STOP}" -ne 1 ]]; then
+        sudo docker compose -f "${ROOT}"/docker-compose/kafka/kafka.yaml --env-file "${ROOT}"/docker-compose/kafka/kafka.env up --build --remove-orphans -d
+        sleep 30s
+        while IFS= read -r line
+            do
+                docker exec -i ${KAFKA_CONTAINER_ID} bash -c "echo "$line" | /opt/kafka/bin/kafka-console-producer.sh --broker-list '${IP_HOST}:19193' --topic test"
+                sleep 1
+            done < ""${ROOT}"/docker-compose/kafka/scripts/test.csv"
+    fi
+fi
+
 if [[ "${RUN_HIVE}" -eq 1 ]]; then
     # hive
     # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"