diff --git a/docker/thirdparties/docker-compose/ranger/ranger-admin/ranger-entrypoint.sh b/docker/thirdparties/docker-compose/ranger/ranger-admin/ranger-entrypoint.sh
new file mode 100644
index 0000000000..8f3898eb99
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger-admin/ranger-entrypoint.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################################################
+# This script will restart all thirdparty containers
+################################################################
+set -ex
+
+cd $RANGER_HOME
+./setup.sh
+echo "Installing Doris Ranger plugins"
+/opt/install_doris_ranger_plugins.sh
+echo "Starting Ranger Admin"
+ranger-admin start
+echo "Installing Doris service definition"
+/opt/install_doris_service_def.sh
+
+# Keep the container running
+tail -f /dev/null
diff --git a/docker/thirdparties/docker-compose/ranger/ranger-mysql/my.cnf b/docker/thirdparties/docker-compose/ranger/ranger-mysql/my.cnf
new file mode 100644
index 0000000000..68fff4d987
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger-mysql/my.cnf
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[mysqld]
+log_bin_trust_function_creators = 1
diff --git a/docker/thirdparties/docker-compose/ranger/ranger-solr/elevate.xml b/docker/thirdparties/docker-compose/ranger/ranger-solr/elevate.xml
new file mode 100644
index 0000000000..25d5cebe4f
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger-solr/elevate.xml
@@ -0,0 +1,38 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/docker/thirdparties/docker-compose/ranger/ranger-solr/managed-schema b/docker/thirdparties/docker-compose/ranger/ranger-solr/managed-schema
new file mode 100644
index 0000000000..c33f6de06b
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger-solr/managed-schema
@@ -0,0 +1,95 @@
+
+
+
+ id
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/ranger/ranger-solr/solrconfig.xml b/docker/thirdparties/docker-compose/ranger/ranger-solr/solrconfig.xml
new file mode 100644
index 0000000000..9791a1ffd3
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger-solr/solrconfig.xml
@@ -0,0 +1,1872 @@
+
+
+
+
+
+
+
+
+ 5.2.0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.data.dir:}
+
+
+
+
+
+
+
+
+ ${solr.hdfs.home:}
+
+ ${solr.hdfs.confdir:}
+
+ ${solr.hdfs.blockcache.enabled:true}
+
+ ${solr.hdfs.blockcache.global:true}
+
+
+
+
+
+
+
+
+
+ true
+ managed-schema
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.lock.type:native}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.ulog.dir:}
+
+
+
+
+ ${solr.autoCommit.maxTime:15000}
+ false
+
+
+
+
+
+ ${solr.autoSoftCommit.maxTime:5000}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1024
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+ 20
+
+
+ 200
+
+
+
+
+
+
+
+
+
+
+
+ static firstSearcher warming in solrconfig.xml
+
+
+
+
+
+ true
+
+
+ 2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ solr-data-config.xml
+
+
+
+
+
+
+
+ explicit
+ 10
+ text
+
+
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+ json
+ true
+ text
+
+
+
+
+
+
+
+ true
+ json
+ true
+
+
+
+
+
+
+
+ explicit
+
+
+ velocity
+ browse
+ layout
+
+
+ edismax
+ *:*
+ 10
+ *,score
+
+
+ on
+ 1
+
+
+
+
+
+
+ text
+ add-unknown-fields-to-the-schema
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+ ignored_
+
+
+ true
+ links
+ ignored_
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ solrpingquery
+
+
+ all
+
+
+
+
+
+
+
+
+ explicit
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ text_general
+
+
+
+
+
+ default
+ text
+ solr.DirectSolrSpellChecker
+
+ internal
+
+ 0.5
+
+ 2
+
+ 1
+
+ 5
+
+ 4
+
+ 0.01
+
+
+
+
+
+ wordbreak
+ solr.WordBreakSolrSpellChecker
+ name
+ true
+ true
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ text
+
+ default
+ wordbreak
+ on
+ true
+ 10
+ 5
+ 5
+ true
+ true
+ 10
+ 5
+
+
+ spellcheck
+
+
+
+
+
+ mySuggester
+ FuzzyLookupFactory
+ DocumentDictionaryFactory
+ cat
+ price
+ string
+
+
+
+
+
+ true
+ 10
+
+
+ suggest
+
+
+
+
+
+
+
+
+ text
+ true
+
+
+ tvComponent
+
+
+
+
+
+
+ lingo
+
+
+ org.carrot2.clustering.lingo.LingoClusteringAlgorithm
+
+
+ clustering/carrot2
+
+
+
+
+ stc
+ org.carrot2.clustering.stc.STCClusteringAlgorithm
+
+
+
+
+ kmeans
+ org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm
+
+
+
+
+
+
+ true
+ true
+
+ name
+
+ id
+
+ features
+
+ true
+
+
+
+ false
+
+
+ edismax
+
+ text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+
+ *:*
+ 10
+ *,score
+
+
+ clustering
+
+
+
+
+
+
+
+
+
+ true
+ false
+
+
+ terms
+
+
+
+
+
+
+
+ string
+ elevate.xml
+
+
+
+
+
+ explicit
+ text
+
+
+ elevator
+
+
+
+
+
+
+
+
+
+
+ 100
+
+
+
+
+
+
+
+ 70
+
+ 0.5
+
+ [-\w ,/\n\"']{20,200}
+
+
+
+
+
+
+ ]]>
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ,,
+ ,,
+ ,,
+ ,,
+ ,]]>
+ ]]>
+
+
+
+
+
+ 10
+ .,!?
+
+
+
+
+
+
+ WORD
+
+
+ en
+ US
+
+
+
+
+
+
+
+
+
+
+ _ttl_
+ +90DAYS
+
+
+ 86400
+ _ttl_
+ _expire_at_
+
+
+ _expire_at_
+
+
+
+
+
+
+
+
+ yyyy-MM-dd'T'HH:mm:ss.SSSZ
+ yyyy-MM-dd'T'HH:mm:ss,SSSZ
+ yyyy-MM-dd'T'HH:mm:ss.SSS
+ yyyy-MM-dd'T'HH:mm:ss,SSS
+ yyyy-MM-dd'T'HH:mm:ssZ
+ yyyy-MM-dd'T'HH:mm:ss
+ yyyy-MM-dd'T'HH:mmZ
+ yyyy-MM-dd'T'HH:mm
+ yyyy-MM-dd HH:mm:ss.SSSZ
+ yyyy-MM-dd HH:mm:ss,SSSZ
+ yyyy-MM-dd HH:mm:ss.SSS
+ yyyy-MM-dd HH:mm:ss,SSS
+ yyyy-MM-dd HH:mm:ssZ
+ yyyy-MM-dd HH:mm:ss
+ yyyy-MM-dd HH:mmZ
+ yyyy-MM-dd HH:mm
+ yyyy-MM-dd
+
+
+
+ key_lower_case
+
+ java.lang.Boolean
+ boolean
+
+
+ java.util.Date
+ tdate
+
+
+ java.lang.Long
+ java.lang.Integer
+ tlong
+
+
+ java.lang.Number
+ tdouble
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ text/plain; charset=UTF-8
+
+
+
+
+ ${velocity.template.base.dir:}
+
+
+
+
+ 5
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *:*
+
+
+
diff --git a/docker/thirdparties/docker-compose/ranger/ranger.yaml.tpl b/docker/thirdparties/docker-compose/ranger/ranger.yaml.tpl
new file mode 100644
index 0000000000..3de94d6fb9
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger.yaml.tpl
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '3.7'
+
+services:
+
+ ranger-admin:
+ image: ghcr.io/takezoe/ranger-docker/ranger-admin:v2.4.0
+ # build:
+ # context: ./ranger-admin
+ # dockerfile: Dockerfile
+ container_name: ${CONTAINER_UID}-ranger-admin
+ ports:
+ - ${RANGER_PORT}:6080
+ networks:
+ - doris--ranger
+ depends_on:
+ ranger-mysql:
+ condition: service_healthy
+ ranger-solr:
+ condition: service_started
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:6080"]
+ interval: 30s
+ timeout: 10s
+ retries: 10
+ volumes:
+ - ./ranger-admin/ranger-entrypoint.sh:/opt/ranger-entrypoint.sh
+ - ./script/install_doris_ranger_plugins.sh:/opt/install_doris_ranger_plugins.sh
+ - ./script/install_doris_service_def.sh:/opt/install_doris_service_def.sh
+
+ entrypoint : ["bash", "-c", "bash /opt/ranger-entrypoint.sh"]
+
+ ranger-mysql:
+ image: mysql:8.0.33
+ container_name: ranger-mysql
+ ports:
+ - ${RANGER_MYSQL_PORT}:3306
+ healthcheck:
+ test: mysqladmin ping -h 127.0.0.1 -u root --password=root && mysql -h 127.0.0.1 -u root --password=root -e "SELECT 1 FROM mysql.innodb_table_stats;"
+ interval: 5s
+ timeout: 60s
+ retries: 120
+ networks:
+ - doris--ranger
+ volumes:
+ - ./ranger-mysql:/etc/mysql/conf.d
+ environment:
+ MYSQL_ROOT_PASSWORD: root
+ MYSQL_USER: rangeradmin
+ MYSQL_PASSWORD: rangeradmin
+ MYSQL_DATABASE: ranger
+
+ ranger-solr:
+ image: solr:8.11.2
+ container_name: ranger-solr
+ ports:
+ - ${RANGER_SOLR_PORT}:8983
+ networks:
+ - doris--ranger
+ volumes:
+ - ./ranger-solr:/opt/solr/server/solr/configsets/ranger_audits/conf
+ entrypoint:
+ - solr-precreate
+ - ranger_audits
+ - /opt/solr/server/solr/configsets/ranger_audits
+
+networks:
+ doris--ranger:
+ ipam:
+ driver: default
+ config:
+ - subnet: 168.45.0.0/24
diff --git a/docker/thirdparties/docker-compose/ranger/ranger_settings.env b/docker/thirdparties/docker-compose/ranger/ranger_settings.env
new file mode 100644
index 0000000000..13dd93d517
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/ranger_settings.env
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export RANGER_SOLAR_PORT=8983
+export RANGER_PORT=6081
+export RANGER_MYSQL_PORT=33061
diff --git a/docker/thirdparties/docker-compose/ranger/script/install_doris_ranger_plugins.sh b/docker/thirdparties/docker-compose/ranger/script/install_doris_ranger_plugins.sh
new file mode 100755
index 0000000000..c3a1cf428b
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/script/install_doris_ranger_plugins.sh
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -ex
+
+if [ ! -d "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris" ]; then
+ mkdir -p "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris"
+fi
+cd "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris"
+curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/mysql-connector-java-8.0.25.jar
+curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/ranger-doris-plugin-3.0.0-SNAPSHOT.jar
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/ranger/script/install_doris_service_def.sh b/docker/thirdparties/docker-compose/ranger/script/install_doris_service_def.sh
new file mode 100755
index 0000000000..c5eeaa6000
--- /dev/null
+++ b/docker/thirdparties/docker-compose/ranger/script/install_doris_service_def.sh
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#!/bin/bash
+set -ex
+
+curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/ranger-servicedef-doris.json
+until curl -f http://localhost:6080; do
+ echo "Waiting for service to be healthy..."
+ sleep 30
+done
+curl -u admin:Ranger1234 -X POST \
+ -H "Accept: application/json" \
+ -H "Content-Type: application/json" \
+ http://localhost:6080/service/plugins/definitions \
+ -d@ranger-servicedef-doris.json
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index 5534afdb06..279346c51b 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -38,12 +38,12 @@ Usage: $0
--reserve-ports reserve host ports by setting 'net.ipv4.ip_local_reserved_ports' to avoid port already bind error
All valid components:
- mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos,oceanbase
+ mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger
"
exit 1
}
DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,oceanbase,kerberos"
-ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul"
+ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul,ranger"
COMPONENTS=$2
HELP=0
STOP=0
@@ -148,7 +148,7 @@ RUN_MARIADB=0
RUN_DB2=0
RUN_KERBEROS=0
RUN_OCENABASE=0
-
+RUN_RANGER=0
RESERVED_PORTS="65535"
for element in "${COMPONENTS_ARR[@]}"; do
@@ -187,6 +187,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
RUN_KERBEROS=1
elif [[ "${element}"x == "oceanbase"x ]];then
RUN_OCEANBASE=1
+ elif [[ "${element}"x == "ranger"x ]]; then
+ RUN_RANGER=1
else
echo "Invalid component: ${element}"
usage
@@ -631,8 +633,20 @@ start_kerberos() {
sleep 2
fi
}
+start_ranger() {
+ echo "RUN_RANGER"
+ export CONTAINER_UID=${CONTAINER_UID}
+ find "${ROOT}/docker-compose/ranger/script" -type f -exec sed -i "s/s3Endpoint/${s3Endpoint}/g" {} \;
+ find "${ROOT}/docker-compose/ranger/script" -type f -exec sed -i "s/s3BucketName/${s3BucketName}/g" {} \;
+ . "${ROOT}/docker-compose/ranger/ranger_settings.env"
+ envsubst <"${ROOT}"/docker-compose/ranger/ranger.yaml.tpl >"${ROOT}"/docker-compose/ranger/ranger.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/ranger/ranger.yaml --env-file "${ROOT}"/docker-compose/ranger/ranger_settings.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo docker compose -f "${ROOT}"/docker-compose/ranger/ranger.yaml --env-file "${ROOT}"/docker-compose/ranger/ranger_settings.env up -d --wait --remove-orphans
+ fi
+}
-echo "starting dockers in parrallel"
+echo "starting dockers in parallel"
reserve_ports
@@ -728,6 +742,11 @@ if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
pids["kerberos"]=$!
fi
+if [[ "${RUN_RANGER}" -eq 1 ]]; then
+ start_ranger > start_ranger.log 2>&1 &
+ pids["ranger"]=$!
+fi
+
echo "waiting all dockers starting done"
for compose in "${!pids[@]}"; do