branch-2.1:[opt](docker) Add ranger docker component (#47697) (#48359)

### What problem does this PR solve?
bp  https://github.com/apache/doris/pull/47697

### Release note

None

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [x] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [x] No.
    - [ ] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
This commit is contained in:
zgxme
2025-02-27 09:47:25 +08:00
committed by GitHub
parent 6da61683d4
commit 676b868d99
10 changed files with 2238 additions and 4 deletions

View File

@ -0,0 +1,34 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
################################################################
# This script will restart all thirdparty containers
################################################################
set -ex
cd $RANGER_HOME
./setup.sh
echo "Installing Doris Ranger plugins"
/opt/install_doris_ranger_plugins.sh
echo "Starting Ranger Admin"
ranger-admin start
echo "Installing Doris service definition"
/opt/install_doris_service_def.sh
# Keep the container running
tail -f /dev/null

View File

@ -0,0 +1,17 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[mysqld]
log_bin_trust_function_creators = 1

View File

@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- If this file is found in the config directory, it will only be
loaded once at startup. If it is found in Solr's data
directory, it will be re-loaded every commit.
See http://wiki.apache.org/solr/QueryElevationComponent for more info
-->
<elevate>
<query text="foo bar">
<doc id="1" />
<doc id="2" />
<doc id="3" />
</query>
<query text="ipod">
<doc id="MA147LL/A" /> <!-- put the actual ipod at the top -->
<doc id="IW-02" exclude="true" /> <!-- exclude this cable -->
</query>
</elevate>

View File

@ -0,0 +1,95 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<schema name="ranger-audit-schema" version="1.6">
<uniqueKey>id</uniqueKey>
<fieldType name="binary" class="solr.BinaryField"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/>
<fieldType name="date" class="solr.TrieDateField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="float" class="solr.TrieFloatField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/>
<fieldType name="int" class="solr.TrieIntField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="key_lower_case" class="solr.TextField" sortMissingLast="true" omitNorms="true">
<analyzer>
<tokenizer class="solr.KeywordTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.LengthFilterFactory" min="0" max="2500"/>
</analyzer>
</fieldType>
<fieldType name="long" class="solr.TrieLongField" docValues="true" precisionStep="0" positionIncrementGap="0"/>
<fieldType name="random" class="solr.RandomSortField" indexed="true"/>
<fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
<fieldType name="tdate" class="solr.TrieDateField" docValues="true" precisionStep="6" positionIncrementGap="0"/>
<fieldType name="tdates" class="solr.TrieDateField" docValues="true" precisionStep="6" multiValued="true" positionIncrementGap="0"/>
<fieldType name="tdouble" class="solr.TrieDoubleField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tdoubles" class="solr.TrieDoubleField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
<fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
</analyzer>
</fieldType>
<fieldType name="tfloat" class="solr.TrieFloatField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tfloats" class="solr.TrieFloatField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
<fieldType name="tint" class="solr.TrieIntField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tints" class="solr.TrieIntField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
<fieldType name="tlong" class="solr.TrieLongField" docValues="true" precisionStep="8" positionIncrementGap="0"/>
<fieldType name="tlongs" class="solr.TrieLongField" docValues="true" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
<field name="_expire_at_" type="tdate" multiValued="false" stored="true" docValues="true"/>
<field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
<field name="_version_" type="long" indexed="false" stored="true"/>
<field name="access" type="key_lower_case" multiValued="false"/>
<field name="action" type="key_lower_case" multiValued="false"/>
<field name="agent" type="key_lower_case" multiValued="false"/>
<field name="agentHost" type="key_lower_case" multiValued="false"/>
<field name="cliIP" type="key_lower_case" multiValued="false"/>
<field name="cliType" type="key_lower_case" multiValued="false"/>
<field name="cluster" type="key_lower_case" multiValued="false"/>
<field name="reqContext" type="key_lower_case" multiValued="true"/>
<field name="enforcer" type="key_lower_case" multiValued="false"/>
<field name="event_count" type="tlong" multiValued="false" docValues="true" default="1"/>
<field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
<field name="evtTime" type="tdate" docValues="true"/>
<field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
<field name="logType" type="key_lower_case" multiValued="false"/>
<field name="policy" type="tlong" docValues="true"/>
<field name="proxyUsers" type="key_lower_case" multiValued="true"/>
<field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
<field name="repo" type="key_lower_case" multiValued="false"/>
<field name="repoType" type="tint" multiValued="false" docValues="true"/>
<field name="req_caller_id" type="key_lower_case" multiValued="false"/>
<field name="req_self_id" type="key_lower_case" multiValued="false"/>
<field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
<field name="reqUser" type="key_lower_case" multiValued="false"/>
<field name="resType" type="key_lower_case" multiValued="false"/>
<field name="resource" type="key_lower_case" multiValued="false"/>
<field name="result" type="tint" multiValued="false"/>
<field name="seq_num" type="tlong" multiValued="false" docValues="true" default="0"/>
<field name="sess" type="key_lower_case" multiValued="false"/>
<field name="tags" type="key_lower_case" multiValued="true"/>
<field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
<field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
<field name="zoneName" type="key_lower_case" multiValued="false"/>
<field name="policyVersion" type="tlong" multiValued="false"/>
</schema>

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '3.7'
services:
ranger-admin:
image: ghcr.io/takezoe/ranger-docker/ranger-admin:v2.4.0
# build:
# context: ./ranger-admin
# dockerfile: Dockerfile
container_name: ${CONTAINER_UID}-ranger-admin
ports:
- ${RANGER_PORT}:6080
networks:
- doris--ranger
depends_on:
ranger-mysql:
condition: service_healthy
ranger-solr:
condition: service_started
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:6080"]
interval: 30s
timeout: 10s
retries: 10
volumes:
- ./ranger-admin/ranger-entrypoint.sh:/opt/ranger-entrypoint.sh
- ./script/install_doris_ranger_plugins.sh:/opt/install_doris_ranger_plugins.sh
- ./script/install_doris_service_def.sh:/opt/install_doris_service_def.sh
entrypoint : ["bash", "-c", "bash /opt/ranger-entrypoint.sh"]
ranger-mysql:
image: mysql:8.0.33
container_name: ranger-mysql
ports:
- ${RANGER_MYSQL_PORT}:3306
healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root --password=root && mysql -h 127.0.0.1 -u root --password=root -e "SELECT 1 FROM mysql.innodb_table_stats;"
interval: 5s
timeout: 60s
retries: 120
networks:
- doris--ranger
volumes:
- ./ranger-mysql:/etc/mysql/conf.d
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_USER: rangeradmin
MYSQL_PASSWORD: rangeradmin
MYSQL_DATABASE: ranger
ranger-solr:
image: solr:8.11.2
container_name: ranger-solr
ports:
- ${RANGER_SOLR_PORT}:8983
networks:
- doris--ranger
volumes:
- ./ranger-solr:/opt/solr/server/solr/configsets/ranger_audits/conf
entrypoint:
- solr-precreate
- ranger_audits
- /opt/solr/server/solr/configsets/ranger_audits
networks:
doris--ranger:
ipam:
driver: default
config:
- subnet: 168.45.0.0/24

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
export RANGER_SOLAR_PORT=8983
export RANGER_PORT=6081
export RANGER_MYSQL_PORT=33061

View File

@ -0,0 +1,24 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/bin/bash
set -ex
if [ ! -d "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris" ]; then
mkdir -p "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris"
fi
cd "${RANGER_HOME}/ews/webapp/WEB-INF/classes/ranger-plugins/doris"
curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/mysql-connector-java-8.0.25.jar
curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/ranger-doris-plugin-3.0.0-SNAPSHOT.jar

View File

@ -0,0 +1,27 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/bin/bash
set -ex
curl -O https://s3BucketName.s3Endpoint/regression/docker/ranger-plugins/ranger-servicedef-doris.json
until curl -f http://localhost:6080; do
echo "Waiting for service to be healthy..."
sleep 30
done
curl -u admin:Ranger1234 -X POST \
-H "Accept: application/json" \
-H "Content-Type: application/json" \
http://localhost:6080/service/plugins/definitions \
-d@ranger-servicedef-doris.json

View File

@ -38,12 +38,12 @@ Usage: $0 <options>
--reserve-ports reserve host ports by setting 'net.ipv4.ip_local_reserved_ports' to avoid port already bind error
All valid components:
mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,kerberos,oceanbase
mysql,pg,oracle,sqlserver,clickhouse,es,hive2,hive3,iceberg,hudi,trino,kafka,mariadb,db2,oceanbase,lakesoul,kerberos,ranger
"
exit 1
}
DEFAULT_COMPONENTS="mysql,es,hive2,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2,oceanbase,kerberos"
ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul"
ALL_COMPONENTS="${DEFAULT_COMPONENTS},hudi,trino,kafka,spark,lakesoul,ranger"
COMPONENTS=$2
HELP=0
STOP=0
@ -148,7 +148,7 @@ RUN_MARIADB=0
RUN_DB2=0
RUN_KERBEROS=0
RUN_OCENABASE=0
RUN_RANGER=0
RESERVED_PORTS="65535"
for element in "${COMPONENTS_ARR[@]}"; do
@ -187,6 +187,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
RUN_KERBEROS=1
elif [[ "${element}"x == "oceanbase"x ]];then
RUN_OCEANBASE=1
elif [[ "${element}"x == "ranger"x ]]; then
RUN_RANGER=1
else
echo "Invalid component: ${element}"
usage
@ -631,8 +633,20 @@ start_kerberos() {
sleep 2
fi
}
start_ranger() {
echo "RUN_RANGER"
export CONTAINER_UID=${CONTAINER_UID}
find "${ROOT}/docker-compose/ranger/script" -type f -exec sed -i "s/s3Endpoint/${s3Endpoint}/g" {} \;
find "${ROOT}/docker-compose/ranger/script" -type f -exec sed -i "s/s3BucketName/${s3BucketName}/g" {} \;
. "${ROOT}/docker-compose/ranger/ranger_settings.env"
envsubst <"${ROOT}"/docker-compose/ranger/ranger.yaml.tpl >"${ROOT}"/docker-compose/ranger/ranger.yaml
sudo docker compose -f "${ROOT}"/docker-compose/ranger/ranger.yaml --env-file "${ROOT}"/docker-compose/ranger/ranger_settings.env down
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -f "${ROOT}"/docker-compose/ranger/ranger.yaml --env-file "${ROOT}"/docker-compose/ranger/ranger_settings.env up -d --wait --remove-orphans
fi
}
echo "starting dockers in parrallel"
echo "starting dockers in parallel"
reserve_ports
@ -728,6 +742,11 @@ if [[ "${RUN_KERBEROS}" -eq 1 ]]; then
pids["kerberos"]=$!
fi
if [[ "${RUN_RANGER}" -eq 1 ]]; then
start_ranger > start_ranger.log 2>&1 &
pids["ranger"]=$!
fi
echo "waiting all dockers starting done"
for compose in "${!pids[@]}"; do