From a607c30ad4e4d1d8af512cf3994cec358de22451 Mon Sep 17 00:00:00 2001 From: AlexYue Date: Wed, 20 Jul 2022 19:03:29 +0800 Subject: [PATCH] [docs] Fe build idea doc (#10996) * [doc](fe): enhance the fe-idea-dev * [doc](fe)add solution for m1 mac compile error Co-authored-by: jackwener --- .../developer/developer-guide/fe-idea-dev.md | 179 +++++++----------- .../developer/developer-guide/fe-idea-dev.md | 179 +++++++----------- 2 files changed, 135 insertions(+), 223 deletions(-) diff --git a/docs/en/developer/developer-guide/fe-idea-dev.md b/docs/en/developer/developer-guide/fe-idea-dev.md index 8b217e3e36..83931d75be 100644 --- a/docs/en/developer/developer-guide/fe-idea-dev.md +++ b/docs/en/developer/developer-guide/fe-idea-dev.md @@ -28,64 +28,86 @@ under the License. ## 1. Environmental Preparation -* Git * JDK1.8+ * IntelliJ IDEA * Maven (Optional, IDEA shipped embedded Maven3) -1. Git clone codebase from https://github.com/apache/incubator-doris.git +Git clone codebase from https://github.com/apache/incubator-doris.git +Use IntelliJ IDEA to open the code `FE` directory -2. Use IntelliJ IDEA to open the code root directory +### Thrift +If your are only interested in FE module, and for some reason you can't or don't want to compile full thirdparty libraries， -3. If your are only interested in FE module, and for some reason you can't or don't want to compile full thirdparty libraries, - the minimum tool required for FE module is `thrift`, so you can manually install `thrift` and copy or create a link of - the executable `thrift` command to `./thirdparty/installed/bin`. - ``` - Doris build against `thrift` 0.13.0 ( note : `Doris` 0.15 and later version build against `thrift` 0.13.0 , the previous version is still `thrift` 0.9.3) +the minimum tool required for FE module is `thrift`, so you can manually install `thrift` and copy or create a link of the executable `thrift` command to `./thirdparty/installed/bin`. + +``` +Doris build against `thrift` 0.13.0 ( note : `Doris` 0.15 and later version build against `thrift` 0.13.0 , the previous version is still `thrift` 0.9.3) + +Windows: + 1. Download：`http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.exe` + 2. Copy：copy the file to `./thirdparty/installed/bin` - Windows: - 1. Download：`http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.exe` - 2. Copy：copy the file to `./thirdparty/installed/bin` - - MacOS: - 1. Download：`brew install thrift@0.13.0` - 2. Establish soft connection： - `mkdir -p ./thirdparty/installed/bin` - # For ARM macOS - `ln -s /opt/homebrew/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` - # For Intel macOS - `ln -s /usr/local/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` - - Note：The error that the version cannot be found may be reported when MacOS execute `brew install thrift@0.13.0`. The solution is execute at the terminal as follows: - 1. `brew tap-new $USER/local-tap` - 2. `brew extract --version='0.13.0' thrift $USER/local-tap` - 3. `brew install thrift@0.13.0` - Reference link: `https://gist.github.com/tonydeng/02e571f273d6cce4230dc8d5f394493c` - ``` +MacOS: + 1. Download：`brew install thrift@0.13.0` + 2. Establish soft connection： + `mkdir -p ./thirdparty/installed/bin` + # For ARM macOS + `ln -s /opt/homebrew/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` + # For Intel macOS + `ln -s /usr/local/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` + +Note：The error that the version cannot be found may be reported when MacOS execute `brew install thrift@0.13.0`. The solution is execute at the terminal as follows: + 1. `brew tap-new $USER/local-tap` + 2. `brew extract --version='0.13.0' thrift $USER/local-tap` + 3. `brew install thrift@0.13.0` +Reference link: `https://gist.github.com/tonydeng/02e571f273d6cce4230dc8d5f394493c` +``` -4. Go to `./fe` folder and run the following maven command to generate sources. +### generate sources - ``` - mvn generate-sources - ``` - - If fails, run following command. - - ``` - mvn clean install -DskipTests - ``` - - You can also use IDE embedded GUI tools to run maven command to generate sources +Go to `./fe` folder and run the following maven command to generate sources. + +``` +mvn generate-sources +``` + +If fails, run following command. + +``` +mvn clean install -DskipTests +``` + +You can also use IDE embedded GUI tools to run maven command to generate sources ![](/images/gen_code.png) If you are developing on the OS which lack of support to run `shell script` and `make` such as Windows, a workround here is generate codes in Linux and copy them back. Using Docker should also be an option. -5. If a help document has not been generated, go to the docs directory and run`sh build_help_zip.sh`， - Then copy help-resource.zip from build to fe/fe-core/target/classes +#### arm mac compile failed + +An error would occur if you generated sources using maven on arm mac. Detailed error messages are as follows. + +``` +[ERROR] Failed to execute goal org.xolstice.maven.plugins:protobuf-maven-plugin:0.6.1:compile (grpc-build) on project apm-network: Unable to resolve artifact: Missing: +[ERROR] 1) com.google.protobuf:protoc:exe:osx-aarch_64:3.14.0 +[ERROR] 1 required artifact is missing. +``` + +Since protobuf v3.14.0 and protoc-gen-grpc-java v1.30.0 don't come up with osx-aarch_64 version, given the advice by [grpc_community], you'd better manually download the corresponding osx_x86 version and then translate them by Rosseta2. + +1. open `doris/fe/fe-core/pom.xml` +2. change `com.google.protobuf:protoc:${protobuf.version}` to `com.google.protobuf:protoc:3.14.0:exe:osx-x86_64` +3. change `io.grpc:protoc-gen-grpc-java:${grpc.version}` to `io.grpc:protoc-gen-grpc-java:1.30.0:exe:osx-x86_64` +4. open terminal and paste `softwareupdate --install-rosetta` + +### help document + +If a help document has not been generated, go to the docs directory and run `sh build_help_zip.sh`， + +Then copy help-resource.zip from build to fe/fe-core/target/classes ## 2. Debug @@ -99,80 +121,9 @@ is generate codes in Linux and copy them back. Using Docker should also be an op ## 3. Custom FE configuration -Copy below content into `conf/fe.conf` and tune it to fit your environment(Note: If developed using`Mac`, since`docker for Mac`does not support`Host`mode,`be`needs to be exposed using`-p` and `fe.conf` `priority_networks` configured to be accessible within the container, such as WIFI Ip). +Copy below content into `conf/fe.conf` and tune it to fit your environment -``` -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -##################################################################### -## The uppercase properties are read and exported by bin/start_fe.sh. -## To see all Frontend configurations, -## see fe/src/org/apache/doris/common/Config.java -##################################################################### - -# the output dir of stderr and stdout -LOG_DIR = ${DORIS_HOME}/log - -DATE = `date +%Y%m%d-%H%M%S` -JAVA_OPTS="-Xmx2048m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE" - -# For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time" - -## -## the lowercase properties are read by main program. -## - -# INFO, WARN, ERROR, FATAL -sys_log_level = INFO - -# store metadata, create it if it is not exist. -# Default value is ${DORIS_HOME}/doris-meta -# meta_dir = ${DORIS_HOME}/doris-meta - -http_port = 8030 -rpc_port = 9020 -query_port = 9030 -edit_log_port = 9010 -mysql_service_nio_enabled = true - -# Choose one if there are more than one ip except loopback address. -# Note that there should at most one ip match this list. -# If no ip match this rule, will choose one randomly. -# use CIDR format, e.g. 10.10.10.0/24 -# Default value is empty. -# priority_networks = 10.10.10.0/24;192.168.0.0/16 - -# Advanced configurations -# log_roll_size_mb = 1024 -# sys_log_dir = ${DORIS_HOME}/log -# sys_log_roll_num = 10 -# sys_log_verbose_modules = -# audit_log_dir = ${DORIS_HOME}/log -# audit_log_modules = slow_query, query -# audit_log_roll_num = 10 -# meta_delay_toleration_second = 10 -# qe_max_connection = 1024 -# max_conn_per_user = 100 -# qe_query_timeout_second = 300 -# qe_slow_log_ms = 5000 - -``` +(Note: If developed using`Mac`, since`docker for Mac`does not support`Host`mode,`be`needs to be exposed using`-p` and `fe.conf` `priority_networks` configured to be accessible within the container, such as WIFI Ip). ## 4. Setting Environment Variables diff --git a/docs/zh-CN/developer/developer-guide/fe-idea-dev.md b/docs/zh-CN/developer/developer-guide/fe-idea-dev.md index 0877b069bd..4f10443c40 100644 --- a/docs/zh-CN/developer/developer-guide/fe-idea-dev.md +++ b/docs/zh-CN/developer/developer-guide/fe-idea-dev.md @@ -28,47 +28,61 @@ under the License. ## 1.环境准备 -JDK1.8+, IntelliJ IDEA +从 https://github.com/apache/incubator-doris.git 下载源码到本地 -1. 从 https://github.com/apache/incubator-doris.git 下载源码到本地 +安装 JDK1.8+ ，使用 IntelliJ IDEA 打开 FE. -2. 使用IntelliJ IDEA 打开代码根目录 +### thrift -3. 如果仅进行fe开发而没有编译过thirdparty，则需要安装thrift，并将thrift 复制或者连接到 `thirdparty/installed/bin` 目录下 +如果仅进行fe开发而没有编译过thirdparty，则需要安装thrift，并将thrift 复制或者连接到 `thirdparty/installed/bin` 目录下 - 安装 `thrift` 0.13.0 版本(注意：`Doris` 0.15 和最新的版本基于 `thrift` 0.13.0 构建, 之前的版本依然使用`thrift` 0.9.3 构建) +安装 `thrift` 0.13.0 版本(注意：`Doris` 0.15 和最新的版本基于 `thrift` 0.13.0 构建, 之前的版本依然使用`thrift` 0.9.3 构建) - Windows: - 1. 下载：`http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.exe` - 2. 拷贝：将文件拷贝至 `./thirdparty/installed/bin` - - MacOS: - 1. 下载：`brew install thrift@0.13.0` - 2. 建立软链接： - `mkdir -p ./thirdparty/installed/bin` - # ARM架构macOS - `ln -s /opt/homebrew/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` - # Intel架构macOS - `ln -s /usr/local/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` - - 注：MacOS执行 `brew install thrift@0.13.0` 可能会报找不到版本的错误，解决方法如下，在终端执行： - 1. `brew tap-new $USER/local-tap` - 2. `brew extract --version='0.13.0' thrift $USER/local-tap` - 3. `brew install thrift@0.13.0` - 参考链接: `https://gist.github.com/tonydeng/02e571f273d6cce4230dc8d5f394493c` +#### Windows 下载 -4. 如果是Mac 或者 Linux 环境 可以通过 如下命令生成自动生成代码： +1. 下载：`http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.exe` +2. 拷贝：将文件拷贝至 `./thirdparty/installed/bin` - ``` - cd fe - mvn generate-sources - ``` +#### MacOS 下载 - 如果出现错误，则执行： +下载：`brew install thrift@0.13.0` - ``` - cd fe && mvn clean install -DskipTests - ``` +注：macOS执行 `brew install thrift@0.13.0` 可能会报找不到版本的错误，解决方法如下，在终端执行： + +1. `brew tap-new $USER/local-tap` +2. `brew extract --version='0.13.0' thrift $USER/local-tap` +3. `brew install thrift@0.13.0` + +参考链接: `https://gist.github.com/tonydeng/02e571f273d6cce4230dc8d5f394493c` + +#### 建立软链接 + +位于 Doris **根**目录下 + +`mkdir -p ./thirdparty/installed/bin` + +ARM架构macOS + +`ln -s /opt/homebrew/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` + +Intel架构macOS + +`ln -s /usr/local/Cellar/thrift@0.13.0/0.13.0/bin/thrift ./thirdparty/installed/bin/thrift` + +### 自动生成代码： + +如果是Mac 或者 Linux 环境 可以通过 如下命令生成 + +``` +cd fe +mvn generate-sources +``` + +如果出现错误，则执行： + +``` +cd fe && mvn clean install -DskipTests +``` 或者通过图形界面运行 maven 命令生成 @@ -76,8 +90,28 @@ JDK1.8+, IntelliJ IDEA 如果使用windows环境可能会有make命令和sh脚本无法执行的情况 可以通过拷贝linux上的 `fe/fe-core/target/generated-sources` 目录拷贝到相应的目录的方式实现，也可以通过docker 镜像挂载本地目录之后，在docker 内部生成自动生成代码，可以参照编译一节 -5. 如果还未生成过help文档，需要跳转到docs目录，执行`sh build_help_zip.sh`， - 然后将build中的help-resource.zip拷贝到fe/fe-core/target/classes中 +#### arm mac compile failed + +如果在m1 mac上进行自动生成代码会出现如下错误 + +``` +[ERROR] Failed to execute goal org.xolstice.maven.plugins:protobuf-maven-plugin:0.6.1:compile (grpc-build) on project apm-network: Unable to resolve artifact: Missing: +[ERROR] 1) com.google.protobuf:protoc:exe:osx-aarch_64:3.14.0 +[ERROR] 1 required artifact is missing. +``` + +ptotobuf3.14.0和protoc-gen-grpc-java1.30.0没有aarch64的版本，根据[grpc社区issue](https://github.com/grpc/grpc-java/issues/7690)的建议下载x86版本并使用rosetta转译 + +1. 打开`doris/fe/fe-core/pom.xml` +2. 将`com.google.protobuf:protoc:${protobuf.version}`修改成`com.google.protobuf:protoc:3.14.0:exe:osx-x86_64` +3. 将`io.grpc:protoc-gen-grpc-java:${grpc.version}`修改成`io.grpc:protoc-gen-grpc-java:1.30.0:exe:osx-x86_64` +4. 打开终端输入`softwareupdate --install-rosetta` + +### help文档 + +如果还未生成过help文档，需要跳转到docs目录，执行`sh build_help_zip.sh`， + +然后将build中的help-resource.zip拷贝到fe/fe-core/target/classes中 ## 2.调试 @@ -91,82 +125,9 @@ JDK1.8+, IntelliJ IDEA ## 3.配置conf/fe.conf -下面是我自己的配置，你可以根据自己的需要进行修改(注意：如果使用`Mac`开发，由于`docker for Mac`不支持`Host`模式，需要使用`-p`方式暴露`be`端口，同时`fe.conf`的`priority_networks`配置为容器内可访问的Ip，例如WIFI的Ip) - -``` -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -##################################################################### -## The uppercase properties are read and exported by bin/start_fe.sh. -## To see all Frontend configurations, -## see fe/src/org/apache/doris/common/Config.java -##################################################################### - -# the output dir of stderr and stdout -LOG_DIR = ${DORIS_HOME}/log - -DATE = `date +%Y%m%d-%H%M%S` -JAVA_OPTS="-Xmx2048m -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$DATE" - -# For jdk 9+, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_9="-Xmx4096m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$DATE:time" - -## -## the lowercase properties are read by main program. -## - -# INFO, WARN, ERROR, FATAL -sys_log_level = INFO - -# store metadata, create it if it is not exist. -# Default value is ${DORIS_HOME}/doris-meta -# meta_dir = ${DORIS_HOME}/doris-meta - -http_port = 8030 -rpc_port = 9020 -query_port = 9030 -edit_log_port = 9010 -mysql_service_nio_enabled = true - -# Choose one if there are more than one ip except loopback address. -# Note that there should at most one ip match this list. -# If no ip match this rule, will choose one randomly. -# use CIDR format, e.g. 10.10.10.0/24 -# Default value is empty. -# priority_networks = 10.10.10.0/24;192.168.0.0/16 - -# Advanced configurations -# log_roll_size_mb = 1024 -# sys_log_dir = ${DORIS_HOME}/log -# sys_log_roll_num = 10 -# sys_log_verbose_modules = -# audit_log_dir = ${DORIS_HOME}/log -# audit_log_modules = slow_query, query -# audit_log_roll_num = 10 -# meta_delay_toleration_second = 10 -# qe_max_connection = 1024 -# max_conn_per_user = 100 -# qe_query_timeout_second = 300 -# qe_slow_log_ms = 5000 - -``` - +配置在 `conf/fe.conf`，你可以根据自己的需要进行修改 +(注意：如果使用`Mac`开发，由于`docker for Mac`不支持`Host`模式，需要使用`-p`方式暴露`be`端口，同时`fe.conf`的`priority_networks`配置为容器内可访问的Ip，例如WIFI的Ip) ## 4.设置环境变量