From afff3b369214a374aefb7d791cdf4dfd0e797ab2 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 15 May 2019 18:35:47 +0800 Subject: [PATCH] Modify dockerfile (#1164) New docker image include new thirdparties --- docker/Docker_build.md | 66 ------------------------------------------ docker/Dockerfile | 26 +++++++---------- docker/README.md | 64 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 82 deletions(-) delete mode 100644 docker/Docker_build.md create mode 100644 docker/README.md diff --git a/docker/Docker_build.md b/docker/Docker_build.md deleted file mode 100644 index 8edcf40a91..0000000000 --- a/docker/Docker_build.md +++ /dev/null @@ -1,66 +0,0 @@ -## Doris Develop Environment based on docker -### 0. Download the latest source code - -``` -git clone https://github.com/apache/incubator-doris.git -cp incubator-doris/docker/Dockerfile . - -``` - -### 1. Build docker image - -```aidl - -docker build -t doris:v1.0 . - --- doris is docker image repository name and base is tag name , you can change them to what you like - -``` - -### 2. Use docker image - -```aidl -docker run -it --name doris doris:v1.0 -// if you want to build your local code ,you should execute: -docker run -it --name test -v **/incubator-doris:/var/local/incubator-doris doris:v1.0 - -// then build source code -cd /var/local/incubator-doris - -sh build.sh -- build project - -//execute unit test -sh run-ut.sh --run -- run unit test - -``` - -### 3. Thirdparty list - -The detail list of thirdparty we mirrored as following: - -* [libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip](http://palo-opensource.gz.bcebos.com/libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip?authorization=bce-auth-v1%2F069fc2786e464e63a5f1183824ddb522%2F2018-06-22T02%3A37%3A48Z%2F-1%2Fhost%2F031b0cc42ab83ca4e0ec3608cba963e95c1ddc46fc70a14457323e2d7960e6ef) -* [openssl-1.0.2k.tar.gz](https://www.openssl.org/source/openssl-1.0.2k.tar.gz) -* [thrift-0.9.3.tar.gz](http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz) -* [llvm-3.4.2.src.tar.gz](http://releases.llvm.org/3.4.2/llvm-3.4.2.src.tar.gz) -* [cfe-3.4.2.src.tar.gz](http://releases.llvm.org/3.4.2/cfe-3.4.2.src.tar.gz) -* [compiler-rt-5.0.0.src.tar.xz](http://releases.llvm.org/5.0.0/compiler-rt-5.0.0.src.tar.xz) -* [protobuf-3.5.1.tar.gz](https://github.com/google/protobuf/archive/v3.5.1.tar.gz) -* [gflags-2.2.0.tar.gz](https://github.com/gflags/gflags/archive/v2.2.0.tar.gz) -* [glog-0.3.3.tar.gz](https://github.com/google/glog/archive/v0.3.3.tar.gz) -* [googletest-release-1.8.0.tar.gz](https://github.com/google/googletest/archive/release-1.8.0.tar.gz) -* [snappy-1.1.4.tar.gz](https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz) -* [gperftools-2.7.tar.gz](https://github.com/gperftools/gperftools/archive/gperftools-2.7.tar.gz) -* [zlib-1.2.11.tar.gz](https://sourceforge.net/projects/libpng/files/zlib/1.2.11/zlib-1.2.11.tar.gz) -* [lz4-1.7.5.tar.gz](https://github.com/lz4/lz4/archive/v1.7.5.tar.gz) -* [bzip2-1.0.6.tar.gz](https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz) -* [lzo-2.10.tar.gz](http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz) -* [rapidjson-1.1.0.tar.gz](https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz) -* [curl-7.54.0.tar.gz](https://curl.haxx.se/download/curl-7.54.0.tar.gz) -* [re2-2017-05-01.tar.gz](https://github.com/google/re2/archive/2017-05-01.tar.gz) -* [boost_1_64_0.tar.gz](https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz) -* [mysql-5.7.18.tar.gz](https://github.com/mysql/mysql-server/archive/mysql-5.7.18.tar.gz) -* [boost_1_59_0.tar.gz](http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz) -* [leveldb-1.20.tar.gz](https://github.com/google/leveldb/archive/v1.20.tar.gz) -* [brpc-0.9.0.tar.gz](https://github.com/brpc/brpc/archive/v0.9.0.tar.gz) -* [rocksdb-5.14.2.tar.gz](https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz) -* [librdkafka-0.11.6-RC5.tar.gz](https://github.com/edenhill/librdkafka/archive/v0.11.6-RC5.tar.gz) diff --git a/docker/Dockerfile b/docker/Dockerfile index 562de6e001..773b09fd28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,8 +24,8 @@ ENV DEFAULT_DIR /var/local ARG GCC_VERSION=7.3.0 ARG GCC_URL=https://mirrors.ustc.edu.cn/gnu/gcc/gcc-${GCC_VERSION} -# install dependencies and build gcc -RUN yum install -y bzip2 wget git gcc-c++ libstdc++-static byacc flex automake libtool binutils-devel bison ncurses-devel make mlocate unzip patch which vim-common redhat-lsb-core zip \ +# install dependencies and build gcc +RUN yum install -y bzip2 wget git gcc-c++ libstdc++-static byacc flex automake libtool binutils-devel bison ncurses-devel make mlocate unzip patch which vim-common redhat-lsb-core zip libcurl-devel \ && updatedb \ && yum -y clean all \ && rm -rf /var/cache/yum \ @@ -39,29 +39,23 @@ RUN yum install -y bzip2 wget git gcc-c++ libstdc++-static byacc flex automake l && make -j$[$(nproc)/4+1] && make install \ && rm -rf /var/local/gcc \ && rm -f /tmp/gcc.tar.gz - + # build cmake ARG CMAKE_VERSION=3.12.3 ARG CMAKE_DOWNLOAD_URL=https://cmake.org/files/v3.12/cmake-${CMAKE_VERSION}.tar.gz RUN mkdir -p /tmp/cmake && curl -fsSL -o /tmp/cmake.tar.gz ${CMAKE_DOWNLOAD_URL} \ && tar -zxf /tmp/cmake.tar.gz -C /tmp/cmake --strip-components=1 \ && cd /tmp/cmake \ - && ./bootstrap --prefix=/usr/local \ - && make -j$[$(nproc)/4+1] \ - && make install \ + && ./bootstrap --system-curl \ + && gmake -j$[$(nproc)/4+1] \ + && gmake install \ && rm -rf /tmp/cmake.tar.gz \ && rm -rf /tmp/cmake # install jdk +COPY ./jdk.rpm ./ RUN touch ${DEFAULT_DIR}/install_jdk.sh \ && echo '#!/bin/bash' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'JAVA_VERSION="8"' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'URL=https://www.oracle.com' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'DOWNLOAD_URL1=${URL}/technetwork/java/javase/downloads/index.html' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'DOWNLOAD_URL2=$(curl -s $DOWNLOAD_URL1 | egrep -o "\/technetwork\/java/\javase\/downloads\/jdk${JAVA_VERSION}-downloads-.*\.html" | head -1)' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'DOWNLOAD_URL3="$(echo ${URL}${DOWNLOAD_URL2}|awk -F\" {"print $1"})"' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'DOWNLOAD_URL4=$(curl -s "$DOWNLOAD_URL3" | egrep -o "https\:\/\/download.oracle\.com\/otn-pub\/java\/jdk\/[7-8]u[0-9]+\-(.*)+\/jdk-[7-8]u[0-9]+(.*)linux-x64.rpm"|tail -n1)' >> ${DEFAULT_DIR}/install_jdk.sh \ - && echo 'wget -q --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" $DOWNLOAD_URL4 -O jdk.rpm > /dev/null 2>&1' >> ${DEFAULT_DIR}/install_jdk.sh \ && echo 'rpm -Uvh jdk.rpm > /dev/null 2>&1' >> ${DEFAULT_DIR}/install_jdk.sh \ && echo 'mv /usr/java/jdk* /usr/java/jdk' >> ${DEFAULT_DIR}/install_jdk.sh \ && echo 'export JAVA_HOME=/usr/java/jdk' >> ${DEFAULT_DIR}/install_jdk.sh \ @@ -88,16 +82,16 @@ RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ && rm -f /tmp/apache-maven.tar.gz \ && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn - + ENV MAVEN_HOME /usr/share/maven - + # build environment WORKDIR ${DEFAULT_DIR} # there is a repo which is included all of thirdparty ENV REPOSITORY_URL=https://doris-incubating-repo.bj.bcebos.com/thirdparty -# clone lastest source code from local dir(please update local dir before docker build), download and build third party +# clone lastest source code, download and build thirdparty COPY incubator-doris ${DEFAULT_DIR}/incubator-doris RUN cd ${DEFAULT_DIR}/incubator-doris && /bin/bash thirdparty/build-thirdparty.sh \ && rm -rf ${DEFAULT_DIR}/incubator-doris/thirdparty/src \ diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..288ef206ef --- /dev/null +++ b/docker/README.md @@ -0,0 +1,64 @@ +## Doris Develop Environment based on docker + +### Preparation + +1. Download the Doris code repo + + ``` + cd /to/your/workspace/ + git clone https://github.com/apache/incubator-doris.git +``` + +2. Copy Dockerfile + + ``` + cd /to/your/workspace/ + cp incubator-doris/docker/Dockerfile ./ + ``` + +3. Download Oracle JDK(1.8+) RPM + + You need to download the Oracle JDK RPM, which can be found [here](https://www.oracle.com/technetwork/java/javase/downloads/index.html). And rename it to `jdk.rpm`. + +After preparation, your workspace should like this: + +``` +. +├── Dockerfile +├── incubator-doris +│   ├── be +│   ├── bin +│   ├── build.sh +│   ├── conf +│   ├── DISCLAIMER +│   ├── docker +│   ├── docs +│   ├── env.sh +│   ├── fe +│   ├── ... +├── jdk.rpm +``` + +### Build docker image + +``` +cd /to/your/workspace/ +docker build -t doris:v1.0 . +``` + +> `doris` is docker image repository name and `v1.0` is tag name, you can change them to whatever you like. + +### Use docker image + +This docker image you just built does not contain Doris source code repo. You need to download it first and map it to the container. (You can just use the one you used to build this image before) + +``` +docker run -it -v /your/local/path/incubator-doris/:/root/incubator-doris/ doris:v1.0 +``` + +Then you can build source code inside the container. + +``` +cd /root/incubator-doris/ +sh build.sh +```