From d471ee5f379634d3a6299bde903cabd2c10b3603 Mon Sep 17 00:00:00 2001
From: lide <mailtolide@sina.com>
Date: Fri, 28 Dec 2018 17:33:45 +0800
Subject: [PATCH] Add md5sum check for third party packages (#480)

---
 thirdparty/download-thirdparty.sh | 52 +++++++++++++++++++++++++------
 thirdparty/vars.sh                | 26 ++++++++++++++++
 2 files changed, 69 insertions(+), 9 deletions(-)

diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 8a8e436aab..fa5510cb52 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -48,10 +48,35 @@ fi
 
 mkdir -p ${TP_DIR}/src
 
-download() {
+md5sum_bin=md5sum
+if ! command -v ${md5sum_bin} >/dev/null 2>&1; then
+    echo "Warn: md5sum is not installed"
+    md5sum_bin=""
+fi
+
+md5sum_func() {
+    local FILENAME=$1
+    local DESC_DIR=$2
+    local MD5SUM=$3
+
+    if [ "$md5sum_bin" == "" ]; then
+       return 0
+    else
+       md5=`md5sum "$DESC_DIR/$FILENAME"`
+       if [ "$md5" != "$MD5SUM  $DESC_DIR/$FILENAME" ]; then
+           echo "$DESC_DIR/$FILENAME md5sum check failed!"
+           return 1
+       fi
+    fi
+
+    return 0
+}
+
+download_func() {
     local FILENAME=$1
     local DOWNLOAD_URL=$2
     local DESC_DIR=$3
+    local MD5SUM=$4
 
     if [ -z "$FILENAME" ]; then
         echo "Error: No file name specified to download"
@@ -70,16 +95,24 @@ download() {
     SUCCESS=0
     for attemp in 1 2; do
         if [ -r "$DESC_DIR/$FILENAME" ]; then
-            echo "Archive $FILENAME already exist."
-            SUCCESS=1
-            break;
+            if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
+                echo "Archive $FILENAME already exist."
+                SUCCESS=1
+                break;
+            fi
+            echo "Archive $FILENAME will be removed and download again."
+            rm -f "$DESC_DIR/$FILENAME"
         else
             echo "Downloading $FILENAME from $DOWNLOAD_URL to $DESC_DIR"
             wget --no-check-certificate $DOWNLOAD_URL -O $DESC_DIR/$FILENAME
             if [ "$?"x == "0"x ]; then
-                SUCCESS=1
-                echo "Success to download $FILENAME"
-                break;
+	        if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
+                    SUCCESS=1
+                    echo "Success to download $FILENAME"
+                    break;
+                fi
+                echo "Archive $FILENAME will be removed and download again."
+                rm -f "$DESC_DIR/$FILENAME"
             else
                 echo "Failed to download $FILENAME. attemp: $attemp"
             fi
@@ -97,12 +130,13 @@ echo "===== Downloading thirdparty archives..."
 for TP_ARCH in ${TP_ARCHIVES[*]}
 do
     NAME=$TP_ARCH"_NAME"
+    MD5SUM=$TP_ARCH"_MD5SUM"
     if test "x$REPOSITORY_URL" = x; then
         URL=$TP_ARCH"_DOWNLOAD"
-        download ${!NAME} ${!URL} $TP_SOURCE_DIR
+        download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM}
     else
         URL="${REPOSITORY_URL}/${!NAME}"
-        download ${!NAME} ${URL} $TP_SOURCE_DIR
+        download_func ${!NAME} ${URL} $TP_SOURCE_DIR ${!MD5SUM}
     fi
 done
 echo "===== Downloading thirdparty archives...done"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index d548305348..e8894cd040 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -58,132 +58,158 @@ export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
 LIBEVENT_DOWNLOAD="http://palo-opensource.gz.bcebos.com/libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip?authorization=bce-auth-v1%2F069fc2786e464e63a5f1183824ddb522%2F2018-06-22T02%3A37%3A48Z%2F-1%2Fhost%2F031b0cc42ab83ca4e0ec3608cba963e95c1ddc46fc70a14457323e2d7960e6ef"
 LIBEVENT_NAME=libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip
 LIBEVENT_SOURCE=libevent-master
+LIBEVENT_MD5SUM="e8b9ba50270ba3b520aec8ff1089f9d7"
 
 # openssl
 OPENSSL_DOWNLOAD="https://www.openssl.org/source/openssl-1.0.2k.tar.gz"
 OPENSSL_NAME=openssl-1.0.2k.tar.gz
 OPENSSL_SOURCE=openssl-1.0.2k
+OPENSSL_MD5SUM="f965fc0bf01bf882b31314b61391ae65"
 
 # thrift
 THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz"
 THRIFT_NAME=thrift-0.9.3.tar.gz
 THRIFT_SOURCE=thrift-0.9.3
+THRIFT_MD5SUM="88d667a8ae870d5adeca8cb7d6795442"
 
 # llvm
 LLVM_DOWNLOAD="http://releases.llvm.org/3.4.2/llvm-3.4.2.src.tar.gz"
 LLVM_NAME=llvm-3.4.2.src.tar.gz
 LLVM_SOURCE=llvm-3.4.2.src
+LLVM_MD5SUM="a20669f75967440de949ac3b1bad439c"
 
 # clang
 CLANG_DOWNLOAD="http://releases.llvm.org/3.4.2/cfe-3.4.2.src.tar.gz"
 CLANG_NAME=cfe-3.4.2.src.tar.gz
 CLANG_SOURCE=cfe-3.4.2.src
+CLANG_MD5SUM="87945973b7c73038871c5f849a818588"
 
 # compiler-rt
 COMPILER_RT_DOWNLOAD="http://releases.llvm.org/5.0.0/compiler-rt-5.0.0.src.tar.xz"
 COMPILER_RT_NAME=compiler-rt-5.0.0.src.tar.xz
 COMPILER_RT_SOURCE=compiler-rt-5.0.0.src
+COMPILER_RT_MD5SUM="da735894133589cbc6052c8ef06b1230"
 
 # protobuf
 PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.5.1.tar.gz"
 PROTOBUF_NAME=protobuf-3.5.1.tar.gz
 PROTOBUF_SOURCE=protobuf-3.5.1
+PROTOBUF_MD5SUM="710f1a75983092c9b45ecef207236104"
 
 # gflags
 GFLAGS_DOWNLOAD="https://github.com/gflags/gflags/archive/v2.2.0.tar.gz"
 GFLAGS_NAME=gflags-2.2.0.tar.gz
 GFLAGS_SOURCE=gflags-2.2.0
+GFLAGS_MD5SUM="b99048d9ab82d8c56e876fb1456c285e"
 
 # glog
 GLOG_DOWNLOAD="https://github.com/google/glog/archive/v0.3.3.tar.gz"
 GLOG_NAME=glog-0.3.3.tar.gz
 GLOG_SOURCE=glog-0.3.3
+GLOG_MD5SUM="c1f86af27bd9c73186730aa957607ed0"
 
 # gtest
 GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.8.0.tar.gz"
 GTEST_NAME=googletest-release-1.8.0.tar.gz
 GTEST_SOURCE=googletest-release-1.8.0
+GTEST_MD5SUM="16877098823401d1bf2ed7891d7dce36"
 
 # snappy
 SNAPPY_DOWNLOAD="https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz"
 SNAPPY_NAME=snappy-1.1.4.tar.gz
 SNAPPY_SOURCE=snappy-1.1.4
+SNAPPY_MD5SUM="c328993b68afe3e5bd87c8ea9bdeb028"
 
 # gperftools
 GPERFTOOLS_DOWNLOAD="https://github.com/gperftools/gperftools/archive/gperftools-2.7.tar.gz"
 GPERFTOOLS_NAME=gperftools-2.7.tar.gz
 GPERFTOOLS_SOURCE=gperftools-gperftools-2.7
+GPERFTOOLS_MD5SUM="797e7b7f6663288e2b90ab664861c61a"
 
 # zlib
 ZLIB_DOWNLOAD="https://sourceforge.net/projects/libpng/files/zlib/1.2.11/zlib-1.2.11.tar.gz"
 ZLIB_NAME=zlib-1.2.11.tar.gz
 ZLIB_SOURCE=zlib-1.2.11 
+ZLIB_MD5SUM="1c9f62f0778697a09d36121ead88e08e"
 
 # lz4
 LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.7.5.tar.gz"
 LZ4_NAME=lz4-1.7.5.tar.gz
 LZ4_SOURCE=lz4-1.7.5
+LZ4_MD5SUM="c9610c5ce97eb431dddddf0073d919b9"
 
 # bzip
 # BZIP_DOWNLOAD="http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz"
 BZIP_DOWNLOAD="https://fossies.org/linux/misc/bzip2-1.0.6.tar.gz"
 BZIP_NAME=bzip2-1.0.6.tar.gz
 BZIP_SOURCE=bzip2-1.0.6
+BZIP_MD5SUM="00b516f4704d4a7cb50a1d97e6e8e15b"
 
 # lzo2
 LZO2_DOWNLOAD="https://github.com/damageboy/lzo2/archive/master.zip"
 LZO2_NAME=lzo2-master.zip
 LZO2_SOURCE=lzo2-master
+LZO2_MD5SUM="0afc2456c8f425778d8a4e976f97d717"
 
 # rapidjson
 RAPIDJSON_DOWNLOAD="https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
 RAPIDJSON_NAME=rapidjson-1.1.0.tar.gz
 RAPIDJSON_SOURCE=rapidjson-1.1.0
+RAPIDJSON_MD5SUM="badd12c511e081fec6c89c43a7027bce"
 
 # curl
 CURL_DOWNLOAD="https://curl.haxx.se/download/curl-7.54.0.tar.gz"
 CURL_NAME=curl-7.54.0.tar.gz
 CURL_SOURCE=curl-7.54.0
+CURL_MD5SUM="18091896d871982cc4c2b307885eacb3"
 
 # RE2
 RE2_DOWNLOAD="https://github.com/google/re2/archive/2017-05-01.tar.gz"
 RE2_NAME=re2-2017-05-01.tar.gz
 RE2_SOURCE=re2-2017-05-01
+RE2_MD5SUM="4aa65a0b22edacb7ddcd7e4aec038dcf"
 
 # boost
 BOOST_DOWNLOAD="https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz"
 BOOST_NAME=boost_1_64_0.tar.gz
 BOOST_SOURCE=boost_1_64_0
+BOOST_MD5SUM="319c6ffbbeccc366f14bb68767a6db79"
 
 # mysql
 MYSQL_DOWNLOAD="https://github.com/mysql/mysql-server/archive/mysql-5.7.18.tar.gz"
 MYSQL_NAME=mysql-5.7.18.tar.gz
 MYSQL_SOURCE=mysql-server-mysql-5.7.18
+MYSQL_MD5SUM="58598b10dce180e4d1fbdd7cf5fa68d6"
 
 # boost for mysql
 BOOST_FOR_MYSQL_DOWNLOAD="http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz"
 BOOST_FOR_MYSQL_NAME=boost_1_59_0.tar.gz
 BOOST_FOR_MYSQL_SOURCE=boost_1_59_0
+BOOST_FOR_MYSQL_MD5SUM="51528a0e3b33d9e10aaa311d9eb451e3"
 
 # leveldb
 LEVELDB_DOWNLOAD="https://github.com/google/leveldb/archive/v1.20.tar.gz"
 LEVELDB_NAME=leveldb-1.20.tar.gz
 LEVELDB_SOURCE=leveldb-1.20
+LEVELDB_MD5SUM="298b5bddf12c675d6345784261302252"
 
 # brpc
 BRPC_DOWNLOAD="https://github.com/brpc/brpc/archive/v0.9.0.tar.gz"
 BRPC_NAME=brpc-0.9.0.tar.gz
 BRPC_SOURCE=brpc-0.9.0
+BRPC_MD5SUM="2e79f413614b99d44083499a81c0db67"
 
 # rocksdb
 ROCKSDB_DOWNLOAD="https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz"
 ROCKSDB_NAME=rocksdb-5.14.2.tar.gz
 ROCKSDB_SOURCE=rocksdb-5.14.2
+ROCKSDB_MD5SUM="b72720ea3b1e9ca9e4ed0febfef65b14"
 
 # librdkafka
 LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/v0.11.6-RC5.tar.gz"
 LIBRDKAFKA_NAME=librdkafka-0.11.6-RC5.tar.gz
 LIBRDKAFKA_SOURCE=librdkafka-0.11.6-RC5
+LIBRDKAFKA_MD5SUM="2e4ecef2df277e55a0144eb6d185e18a"
 
 # all thirdparties which need to be downloaded is set in array TP_ARCHIVES
 export TP_ARCHIVES="LIBEVENT OPENSSL THRIFT LLVM CLANG COMPILER_RT PROTOBUF GFLAGS GLOG GTEST RAPIDJSON SNAPPY GPERFTOOLS ZLIB LZ4 BZIP LZO2 CURL RE2 BOOST MYSQL BOOST_FOR_MYSQL LEVELDB BRPC ROCKSDB LIBRDKAFKA"