sort out the fs_brokers dir (#98)

* sort out the fs_broker dir
* modify install.md

docs/admin_guide/install.md

@@ -104,7 +104,7 @@ Palo 主要包括 Frontend（FE）和 Backend（BE）两个进程。其中 FE
 
 ### 3.3 （可选）FS_Broker 部署
 
-broker 以插件的形式，独立于 Palo 部署。如果需要从第三方存储系统导入数据，需要部署相应的 broker，默认提供了读取HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的，建议每一个 FE 和 BE 节点都部署一个 broker。
+broker 以插件的形式，独立于 Palo 部署。如果需要从第三方存储系统导入数据，需要部署相应的 broker，默认提供了读取 HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的，建议每一个 FE 和 BE 节点都部署一个 broker。
 
 * 拷贝源码 fs_broker 的 output 目录下的相应 broker 目录到需要部署的所有节点上。建议和 BE 或者 FE 目录保持同级。
 
@@ -114,7 +114,7 @@ broker 以插件的形式，独立于 Palo 部署。如果需要从第三方存
 
  * 启动broker
 
-    sh bin/start_hdfs_broker.sh 启动hdfs的broker，其他类型的broker请进入相应的目录予以启动
+    sh bin/start_broker.sh 启动broker。
 
 * 添加broker
 

fs_brokers/apache_hdfs_broker/bin/start_broker.sh

@@ -21,9 +21,6 @@ curdir=`cd "$curdir"; pwd`
 export BROKER_HOME=`cd "$curdir/.."; pwd`
 export PID_DIR=`cd "$curdir"; pwd`
 
-#
-# JAVA_OPTS
-# BROKER_LOG_DIR
 export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8"
 export BROKER_LOG_DIR="$BROKER_HOME/log"
 # export JAVA_HOME="/usr/java/jdk1.8.0_131"
@@ -62,12 +59,6 @@ if [ ! -d $BROKER_LOG_DIR ]; then
     mkdir -p $BROKER_LOG_DIR
 fi
 
-if [ ! -f /bin/limit3 ]; then
-    LIMIT=
-else
-    LIMIT="/bin/limit3 -c 0 -n 65536"
-fi
-    
 nohup $LIMIT $JAVA $JAVA_OPTS com.baidu.palo.broker.hdfs.BrokerBootstrap "$@" >$BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 </dev/null &
 
 echo $! > $pidfile

fs_brokers/apache_hdfs_broker/build.sh

@@ -35,7 +35,7 @@ export BROKER_HOME=$ROOT
 
 # Every time, build deps
 DEPS_DIR=${BROKER_HOME}/deps
-cd ${DEPS_DIR} && sh build_deps.sh
+cd ${DEPS_DIR} && sh build.sh
 cd ${BROKER_HOME}
 
 # export all variable need by other module

fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/BrokerBootstrap.java

@@ -43,13 +43,14 @@ public class BrokerBootstrap {
             System.setProperty("BROKER_LOG_DIR", System.getenv("BROKER_LOG_DIR"));
             PropertyConfigurator.configure(brokerHome + "/conf/log4j.properties");
             Logger logger = Logger.getLogger(BrokerBootstrap.class);
-            logger.debug("starting apache hdfs broker....");
+            logger.info("starting apache hdfs broker....");
             new BrokerConfig().init(brokerHome + "/conf/apache_hdfs_broker.conf");
 
             TProcessor tprocessor = new TPaloBrokerService.Processor<TPaloBrokerService.Iface>(
                     new HDFSBrokerServiceImpl());
             ThriftServer server = new ThriftServer(BrokerConfig.broker_ipc_port, tprocessor);
             server.start();
+            logger.info("starting apache hdfs broker....succeed");
             while (true) {
                 Thread.sleep(2000);
             }

fs_brokers/baidu_bos_broker/bin/start_broker.sh

@@ -21,11 +21,8 @@ curdir=`cd "$curdir"; pwd`
 export BROKER_HOME=`cd "$curdir/.."; pwd`
 export PID_DIR=`cd "$curdir"; pwd`
 
-#
-# JAVA_OPTS
-# LOG_DIR
 export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8"
-export LOG_DIR="$BROKER_HOME/log"
+export BROKER_LOG_DIR="$BROKER_HOME/log"
 # export JAVA_HOME="/usr/java/jdk1.8.0_131"
 # java
 if [ "$JAVA_HOME" = "" ]; then
@@ -49,7 +46,7 @@ while read line; do
     fi
 done < $BROKER_HOME/conf/baidu_bos_broker.conf
 
-pidfile=$PID_DIR/bos_broker.pid
+pidfile=$PID_DIR/baidu_bos_broker.pid
 
 if [ -f $pidfile ]; then
     if kill -0 `cat $pidfile` > /dev/null 2>&1; then
@@ -58,16 +55,10 @@ if [ -f $pidfile ]; then
     fi
 fi
 
-if [ ! -d $LOG_DIR ]; then
-    mkdir -p $LOG_DIR
+if [ ! -d $BROKER_LOG_DIR ]; then
+    mkdir -p $BROKER_LOG_DIR
 fi
 
-if [ ! -f /bin/limit3 ]; then
-    LIMIT=
-else
-    LIMIT="/bin/limit3 -c 0 -n 65536"
-fi
-
-nohup $LIMIT $JAVA $JAVA_OPTS com.baidu.palo.broker.bos.BrokerBootstrap "$@" >$LOG_DIR/bos_broker.out 2>&1 </dev/null &
+nohup $LIMIT $JAVA $JAVA_OPTS com.baidu.palo.broker.bos.BrokerBootstrap "$@" >$BROKER_LOG_DIR/baidu_bos_broker.out 2>&1 </dev/null &
 
 echo $! > $pidfile

fs_brokers/baidu_bos_broker/bin/stop_broker.sh

@@ -29,7 +29,7 @@ while read line; do
     fi
 done < $BROKER_HOME/conf/baidu_bos_broker.conf
 
-pidfile=$PID_DIR/bos_broker.pid
+pidfile=$PID_DIR/baidu_bos_broker.pid
 
 if [ -f $pidfile ]; then
    pid=`cat $pidfile`

fs_brokers/baidu_bos_broker/build.sh

@@ -35,12 +35,12 @@ export BROKER_HOME=$ROOT
 
 # Every time, build deps
 DEPS_DIR=${BROKER_HOME}/deps
-cd ${DEPS_DIR} && sh build_deps.sh
+cd ${DEPS_DIR} && sh build.sh
 cd ${BROKER_HOME}
 
 # export all variable need by other module
 export PATH=${DEPS_DIR}/bin:$PATH
 ANT_HOME=${DEPS_DIR}/ant
 export PATH=${ANT_HOME}/bin:$PATH
-ant release
+ant output
 exit

fs_brokers/baidu_bos_broker/deps/build.sh

@@ -23,7 +23,7 @@ DEPSDIR=`dirname "$0"`
 DEPSDIR=`cd ${DEPSDIR}; pwd`
 
 CURDIR=`pwd`
-if [ ! -f ${DEPSDIR}/bin/thrift ]
+if [ ! -f ${DEPSDIR}/bin/bce-java-sdk-0.9.1-internal.jar ]
 then
     echo "***********************************"
     echo " Downloading dependency libraries "
@@ -31,9 +31,16 @@ then
     cd ${DEPSDIR}
     # Check out depends
     # extract archive
-    tar xzf baidu_bos_broker_deps.tar.gz
+    tar xzf baidu_bos_broker_java_libraries.tar.gz
     echo "Unpacking dependency libraries...Done "
 else
     echo "Depends already exists."
 fi
+
+if [ ! -f bin/thrift ];then
+	echo "thrift is not found."
+	echo "You need to copy thrift binary file from 'thirdparty/installed/bin/thrift' to $CURDIR"
+	exit 1
+fi
+
 cd ${CURDIR}