hcq/openGauss-OM
hcq/openGauss-OM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

om分仓适配

Browse Source
This commit is contained in:
gyt0221 2020-12-18 10:00:11 +08:00
parent a2679f0cf7
commit 043fed1c43
16 changed files with 434 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
build.sh Normal file
View File

@ -0,0 +1,255 @@
#!/bin/bash
declare binarylib_dir='None'
declare module_name="openGauss"
declare version_number='1.0.1'
ROOT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
echo "ROOT_DIR : $ROOT_DIR"
declare ERR_MKGS_FAILED=1
declare LOG_FILE="${ROOT_DIR}/build.log"
declare PKG_DIR="${ROOT_DIR}/package"
declare PKG_TMP_DIR="${ROOT_DIR}/package/temp"
declare version_string="${module_name}-${version_number}"
#########################################################################
##read command line paramenters
#######################################################################
function print_help()
{
echo "Usage: $0 [OPTION]
-h|--help show help information
-3rd|--binarylib_dir the parent directory of binarylibs
"
}
while [ $# -gt 0 ]; do
case "$1" in
-h|--help)
print_help
exit 1
;;
-3rd|--binarylib_dir)
if [ "$2"X = X ]; then
echo "no given binarylib directory values"
exit 1
fi
binarylib_dir=$2
shift 2
;;
*)
echo "Internal Error: option processing error: $1" 1>&2
echo "please input right paramtenter, the following command may help you"
echo "./build.sh --help or ./build.sh -h"
exit 1
esac
done
PLAT_FORM_STR=$(sh "${ROOT_DIR}/build/get_PlatForm_str.sh")
if [ "${PLAT_FORM_STR}"x == "Failed"x ]; then
echo "We only support openEuler(aarch64), EulerOS(aarch64), CentOS platform."
exit 1;
fi
PLATFORM=32
bit=$(getconf LONG_BIT)
if [ "$bit" -eq 64 ]; then
PLATFORM=64
fi
if [ X$(echo $PLAT_FORM_STR | grep "centos") != X"" ]; then
dist_version="CentOS"
elif [ X$(echo $PLAT_FORM_STR | grep "openeuler") != X"" ]; then
dist_version="openEuler"
elif [ X$(echo $PLAT_FORM_STR | grep "euleros") != X"" ]; then
dist_version="EulerOS"
else
echo "We only support openEuler(aarch64), EulerOS(aarch64), CentOS platform."
echo "Kernel is $kernel"
exit 1
fi
declare package_pre_name="${version_string}-${dist_version}-${PLATFORM}bit-om"
declare package_name="${package_pre_name}.tar.gz"
declare sha256_name="${package_pre_name}.sha256"
if [ ${binarylib_dir} != 'None' ] && [ -d ${binarylib_dir} ]; then
BINARYLIBS_PATH="${binarylib_dir}/dependency/${PLAT_FORM_STR}"
BUILD_TOOLS_PATH="${binarylib_dir}/buildtools/${PLAT_FORM_STR}"
else
BINARYLIBS_PATH="${ROOT_DIR}/binarylibs/dependency/${PLAT_FORM_STR}"
BUILD_TOOLS_PATH="${ROOT_DIR}/binarylibs/buildtools/${PLAT_FORM_STR}"
fi
log()
{
echo "[makegaussdb] $(date +%y-%m-%d' '%T): $@"
echo "[makegaussdb] $(date +%y-%m-%d' '%T): $@" >> "$LOG_FILE" 2>&1
}
die()
{
log "$@"
echo "$@"
exit $ERR_MKGS_FAILED
}
function env_check()
{
if [ -d "$PKG_DIR" ]; then
rm -rf ${PKG_DIR}
fi
mkdir -p ${PKG_TMP_DIR}
if [ -d "$LOG_FILE" ]; then
rm -rf $LOG_FILE
fi
if [ $? -eq 0 ]; then
echo "Everything is ready."
else
echo "clean enviroment failed."
exit 1
fi
}
function copy_script_file()
{
cp -rf $ROOT_DIR/script $PKG_TMP_DIR/ &&
cp -rf $ROOT_DIR/other/transfer.py $PKG_TMP_DIR/script/ &&
find $PKG_TMP_DIR/script/ -type f -print0 | xargs -0 -n 10 -r dos2unix > /dev/null 2>&1 &&
find $PKG_TMP_DIR/script/gspylib/inspection/ -name d2utmp* -print0 | xargs -0 rm -rf &&
cp -rf $ROOT_DIR/script/gspylib/inspection/lib/checknetspeed/speed_test* $PKG_TMP_DIR/script/gspylib/inspection/lib/checknetspeed/ &&
cp -rf $ROOT_DIR/script/gspylib/inspection/lib/*.png $PKG_TMP_DIR/script/gspylib/inspection/lib/ &&
if [ $? -ne 0 ]; then
die "cp -r $ROOT_DIR/script $PKG_TMP_DIR failed "
fi
chmod -R +x $PKG_TMP_DIR/script/
cp -rf $ROOT_DIR/simpleInstall $PKG_TMP_DIR/
if [ $? -ne 0 ]; then
die "cp -r $ROOT_DIR/simpleInstall $PKG_TMP_DIR/ failed "
fi
}
function version_cfg()
{
gitversion=$(git log | grep commit | head -1 | awk '{print $2}' | cut -b 1-8)
commits=$(git log | grep "See in merge request" | wc -l)
mrid=$(git log | grep "See in merge request" | head -1 | awk -F! '{print $2}' | grep -o '[0-9]\+')
om_version="(openGauss OM 1.0.1 build $gitversion) compiled at `date -d today +\"%Y-%m-%d %H:%M:%S\"` commit $commits last mr $mrid"
version_file=${PKG_TMP_DIR}/version.cfg
touch ${version_file}
echo "${module_name}-${version_number}">${version_file}
echo "92.072" >>${version_file}
echo "${gitversion}" >>${version_file}
if [ -f ${PKG_TMP_DIR}/script/gspylib/common/VersionInfo.py ] ; then
sed -i -e "s/COMMON_VERSION = \"Gauss200 OM VERSION\"/COMMON_VERSION = \"$(echo ${om_version})\"/g" -e "s/__GAUSS_PRODUCT_STRING__/$module_name/g" ${PKG_TMP_DIR}/script/gspylib/common/VersionInfo.py
if [ $? -ne 0 ]; then
die "Failed to replace OM tools version number."
fi
else
sed -i "s/COMMON_VERSION = \"Gauss200 OM VERSION\"/COMMON_VERSION = \"$(echo ${om_version})\"/g" ${PKG_TMP_DIR}/script/gspylib/os/gsOSlib.py
if [ $? -ne 0 ]; then
die "Failed to replace OM tools version number."
fi
fi
}
function clib_copy()
{
rm -rf $PKG_TMP_DIR/script/gspylib/clib
mkdir -p $PKG_TMP_DIR/script/gspylib/clib
cp $BUILD_TOOLS_PATH/gcc8.2/gcc/lib64/libstdc++.so.6 $PKG_TMP_DIR/script/gspylib/clib
cp $BINARYLIBS_PATH/openssl/comm/lib/libssl.so.1.1 $PKG_TMP_DIR/script/gspylib/clib
cp $BINARYLIBS_PATH/openssl/comm/lib/libcrypto.so.1.1 $PKG_TMP_DIR/script/gspylib/clib
#cp $BUILD_DIR/bin/encrypt $BUILD_DIR/script/gspylib/clib
}
function lib_copy()
{
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/output/log/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/output/nodes/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/asn1crypto/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/bcrypt/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/cffi/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/cryptography/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/idna/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/nacl/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/pyasn1/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/pycparser/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/OpenSSL/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/psutil/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/netifaces/ &&
mkdir -p ${PKG_TMP_DIR}/script/gspylib/inspection/lib/paramiko/ &&
cp -rf ${BINARYLIBS_PATH}/install_tools/asn1crypto/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/bcrypt/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/cffi/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/cryptography/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/idna/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/nacl/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/pyasn1/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/pycparser/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/OpenSSL/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/ipaddress.py ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/six.py ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/_cffi_backend.py ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/_cffi_backend.so* ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/psutil/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/netifaces/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
cp -rf ${BINARYLIBS_PATH}/install_tools/paramiko/ ${PKG_TMP_DIR}/script/gspylib/inspection/lib/
mkdir -p ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/asn1crypto ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/bcrypt ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/cffi ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/cryptography ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/idna ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/nacl ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/pyasn1 ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/pycparser ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/OpenSSL ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/ipaddress.py ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/six.py ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/_cffi_backend.py ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/_cffi_backend.so* ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/paramiko ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/psutil ${PKG_TMP_DIR}/lib
cp -rf ${BINARYLIBS_PATH}/install_tools/netifaces ${PKG_TMP_DIR}/lib
}
function main()
{
# 1. clean install path and log file
env_check
# 2. copy script file
copy_script_file
# 3. copy clib file
clib_copy
# 4. copy lib file
lib_copy
# 5. make version file
version_cfg
cd $PKG_TMP_DIR
tar -zvcf "${package_name}" ./* >>"$LOG_FILE" 2>&1
if [ $? -ne 0 ]; then
die "$package_command ${package_name} failed"
fi
sha256sum "${package_name}" | awk -F" " '{print $1}' > "$sha256_name"
if [ $? -ne 0 ]; then
die "generate sha256 file failed."
fi
mv $package_name $sha256_name ../
cd $PKG_DIR
rm -rf $PKG_TMP_DIR
echo "success!"
}
main
exit 0

37
build/get_PlatForm_str.sh Normal file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# *************************************************************************
# Copyright: (c) Huawei Technologies Co., Ltd. 2020. All rights reserved
#
# description: the script is to get platform string value
# date: 2020-06-01
# version: 1.0
# history:
#
# *************************************************************************
function get_os_str() {
if [ -f "/etc/os-release" ]; then
os_name=$(source /etc/os-release; echo $ID)
else
echo "Can not get /etc/os-release file, please check it!"
exit 1
fi
cpu_arc=$(uname -p)
if [ "$os_name"x = "centos"x ] && [ "$cpu_arc"x = "x86_64"x ]; then
os_str=centos7.6_x86_64
elif [ "$os_name"x = "euleros"x ] && [ "$cpu_arc"x = "aarch64"x ]; then
os_str=euleros2.0_sp8_aarch64
elif [ "$os_name"x = "openEuler"x ] && [ "$cpu_arc"x = "aarch64"x ]; then
os_str=openeuler_aarch64
elif [ "$os_name"x = "openEuler"x ] && [ "$cpu_arc"x = "x86_64"x ]; then
os_str=openeuler_x86_64
else
os_str="Failed"
fi
echo $os_str
}
get_os_str

20
script/gs_backup
View File

@ -51,6 +51,7 @@ class Backup(ParallelBaseOM):
self.backupDir = ""
self.isParameter = False
self.isBinary = False
self.isForce = False
####################################################################################
# Help context. U:R:oC:v:
@ -66,6 +67,7 @@ Usage:
[--binary] [--all] [-l LOGFILE]
gs_backup -t restore --backup-dir=BACKUPDIR [-h HOSTNAME] [--parameter]
[--binary] [--all] [-l LOGFILE]
[--force]
General options:
-t Operation type. It can be backup or restore.
@ -78,6 +80,8 @@ General options:
(This option is used by default.)
--binary Back up or restore binary files only.
--all Back up or restore both parameter files and binary files.
--force Force to restore binary files even if the
cluster_static_config is lost
-l Path of log file.
-?, --help Show help information for this utility,
and exit the command line mode.
@ -90,13 +94,17 @@ General options:
def checkAction(self):
"""
function: check action
if action if null, throw error
if action not in (ACTION_BACKUP, ACTION_RESTORE), throw error
input : NA
output: NA
"""
if (self.action == ""):
GaussLog.exitWithError(
ErrorCode.GAUSS_500["GAUSS_50001"] % 't' + ".")
if self.action not in (ACTION_BACKUP, ACTION_RESTORE):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % 't'
+ " The value of the '-t' parameter :"
" backup or restore.")
def checkUserParameter(self):
"""
@ -158,12 +166,14 @@ General options:
sys.exit(0)
# parse --all parameter
backupAll = False
forceRestore = False
parameter_map = {"action": self.action,
"backupDir": self.backupDir,
"isBinary": self.isBinary,
"isParameter": self.isParameter,
"logFile": self.logFile,
"all": backupAll}
"all": backupAll,
"force": forceRestore}
parameter_keys = parameter_map.keys()
for key in parameter_keys:
@ -175,6 +185,7 @@ General options:
self.isBinary = parameter_map["isBinary"]
self.isParameter = parameter_map["isParameter"]
self.logFile = parameter_map["logFile"]
self.isForce = parameter_map["force"]
if (parameter_map["all"]):
self.isBinary = True
@ -206,10 +217,7 @@ General options:
"and '--all' were not specified." +
" Only parameter files will be backed up.")
self.isParameter = True
if self.action not in (ACTION_BACKUP, ACTION_RESTORE):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % 't'
+ " The value of the '-t' parameter :"
" backup or restore.")
if __name__ == '__main__':

13
script/gs_preinstall
View File

@ -332,6 +332,17 @@ General options:
except Exception as e:
GaussLog.exitWithError(str(e))
# decompress version.cfg from bz2
def decompressVersioncfg(self):
package_path = os.path.dirname(os.path.realpath(__file__))
toolpath = package_path + "/../"
cmd = "cd %s && tar -xpf openGauss-*.tar.bz2 ./version.cfg " % toolpath
(status, output) = subprocess.getstatusoutput(cmd)
if status != 0:
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50217"]
% "version.cfg" + "The cmd is %s. " % cmd +
"The output is %s." % output)
# init global variables
def initGlobals(self):
"""
@ -465,6 +476,8 @@ if __name__ == '__main__':
preinstall = Preinstall()
# set LD_LIBRARY_PATH
preinstall.setLibPath()
# decompress version.cfg
preinstall.decompressVersioncfg()
# parse cmd lines
preinstall.parseCommandLine()
# check parameters

2
script/gspylib/common/Common.py
View File

@ -1060,7 +1060,7 @@ class DefaultValue():
g_file.removeFile(inspectToolsCffiPath)
# copy the correct version
newPythonDependCryptoPath = "%s_UCS%d_%s" % (omToolsCffiPath,
flagNum,version)
flagNum, version)
if os.path.exists(newPythonDependCryptoPath):
g_file.cpFile(newPythonDependCryptoPath, omToolsCffiPath,
"shell")

41
script/gspylib/common/DbClusterInfo.py
View File

@ -1410,9 +1410,9 @@ class dbClusterInfo():
"""
try:
with open(staticConfigFile, "rb") as fp:
info = fp.read(32)
info = fp.read(28)
(crc, lenth, version, currenttime, nodeNum,
localNodeId) = struct.unpack("=qIIqiI", info)
localNodeId) = struct.unpack("=IIIqiI", info)
except Exception as e:
raise Exception(
ErrorCode.GAUSS_512["GAUSS_51236"] + " Error: \n%s." % str(e))
@ -1935,7 +1935,7 @@ class dbClusterInfo():
"failed to connect") >= 0:
continue
else:
output='\n'.join(output.split('\n')[1:])
output = '\n'.join(output.split('\n')[1:])
else:
cmd = "gsql -m -d postgres -p %s -c \"%s\"" % (
dnInst.port, secondSql)
@ -2151,6 +2151,7 @@ class dbClusterInfo():
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"]
% cmd + " Error: \n%s" % output)
return output.split("\n")[0]
def __getStatusByOM(self, user):
"""
function :Get the environment parameter.
@ -2220,9 +2221,9 @@ class dbClusterInfo():
try:
# read static_config_file
fp = open(staticConfigFile, "rb")
info = fp.read(32)
info = fp.read(28)
(crc, lenth, version, currenttime, nodeNum,
localNodeId) = struct.unpack("=qIIqiI", info)
localNodeId) = struct.unpack("=IIIqiI", info)
self.version = version
self.installTime = currenttime
self.localNodeId = localNodeId
@ -2283,8 +2284,8 @@ class dbClusterInfo():
input : file
output : Object
"""
info = fp.read(76)
(crc, nodeId, nodeName) = struct.unpack("=qI64s", info)
info = fp.read(72)
(crc, nodeId, nodeName) = struct.unpack("=II64s", info)
nodeName = nodeName.decode().strip('\x00')
dbNode = dbNodeInfo(nodeId, nodeName)
info = fp.read(68)
@ -2571,9 +2572,9 @@ class dbClusterInfo():
try:
# read cluster info from static config file
fp = open(staticConfigFile, "rb")
info = fp.read(32)
info = fp.read(28)
(crc, lenth, version, currenttime, nodeNum,
localNodeId) = struct.unpack("=qIIqiI", info)
localNodeId) = struct.unpack("=IIIqiI", info)
if (version <= 100):
raise Exception(ErrorCode.GAUSS_516["GAUSS_51637"]
% ("cluster static config version[%s]"
@ -4589,7 +4590,7 @@ class dbClusterInfo():
info += struct.pack("I", localNodeId)
crc = binascii.crc32(info)
info = struct.pack("q", crc) + info
info = struct.pack("I", crc) + info
fp.write(info)
for dbNode in dbNodes:
@ -4648,7 +4649,7 @@ class dbClusterInfo():
info += struct.pack("I", 0)
crc = binascii.crc32(info)
return struct.pack("q", crc) + info
return struct.pack("I", crc) + info
def __packNodeInfoForLC(self, dbNode):
"""
@ -4671,7 +4672,7 @@ class dbClusterInfo():
info += struct.pack("I", 0)
crc = binascii.crc32(info)
return struct.pack("q", crc) + info
return struct.pack("I", crc) + info
def __packEtcdInfo(self, dbNode):
"""
@ -6091,7 +6092,7 @@ class dbClusterInfo():
# node count
info += struct.pack("I", len(self.dbNodes))
crc = binascii.crc32(info)
info = struct.pack("q", crc) + info
info = struct.pack("I", crc) + info
fp.write(info)
primaryDnNum = 0
for dbNode in self.dbNodes:
@ -6194,7 +6195,7 @@ class dbClusterInfo():
info += struct.pack("I", 0)
info += struct.pack("I", 0)
crc = binascii.crc32(info)
return (primaryNum, struct.pack("q", crc) + info)
return (primaryNum, struct.pack("I", crc) + info)
def __getClusterSwitchTime(self, dynamicConfigFile):
"""
@ -6206,9 +6207,9 @@ class dbClusterInfo():
fp = None
try:
fp = open(dynamicConfigFile, "rb")
info = fp.read(28)
info = fp.read(24)
(crc, lenth, version, switchTime, nodeNum) = \
struct.unpack("=qIIqi", info)
struct.unpack("=IIIqi", info)
fp.close()
except Exception as e:
if fp:
@ -6344,9 +6345,9 @@ class dbClusterInfo():
dynamicConfigFile = self.__getDynamicConfig(user)
# read dynamic_config_file
fp = open(dynamicConfigFile, "rb")
info = fp.read(28)
info = fp.read(24)
(crc, lenth, version, currenttime, nodeNum) = \
struct.unpack("=qIIqi", info)
struct.unpack("=IIIqi", info)
totalMaterDnNum = 0
for i in range(nodeNum):
offset = (fp.tell() // PAGE_SIZE + 1) * PAGE_SIZE
@ -6365,8 +6366,8 @@ class dbClusterInfo():
dynamicConfigFile + " Error:\n" + str(e))
def __unpackDynamicNodeInfo(self, fp):
info = fp.read(76)
(crc, nodeId, nodeName) = struct.unpack("=qI64s", info)
info = fp.read(72)
(crc, nodeId, nodeName) = struct.unpack("=II64s", info)
nodeName = nodeName.decode().strip('\x00')
dbNode = dbNodeInfo(nodeId, nodeName)
info = fp.read(4)

2
script/gspylib/common/ParameterParsecheck.py
View File

@ -81,7 +81,7 @@ gs_check = ["-?", "--help", "-V", "--version", "-e:", "-i:",
gs_sshexkey = ["-?", "--help", "-V", "--version",
"-f:", "--skip-hostname-set", "-l:"]
gs_backup = ["-?", "--help", "-V", "--version", "--backup-dir=",
"--parameter",
"--parameter", "--force",
"--binary", "--all", "-l:", "-h:", "-t:", "-X:"]
gs_collector = ["-?", "--help", "-V", "--version", "--begin-time=",
"--end-time=",

6
script/gspylib/component/Kernel/DN_OLAP/DN_OLAP.py
View File

@ -219,9 +219,6 @@ class DN_OLAP(Kernel):
elif len(azNames) == 3 and totalnum in (5, 6, 7):
tmpDNDict["synchronous_standby_names"] = \
"'ANY 3(%s,%s,%s)'" % (azNames[0], azNames[1], azNames[2])
if len(peerInsts) > 4:
if "synchronous_standby_names" in tmpDNDict:
del tmpDNDict['synchronous_standby_names']
if (self.clusterType == DefaultValue.CLUSTER_TYPE_SINGLE):
tmpDNDict["replication_type"] = "2"
@ -310,9 +307,6 @@ class DN_OLAP(Kernel):
tmpDict1 = {}
tmpDict1[connInfo] = "'%s'" % connInfo1[i]
self.setGucConfig(tmpDict1)
if "availablezone" in tmpDict1[connInfo]:
tempazname = tmpDict1[connInfo].split("=")[-1].strip("'")
#if "availablezone" in str(connInfo1):
self.setGucConfig({"available_zone": "'%s'" %
self.instInfo.azName})
else:

1
script/gspylib/component/Kernel/Kernel.py
View File

@ -132,6 +132,7 @@ class Kernel(BaseComponent):
if (status != 0):
raise Exception(ErrorCode.GAUSS_514["GAUSS_51400"] % cmd +
" Error: \n%s " % output)
def build_cascade(self, buidMode="full", standByBuildTimeout=300):
"""
"""

5
script/gspylib/os/gsplatform.py
View File

@ -100,7 +100,8 @@ EULEROS = "euleros"
KYLIN = "kylin"
OPENEULER = "openeuler"
ASIANUX = "asianux"
SUPPORT_WHOLE_PLATFORM_LIST = [SUSE, REDHAT, CENTOS, EULEROS, OPENEULER, KYLIN, ASIANUX]
SUPPORT_WHOLE_PLATFORM_LIST = [SUSE, REDHAT, CENTOS, EULEROS,
OPENEULER, KYLIN, ASIANUX]
# RedhatX platform
SUPPORT_RHEL_SERIES_PLATFORM_LIST = [REDHAT, CENTOS, "kylin", "asianux"]
SUPPORT_RHEL6X_VERSION_LIST = ["6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "10"]
@ -129,7 +130,6 @@ PAK_OPENEULER = "openEuler"
PAK_REDHAT = "RedHat"
PAK_ASIANUX = "asianux"
#######################################################
_supported_dists = (
'SuSE', 'debian', 'fedora', 'redhat', 'centos', 'euleros', "openEuler",
@ -1998,4 +1998,3 @@ class UserPlatform():
# global platform class
g_Platform = UserPlatform().userPlatform

26
script/impl/backup/OLAP/BackupImplOLAP.py
View File

@ -48,26 +48,31 @@ class BackupImplOLAP(BackupImpl):
output: NA
"""
self.context.logger.log("Parsing configuration files.")
if self.context.isForce and self.context.nodename != "" \
and self.context.action == BackupImpl.ACTION_RESTORE:
self.context.initSshTool([self.context.nodename],
DefaultValue.TIMEOUT_PSSH_BACKUP)
self.context.logger.log(
"Successfully init restore nodename: %s."
% self.context.nodename)
return
try:
self.context.initClusterInfoFromStaticFile(self.context.user)
nodeNames = self.context.clusterInfo.getClusterNodeNames()
if (self.context.nodename == ""):
if self.context.nodename == "":
self.context.nodename = nodeNames
else:
remoteNode = self.context.clusterInfo.getDbNodeByName(
self.context.nodename)
if (remoteNode is None):
if remoteNode is None:
raise Exception(ErrorCode.GAUSS_512["GAUSS_51209"] % (
"the node", self.context.nodename))
self.context.nodename = [self.context.nodename]
if (self.context.action == BackupImpl.ACTION_RESTORE):
self.context.initSshTool(self.context.nodename,
DefaultValue.TIMEOUT_PSSH_BACKUP)
else:
self.context.initSshTool(nodeNames,
DefaultValue.TIMEOUT_PSSH_BACKUP)
self.context.initSshTool(self.context.nodename,
DefaultValue.TIMEOUT_PSSH_BACKUP)
except Exception as e:
raise Exception(str(e))
@ -95,6 +100,7 @@ class BackupImplOLAP(BackupImpl):
cmd += " -p"
if self.context.isBinary:
cmd += " -b"
self.context.logger.debug("Remote backup command is %s." % cmd)
try:
@ -208,7 +214,7 @@ class BackupImplOLAP(BackupImpl):
"""
self.context.logger.log("Performing remote restoration.")
cmd = "%s -U %s -l %s --ingore_miss" % (
cmd = "%s -U %s -l %s " % (
OMCommand.getLocalScript("Local_Restore"),
self.context.user,
self.context.localLog)
@ -218,6 +224,8 @@ class BackupImplOLAP(BackupImpl):
cmd += " -p"
if self.context.isBinary:
cmd += " -b"
if self.context.isForce:
cmd += " -f"
self.context.logger.debug("Remote restoration command: %s." % cmd)
try:

21
script/local/Install.py
View File

@ -368,6 +368,27 @@ class Install(LocalBaseOM):
self.logger.logExit(ErrorCode.GAUSS_502["GAUSS_50217"]
% tarFile + " Error: \n%s" % str(output))
# mv $GPHOME/script/transfer.py to $GAUSSHOME/bin/
dirName = os.path.dirname(os.path.realpath(__file__))
transferFile = dirName + "/../../script/transfer.py"
if os.path.exists(transferFile):
g_file.cpFile(transferFile, self.installPath + "/bin/")
g_file.removeFile(transferFile)
# cp $GPHOME/script to $GAUSSHOME/bin/
g_file.cpFile(dirName + "/../../script",
self.installPath + "/bin/")
# cp $GAUSSHOME/bin/script/gspylib/etc/sql/pmk to /share/postgresql
destPath = self.installPath + "/share/postgresql/"
pmkPath = self.installPath + "/bin/script/gspylib/etc/sql/"
pmkFile = pmkPath + "pmk_schema.sql"
if os.path.exists(pmkFile):
g_file.cpFile(pmkFile, destPath)
pmkSingeInstFile = pmkPath + "pmk_schema_single_inst.sql"
if os.path.exists(pmkSingeInstFile):
g_file.cpFile(pmkSingeInstFile, destPath)
# change owner for tar file.
g_file.changeOwner(self.user, self.installPath, True)
self.logger.log("Successfully decompressed bin file.")

1
script/local/LocalCheck.py
View File

@ -22,7 +22,6 @@ import os
import sys
import getopt
import subprocess
import platform
import glob
import xml.etree.cElementTree as ETree

7
script/local/LocalCollect.py
View File

@ -1301,12 +1301,13 @@ def core_copy():
g_jobInfo.successTask.append("find core files")
isEmpty = 1
for core in cores:
if len(str(core.split("/")[-1]).split("-")) < 2:
tempName = str(core.split("/")[-1])
if not tempName.startswith("core-"):
g_logger.debug(
"WARNING: core file %s is not match core-e-p-t." % (
str(core.split("/")[-1])))
continue
p = str(core.split("/")[-1]).split("-")[1]
p = tempName.split("-")[1]
if "".join(p).lower() in ",".join(g_opts.content).lower():
p_stack = "%s_stack" % p
cmdList = []
@ -1619,7 +1620,7 @@ def parseConfig():
"""
if g_opts.config != "":
d = json.loads(g_opts.config)
g_opts.content = d['Content'].split(",")
g_opts.content = list(filter(None, d['Content'].split(",")))
def main():

35
script/local/Restore.py
View File

@ -40,6 +40,7 @@ HOSTNAME = DefaultValue.GetHostIpOrName()
# init global paramter
g_clusterUser = ""
g_ignoreMiss = False
g_forceRestore = False
g_staticFile = ""
@ -110,12 +111,16 @@ class LocalRestore(LocalBaseOM):
try:
self.clusterInfo = dbClusterInfo()
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
hostName = DefaultValue.GetHostIpOrName()
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if (self.dbNodeInfo is None):
self.logger.logExit(
ErrorCode.GAUSS_516["GAUSS_51619"] % hostName)
gaussHome = os.getenv("GAUSSHOME")
if g_forceRestore and self.restoreBin:
self.clusterInfo.appPath = gaussHome
else:
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
hostName = DefaultValue.GetHostIpOrName()
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if self.dbNodeInfo is None:
self.logger.logExit(
ErrorCode.GAUSS_516["GAUSS_51619"] % hostName)
# Getting local installation path for restoration.
self.logger.log("Getting local installation path for restoration.")
self.installPath = os.path.realpath(self.clusterInfo.appPath)
@ -214,6 +219,16 @@ class LocalRestore(LocalBaseOM):
if self.restorePara:
self.logger.log("Restoring parameter files.")
# Re-obtaining clusterInfo because the restoreBin succeeded
if self.dbNodeInfo is None:
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
hostName = DefaultValue.GetHostIpOrName()
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(
hostName)
if self.dbNodeInfo is None:
self.logger.logExit(
ErrorCode.GAUSS_516["GAUSS_51619"] % hostName)
# Restoring parameter files.
try:
# decompress tar file
@ -317,7 +332,6 @@ class LocalRestore(LocalBaseOM):
output: NA
"""
storedParaFileNum = len(os.listdir(temp_dir)) - 1
for inst in self.dbNodeInfo.datanodes:
self.__checkSingleParaFile(inst, temp_dir, paraFileList)
if ((storedParaFileNum > len(paraFileList)) -
@ -449,10 +463,10 @@ def main():
"""
try:
opts, args = getopt.getopt(sys.argv[1:], "U:P:l:pbhis:", \
opts, args = getopt.getopt(sys.argv[1:], "U:P:l:pbhifs:",
["position=", "parameter", "binary_file",
"logpath=", "help", "ingore_miss",
"static_file="])
"force", "static_file="])
except getopt.GetoptError as e:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % e.msg)
if (len(args) > 0):
@ -462,6 +476,7 @@ def main():
global g_clusterUser
global g_ignoreMiss
global g_staticFile
global g_forceRestore
restoreDir = ""
restorePara = False
restoreBin = False
@ -485,6 +500,8 @@ def main():
g_staticFile = value.strip()
elif (key == "-l" or key == "--logpath"):
logFile = value
elif (key == "-f" or key == "--force"):
g_forceRestore = True
else:
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % value)

34
simpleInstall/install.sh
View File

@ -98,30 +98,38 @@ function fn_get_openGauss_tar()
echo "We only support CentOS+x86, openEuler+arm and openEuler+x86 by now."
return 1
fi
if [ "`find $cur_path/../ -maxdepth 1 -name "openGauss-1.0.1*tar.gz"`" == "" ]
cd "$install_tar"
if [ "`find $cur_path/../ -maxdepth 1 -name "openGauss-1.0.1-*"|wc -l`" -lt "3" ]
then
cd "$install_tar"
if [ "`find . -maxdepth 1 -name "openGauss-1.0.1*tar.gz"`" == "" ]
if [ "`find . -name "openGauss-1.0.1-*"|wc -l`" -lt "3" ]
then
url="https://opengauss.obs.cn-south-1.myhuaweicloud.com/1.0.1/${system_arch}/openGauss-1.0.1-${system_name}-64bit.tar.gz"
url="https://opengauss.obs.cn-south-1.myhuaweicloud.com/1.0.1/${system_arch}/openGauss_1.0.1_PACKAGES_RELEASE.tar.gz"
echo "Downloading openGauss tar from official website at ${install_tar}"
wget $url --timeout=30 --tries=3
wget $url --timeout=30 --tries=3 && tar -zxvf openGauss_1.0.1_PACKAGES_RELEASE.tar.gz
if [ $? -ne 0 ]
then
echo "wget error."
echo "wget error. The $install_tar need openGauss-1.0.1-${system_name}-64bit-om.tar.gz"
echo "wget error. The $install_tar need openGauss-1.0.1-${system_name}-64bit.sha256"
echo "wget error. The $install_tar need openGauss-1.0.1-${system_name}-64bit.tar.bz2"
return 1
else
echo "wget success."
fi
fi
else
cp "$cur_path/../openGauss-1.0.1-${system_name}-64bit.tar.gz" "$install_tar"
if [ $? -ne 0 ]
if [ "`find . -name "openGauss-1.0.1-*"|wc -l`" -lt "3" ]
then
echo "copy Installation package error."
return 1
else
echo "copy Installation package success."
cp "$cur_path/../openGauss-1.0.1-${system_name}-64bit-om.tar.gz" \
"$cur_path/../openGauss-1.0.1-${system_name}-64bit.tar.bz2" \
"$cur_path/../openGauss-1.0.1-${system_name}-64bit.sha256" "$install_tar"
if [ $? -ne 0 ]
then
echo "copy Installation package error."
return 1
else
echo "copy Installation package success."
fi
fi
fi
return 0
@ -261,7 +269,7 @@ function fn_tar()
echo "Get openGauss Installation package success."
fi
cd "${install_tar}"
tar -zxf "openGauss-1.0.1-${system_name}-64bit.tar.gz"
tar -zxf "openGauss-1.0.1-${system_name}-64bit-om.tar.gz"
if [ $? -ne 0 ]
then
echo "tar package error."