Files
openGauss-OM/script/gs_preinstall

542 lines
21 KiB
Python

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
#############################################################################
# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
#
# openGauss is licensed under Mulan PSL v2.
# You can use this software according to the terms
# and conditions of the Mulan PSL v2.
# You may obtain a copy of Mulan PSL v2 at:
#
# http://license.coscl.org.cn/MulanPSL2
#
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
# WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
# ----------------------------------------------------------------------------
# Description : gs_preinstall is a utility to create an installation
# environment for a cluster.
#############################################################################
import os
import pwd
import sys
import grp
import subprocess
from gspylib.common.CheckPythonVersion import check_python_version, \
check_python_compiler_option, check_os_and_package_arch
check_os_and_package_arch()
check_python_version()
check_python_compiler_option()
from base_utils.os.file_util import FileUtil
if "--unused-third-party" in sys.argv:
package_path = os.path.dirname(os.path.realpath(__file__))
lib_path = os.path.join(package_path, "..", "lib")
clib_files = os.path.join(package_path, "gspylib/clib/*.so*")
FileUtil.cleanDirectoryContent(lib_path)
FileUtil.removeFile(clib_files)
# use system pip dependecies
import psutil
import netifaces
import cryptography
import paramiko
from gspylib.common.GaussLog import GaussLog
from gspylib.common.Common import DefaultValue
from gspylib.common.ErrorCode import ErrorCode
from gspylib.common.ParallelBaseOM import ParallelBaseOM
from gspylib.common.ParameterParsecheck import Parameter
from impl.preinstall.OLAP.PreinstallImplOLAP import PreinstallImplOLAP
from gspylib.threads.SshTool import SshTool
from domain_utils.cluster_file.cluster_config_file import ClusterConfigFile
from domain_utils.cluster_file.cluster_dir import ClusterDir
from domain_utils.cluster_file.profile_file import ProfileFile
from domain_utils.cluster_file.version_info import VersionInfo
from domain_utils.cluster_os.cluster_user import ClusterUser
from base_utils.os.net_util import NetUtil
from base_utils.os.file_util import FileUtil
from domain_utils.domain_common.cluster_constants import ClusterConstants
from base_utils.os.user_util import UserUtil
#############################################################################
# Global variables
#############################################################################
userNameFirtChar = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
class Preinstall(ParallelBaseOM):
def __init__(self):
ParallelBaseOM.__init__(self)
self.password = ""
self.envParams = []
self.rootUser = ""
self.rootPasswd = ""
self.createUserSshTrust = True
self.clusterToolPath = ""
self.needFixOwnerPaths = []
self.preMode = False
self.skipOSSet = False
self.skipHostnameSet = False
self.passwordsec = ""
self.corePath = ""
self.is_new_root_path = False
self.ips = ""
self.root_ssh_agent_flag = False
self.root_delete_flag = False
self.user_ssh_agent_flag = False
self.enable_dss = ""
self.dss_vg_info = ""
self.dss_vgname = ""
def usage(self):
"""
gs_preinstall is a utility to create an installation environment for a cluster.
Usage:
gs_preinstall -? | --help
gs_preinstall -V | --version
gs_preinstall -U USER -G GROUP -X XMLFILE
[-L] [--skip-os-set] [--env-var="ENVVAR" [...]]
[--sep-env-file=ENVFILE] [--skip-hostname-set] [-l LOGFILE]
[--non-interactive] [--delete-root-trust] [--unused-third-party]
General options:
-U Cluster user.
-G Group of the cluster user.
-X Path of the XML configuration file.
-L Only perform preinstallation on local
nodes.
--skip-os-set Whether to skip OS parameter setting.
(The default value is set.)
--env-var="ENVVAR" OS user environment variables.
--sep-env-file=ENVFILE Path of the MPP environment file.
--skip-hostname-set Whether to skip hostname setting.
(The default value is set.)
-l Path of log file.
-?, --help Show help information for this
utility, and exit the command line mode.
-V, --version Show version information.
--non-interactive Pre-execution of non-secure mode.
If it is not specified, you can choose
whether create the SSH trust for root
user or cluster user.
If it is specified, you must ensure the
SSH trust for root user and cluster
user have been created.
--delete-root-trust Whether to delete root trust.
(The default value is not deleted)
--unused-third-party Whether to use om's third-party.
(The default value is used)
"""
print(self.usage.__doc__)
# get parameter from command
def parseCommandLine(self):
"""
function: Parse command line and save to global variable
input: NA
output: NA
"""
# init the ParaObj
ParaObj = Parameter()
ParaDict = ParaObj.ParameterCommandLine("preinstall")
# parameter -h or -?
if (ParaDict.__contains__("helpFlag")):
self.usage()
sys.exit(0)
# Resolves command line arguments
# parameter -U
if (ParaDict.__contains__("user")):
self.user = ParaDict.get("user")
DefaultValue.checkPathVaild(self.user)
# parameter -G
if (ParaDict.__contains__("group")):
self.group = ParaDict.get("group")
# parameter -X
if (ParaDict.__contains__("confFile")):
self.xmlFile = ParaDict.get("confFile")
# parameter -L
if (ParaDict.__contains__("localMode")):
self.localMode = ParaDict.get("localMode")
# parameter -l
if (ParaDict.__contains__("logFile")):
self.logFile = ParaDict.get("logFile")
# parameter --env-var
if (ParaDict.__contains__("envparams")):
self.envParams = ParaDict.get("envparams")
# parameter --sep-env-file
if (ParaDict.__contains__("mpprcFile")):
self.mpprcFile = ParaDict.get("mpprcFile")
DefaultValue.checkPathVaild(self.mpprcFile)
# parameter --skip-hostname-set
if (ParaDict.__contains__("skipHostnameSet")):
self.skipHostnameSet = ParaDict.get("skipHostnameSet")
# parameter --skip-os-set
if (ParaDict.__contains__("skipOSSet")):
self.skipOSSet = ParaDict.get("skipOSSet")
# parameter --non-interactive
if (ParaDict.__contains__("preMode")):
self.preMode = ParaDict.get("preMode")
# parameter --delete-root-trust
if (ParaDict.__contains__("root_delete_flag")):
self.root_delete_flag = ParaDict.get("root_delete_flag")
def checkUserParameter(self):
"""
"""
if (self.user == ""):
GaussLog.exitWithError(
ErrorCode.GAUSS_500["GAUSS_50001"] % 'U' + ".")
elif (":" in self.user):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % 'U')
# check if user exists
cmd = "cat /etc/passwd|grep -v nologin|grep -v halt|" \
"grep -v shutdown|awk -F: '{ print $1 }'|" \
" grep '^%s$' 2>/dev/null" % self.user
status = subprocess.getstatusoutput(cmd)[0]
if status == 0:
if pwd.getpwnam(self.user).pw_uid == 0:
# user exists and uid is 0, exit.
GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50302"])
# check the local user and the localmode,
# if user not exist exit with error
if (self.localMode):
try:
DefaultValue.getUserId(self.user)
except Exception as e:
GaussLog.exitWithError(str(e))
def checkUserAndGroup(self):
"""
"""
if (self.localMode):
usergroup = grp.getgrgid(pwd.getpwnam(self.user).pw_gid).gr_name
if (self.group != usergroup):
GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50305"]
+ "User:Group[%s:%s]"
% (self.user, self.group))
def check_config_content(self, g_nodeInfo):
UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbAppPath", self.xmlFile))
UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbToolPath", self.xmlFile))
UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("tmpMppdbPath", self.xmlFile))
UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbLogPath", self.xmlFile))
UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("corePath", self.xmlFile))
# check cm
UserUtil.check_path_owner(g_nodeInfo.cmDataDir)
for cmaInst in g_nodeInfo.cmagents:
UserUtil.check_path_owner(cmaInst.datadir)
for cmsInst in g_nodeInfo.cmservers:
UserUtil.check_path_owner(cmsInst.datadir)
# check dn
for dnInst in g_nodeInfo.datanodes:
UserUtil.check_path_owner(dnInst.datadir)
if (len(dnInst.ssdDir) != 0):
UserUtil.check_path_owner(dnInst.ssdDir)
# check dn xlog
for dnInst in g_nodeInfo.datanodes:
if dnInst.xlogdir != '':
UserUtil.check_path_owner(dnInst.xlogdir)
def checkEnvValueParameter(self):
"""
"""
for param in self.envParams:
# check environmental variables vaild
illegal = ["|", ";", "&", "$", ">", "<", "`", "\\", "!", "\n"]
if any(ill_char in param for ill_char in illegal):
GaussLog.exitWithError(
ErrorCode.GAUSS_500["GAUSS_50004"] % "-env-var" +
" There are illegal characters in the parameter.")
def checkLogFile(self):
"""
"""
if (self.logFile == ""):
self.logFile = self.getPreOMLogPath(
ClusterConstants.PREINSTALL_LOG_FILE, self.xmlFile)
if (not os.path.isabs(self.logFile)):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50213"]
% self.logFile)
UserUtil.check_path_owner(self.logFile)
def checkMpprcFile(self):
"""
"""
if (self.mpprcFile == ""):
return
if (not os.path.isabs(self.mpprcFile)):
GaussLog.exitWithError(ErrorCode.GAUSS_512["GAUSS_51206"]
% self.mpprcFile)
# check mpprc file path
mpprcFilePath = os.path.normpath(self.mpprcFile)
if os.path.islink(mpprcFilePath):
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_51251"] % mpprcFilePath)
if (mpprcFilePath == "/home/%s" % self.user):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % \
'-sep-env-file' + " The file [%s] can not"
" be a reserved home "
"directory."
% self.mpprcFile)
if (os.path.isdir(self.mpprcFile)):
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % \
'-sep-env-file' + " The file [%s] can not "
"be a directory."
% self.mpprcFile)
ProfileFile.checkMpprcFileChange(self.mpprcFile, "", self.mpprcFile)
(checkstatus, checkoutput) = ProfileFile.check_env_file(self.mpprcFile)
if (not checkstatus):
if (self.mpprcFile != ""):
envfile = self.mpprcFile + " and /etc/profile"
else:
envfile = "/etc/profile and ~/.bashrc"
GaussLog.exitWithError(ErrorCode.GAUSS_518["GAUSS_51808"] % \
checkoutput + "Please check %s." % envfile)
def checkParameter(self):
"""
function: Check parameter from command line
input: NA
output: NA
"""
ClusterUser.checkGroupParameter(self.user, self.group)
# remove HOST_IP info with /etc/profile and environ
cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile"
(status, output) = subprocess.getstatusoutput(cmd)
if status != 0:
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50205"]
% ClusterConstants.ETC_PROFILE + "The cmd is %s" % cmd)
if "HOST_IP" in os.environ.keys():
os.environ.pop("HOST_IP")
# check config file
ClusterConfigFile.checkConfigFile(self.xmlFile)
# check user info
self.checkUserParameter()
# check user group match
self.checkUserAndGroup()
self.initClusterInfo()
# check config content
hostName = NetUtil.GetHostIpOrName()
g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName)
if (g_nodeInfo is None):
GaussLog.exitWithError(ErrorCode.GAUSS_516["GAUSS_51620"] % "local" +
" It is not a host name %s." % hostName)
# check env-val
self.checkEnvValueParameter()
# check mpprc file
self.checkMpprcFile()
# check log file
self.checkLogFile()
# set LD_LIBRARY_PATH add local lib
def setLibPath(self):
package_path = os.path.dirname(os.path.realpath(__file__))
ld_path = package_path + "/gspylib/clib"
rerun = True
if 'LD_LIBRARY_PATH' not in os.environ:
os.environ['LD_LIBRARY_PATH'] = ld_path
elif not os.environ.get('LD_LIBRARY_PATH').startswith(ld_path):
os.environ['LD_LIBRARY_PATH'] = \
ld_path + ":" + os.environ['LD_LIBRARY_PATH']
else:
rerun = False
if rerun:
try:
os.execve(os.path.realpath(__file__), sys.argv, os.environ)
except Exception as e:
GaussLog.exitWithError(str(e))
# decompress version.cfg from bz2
def decompressVersioncfg(self):
bin_cmd = 'tar -xpf `head -1 version.cfg`*.tar.bz2 ./version.cfg'
cm_cmd = 'tar -xpf `head -1 version.cfg`*cm.tar.gz'
cmd = self.get_dec_package_cmd(bin_cmd, cm_cmd)
status, _ = subprocess.getstatusoutput(cmd)
if status != 0:
bin_cmd = 'tar -xpf `ls openGauss*.tar.bz2|tail -1` ./version.cfg'
cm_cmd = 'tar -xpf `ls openGauss*cm.tar.gz|tail -1`'
cmd = self.get_dec_package_cmd(bin_cmd, cm_cmd)
status, output = subprocess.getstatusoutput(cmd)
if status != 0:
GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50217"] %
"version.cfg" + "The cmd is %s. " % cmd +
"The output is %s." % output)
def get_dec_package_cmd(self, bin_cmd, cm_cmd):
root = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
clib = os.path.join(root, "script/gspylib/clib")
bin_files = ['./bin/encrypt']
clib_app = os.path.realpath(
os.path.join(f'{clib}',
f"dss_app_$(cat {root}/version.cfg | tail -n 1)"))
dss_files = []
cm_files = []
if self.clusterInfo.enable_dss == 'on':
cm_files = ['bin/cm_persist']
dss_files = [
'./bin/perctrl', './bin/dsscmd', './lib/libdssapi.so',
'./bin/dss_clear.sh'
]
else:
cm_files = []
cmd = 'cd {} && '.format(root)
cmd += '{} {} && '.format(bin_cmd, ' '.join(bin_files + dss_files))
if cm_files:
cmd += '{} {} && '.format(cm_cmd, ' '.join(cm_files))
cmd += 'mkdir -p {0} -m u=rwx && '.format(clib_app)
cmd += 'mv {} {} && '.format(' '.join(cm_files + dss_files), clib_app)
cmd += '\mv {} {} && '.format(' '.join(bin_files), clib)
cmd += 'cd {} && rm -rf bin'.format(root)
return cmd
# init global variables
def initGlobals(self):
"""
function: init global parameters
input: NA
output: NA
"""
# init the log file
self.initLogger("gs_preinstall")
# get the clusterToolPath
self.clusterToolPath = ClusterDir.getPreClusterToolPath(self.xmlFile)
temp_nodes = ClusterConfigFile.getOneClusterConfigItem("nodeNames", self.xmlFile)
if len(temp_nodes.split(',')) < 2:
self.isSingle = True
os.environ[ClusterConstants.TOOL_PATH_ENV] = self.clusterToolPath
self.logger.log("Parsing the configuration file.", "addStep")
try:
# parse the configuration file
self.sshTool = SshTool(self.clusterInfo.getClusterNodeNames(),
self.logFile,
DefaultValue.TIMEOUT_PSSH_PREINSTALL)
except Exception as e:
self.logger.logExit(str(e))
# check the local hostname
if NetUtil.GetHostIpOrName() not in \
self.clusterInfo.getClusterNodeNames():
self.logger.logExit(ErrorCode.GAUSS_516["GAUSS_51619"]
% NetUtil.GetHostIpOrName())
self.logger.log("Successfully parsed the configuration file.",
"constant")
# check expect for cm/create trust
def check_expect(self):
"""
function: check expect
input: NA
output: NA
"""
temp_nodes = ClusterConfigFile.getOneClusterConfigItem("nodeNames", self.xmlFile)
if len(temp_nodes.split(',')) > 1:
cmd = "echo exit|expect"
(status, _) = subprocess.getstatusoutput(cmd)
if status != 0:
GaussLog.exitWithError(ErrorCode.GAUSS_514["GAUSS_51405"] % "expect")
def getPreOMLogPath(self, logName, xml):
"""
function: get the OM log path
input: logName, xml
output: fullLogPath
"""
try:
fullLogPath = ""
# get the log path
configedLogPath = ClusterConfigFile.getOneClusterConfigItem("gaussdbLogPath",
xml)
DefaultValue.checkPathVaild(configedLogPath)
# check gaussdbLogPath is not null
if configedLogPath == "":
fullLogPath = "%s/%s/om/%s" % (
ClusterConstants.GAUSSDB_DIR, self.user, logName)
else:
fullLogPath = "%s/%s/om/%s" % (
os.path.normpath(configedLogPath), self.user, logName)
UserUtil.check_path_owner(fullLogPath)
return fullLogPath
except Exception as e:
GaussLog.exitWithError(str(e))
def change_lib_path(self):
"""
if gs_preinstall current path is /root/gauss_om/username,
so change its lib path
:return:
"""
gsom_path = os.path.realpath(
os.path.join(os.path.realpath(__file__), "../../../"))
package_path = os.path.dirname(os.path.realpath(__file__))
lib_path = os.path.join(package_path, "..", "lib")
sys.path.insert(0, lib_path)
if gsom_path == DefaultValue.ROOT_SCRIPTS_PATH:
self.is_new_root_path = True
def clearHistTimeFormat():
cmd = "sed -i '/HISTTIMEFORMAT=/d' /etc/profile"
(status, output) = subprocess.getstatusoutput(cmd)
if status != 0:
GaussLog.exitWithError("Clear HISTTIMEFORMAT from /etc/profile "
"failed.\nError: %s\nThe cmd is: %s\n" %
(output,cmd))
if __name__ == '__main__':
"""
main function
"""
# check if user is root
if os.getuid() != 0:
GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50104"])
clearHistTimeFormat()
try:
# Objectize class
preinstall = Preinstall()
# set LD_LIBRARY_PATH
preinstall.setLibPath()
# parse cmd lines
preinstall.parseCommandLine()
# check parameters
preinstall.checkParameter()
# check expect
preinstall.check_expect()
# init global variables
preinstall.initGlobals()
# decompress version.cfg
preinstall.decompressVersioncfg()
preinstall.change_lib_path()
impl = PreinstallImplOLAP(preinstall)
# Perform the whole extand process
impl.run()
except Exception as e:
GaussLog.exitWithError(str(e))