#!/usr/bin/env python3 # -*- coding:utf-8 -*- ############################################################################# # Copyright (c) 2020 Huawei Technologies Co.,Ltd. # # openGauss is licensed under Mulan PSL v2. # You can use this software according to the terms # and conditions of the Mulan PSL v2. # You may obtain a copy of Mulan PSL v2 at: # # http://license.coscl.org.cn/MulanPSL2 # # THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, # WITHOUT WARRANTIES OF ANY KIND, # EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. # ---------------------------------------------------------------------------- # Description : gs_preinstall is a utility to create an installation # environment for a cluster. ############################################################################# import os import pwd import sys import grp import subprocess from gspylib.common.CheckPythonVersion import check_python_version, \ check_python_compiler_option, check_os_and_package_arch check_os_and_package_arch() check_python_version() check_python_compiler_option() from base_utils.os.file_util import FileUtil if "--unused-third-party" in sys.argv: package_path = os.path.dirname(os.path.realpath(__file__)) lib_path = os.path.join(package_path, "..", "lib") clib_files = os.path.join(package_path, "gspylib/clib/*.so*") FileUtil.cleanDirectoryContent(lib_path) FileUtil.removeFile(clib_files) # use system pip dependecies import psutil import netifaces import cryptography import paramiko from gspylib.common.GaussLog import GaussLog from gspylib.common.Common import DefaultValue from gspylib.common.ErrorCode import ErrorCode from gspylib.common.ParallelBaseOM import ParallelBaseOM from gspylib.common.ParameterParsecheck import Parameter from impl.preinstall.OLAP.PreinstallImplOLAP import PreinstallImplOLAP from gspylib.threads.SshTool import SshTool from domain_utils.cluster_file.cluster_config_file import ClusterConfigFile from domain_utils.cluster_file.cluster_dir import ClusterDir from domain_utils.cluster_file.profile_file import ProfileFile from domain_utils.cluster_file.version_info import VersionInfo from domain_utils.cluster_os.cluster_user import ClusterUser from base_utils.os.net_util import NetUtil from base_utils.os.file_util import FileUtil from domain_utils.domain_common.cluster_constants import ClusterConstants from base_utils.os.user_util import UserUtil ############################################################################# # Global variables ############################################################################# userNameFirtChar = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'] class Preinstall(ParallelBaseOM): def __init__(self): ParallelBaseOM.__init__(self) self.password = "" self.envParams = [] self.rootUser = "" self.rootPasswd = "" self.createUserSshTrust = True self.clusterToolPath = "" self.needFixOwnerPaths = [] self.preMode = False self.skipOSSet = False self.skipHostnameSet = False self.passwordsec = "" self.corePath = "" self.is_new_root_path = False self.ips = "" self.root_ssh_agent_flag = False self.root_delete_flag = False self.user_ssh_agent_flag = False self.enable_dss = "" self.dss_vg_info = "" self.dss_vgname = "" def usage(self): """ gs_preinstall is a utility to create an installation environment for a cluster. Usage: gs_preinstall -? | --help gs_preinstall -V | --version gs_preinstall -U USER -G GROUP -X XMLFILE [-L] [--skip-os-set] [--env-var="ENVVAR" [...]] [--sep-env-file=ENVFILE] [--skip-hostname-set] [-l LOGFILE] [--non-interactive] [--delete-root-trust] [--unused-third-party] General options: -U Cluster user. -G Group of the cluster user. -X Path of the XML configuration file. -L Only perform preinstallation on local nodes. --skip-os-set Whether to skip OS parameter setting. (The default value is set.) --env-var="ENVVAR" OS user environment variables. --sep-env-file=ENVFILE Path of the MPP environment file. --skip-hostname-set Whether to skip hostname setting. (The default value is set.) -l Path of log file. -?, --help Show help information for this utility, and exit the command line mode. -V, --version Show version information. --non-interactive Pre-execution of non-secure mode. If it is not specified, you can choose whether create the SSH trust for root user or cluster user. If it is specified, you must ensure the SSH trust for root user and cluster user have been created. --delete-root-trust Whether to delete root trust. (The default value is not deleted) --unused-third-party Whether to use om's third-party. (The default value is used) """ print(self.usage.__doc__) # get parameter from command def parseCommandLine(self): """ function: Parse command line and save to global variable input: NA output: NA """ # init the ParaObj ParaObj = Parameter() ParaDict = ParaObj.ParameterCommandLine("preinstall") # parameter -h or -? if (ParaDict.__contains__("helpFlag")): self.usage() sys.exit(0) # Resolves command line arguments # parameter -U if (ParaDict.__contains__("user")): self.user = ParaDict.get("user") DefaultValue.checkPathVaild(self.user) # parameter -G if (ParaDict.__contains__("group")): self.group = ParaDict.get("group") # parameter -X if (ParaDict.__contains__("confFile")): self.xmlFile = ParaDict.get("confFile") # parameter -L if (ParaDict.__contains__("localMode")): self.localMode = ParaDict.get("localMode") # parameter -l if (ParaDict.__contains__("logFile")): self.logFile = ParaDict.get("logFile") # parameter --env-var if (ParaDict.__contains__("envparams")): self.envParams = ParaDict.get("envparams") # parameter --sep-env-file if (ParaDict.__contains__("mpprcFile")): self.mpprcFile = ParaDict.get("mpprcFile") DefaultValue.checkPathVaild(self.mpprcFile) # parameter --skip-hostname-set if (ParaDict.__contains__("skipHostnameSet")): self.skipHostnameSet = ParaDict.get("skipHostnameSet") # parameter --skip-os-set if (ParaDict.__contains__("skipOSSet")): self.skipOSSet = ParaDict.get("skipOSSet") # parameter --non-interactive if (ParaDict.__contains__("preMode")): self.preMode = ParaDict.get("preMode") # parameter --delete-root-trust if (ParaDict.__contains__("root_delete_flag")): self.root_delete_flag = ParaDict.get("root_delete_flag") def checkUserParameter(self): """ """ if (self.user == ""): GaussLog.exitWithError( ErrorCode.GAUSS_500["GAUSS_50001"] % 'U' + ".") elif (":" in self.user): GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % 'U') # check if user exists cmd = "cat /etc/passwd|grep -v nologin|grep -v halt|" \ "grep -v shutdown|awk -F: '{ print $1 }'|" \ " grep '^%s$' 2>/dev/null" % self.user status = subprocess.getstatusoutput(cmd)[0] if status == 0: if pwd.getpwnam(self.user).pw_uid == 0: # user exists and uid is 0, exit. GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50302"]) # check the local user and the localmode, # if user not exist exit with error if (self.localMode): try: DefaultValue.getUserId(self.user) except Exception as e: GaussLog.exitWithError(str(e)) def checkUserAndGroup(self): """ """ if (self.localMode): usergroup = grp.getgrgid(pwd.getpwnam(self.user).pw_gid).gr_name if (self.group != usergroup): GaussLog.exitWithError(ErrorCode.GAUSS_503["GAUSS_50305"] + "User:Group[%s:%s]" % (self.user, self.group)) def check_config_content(self, g_nodeInfo): UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbAppPath", self.xmlFile)) UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbToolPath", self.xmlFile)) UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("tmpMppdbPath", self.xmlFile)) UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("gaussdbLogPath", self.xmlFile)) UserUtil.check_path_owner(ClusterConfigFile.getOneClusterConfigItem("corePath", self.xmlFile)) # check cm UserUtil.check_path_owner(g_nodeInfo.cmDataDir) for cmaInst in g_nodeInfo.cmagents: UserUtil.check_path_owner(cmaInst.datadir) for cmsInst in g_nodeInfo.cmservers: UserUtil.check_path_owner(cmsInst.datadir) # check dn for dnInst in g_nodeInfo.datanodes: UserUtil.check_path_owner(dnInst.datadir) if (len(dnInst.ssdDir) != 0): UserUtil.check_path_owner(dnInst.ssdDir) # check dn xlog for dnInst in g_nodeInfo.datanodes: if dnInst.xlogdir != '': UserUtil.check_path_owner(dnInst.xlogdir) def checkEnvValueParameter(self): """ """ for param in self.envParams: # check environmental variables vaild illegal = ["|", ";", "&", "$", ">", "<", "`", "\\", "!", "\n"] if any(ill_char in param for ill_char in illegal): GaussLog.exitWithError( ErrorCode.GAUSS_500["GAUSS_50004"] % "-env-var" + " There are illegal characters in the parameter.") def checkLogFile(self): """ """ if (self.logFile == ""): self.logFile = self.getPreOMLogPath( ClusterConstants.PREINSTALL_LOG_FILE, self.xmlFile) if (not os.path.isabs(self.logFile)): GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50213"] % self.logFile) UserUtil.check_path_owner(self.logFile) def checkMpprcFile(self): """ """ if (self.mpprcFile == ""): return if (not os.path.isabs(self.mpprcFile)): GaussLog.exitWithError(ErrorCode.GAUSS_512["GAUSS_51206"] % self.mpprcFile) # check mpprc file path mpprcFilePath = os.path.normpath(self.mpprcFile) if os.path.islink(mpprcFilePath): GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_51251"] % mpprcFilePath) if (mpprcFilePath == "/home/%s" % self.user): GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % \ '-sep-env-file' + " The file [%s] can not" " be a reserved home " "directory." % self.mpprcFile) if (os.path.isdir(self.mpprcFile)): GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50004"] % \ '-sep-env-file' + " The file [%s] can not " "be a directory." % self.mpprcFile) ProfileFile.checkMpprcFileChange(self.mpprcFile, "", self.mpprcFile) (checkstatus, checkoutput) = ProfileFile.check_env_file(self.mpprcFile) if (not checkstatus): if (self.mpprcFile != ""): envfile = self.mpprcFile + " and /etc/profile" else: envfile = "/etc/profile and ~/.bashrc" GaussLog.exitWithError(ErrorCode.GAUSS_518["GAUSS_51808"] % \ checkoutput + "Please check %s." % envfile) def checkParameter(self): """ function: Check parameter from command line input: NA output: NA """ ClusterUser.checkGroupParameter(self.user, self.group) # remove HOST_IP info with /etc/profile and environ cmd = "sed -i '/^export[ ]*HOST_IP=/d' /etc/profile" (status, output) = subprocess.getstatusoutput(cmd) if status != 0: GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50205"] % ClusterConstants.ETC_PROFILE + "The cmd is %s" % cmd) if "HOST_IP" in os.environ.keys(): os.environ.pop("HOST_IP") # check config file ClusterConfigFile.checkConfigFile(self.xmlFile) # check user info self.checkUserParameter() # check user group match self.checkUserAndGroup() self.initClusterInfo() # check config content hostName = NetUtil.GetHostIpOrName() g_nodeInfo = self.clusterInfo.getDbNodeByName(hostName) if (g_nodeInfo is None): GaussLog.exitWithError(ErrorCode.GAUSS_516["GAUSS_51620"] % "local" + " It is not a host name %s." % hostName) # check env-val self.checkEnvValueParameter() # check mpprc file self.checkMpprcFile() # check log file self.checkLogFile() # set LD_LIBRARY_PATH add local lib def setLibPath(self): package_path = os.path.dirname(os.path.realpath(__file__)) ld_path = package_path + "/gspylib/clib" rerun = True if 'LD_LIBRARY_PATH' not in os.environ: os.environ['LD_LIBRARY_PATH'] = ld_path elif not os.environ.get('LD_LIBRARY_PATH').startswith(ld_path): os.environ['LD_LIBRARY_PATH'] = \ ld_path + ":" + os.environ['LD_LIBRARY_PATH'] else: rerun = False if rerun: try: os.execve(os.path.realpath(__file__), sys.argv, os.environ) except Exception as e: GaussLog.exitWithError(str(e)) # decompress version.cfg from bz2 def decompressVersioncfg(self): bin_cmd = 'tar -xpf `head -1 version.cfg`*.tar.bz2 ./version.cfg' cm_cmd = 'tar -xpf `head -1 version.cfg`*cm.tar.gz' cmd = self.get_dec_package_cmd(bin_cmd, cm_cmd) status, _ = subprocess.getstatusoutput(cmd) if status != 0: bin_cmd = 'tar -xpf `ls openGauss*.tar.bz2|tail -1` ./version.cfg' cm_cmd = 'tar -xpf `ls openGauss*cm.tar.gz|tail -1`' cmd = self.get_dec_package_cmd(bin_cmd, cm_cmd) status, output = subprocess.getstatusoutput(cmd) if status != 0: GaussLog.exitWithError(ErrorCode.GAUSS_502["GAUSS_50217"] % "version.cfg" + "The cmd is %s. " % cmd + "The output is %s." % output) def get_dec_package_cmd(self, bin_cmd, cm_cmd): root = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') clib = os.path.join(root, "script/gspylib/clib") bin_files = ['./bin/encrypt'] clib_app = os.path.realpath( os.path.join(f'{clib}', f"dss_app_$(cat {root}/version.cfg | tail -n 1)")) dss_files = [] cm_files = [] if self.clusterInfo.enable_dss == 'on': cm_files = ['bin/cm_persist'] dss_files = [ './bin/perctrl', './bin/dsscmd', './lib/libdssapi.so', './bin/dss_clear.sh' ] else: cm_files = [] cmd = 'cd {} && '.format(root) cmd += '{} {} && '.format(bin_cmd, ' '.join(bin_files + dss_files)) if cm_files: cmd += '{} {} && '.format(cm_cmd, ' '.join(cm_files)) cmd += 'mkdir -p {0} -m u=rwx && '.format(clib_app) cmd += 'mv {} {} && '.format(' '.join(cm_files + dss_files), clib_app) cmd += '\mv {} {} && '.format(' '.join(bin_files), clib) cmd += 'cd {} && rm -rf bin'.format(root) return cmd # init global variables def initGlobals(self): """ function: init global parameters input: NA output: NA """ # init the log file self.initLogger("gs_preinstall") # get the clusterToolPath self.clusterToolPath = ClusterDir.getPreClusterToolPath(self.xmlFile) temp_nodes = ClusterConfigFile.getOneClusterConfigItem("nodeNames", self.xmlFile) if len(temp_nodes.split(',')) < 2: self.isSingle = True os.environ[ClusterConstants.TOOL_PATH_ENV] = self.clusterToolPath self.logger.log("Parsing the configuration file.", "addStep") try: # parse the configuration file self.sshTool = SshTool(self.clusterInfo.getClusterNodeNames(), self.logFile, DefaultValue.TIMEOUT_PSSH_PREINSTALL) except Exception as e: self.logger.logExit(str(e)) # check the local hostname if NetUtil.GetHostIpOrName() not in \ self.clusterInfo.getClusterNodeNames(): self.logger.logExit(ErrorCode.GAUSS_516["GAUSS_51619"] % NetUtil.GetHostIpOrName()) self.logger.log("Successfully parsed the configuration file.", "constant") # check expect for cm/create trust def check_expect(self): """ function: check expect input: NA output: NA """ temp_nodes = ClusterConfigFile.getOneClusterConfigItem("nodeNames", self.xmlFile) if len(temp_nodes.split(',')) > 1: cmd = "echo exit|expect" (status, _) = subprocess.getstatusoutput(cmd) if status != 0: GaussLog.exitWithError(ErrorCode.GAUSS_514["GAUSS_51405"] % "expect") def getPreOMLogPath(self, logName, xml): """ function: get the OM log path input: logName, xml output: fullLogPath """ try: fullLogPath = "" # get the log path configedLogPath = ClusterConfigFile.getOneClusterConfigItem("gaussdbLogPath", xml) DefaultValue.checkPathVaild(configedLogPath) # check gaussdbLogPath is not null if configedLogPath == "": fullLogPath = "%s/%s/om/%s" % ( ClusterConstants.GAUSSDB_DIR, self.user, logName) else: fullLogPath = "%s/%s/om/%s" % ( os.path.normpath(configedLogPath), self.user, logName) UserUtil.check_path_owner(fullLogPath) return fullLogPath except Exception as e: GaussLog.exitWithError(str(e)) def change_lib_path(self): """ if gs_preinstall current path is /root/gauss_om/username, so change its lib path :return: """ gsom_path = os.path.realpath( os.path.join(os.path.realpath(__file__), "../../../")) package_path = os.path.dirname(os.path.realpath(__file__)) lib_path = os.path.join(package_path, "..", "lib") sys.path.insert(0, lib_path) if gsom_path == DefaultValue.ROOT_SCRIPTS_PATH: self.is_new_root_path = True def clearHistTimeFormat(): cmd = "sed -i '/HISTTIMEFORMAT=/d' /etc/profile" (status, output) = subprocess.getstatusoutput(cmd) if status != 0: GaussLog.exitWithError("Clear HISTTIMEFORMAT from /etc/profile " "failed.\nError: %s\nThe cmd is: %s\n" % (output,cmd)) if __name__ == '__main__': """ main function """ # check if user is root if os.getuid() != 0: GaussLog.exitWithError(ErrorCode.GAUSS_501["GAUSS_50104"]) clearHistTimeFormat() try: # Objectize class preinstall = Preinstall() # set LD_LIBRARY_PATH preinstall.setLibPath() # parse cmd lines preinstall.parseCommandLine() # check parameters preinstall.checkParameter() # check expect preinstall.check_expect() # init global variables preinstall.initGlobals() # decompress version.cfg preinstall.decompressVersioncfg() preinstall.change_lib_path() impl = PreinstallImplOLAP(preinstall) # Perform the whole extand process impl.run() except Exception as e: GaussLog.exitWithError(str(e))