529 lines
20 KiB
Python
529 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding:utf-8 -*-
|
|
#############################################################################
|
|
# Copyright (c) 2020 Huawei Technologies Co.,Ltd.
|
|
#
|
|
# openGauss is licensed under Mulan PSL v2.
|
|
# You can use this software according to the terms
|
|
# and conditions of the Mulan PSL v2.
|
|
# You may obtain a copy of Mulan PSL v2 at:
|
|
#
|
|
# http://license.coscl.org.cn/MulanPSL2
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OF ANY KIND,
|
|
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
# See the Mulan PSL v2 for more details.
|
|
# ----------------------------------------------------------------------------
|
|
# Description : Restore.py is a local utility to
|
|
# restore binary file and parameter file.
|
|
#############################################################################
|
|
import subprocess
|
|
import getopt
|
|
import os
|
|
import sys
|
|
|
|
sys.path.append(sys.path[0] + "/../")
|
|
from gspylib.common.ParameterParsecheck import Parameter
|
|
from gspylib.common.GaussLog import GaussLog
|
|
from gspylib.common.Common import DefaultValue
|
|
from gspylib.common.ErrorCode import ErrorCode
|
|
from gspylib.common.LocalBaseOM import LocalBaseOM
|
|
from gspylib.common.DbClusterInfo import dbClusterInfo
|
|
from gspylib.os.gsfile import g_file
|
|
from domain_utils.cluster_file.cluster_log import ClusterLog
|
|
from base_utils.os.env_util import EnvUtil
|
|
from base_utils.os.file_util import FileUtil
|
|
from base_utils.os.net_util import NetUtil
|
|
from domain_utils.domain_common.cluster_constants import ClusterConstants
|
|
from base_utils.common.constantsbase import ConstantsBase
|
|
from domain_utils.cluster_os.cluster_user import ClusterUser
|
|
|
|
# init config file parameter
|
|
POSTGRESQL_CONF = "postgresql.conf"
|
|
POSTGRESQL_HBA_CONF = "pg_hba.conf"
|
|
HOSTNAME = NetUtil.GetHostIpOrName()
|
|
# init global paramter
|
|
g_clusterUser = ""
|
|
g_ignoreMiss = False
|
|
g_forceRestore = False
|
|
g_staticFile = ""
|
|
|
|
|
|
class LocalRestore(LocalBaseOM):
|
|
'''
|
|
classdocs
|
|
'''
|
|
|
|
def __init__(self, logFile="", user="", restoreDir="", restorePara=False,
|
|
restoreBin=False):
|
|
"""
|
|
function: Constructor
|
|
input : logFile, user, restoreDir, restorePara, restoreBin
|
|
output: NA
|
|
"""
|
|
LocalBaseOM.__init__(self, logFile, user)
|
|
self.restoreDir = restoreDir
|
|
self.restorePara = restorePara
|
|
self.restoreBin = restoreBin
|
|
|
|
self.installPath = ""
|
|
self.binExtractName = ""
|
|
self.group = ""
|
|
self.dbNodeInfo = None
|
|
self.clusterInfo = None
|
|
self.__hostNameFile = None
|
|
|
|
# #static parameter
|
|
# Use binary_$hostname/parameter_$hostname to
|
|
# confirm the backup asked that
|
|
self.binTarName = "binary_%s.tar" % HOSTNAME
|
|
self.paraTarName = "parameter_%s.tar" % HOSTNAME
|
|
self.hostnameFileName = "HOSTNAME"
|
|
|
|
##########################################################################
|
|
# This is the main restore flow.
|
|
##########################################################################
|
|
|
|
def run(self):
|
|
"""
|
|
function: 1.parse the configuration file
|
|
2.check restored directory
|
|
3.restore files
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
try:
|
|
self.logger.log("Executing the local restoration")
|
|
self.parseConfigFile()
|
|
self.checkRestoreDir()
|
|
self.doRestore()
|
|
self.logger.log("Successfully execute the local restoration.")
|
|
self.logger.closeLog()
|
|
sys.exit(0)
|
|
except Exception as e:
|
|
raise Exception(str(e))
|
|
|
|
def parseConfigFile(self):
|
|
"""
|
|
function: parse the configuration file:
|
|
1.get local installation path for restoration
|
|
2.Obtain user and group for restoration
|
|
3.Obtain the local node information for restoration
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
self.logger.log("Parsing the configuration file.")
|
|
|
|
try:
|
|
self.clusterInfo = dbClusterInfo()
|
|
gaussHome = EnvUtil.getEnvironmentParameterValue("GAUSSHOME", self.user)
|
|
if g_forceRestore and self.restoreBin:
|
|
self.clusterInfo.appPath = gaussHome
|
|
else:
|
|
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
|
|
hostName = NetUtil.GetHostIpOrName()
|
|
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(hostName)
|
|
if self.dbNodeInfo is None:
|
|
self.logger.logExit(
|
|
ErrorCode.GAUSS_516["GAUSS_51619"] % hostName)
|
|
# Getting local installation path for restoration.
|
|
self.logger.log("Getting local installation path for restoration.")
|
|
self.installPath = os.path.realpath(self.clusterInfo.appPath)
|
|
self.binExtractName = self.installPath.split("/")[-1]
|
|
self.logger.debug(
|
|
"Local installation path: %s." % self.installPath)
|
|
except Exception as e:
|
|
raise Exception(str(e))
|
|
|
|
self.logger.log("Successfully parsed the configuration file.")
|
|
|
|
def checkRestoreDir(self):
|
|
"""
|
|
function: check restored directory
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
self.logger.log("Checking restored directory.")
|
|
|
|
try:
|
|
if (not os.path.exists(self.restoreDir) or len(
|
|
os.listdir(self.restoreDir)) == 0):
|
|
if (g_ignoreMiss):
|
|
self.logger.log(
|
|
"Restored directory does not exist or is empty.")
|
|
sys.exit(0)
|
|
else:
|
|
raise Exception(ErrorCode.GAUSS_502[
|
|
"GAUSS_50228"] % "restored directory" +
|
|
" Error: \n%s" % self.restoreDir)
|
|
except Exception as e:
|
|
raise Exception(str(e))
|
|
|
|
self.logger.log("Successfully checked restored directory.")
|
|
|
|
def doRestore(self):
|
|
"""
|
|
function: restore files
|
|
Restoring binary files:
|
|
1.decompress tar file
|
|
2.Check binary files
|
|
3.Create installation path
|
|
4.Restore binary files to install path
|
|
Restoring parameter files:
|
|
1.decompress tar file
|
|
2.delete temporary directory
|
|
3.extract parameter files to the temporary directory
|
|
4.check hostname and parameter
|
|
5.Restore parameter files
|
|
6.Remove the temporary directory
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
self.logger.log("Restoring files.")
|
|
|
|
if self.restoreBin:
|
|
self.logger.log("Restoring binary files.")
|
|
try:
|
|
# decompress tar file
|
|
self.decompressTarFile("binary")
|
|
|
|
# Checking binary files
|
|
self.logger.debug("Checking if binary files exist.")
|
|
tarName = os.path.join(self.restoreDir, self.binTarName)
|
|
if (not os.path.exists(tarName)):
|
|
raise Exception(
|
|
ErrorCode.GAUSS_502["GAUSS_50201"] % "Binary files")
|
|
|
|
# Creating installation path
|
|
self.logger.debug(
|
|
"Creating installation path if did not exist.")
|
|
if (not os.path.exists(self.installPath)):
|
|
os.makedirs(self.installPath,
|
|
ConstantsBase.KEY_DIRECTORY_PERMISSION)
|
|
|
|
# Restore binary files to install path.
|
|
self.logger.debug("Restore binary files to install path.")
|
|
FileUtil.cleanDirectoryContent(self.installPath)
|
|
cmd = g_file.SHELL_CMD_DICT["decompressTarFile"] % (
|
|
self.restoreDir, tarName)
|
|
cmd += " && "
|
|
cmd += g_file.SHELL_CMD_DICT["copyFile"] % (
|
|
"'%s'/*" % self.binExtractName, self.installPath)
|
|
self.logger.debug(
|
|
"Command for restoring binary files:%s." % cmd)
|
|
(status, output) = subprocess.getstatusoutput(cmd)
|
|
if (status != 0):
|
|
raise Exception(ErrorCode.GAUSS_502["GAUSS_50220"] % (
|
|
"binary files to install path[%s]" % \
|
|
self.installPath) + " Error: \n%s" % output)
|
|
FileUtil.removeDirectory(
|
|
os.path.join(self.restoreDir, self.binExtractName))
|
|
except Exception as e:
|
|
raise Exception(str(e))
|
|
self.logger.log("Successfully restored binary files.")
|
|
|
|
if self.restorePara:
|
|
self.logger.log("Restoring parameter files.")
|
|
# Re-obtaining clusterInfo because the restoreBin succeeded
|
|
if self.dbNodeInfo is None:
|
|
self.clusterInfo.initFromStaticConfig(self.user, g_staticFile)
|
|
hostName = NetUtil.GetHostIpOrName()
|
|
self.dbNodeInfo = self.clusterInfo.getDbNodeByName(
|
|
hostName)
|
|
if self.dbNodeInfo is None:
|
|
self.logger.logExit(
|
|
ErrorCode.GAUSS_516["GAUSS_51619"] % hostName)
|
|
|
|
# Restoring parameter files.
|
|
try:
|
|
# decompress tar file
|
|
self.decompressTarFile("parameter")
|
|
# delete temporary directory
|
|
self.logger.debug(
|
|
"Delete temporary directory if it has existed.")
|
|
temp_dir = os.path.join(self.restoreDir,
|
|
"parameter_%s" % HOSTNAME)
|
|
if (os.path.exists(temp_dir)):
|
|
FileUtil.removeDirectory(temp_dir)
|
|
|
|
# extract parameter files to the temporary directory
|
|
self.logger.debug(
|
|
"Extract parameter files to the temporary directory.")
|
|
tarName = os.path.join(self.restoreDir, self.paraTarName)
|
|
if (not os.path.exists(tarName)):
|
|
if (g_ignoreMiss):
|
|
self.logger.error(ErrorCode.GAUSS_502[
|
|
"GAUSS_50201"]
|
|
% "parameter files")
|
|
sys.exit(0)
|
|
else:
|
|
raise Exception(ErrorCode.GAUSS_502[
|
|
"GAUSS_50201"] % "parameter files")
|
|
|
|
cmd = g_file.SHELL_CMD_DICT["decompressTarFile"] % (
|
|
self.restoreDir, tarName)
|
|
(status, output) = subprocess.getstatusoutput(cmd)
|
|
if (status != 0):
|
|
raise Exception(ErrorCode.GAUSS_514[
|
|
"GAUSS_51400"] % cmd
|
|
+ " Error: \n%s" % output)
|
|
|
|
# check hostname
|
|
self.logger.debug("Checking hostname.")
|
|
self.__checkHostName(
|
|
"%s/%s" % (temp_dir, self.hostnameFileName))
|
|
# check parameter
|
|
self.logger.debug("Checking parameter files.")
|
|
paraFileList = []
|
|
self.__checkParaFiles(temp_dir, paraFileList)
|
|
|
|
self.logger.debug("Restoring parameter files.")
|
|
paraFileNum = len(paraFileList)
|
|
for i in range(paraFileNum):
|
|
tarFileName, paraFilePath = paraFileList[i].split('|')
|
|
FileUtil.cpFile(os.path.join(temp_dir, tarFileName),
|
|
paraFilePath)
|
|
|
|
self.logger.debug("Remove the temporary directory.")
|
|
FileUtil.removeDirectory(temp_dir)
|
|
except Exception as e:
|
|
FileUtil.removeDirectory(temp_dir)
|
|
raise Exception(str(e))
|
|
self.logger.log("Successfully restored parameter files.")
|
|
|
|
self.logger.log("Successfully restored files.")
|
|
|
|
def decompressTarFile(self, flag):
|
|
"""
|
|
function: Decompress package on restore node
|
|
input : flag
|
|
output: NA
|
|
"""
|
|
tarFile = "%s/%s.tar" % (self.restoreDir, flag)
|
|
if (not os.path.exists(tarFile)):
|
|
return
|
|
# Decompress package on restore node
|
|
cmd = g_file.SHELL_CMD_DICT["decompressTarFile"] % (
|
|
self.restoreDir, tarFile)
|
|
(status, output) = subprocess.getstatusoutput(cmd)
|
|
if (status != 0):
|
|
raise Exception(ErrorCode.GAUSS_502[
|
|
"GAUSS_50217"] % tarFile
|
|
+ " Error: \n%s." % output
|
|
+ "The cmd is %s " % cmd)
|
|
|
|
def __checkHostName(self, hostnameFile):
|
|
"""
|
|
function: make sure the hostname stored in tar files
|
|
input : hostnameFile
|
|
output: NA
|
|
"""
|
|
# make sure the hostname stored in tar files
|
|
localHostName = NetUtil.GetHostIpOrName()
|
|
with open(hostnameFile, 'r') as self.__hostNameFile:
|
|
storedHostName = self.__hostNameFile.read()
|
|
storedHostName.strip('\n')
|
|
if (((localHostName > storedHostName) - (
|
|
localHostName < storedHostName)) != 0):
|
|
raise Exception(ErrorCode.GAUSS_516["GAUSS_51637"] % \
|
|
(("Local hostname [%s]",
|
|
"the hostname [%s] stored in tar files") % (
|
|
localHostName, storedHostName)))
|
|
|
|
def __checkParaFiles(self, temp_dir, paraFileList):
|
|
"""
|
|
function: check parameter file
|
|
input : temp_dir, paraFileList
|
|
output: NA
|
|
"""
|
|
storedParaFileNum = len(os.listdir(temp_dir)) - 1
|
|
for inst in self.dbNodeInfo.datanodes:
|
|
self.__checkSingleParaFile(inst, temp_dir, paraFileList)
|
|
if ((storedParaFileNum > len(paraFileList)) -
|
|
(storedParaFileNum < len(paraFileList))) != 0:
|
|
raise Exception(ErrorCode.GAUSS_516["GAUSS_51637"] %
|
|
("number of parameter files",
|
|
"the number of files requested"))
|
|
|
|
def __checkSingleParaFile(self, inst, temp_dir, paraFileList):
|
|
"""
|
|
function: check single parameter file
|
|
input : inst, temp_dir, paraFileList
|
|
output: NA
|
|
"""
|
|
# makesure instance exist
|
|
if (not os.path.exists(inst.datadir)):
|
|
if (g_ignoreMiss):
|
|
self.logger.log(
|
|
"Data directory [%s] of instance [%s]does not exist." % (
|
|
inst.datadir, str(inst)))
|
|
return
|
|
else:
|
|
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % \
|
|
("Data directory [%s] of instance [%s]" % (
|
|
inst.datadir, str(inst))))
|
|
# get all parameter file path into paraFileMap
|
|
paraFileMap = {}
|
|
if inst.instanceRole == DefaultValue.INSTANCE_ROLE_DATANODE:
|
|
paraFileMap[POSTGRESQL_CONF] = \
|
|
os.path.join(inst.datadir, POSTGRESQL_CONF)
|
|
paraFileMap[POSTGRESQL_HBA_CONF] = \
|
|
os.path.join(inst.datadir, POSTGRESQL_HBA_CONF)
|
|
else:
|
|
raise Exception(ErrorCode.GAUSS_516["GAUSS_51204"] % (
|
|
"specified", inst.instanceRole))
|
|
|
|
for key in paraFileMap:
|
|
backupFileName = "%d_%s" % (inst.instanceId, key)
|
|
if (not os.path.exists(os.path.join(temp_dir, backupFileName))):
|
|
if (g_ignoreMiss):
|
|
self.logger.log(
|
|
"The file of %s does not exist." % backupFileName)
|
|
return
|
|
else:
|
|
raise Exception(
|
|
ErrorCode.GAUSS_502["GAUSS_50201"] % backupFileName)
|
|
newRecord = "%s|%s" % (backupFileName, paraFileMap[key])
|
|
paraFileList.append(newRecord)
|
|
|
|
|
|
##############################################################################
|
|
# Help context. U:R:oC:v:
|
|
##############################################################################
|
|
def usage():
|
|
"""
|
|
function: usage
|
|
input : NA
|
|
output : NA
|
|
"""
|
|
print(
|
|
"Restore.py is a local utility to restore binary file "
|
|
"and parameter file.")
|
|
print(" ")
|
|
print("Usage:")
|
|
print("python3 Restore.py --help")
|
|
print(" ")
|
|
print("Common options:")
|
|
print(" -U the user of cluster.")
|
|
print(" -P, --position=RESTOREPATH the restore directory.")
|
|
print(" -p, --parameter restore parameter files.")
|
|
print(" -b, --binary_file restore binary files.")
|
|
print(" -i, --ingore_miss ignore Backup entity miss.")
|
|
print(" -s, --static_file static configuration files.")
|
|
print(" -l, --logpath=LOGPATH the log directory.")
|
|
print(" -h, --help show this help, then exit.")
|
|
print(" ")
|
|
|
|
|
|
def checkUserExist():
|
|
"""
|
|
function: check user exists
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
if (g_clusterUser == ""):
|
|
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % "U" + ".")
|
|
ClusterUser.checkUser(g_clusterUser, False)
|
|
|
|
|
|
def checkRestorePara(restorePara, restoreBin):
|
|
"""
|
|
function: check restore parameter
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
if not restorePara and not restoreBin:
|
|
GaussLog.exitWithError(
|
|
ErrorCode.GAUSS_500["GAUSS_50001"] % "p or -b" + ".")
|
|
|
|
|
|
def checkRestoreDir(restoreDir):
|
|
"""
|
|
function: check restore directory
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
if (restoreDir == ""):
|
|
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50001"] % "P" + ".")
|
|
|
|
|
|
def main():
|
|
"""
|
|
function: main function
|
|
input : NA
|
|
output: NA
|
|
"""
|
|
|
|
try:
|
|
opts, args = getopt.getopt(sys.argv[1:], "U:P:l:pbhifs:",
|
|
["position=", "parameter", "binary_file",
|
|
"logpath=", "help", "ingore_miss",
|
|
"force", "static_file="])
|
|
except getopt.GetoptError as e:
|
|
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % e.msg)
|
|
if (len(args) > 0):
|
|
GaussLog.exitWithError(
|
|
ErrorCode.GAUSS_500["GAUSS_50000"] % str(args[0]))
|
|
|
|
global g_clusterUser
|
|
global g_ignoreMiss
|
|
global g_staticFile
|
|
global g_forceRestore
|
|
restoreDir = ""
|
|
restorePara = False
|
|
restoreBin = False
|
|
logFile = ""
|
|
|
|
for key, value in opts:
|
|
if (key == "-h" or key == "--help"):
|
|
usage()
|
|
sys.exit(0)
|
|
elif (key == "-U"):
|
|
g_clusterUser = value.strip()
|
|
elif (key == "-P" or key == "--position"):
|
|
restoreDir = value.strip()
|
|
elif (key == "-p" or key == "--parameter"):
|
|
restorePara = True
|
|
elif (key == "-b" or key == "--binary_file"):
|
|
restoreBin = True
|
|
elif (key == "-i" or key == "--ingore_miss"):
|
|
g_ignoreMiss = True
|
|
elif (key == "-s" or key == "--static_file"):
|
|
g_staticFile = value.strip()
|
|
elif (key == "-l" or key == "--logpath"):
|
|
logFile = value
|
|
elif (key == "-f" or key == "--force"):
|
|
g_forceRestore = True
|
|
else:
|
|
GaussLog.exitWithError(ErrorCode.GAUSS_500["GAUSS_50000"] % value)
|
|
|
|
Parameter.checkParaVaild(key, value)
|
|
|
|
if (g_ignoreMiss):
|
|
gaussHome = EnvUtil.getEnv("GAUSSHOME")
|
|
if not gaussHome:
|
|
return
|
|
|
|
# check if user exist and is the right user
|
|
checkUserExist()
|
|
# check log file
|
|
logFile = ClusterLog.checkLogFile(logFile, g_clusterUser, "",
|
|
ClusterConstants.LOCAL_LOG_FILE)
|
|
# check -p and -b
|
|
checkRestorePara(restorePara, restoreBin)
|
|
# check -P
|
|
checkRestoreDir(restoreDir)
|
|
|
|
try:
|
|
LocalRestorer = LocalRestore(logFile, g_clusterUser, restoreDir,
|
|
restorePara, restoreBin)
|
|
LocalRestorer.run()
|
|
except Exception as e:
|
|
GaussLog.exitWithError(str(e))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|