!753 om支持ipv6相关

Merge pull request !753 from 赵晨/master
This commit is contained in:
opengauss_bot 2024-07-30 01:52:28 +00:00 committed by Gitee
commit 13258f0d95
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
31 changed files with 257 additions and 157 deletions

View File

@ -204,7 +204,7 @@ class CmdUtil(object):
if int(packet_size) != int(56):
opts = " -s " + str(packet_size)
return CmdUtil.findCmdInPath('ping') + BLANK_SPACE + host + " -c " + \
count + " -i " + interval + opts
count + " -i " + interval + opts
@staticmethod
def getWcCmd():

View File

@ -25,6 +25,7 @@ import subprocess
import sys
import _thread
import time
import ipaddress
from gspylib.common.ErrorCode import ErrorCode
from base_utils.os.cmd_util import CmdUtil
@ -84,6 +85,12 @@ g_lock = _thread.allocate_lock()
class NetUtil(object):
"""net util"""
ADDRESS_FAMILY_INDEX = 4
IP_ADDRESS_INDEX = 0
NET_IPV6 = "ipv6"
NET_IPV4 = "ipv4"
IPV4_SUBMASK_LEN = 32
IPV6_SUBMASK_LEN = 128
@staticmethod
def GetHostIpOrName():
@ -99,7 +106,11 @@ class NetUtil(object):
if host_ip is not None and NetUtil.isIpValid(host_ip):
return host_ip
try:
host_ip = socket.gethostbyname(socket.gethostname())
# Obtain the address of the local host
addr_info = socket.getaddrinfo(socket.gethostname(), None)
for info in addr_info:
# Extract IPv4 or IPv6 addresses from address information
host_ip = info[NetUtil.ADDRESS_FAMILY_INDEX][NetUtil.IP_ADDRESS_INDEX]
except Exception as e:
raise e
return host_ip
@ -126,13 +137,39 @@ class NetUtil(object):
input : String
output : bool
"""
Valid = re.match(r"^(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|"
r"[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9]["
r"0-9]"
r"|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9]"
r"[0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|"
r"[1-9][0-9]|[0-9])$", ip_address)
return Valid and Valid.group() == ip_address
try:
ipaddress.ip_address(ip_address)
return True
except ValueError:
return False
@staticmethod
def get_ip_version(hostname):
try:
ip = ipaddress.ip_address(hostname)
# If hostname is a valid IP address (both IPv4 and IPv6)
if(ip.version == 4):
return NetUtil.NET_IPV4
if(ip.version == 6):
return NetUtil.NET_IPV6
except ValueError:
# hostname may be a hostname or an unvalid ip
return ""
@staticmethod
def get_sockects(ip):
if NetUtil.get_ip_version(ip) == NetUtil.NET_IPV6:
sockets = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
sockets = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
return sockets
@staticmethod
def get_submask_len(ip_address):
if NetUtil.get_ip_version(ip_address) == NetUtil.NET_IPV6:
return NetUtil.IPV6_SUBMASK_LEN
else:
return NetUtil.IPV4_SUBMASK_LEN
@staticmethod
def executePingCmd(ip_address):
@ -163,12 +200,14 @@ class NetUtil(object):
return g_failed_address_list
@staticmethod
def getAllNetworkIp():
def getAllNetworkIp(ip_type=NET_IPV4):
"""
function: get All network ip
"""
if ip_type == "":
raise ValueError("This is the host name, not an ip")
network_info_list = []
mapping_list = NetUtil.getIpAddressAndNICList()
mapping_list = NetUtil.getIpAddressAndNICList(ip_type)
for onelist in mapping_list:
data = NetworkInfo()
# NIC number
@ -179,7 +218,7 @@ class NetUtil(object):
return network_info_list
@staticmethod
def getIpAddressAndNICList(ip_type="ipv4"):
def getIpAddressAndNICList(ip_type=NET_IPV4):
"""
function: get ip address and nicList
input: ip_type
@ -188,13 +227,13 @@ class NetUtil(object):
return list(NetUtil.getIpAddressAndNIC(ip_type))
@staticmethod
def getIpAddressAndNIC(ip_type="ipv4"):
def getIpAddressAndNIC(ip_type=NET_IPV4):
"""
function: get ip address and nic
input: ip_type
output: NA
"""
if ip_type == "ipv4":
if ip_type == NetUtil.NET_IPV4:
key = AF_INET
else:
key = AF_INET6
@ -261,13 +300,13 @@ class NetUtil(object):
return bond_info
@staticmethod
def getNetworkMaskByNICNum(network_card_num, ip_type="ipv4"):
def getNetworkMaskByNICNum(network_card_num, ip_type=NET_IPV4):
"""
function: get Network Mask By NICNum
input: network_card_num, ip_type
output: str
"""
if ip_type == "ipv4":
if ip_type == NetUtil.NET_IPV4:
return ifaddresses(network_card_num)[AF_INET][0]["netmask"]
else:
return ifaddresses(network_card_num)[AF_INET6][0]["netmask"]
@ -338,12 +377,14 @@ class NetUtil(object):
return network_conf_file
@staticmethod
def getAllNetworkInfo():
def getAllNetworkInfo(ip_type=NET_IPV4):
"""
function: get all network info
"""
if ip_type == "":
raise ValueError("This is the host name,not an ip.")
network_info_list = []
mapping_list = NetUtil.getIpAddressAndNICList()
mapping_list = NetUtil.getIpAddressAndNICList(ip_type)
for one_list in mapping_list:
data = NetworkInfo()
# NIC number
@ -361,7 +402,7 @@ class NetUtil(object):
# network mask
try:
data.networkMask = NetUtil.getNetworkMaskByNICNum(
data.NICNum)
data.NICNum,ip_type)
except Exception:
data.networkMask = ""
@ -425,7 +466,10 @@ class NetUtil(object):
if host_ip is not None:
if NetUtil.isIpValid(host_ip):
return host_ip
host_ip = socket.gethostbyname(socket.gethostname())
addr_info = socket.getaddrinfo(socket.gethostname(), None)
for info in addr_info:
# 从地址信息中提取 IPv4 或 IPv6 地址
host_ip = info[NetUtil.ADDRESS_FAMILY_INDEX][NetUtil.IP_ADDRESS_INDEX]
except Exception as e:
raise Exception(str(e))
return host_ip
@ -507,10 +551,10 @@ class NetUtil(object):
net_work_num = ""
net_work_info = psutil.net_if_addrs()
for nic_num in list(net_work_info.keys()):
net_info = net_work_info[nic_num][0]
if net_info.address == ip_address:
net_work_num = nic_num
break
for net_info in net_work_info[nic_num]:
if net_info.address == ip_address:
net_work_num = nic_num
break
if net_work_num == "":
raise Exception(ErrorCode.GAUSS_506["GAUSS_50604"] % ip_address)
return net_work_num

View File

@ -27,7 +27,11 @@ class SecurityChecker(object):
INJECTION_CHAR_LIST = ["|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", "{", "}", "(", ")",
"[", "]", "~", "*", "?", " ", "!", "\n"]
PWD_VALIDATION_PATTERN = r'^[A-Za-z0-9~!@#%^*\-_=+?,]+$'
IP_PATTERN = r'^((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)$'
IP_PATTERN = (
r'^(?:(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)(?:\.(?!$)|$)){4}$|' # IPv4
r'^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|' # IPv6 full
r'^::1$|^::$' # IPv6 loopback and unspecified
)
@staticmethod
def check_injection_char(check_value):

View File

@ -473,7 +473,11 @@ def get_localhost_name():
def get_localhost_ip():
return socket.gethostbyname(get_localhost_name())
addr_info = socket.getaddrinfo(get_localhost_name(), None)
for info in addr_info:
# 从地址信息中提取 IPv4 或 IPv6 地址
host_ip = info[NetUtil.ADDRESS_FAMILY_INDEX][NetUtil.IP_ADDRESS_INDEX]
return host_ip
class PriStandbyIpStatus(TemplateStatus):

View File

@ -108,7 +108,7 @@ General options:
"""
try:
for name in node_names:
socket.gethostbyname(name)
addr_info = socket.getaddrinfo(name, None)
except Exception as e:
GaussLog.printMessage(
"[Warning] Name or service not known on node [%s]." % name)

View File

@ -499,6 +499,7 @@ General options:
" Error: \nThe /etc/hosts does not exist.")
(status, output) = GrepUtil.getGrepValue("-v", " #Gauss.* IP Hosts Mapping",
'/etc/hosts')
result["::1"] = "localhost"
result["127.0.0.1"] = "localhost"
FileUtil.createFile(tmpHostIpName)
FileUtil.changeMode(DefaultValue.KEY_FILE_MODE, tmpHostIpName)
@ -1057,10 +1058,14 @@ General options:
input : hostname
output: NA
'''
bashrc_file = os.path.join(pwd.getpwuid(os.getuid()).pw_dir, ".bashrc")
cmd = 'source %s;%s;' % (bashrc_file, SYSTEM_SSH_ENV)
hostIp = hostname
if NetUtil.get_ip_version(hostname) == NetUtil.NET_IPV6:
hostIp = f"[{hostname}]"
cmd += ('/usr/bin/scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' +
'%s %s %s:.ssh/' % (self.id_rsa_fname, self.id_rsa_pub_fname, hostname))
'%s %s %s:.ssh/' % (self.id_rsa_fname, self.id_rsa_pub_fname, hostIp))
cmd += ''' && temp_auth=$(grep '#OM' %s)''' \
''' && /usr/bin/ssh %s "sed -i '/#OM/d' %s; echo '${temp_auth}' >> %s"''' % (
self.authorized_keys_fname, hostname, self.authorized_keys_fname,
@ -1198,8 +1203,9 @@ General options:
input : hostname
output: NA
"""
ipadds = DefaultValue.get_local_ips(NetUtil.get_ip_version(hostname))
if (hostname == self.localHost or
hostname in DefaultValue.get_local_ips()):
hostname in ipadds):
return
bashrc_file = os.path.join(pwd.getpwuid(os.getuid()).pw_dir, ".bashrc")
@ -1345,8 +1351,9 @@ General options:
localDirPath = os.path.dirname(os.path.realpath(__file__))
shell_file = os.path.join(localDirPath, "./local/ssh-agent.sh")
for ip in ips:
ipadds = DefaultValue.get_local_ips(NetUtil.get_ip_version(ip))
if (ip == self.localHost or
ip in DefaultValue.get_local_ips()):
ip in ipadds):
continue
session = self.get_ssh_session(ip)
DefaultValue.register_remote_ssh_agent(session, ip, self.logger)
@ -1400,9 +1407,13 @@ General options:
def copy_shell_to_remote_node(self, shell_file, hostname):
# scp ssh_protect to remote node
if NetUtil.get_ip_version(hostname) == NetUtil.NET_IPV6:
hostname = f"[{hostname}]"
cmd = 'source ~/.bashrc;%s;' % SYSTEM_SSH_ENV
cmd += ('/usr/bin/scp -q -o "BatchMode yes" -o "NumberOfPasswordPrompts 0" ' + '%s %s:.ssh/' % (
shell_file, hostname))
(status, output) = subprocess.getstatusoutput(cmd)
if status != 0:
raise Exception(

View File

@ -721,8 +721,10 @@ class DefaultValue():
return hostIp
# get local host by os function
hostIp = socket.gethostbyname(hostname)
addr_info = socket.getaddrinfo(hostname, None)
for info in addr_info:
# Extract IPv4 or IPv6 addresses from address information
hostIp = info[NetUtil.ADDRESS_FAMILY_INDEX][NetUtil.IP_ADDRESS_INDEX]
# due to two loopback address in ubuntu, 127.0.1.1 are choosed by hostname.
# there is need to choose 127.0.0.1
version = LinuxDistro.linux_distribution()[1].split('/')[0]
@ -2558,13 +2560,13 @@ class DefaultValue():
return instances
@staticmethod
def get_local_ips():
def get_local_ips(ip_type):
"""
get local node all ips
:return:
"""
# eg "ip_mappings: [('lo', '127.0.0.1'), ('eth1', '10.10.10.10')]"
ip_mappings = NetUtil.getIpAddressAndNICList()
ip_mappings = NetUtil.getIpAddressAndNICList(ip_type)
local_ips = []
for ip_info in ip_mappings:
local_ips.append(ip_info[1])

View File

@ -29,7 +29,7 @@ import pwd
import copy
import socket
import json
import ipaddress
sys.path.append(os.path.split(os.path.realpath(__file__))[0] + "/../../")
from gspylib.common.ErrorCode import ErrorCode
@ -2950,7 +2950,7 @@ class dbClusterInfo():
ipConfigItem, ipCheckMap[ipConfigItem])
raise Exception(ErrorCode.GAUSS_512["GAUSS_51220"] % (
"with cm and etcd") + errMsg)
# create a dictionary
# create a dictionary
nodeipport[dbNode.name] = [nodeips, nodeports]
# check port and ip
self.__checkPortandIP(nodeips, nodeports, dbNode.name)
@ -3951,19 +3951,10 @@ class dbClusterInfo():
input : String
output : NA
"""
IpValid = re.match(
"^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|["
"1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|["
"1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{"
"1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}["
"0-9]{1}|[0-9])$",
ip)
if IpValid:
if (IpValid.group() == ip):
return True
else:
return False
else:
try:
ipaddress.ip_address(ip)
return True
except ValueError:
return False
def __isPortValid(self, port):
@ -4639,7 +4630,8 @@ class dbClusterInfo():
"The cmd is %s" % cmd)
output_list = self.__getStatusByOM(user)
output_num = 0
pattern = re.compile("(\d+) (.*) (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*)")
# The purpose of this regular expression is to match text lines containing IPv4 or IPv6 addresses.
pattern = re.compile(r'(\d+) (.*) ((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[0-9a-fA-F]{0,4}(?::[0-9a-fA-F]{0,4})*::?(?:[0-9a-fA-F]{0,4})?)) (.*)')
if not self.hasNoCm():
output_list = [i for i in output_list if i]
output_list = output_list[-1].split('|')

View File

@ -114,6 +114,7 @@ class ErrorCode():
###########################################################################
# permission
###########################################################################
GAUSS_501 = {
'GAUSS_50100': "[GAUSS-50100] : The %s is not readable for %s.",
'GAUSS_50101': "[GAUSS-50101] : The %s is not executable for %s.",

View File

@ -128,7 +128,7 @@ class BaseComponent(object):
# Verify that the port is occupied
for ip in ipList:
sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sk = NetUtil.get_sockects(ip)
sk.settimeout(TIME_OUT)
# retry port 4 times

View File

@ -34,10 +34,15 @@ from domain_utils.cluster_os.cluster_user import ClusterUser
from base_utils.os.grep_util import GrepUtil
from base_utils.os.user_util import UserUtil
from gspylib.component.DSS.dss_checker import DssConfig
from base_utils.os.net_util import NetUtil
METHOD_TRUST = "trust"
METHOD_SHA = "sha256"
# IPv4 subnet mask length
IPv4_SUBNET_MASK_LENGTH = 32
# IPv6 subnet mask length
IPv6_SUBNET_MASK_LENGTH = 128
MAX_PARA_NUMBER = 1000
INSTANCE_TYPE_UNDEFINED = -1
MASTER_INSTANCE = 0
@ -495,25 +500,26 @@ class DN_OLAP(Kernel):
pg_user = ClusterUser.get_pg_user()
for ip_address in clusterAllIpList:
i += 1
subnet_length = NetUtil.get_submask_len(ip_address)
# Set the initial user and initial database access permissions
if principal is None:
if ip_address.startswith("floatIp"):
guc_paras_str += "-h \"host all all %s/32 %s\" " % \
(float_ips[ip_address], METHOD_SHA)
guc_paras_str += "-h \"host all all %s/%s %s\" " % \
(float_ips[ip_address], subnet_length, METHOD_SHA)
else:
guc_paras_str += "-h \"host all %s %s/32 %s\" " % \
(pg_user, ip_address, METHOD_TRUST)
guc_paras_str += "-h \"host all all %s/32 %s\" " % \
(ip_address, METHOD_SHA)
guc_paras_str += "-h \"host all %s %s/%s %s\" " % \
(pg_user, ip_address, subnet_length, METHOD_TRUST)
guc_paras_str += "-h \"host all all %s/%s %s\" " % \
(ip_address, subnet_length, METHOD_SHA)
else:
if ip_address.startswith("floatIp"):
guc_paras_str += "-h \"host all all %s/32 %s\" " % \
(float_ips[ip_address], METHOD_SHA)
guc_paras_str += "-h \"host all all %s/%s %s\" " % \
(float_ips[ip_address], subnet_length, METHOD_SHA)
else:
guc_paras_str += "-h \"host all %s %s/32 gss include_realm=1 " \
" krb_realm=%s\" " % (pg_user, ip_address, principal)
guc_paras_str += "-h \"host all all %s/32 %s\" " % \
(ip_address, METHOD_SHA)
guc_paras_str += "-h \"host all %s %s/%s gss include_realm=1 " \
" krb_realm=%s\" " % (pg_user, ip_address, subnet_length, principal)
guc_paras_str += "-h \"host all all %s/%s %s\" " % \
(ip_address, subnet_length, METHOD_SHA)
if (i % MAX_PARA_NUMBER == 0):
GUCParasStrList.append(guc_paras_str)
i = 0
@ -522,10 +528,11 @@ class DN_OLAP(Kernel):
streaming_dn_ips = self.get_streaming_relate_dn_ips(self.instInfo)
if streaming_dn_ips:
for dn_ip in streaming_dn_ips:
guc_paras_str += "-h \"host all %s %s/32 %s\" " \
% (pg_user, dn_ip, METHOD_TRUST)
guc_paras_str += "-h \"host all all %s/32 %s\" " \
% (dn_ip, METHOD_SHA)
subnet_length = NetUtil.get_submask_len(ip_address)
guc_paras_str += "-h \"host all %s %s/%s %s\" " \
% (pg_user, dn_ip, subnet_length, METHOD_TRUST)
guc_paras_str += "-h \"host all all %s/%s %s\" " \
% (dn_ip, subnet_length, METHOD_SHA)
ip_segment = '.'.join(dn_ip.split('.')[:2]) + ".0.0/16"
guc_paras_str += "-h \"host replication all %s sha256\" " % ip_segment

View File

@ -32,6 +32,7 @@ from base_utils.os.env_util import EnvUtil
from base_utils.os.file_util import FileUtil
from base_utils.security.security_checker import SecurityChecker
from domain_utils.cluster_os.cluster_user import ClusterUser
from base_utils.os.net_util import NetUtil
MAX_PARA_NUMBER = 1000
@ -531,8 +532,9 @@ class Kernel(BaseComponent):
pg_user = ClusterUser.get_pg_user()
for ipAddress in ipAddressList:
i += 1
GUCParasStr += " -h \"host all all %s/32\"" % (ipAddress)
GUCParasStr += " -h \"host all %s %s/32\"" % (pg_user, ipAddress)
submask_length = NetUtil.get_submask_len(ipAddress)
GUCParasStr += " -h \"host all all %s/%s\"" % (ipAddress, submask_length)
GUCParasStr += " -h \"host all %s %s/%s\"" % (pg_user, ipAddress, submask_length)
if i * 2 % MAX_PARA_NUMBER == 0:
GUCParasStrList.append(GUCParasStr)
i = 0

View File

@ -451,6 +451,8 @@ def sendFile(fileName, host, user, path, passwd=None):
t.close()
else:
if "HOST_IP" not in list(os.environ.keys()):
if NetUtil.get_ip_version(host) == NetUtil.NET_IPV6:
host = "[" + host + "]"
host = "%s@%s" % (user, host)
cmd = "pscp -H %s '%s' %s" % (host, fileName, path)
if (os.getuid() == 0):
@ -581,7 +583,7 @@ def is_local_node(host):
"""
if (host == NetUtil.GetHostIpOrName()):
return True
allNetworkInfo = NetUtil.getAllNetworkIp()
allNetworkInfo = NetUtil.getAllNetworkIp(NetUtil.get_ip_version(host))
for network in allNetworkInfo:
if (host == network.ipAddress):
return True

View File

@ -41,7 +41,7 @@ class CheckBond(BaseItem):
serviceIP = self.ipAddr
else:
serviceIP = SharedFuncs.getIpByHostName(self.host)
networkCards = NetUtil.getAllNetworkInfo()
networkCards = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(serviceIP))
for network in networkCards:
if (network.ipAddress == serviceIP):
networkCardNum = network.NICNum

View File

@ -53,7 +53,7 @@ class CheckMTU(BaseItem):
else:
backIP = SharedFuncs.getIpByHostName(self.host)
# Get the network card number
networkCards = NetUtil.getAllNetworkInfo()
networkCards = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(backIP))
for network in networkCards:
if network.ipAddress == backIP:
networkCardNum = network.NICNum

View File

@ -48,7 +48,7 @@ class CheckMultiQueue(BaseItem):
else:
backIP = SharedFuncs.getIpByHostName(self.host)
# Get the network card number
allNetworkInfo = NetUtil.getAllNetworkInfo()
allNetworkInfo = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(backIP))
for network in allNetworkInfo:
if network.ipAddress == backIP:
networkNum = network.NICNum

View File

@ -197,7 +197,8 @@ class CheckNetSpeed(BaseItem):
global MaxDelayFailFlag
network_card_num = ""
serviceIP = SharedFuncs.getIpByHostName(self.host)
for network in NetUtil.getAllNetworkInfo():
allNetworkInfo = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(serviceIP))
for network in allNetworkInfo:
if (network.ipAddress == serviceIP):
network_card_num = network.NICNum
break

View File

@ -37,8 +37,7 @@ class CheckRXTX(BaseItem):
backIP = LocalNodeInfo.backIps[0]
else:
backIP = SharedFuncs.getIpByHostName(self.host)
allNetworkInfo = NetUtil.getAllNetworkInfo()
allNetworkInfo = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(backIp))
for network in allNetworkInfo:
if (network.ipAddress == backIP):
networkNum = network.NICNum

View File

@ -20,6 +20,8 @@ import socket
import time
import sys
from base_utils.os.net_util import NetUtil
listen_ip = "localhost"
listen_port = 31111
run_mode = 0 # 0:connect, 1:send, 2:recv
@ -29,7 +31,7 @@ def send_main():
global listen_ip
global listen_port
buf = "this is a test !" * 512 # buf 8192 block
sockets = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockets = NetUtil.get_sockects(listen_ip)
sockets.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
print(listen_ip+":"+listen_port)
while(sockets.connect_ex((listen_ip, int(listen_port))) != 0):
@ -52,7 +54,7 @@ def recv_main():
try:
global listen_ip
global listen_port
sockets = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockets = NetUtil.get_sockects(listen_ip)
sockets.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
sockets.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sockets.bind((listen_ip, int(listen_port)))
@ -69,9 +71,8 @@ def recv_main():
break
except Exception as e:
print(str(e))
def connect_main():
sockets = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockets = NetUtil.get_sockects(listen_ip)
sockets.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
if sockets.connect_ex((listen_ip, int(listen_port))) != 0:
print("Failed to connect %s:%d on %s mode:%m.\n",

View File

@ -76,26 +76,23 @@ class Network():
"""
function: Init the Network options
"""
NET_IPV6 = "ipv6"
NET_IPV4 = "ipv4"
def __init__(self):
pass
def isIpValid(self, ipAddress):
def isIpValid(self, ip_address):
"""
function : check if the input ip address is valid
input : String
output : bool
"""
Valid = re.match("^(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|"
"[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9]["
"0-9]"
"|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9]"
"[0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|"
"[1-9][0-9]|[0-9])$", ipAddress)
if (Valid):
if (Valid.group() == ipAddress):
return True
return False
try:
ipaddress.ip_address(ip_address)
return True
except ValueError:
return False
def executePingCmd(self, ipAddress):
"""
@ -123,12 +120,14 @@ class Network():
parallelTool.parallelExecute(self.executePingCmd, ipAddressList)
return g_failedAddressList
def getAllNetworkIp(self):
def getAllNetworkIp(self,ip_type=NET_IPV4):
"""
function: get All network ip
"""
if ip_type == "":
raise ValueError("This is the host name,not an ip.")
networkInfoList = []
mappingList = g_Platform.getIpAddressAndNICList()
mappingList = g_Platform.getIpAddressAndNICList(ip_type)
for onelist in mappingList:
data = networkInfo()
# NIC number
@ -144,12 +143,14 @@ class Network():
"""
return psutil.net_if_stats()[networkCardNum].mtu
def getAllNetworkInfo(self):
def getAllNetworkInfo(self,ip_type=NET_IPV4):
"""
function: get all network info
"""
if ip_type == "":
raise ValueError("This is the host name,not an ip.")
networkInfoList = []
mappingList = g_Platform.getIpAddressAndNICList()
mappingList = g_Platform.getIpAddressAndNICList(ip_type)
for oneList in mappingList:
data = networkInfo()
# NIC number
@ -167,7 +168,7 @@ class Network():
# network mask
try:
data.networkMask = g_Platform.getNetworkMaskByNICNum(
data.NICNum)
data.NICNum,ip_type)
except Exception:
data.networkMask = ""

View File

@ -46,6 +46,8 @@ from netifaces import interfaces, ifaddresses, AF_INET, AF_INET6
ISCONFIGURETRUE = "# isConfigure = TRUE"
ISCONFIGUREFALSE = "# isConfigure = FALSE"
SESSIONTIMEOUT = 300
NET_IPV6 = "ipv6"
NET_IPV4 = "ipv4"
# ---------------platforms--------------------
# global variable for our platform
@ -687,7 +689,7 @@ class GenericPlatform:
if int(packetSize) != int(56):
opts = " -s " + str(packetSize)
return findCmdInPath('ping') + BLANK_SPACE + host + " -c " + \
count + " -i " + interval + opts
count + " -i " + interval + opts
def getWcCmd(self):
"""
@ -1361,13 +1363,13 @@ class LinuxPlatform(GenericPlatform):
else:
return True
def getIpAddressAndNIC(self, ipType="ipv4"):
def getIpAddressAndNIC(self, ipType=NET_IPV4):
"""
function: get ip address and nic
input: ipType
output: NA
"""
if ipType == "ipv4":
if ipType == NET_IPV4:
key = AF_INET
else:
key = AF_INET6
@ -1377,7 +1379,7 @@ class LinuxPlatform(GenericPlatform):
ipAddress = ifaddresses(iface)[key][0]['addr']
yield (iface, ipAddress)
def getIpAddressAndNICList(self, ipType="ipv4"):
def getIpAddressAndNICList(self, ipType=NET_IPV4):
"""
function: get ip address and nicList
input: ipType
@ -1385,7 +1387,7 @@ class LinuxPlatform(GenericPlatform):
"""
return list(self.getIpAddressAndNIC(ipType))
def getNetworkNumByIPAddr(self, ipAddress, ipType="ipv4"):
def getNetworkNumByIPAddr(self, ipAddress, ipType=NET_IPV4):
"""
function: get netWork num by IP addr
input: ipAddress, ipType
@ -1482,13 +1484,13 @@ class LinuxPlatform(GenericPlatform):
bondInfo = "BondMode Null"
return bondInfo
def getNetworkMaskByNICNum(self, networkCardNum, ipType="ipv4"):
def getNetworkMaskByNICNum(self, networkCardNum, ipType=NET_IPV4):
"""
function: get Network Mask By NICNum
input: networkCardNum, ipType
output: str
"""
if ipType == "ipv4":
if ipType == NET_IPV4:
return ifaddresses(networkCardNum)[AF_INET][0]["netmask"]
else:
return ifaddresses(networkCardNum)[AF_INET6][0]["netmask"]

View File

@ -737,7 +737,7 @@ class SshTool():
outputCollect = ""
localMode = False
resultMap = {}
ssh_hosts = []
sshHosts = []
if hostList is None:
hostList = []
try:
@ -758,19 +758,24 @@ class SshTool():
pscppre = "python3 %s/script/gspylib/pssh/bin/pscp" % gp_home
if len(hostList) == 0:
ssh_hosts = copy.deepcopy(self.hostNames)
hostSshs = copy.deepcopy(self.hostNames)
else:
ssh_hosts = copy.deepcopy(hostList)
if len(ssh_hosts) == 1 and ssh_hosts[0] == socket.gethostname() and \
hostSshs = copy.deepcopy(hostList)
for host in hostSshs:
if NetUtil.get_ip_version(host) == NetUtil.NET_IPV6:
sshHosts.append("[" + host + "]")
else:
sshHosts.append(host)
if len(sshHosts) == 1 and sshHosts[0] == socket.gethostname() and \
srcFile != targetDir and \
srcFile != os.path.join(targetDir, os.path.split(srcFile)[1]):
localMode = True
scpCmd = "cp -r %s %s" % (srcFile, targetDir)
else:
# cp file on local node
if socket.gethostname() in ssh_hosts:
localhost_idx = ssh_hosts.index(socket.gethostname())
ssh_hosts.pop(localhost_idx)
if socket.gethostname() in sshHosts:
localhost_idx = sshHosts.index(socket.gethostname())
sshHosts.pop(localhost_idx)
cpcmd = "cp -r %s %s" % (srcFile, targetDir)
if srcFile != targetDir and srcFile != os.path.join(targetDir, os.path.basename(srcFile)):
(status, output) = subprocess.getstatusoutput(cpcmd)
@ -778,12 +783,12 @@ class SshTool():
resultMap[socket.gethostname()] = DefaultValue.SUCCESS
else:
resultMap[socket.gethostname()] = DefaultValue.FAILURE
if not ssh_hosts:
if not sshHosts:
return
scpCmd = "%s -r -v -t %s -p %s -H %s -o %s -e %s %s %s" \
" 2>&1 | tee %s" % (pscppre, self.__timeout,
parallel_num,
" -H ".join(ssh_hosts),
" -H ".join(sshHosts),
self.__outputPath,
self.__errorPath, srcFile,
targetDir, self.__resultFile)
@ -811,38 +816,38 @@ class SshTool():
if localMode:
dir_permission = 0o700
if status == 0:
resultMap[ssh_hosts[0]] = DefaultValue.SUCCESS
outputCollect = "[%s] %s:\n%s" % ("SUCCESS", ssh_hosts[0],
resultMap[sshHosts[0]] = DefaultValue.SUCCESS
outputCollect = "[%s] %s:\n%s" % ("SUCCESS", sshHosts[0],
SensitiveMask.mask_pwd(output))
if not os.path.exists(self.__outputPath):
os.makedirs(self.__outputPath, mode=dir_permission)
file_path = os.path.join(self.__outputPath, ssh_hosts[0])
file_path = os.path.join(self.__outputPath, sshHosts[0])
FileUtil.createFileInSafeMode(file_path)
with open(file_path, "w") as fp:
fp.write(SensitiveMask.mask_pwd(output))
fp.flush()
fp.close()
else:
resultMap[ssh_hosts[0]] = DefaultValue.FAILURE
outputCollect = "[%s] %s:\n%s" % ("FAILURE", ssh_hosts[0],
resultMap[sshHosts[0]] = DefaultValue.FAILURE
outputCollect = "[%s] %s:\n%s" % ("FAILURE", sshHosts[0],
SensitiveMask.mask_pwd(output))
if not os.path.exists(self.__errorPath):
os.makedirs(self.__errorPath, mode=dir_permission)
file_path = os.path.join(self.__errorPath, ssh_hosts[0])
file_path = os.path.join(self.__errorPath, sshHosts[0])
FileUtil.createFileInSafeMode(file_path)
with open(file_path, "w") as fp:
fp.write(SensitiveMask.mask_pwd(output))
fp.flush()
fp.close()
else:
resultMap, outputCollect = self.parseSshResult(ssh_hosts)
resultMap, outputCollect = self.parseSshResult(sshHosts)
except Exception as e:
self.clenSshResultFiles()
raise Exception(str(e))
for host in ssh_hosts:
for host in sshHosts:
if resultMap.get(host) != DefaultValue.SUCCESS:
raise Exception(ErrorCode.GAUSS_502["GAUSS_50216"]
% ("file [%s]" % srcFile) +

View File

@ -344,9 +344,10 @@ class DoradoDisasterRecoveryBase(StreamingBase):
remote_ips = self.__get_remote_ips()
for remote_ip in remote_ips:
submask_length = NetUtil.get_submask_len(remote_ip)
cmd = "source %s ; gs_guc set -Z datanode -N all -I all -h " \
"\"host all all %s/32 trust\"" \
% (self.mpp_file, remote_ip)
"\"host all all %s/%s trust\"" \
% (self.mpp_file, remote_ip, submask_length)
self.logger.debug("Update pg_hba.conf with cmd: %s" % cmd)
status, output = CmdUtil.retryGetstatusoutput(cmd)
if status != 0:

View File

@ -660,15 +660,22 @@ class OperCommon:
self.logger.log(
"[gs_dropnode]Start of set pg_hba config file on %s." % host)
cmd = 'source %s;' % envProfile
if len(pgHbaValue):
if not flagRollback:
for i in pgHbaValue[:-1].split('|'):
v = i[0:i.find('/32') + 3]
cmd += "gs_guc set -N %s -I all -h '%s';" % (host, v)
if flagRollback:
for i in pgHbaValue[:-1].split('|'):
cmd += "gs_guc set -N %s -I all -h '%s';" \
% (host, i.strip())
ip_entries = pgHbaValue[:-1].split('|')
for entry in ip_entries:
entry = entry.strip()
if not flagRollback:
if NetUtil.get_ip_version(entry) == NetUtil.NET_IPV4:
v = entry[0:entry.find('/32') + 3]
cmd += "gs_guc set -N %s -I all -h '%s';" % (host, v)
elif NetUtil.get_ip_version(entry) == NetUtil.NET_IPV6:
v = entry[0:entry.find('/128') + 4]
cmd += "gs_guc set -N %s -I all -h '%s';" % (host, v)
elif NetUtil.get_ip_version(entry) == "":
raise ValueError(f"Invalid IP address format: {entry}")
else:
cmd += "gs_guc set -N %s -I all -h '%s';" % (host, entry)
(status, output) = subprocess.getstatusoutput(cmd)
result_v = re.findall(r'Failed instances: (\d)\.', output)
if status:

View File

@ -48,6 +48,7 @@ from domain_utils.cluster_file.cluster_dir import ClusterDir
from base_utils.os.env_util import EnvUtil
from base_utils.os.cmd_util import CmdUtil
from domain_utils.cluster_file.version_info import VersionInfo
from base_utils.os.net_util import NetUtil
#boot/build mode
MODE_PRIMARY = "primary"
@ -566,9 +567,9 @@ class ExpansionImpl():
GaussLog.exitWithError(ErrorCode.GAUSS_516["GAUSS_51600"])
instances = re.split('(?:\|)|(?:\n)', outputCollect)
self.existingHosts = []
pattern = re.compile('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*')
pattern_ip = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b|\b(?:[0-9a-fA-F]{0,4}(?::[0-9a-fA-F]{0,4}){7})\b')
for inst in instances:
existing_hosts_ip = pattern.findall(inst)
existing_hosts_ip = pattern_ip.findall(inst)
if len(existing_hosts_ip) != 0:
self.existingHosts.append(existing_hosts_ip[0])
self.existingHosts = list(set(self.existingHosts))
@ -585,11 +586,11 @@ class ExpansionImpl():
primaryHost = self.getPrimaryHostName()
result = self.commonGsCtl.queryOmCluster(primaryHost, self.envFile)
instances = re.split('(?:\|)|(?:\n)', result)
pattern = re.compile('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*')
pattern_ip = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b|\b(?:[0-9a-fA-F]{0,4}(?::[0-9a-fA-F]{0,4}){7})\b')
host_ip = []
for ins in instances:
if re.findall('Primary', ins):
host_ip = pattern.findall(ins)
host_ip = pattern_ip.findall(ins)
break
# update xml config node info
primary_host_ip = "".join(host_ip)
@ -670,8 +671,10 @@ gs_guc set -D {dn} -c "available_zone='{azName}'"
node = self.context.clusterInfo.getDbNodeByName(name)
for inst in node.datanodes:
for float_ip in inst.float_ips:
cmd += " -h 'host all all %s/32 sha256'" % \
self.context.clusterInfo.float_ips[float_ip]
ip_address = self.context.clusterInfo.float_ips[float_ip]
# Check whether the IP address is ipv4 or ipv6 and obtain the corresponding mask length
submask_length = NetUtil.get_submask_len(ip_address)
cmd += " -h 'host all all %s/%s sha256'" % (ip_address, submask_length)
return cmd
def addTrust(self):
@ -690,12 +693,14 @@ gs_guc set -D {dn} -c "available_zone='{azName}'"
cmd = "source %s;gs_guc set -D %s" % (self.envFile, dataNode)
if hostExec in self.existingHosts:
for hostParam in self.context.newHostList:
cmd += " -h 'host all %s %s/32 trust'" % (self.user, hostParam)
submask_length = NetUtil.get_submask_len(hostParam)
cmd += " -h 'host all %s %s/%s trust'" % (self.user, hostParam, submask_length)
cmd += self.get_add_float_ip_cmd(hostParam)
else:
for hostParam in allHosts:
if hostExec != hostParam:
cmd += " -h 'host all %s %s/32 trust'" % (self.user, hostParam)
submask_length = NetUtil.get_submask_len(hostParam)
cmd += " -h 'host all %s %s/%s trust'" % (self.user, hostParam, submask_length)
cmd += self.get_add_float_ip_cmd(hostParam)
self.logger.debug("[%s] trustCmd:%s" % (hostExec, cmd))
sshTool = SshTool([hostExec])
@ -1391,8 +1396,12 @@ remoteservice={remoteservice}'"\
"""
backips = self.context.newHostList
for backip in backips:
ck_net_delay = "ping -s 8192 -c 5 -i 0.3 %s | "\
"awk -F / '{print $5}'| awk '{print $1}'" % backip
if NetUtil.get_ip_version(backip) == NetUtil.NET_IPV6:
ck_net_delay = "ping6 -s 8192 -c 5 -i 0.3 %s | " \
"awk -F / '{print $5}'| awk '{print $1}'" % backip
else:
ck_net_delay = "ping -s 8192 -c 5 -i 0.3 %s | "\
"awk -F / '{print $5}'| awk '{print $1}'" % backip
(status, output) = subprocess.getstatusoutput(ck_net_delay)
if status == 0:
try:
@ -1467,7 +1476,7 @@ remoteservice={remoteservice}'"\
nodeStates = re.split('(?:\|)|(?:\n)', allNodesState)
dataNodes = {}
for nodeState in nodeStates:
pattern = re.compile("[ ]+[^ ]+[ ]+(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[ ]+[^ ]+[ ]+[^ ]+[ ]+([^ ]+)[ ]+")
pattern = re.compile(r"[ ]+[^ ]+[ ]+((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[0-9a-fA-F:]+))[ ]+[^ ]+[ ]+[^ ]+[ ]+([^ ]+)[ ]+")
result = pattern.findall(nodeState)
if len(result) != 0:
result = result[0]

View File

@ -421,14 +421,17 @@ class ExpansionImplWithCm(ExpansionImpl):
"""
Config pg_hba.conf on new nodes
"""
new_node = self._get_new_node_by_back_ip(host_ip)
new_inst = new_node.datanodes[0]
cmd = "source {0};gs_guc set -D {1}".format(self.envFile, new_inst.datadir)
cmd += " -h 'host all %s %s/32 trust'" % (self.user, host_ip)
cmd += " -h 'host all all %s/32 sha256'" % host_ip
submask_length = NetUtil.get_submask_len(host_ip)
cmd += " -h 'host all %s %s/%s trust'" % (self.user, host_ip, submask_length)
cmd += " -h 'host all all %s/%s sha256'" % (host_ip, submask_length)
if self.xml_cluster_info.float_ips:
cmd += " -h 'host all all %s/32 sha256'" % \
self.xml_cluster_info.float_ips[new_inst.float_ips[0]]
submask_length = NetUtil.get_submask_len(self.xml_cluster_info.float_ips[new_inst.float_ips[0]])
cmd += " -h 'host all all %s/%s sha256'" % \
(self.xml_cluster_info.float_ips[new_inst.float_ips[0]], submask_length)
self.logger.log("Ready to perform command on node [{0}]. "
"Command is : {1}".format(new_node.name, cmd))
CmdExecutor.execCommandWithMode(cmd, self.ssh_tool, host_list=[new_node.name])

View File

@ -170,6 +170,7 @@ class PreinstallImpl:
if self.context.localMode or self.context.isSingle:
if not self.context.skipHostnameSet:
self.writeLocalHosts({"127.0.0.1": "localhost"})
self.writeLocalHosts({"::1": "localhost"})
return
try:
# save the sshIps

View File

@ -954,10 +954,10 @@ class StreamingBase(object):
if output.count(f"replconninfo{idx}=''") >= 2:
continue
ret = re.search(
r"replconninfo%s='localhost=(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})"
r"replconninfo%s='localhost=((?:[0-9]{1,3}\.){3}[0-9]{1,3}|(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4})"
r" localport=(\d{4,5}) localheartbeatport=(\d{4,5}) "
r"localservice=(\d{4,5}) "
r"remotehost=(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) "
r"replconninfo%s='localhost=((?:[0-9]{1,3}\.){3}[0-9]{1,3}|(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4})"
r"remoteport=(\d{4,5}) remoteheartbeatport=(\d{4,5}) "
r"remoteservice=(\d{4,5})" % idx, output)
if not ret:
@ -1881,10 +1881,10 @@ class StreamingBase(object):
continue
elif "iscrossregion=false" in output.lower():
ret = re.search(
r"replconninfo%s='localhost=(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})"
r"replconninfo%s='localhost=((?:[0-9]{1,3}\.){3}[0-9]{1,3}|(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4})"
r" localport=(\d{4,5}) localheartbeatport=(\d{4,5}) "
r"localservice=(\d{4,5}) "
r"remotehost=(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) "
r"replconninfo%s='localhost=((?:[0-9]{1,3}\.){3}[0-9]{1,3}|(?:[a-f0-9]{1,4}:){7}[a-f0-9]{1,4})"
r"remoteport=(\d{4,5}) remoteheartbeatport=(\d{4,5}) "
r"remoteservice=(\d{4,5})" % idx, output)
if not ret:

View File

@ -457,27 +457,28 @@ class Kerberos():
self.__allIps += inst.haIps
self.__allIps += inst.listenIps
# set local ip 127.0.0.1
self.__allIps += ['127.0.0.1']
self.__allIps += ['127.0.0.1', '::1']
# get all ips. Remove the duplicates ips
self.__allIps = DefaultValue.Deduplication(self.__allIps)
# build ip string list
#set Kerberos ip
# set Kerberos ip
principals = g_opts.principal.split("/")
principals = principals[1].split("@")
# Every 1000 records merged into one"
ipstring = ""
for ip in self.__allIps:
if not isUninstall:
submask_length = NetUtil.get_submask_len(ip)
ipstring += " -h 'host all all " \
" %s/32 gss " \
" %s/%s gss " \
"include_realm=1 krb_realm=%s'" % \
(ip, principals[1])
(ip, submask_length, principals[1])
else:
ipstring += " -h 'host all all " \
" %s/32 %s'" % (ip, METHOD_TRUST)
" %s/%s %s'" % (ip, submask_length, METHOD_TRUST)
if ipstring != "":
self.__IpStringList.append(ipstring)
#write config hba
# write config hba
self.__writeConfigHba()
except Exception as e:
raise Exception(ErrorCode.GAUSS_530["GAUSS_53024"]

View File

@ -2183,7 +2183,7 @@ Common options:
for backIp in g_nodeInfo.backIps:
# Get backIP subnet mask
subnetMask = ""
allNetworkInfo = NetUtil.getAllNetworkInfo()
allNetworkInfo = NetUtil.getAllNetworkInfo(NetUtil.get_ip_version(backIp))
for network in allNetworkInfo:
if backIp == network.ipAddress:
subnetMask = network.networkMask

View File

@ -162,7 +162,7 @@ class Resetreplconninfo():
"""
output_list = self.__getStatusByOM()
output_num = 0
pattern = re.compile("(\d+) (.*) (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (.*)")
pattern = re.compile(r'(\d+) (.*) ((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[0-9a-fA-F]{0,4}(?::[0-9a-fA-F]{0,4})*::?(?:[0-9a-fA-F]{0,4})?)) (.*)')
has_cm_component = True if [i for i in output_list if "CMServer State" in i] else False
if has_cm_component:
output_list = [i for i in output_list if i]