coolany
876
script/base_utils/os/file_util.py
Normal file
876
script/base_utils/os/file_util.py
Normal file
@ -0,0 +1,876 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
#############################################################################
|
||||
# Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd.
|
||||
# Portions Copyright (c) 2007 Agendaless Consulting and Contributors.
|
||||
#
|
||||
# openGauss is licensed under Mulan PSL v2.
|
||||
# You can use this software according to the terms
|
||||
# and conditions of the Mulan PSL v2.
|
||||
# You may obtain a copy of Mulan PSL v2 at:
|
||||
#
|
||||
# http://license.coscl.org.cn/MulanPSL2
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OF ANY KIND,
|
||||
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
||||
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
||||
# See the Mulan PSL v2 for more details.
|
||||
# Description : file_util.py for file utility.
|
||||
#############################################################################
|
||||
import os
|
||||
import _thread
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import pwd
|
||||
import grp
|
||||
from subprocess import PIPE
|
||||
|
||||
from base_utils.common.constantsbase import ConstantsBase
|
||||
from gspylib.common.ErrorCode import ErrorCode
|
||||
from base_utils.os.cmd_util import CmdUtil
|
||||
from base_utils.os.env_util import EnvUtil
|
||||
from base_utils.os.user_util import UserUtil
|
||||
from base_utils.common.fast_popen import FastPopen
|
||||
from base_utils.security.sensitive_mask import SensitiveMask
|
||||
|
||||
|
||||
class FileUtil(object):
|
||||
"""file utility"""
|
||||
|
||||
@staticmethod
|
||||
def cleanTmpFile(path, file_handler=None):
|
||||
"""
|
||||
function : close and remove temporary file
|
||||
input : String,file
|
||||
output : NA
|
||||
"""
|
||||
if file_handler:
|
||||
file_handler.close()
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
@staticmethod
|
||||
def createFileInSafeMode(file_path, mode=stat.S_IWUSR | stat.S_IRUSR):
|
||||
"""
|
||||
Call this method before open(file_path) functions,
|
||||
if it may create a new file.
|
||||
This method guarantees a 0o600 file is created
|
||||
instead of an arbitrary one.
|
||||
"""
|
||||
if os.path.exists(file_path):
|
||||
return
|
||||
try:
|
||||
os.mknod(file_path, mode)
|
||||
except IOError as error:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50206"] % file_path +
|
||||
" Error:\n%s." % str(error))
|
||||
|
||||
@staticmethod
|
||||
def get_permission_value(file_path):
|
||||
"""
|
||||
function : Obtaining Permissions on a File or Directory
|
||||
input : String
|
||||
output : String
|
||||
"""
|
||||
if not os.path.exists(file_path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % file_path)
|
||||
try:
|
||||
value = oct(os.stat(file_path).st_mode)[-3:]
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] %
|
||||
("permission with %s" % file_path) + " Error:\n%s" % str(e))
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def check_file_permission(filename,
|
||||
check_read_permission=False,
|
||||
check_write_permission=False,
|
||||
check_execute_permission=False):
|
||||
"""
|
||||
Function : check file: 1.exist 2. isfile 3. permission
|
||||
Note : 1.You must check that the file exist and is a file.
|
||||
2.You can choose whether to check the file's
|
||||
permission:readable/writable/executable.
|
||||
input : filename, isread, iswrite, isexecute
|
||||
output : True
|
||||
"""
|
||||
if not os.path.exists(filename):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50236"] % filename)
|
||||
if not os.path.isfile(filename):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % filename)
|
||||
if check_read_permission:
|
||||
if not os.access(filename, os.R_OK):
|
||||
raise Exception(ErrorCode.GAUSS_501["GAUSS_50100"] % (
|
||||
filename, "the user") +
|
||||
" Error:\n%s: Permission denied." % filename)
|
||||
if check_write_permission:
|
||||
if not os.access(filename, os.W_OK):
|
||||
raise Exception(ErrorCode.GAUSS_501["GAUSS_50102"] % (
|
||||
filename, "the user") +
|
||||
" Error:\n%s: Permission denied." % filename)
|
||||
if check_execute_permission:
|
||||
if not os.access(filename, os.X_OK):
|
||||
raise Exception(ErrorCode.GAUSS_501[
|
||||
"GAUSS_50101"] % (filename, "the user") +
|
||||
" Error:\n%s: Permission denied." % filename)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def createFile(path, overwrite=True, mode=None):
|
||||
"""
|
||||
function: create file and set the permission
|
||||
input:
|
||||
path: the file path.
|
||||
overwrite: if file already exists and this parameter is true,
|
||||
we can overwrtie it.
|
||||
mode: Specify file permissions, type is int and start with 0.
|
||||
ex: 0700
|
||||
output:
|
||||
return true or false.
|
||||
"""
|
||||
try:
|
||||
if overwrite:
|
||||
cmd = CmdUtil.getCreateFileCmd(path)
|
||||
if mode:
|
||||
cmd += "; %s" % CmdUtil.getChmodCmd(str(mode), path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + " The cmd is %s" % cmd)
|
||||
else:
|
||||
# create file by python API
|
||||
if mode:
|
||||
os.mknod(path, mode)
|
||||
else:
|
||||
os.mknod(path)
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50206"] % path +
|
||||
" Error:\n%s" % str(e))
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def removeFile(path, cmd_type="shell"):
|
||||
"""
|
||||
function: remove a file
|
||||
input: the path of file(include file name)
|
||||
output: return true or false
|
||||
"""
|
||||
if cmd_type in ["python", "python3"]:
|
||||
# no file need remove.
|
||||
if not os.path.exists(path):
|
||||
return True
|
||||
# check if is a file.
|
||||
if not os.path.isfile(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % path)
|
||||
try:
|
||||
# remove file.
|
||||
os.remove(path)
|
||||
except Exception:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50207"] % path)
|
||||
else:
|
||||
# Support* for fuzzy matching
|
||||
if "*" in path:
|
||||
path = FileUtil.withAsteriskPath(path)
|
||||
cmd = CmdUtil.getRemoveCmd('file') + path
|
||||
else:
|
||||
cmd = CmdUtil.getRemoveCmd('file') + "'" + path + "'"
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50207"] % path
|
||||
+ " Error:\n%s" % output
|
||||
+ " The cmd is %s" % cmd)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def moveFile(src, dest, overwrite=True):
|
||||
"""
|
||||
function: move a file
|
||||
input:
|
||||
src: the dir of file
|
||||
dest: the dir which want to move
|
||||
output:
|
||||
return true or false
|
||||
"""
|
||||
# check if can overwrite
|
||||
if os.path.exists(dest) and not overwrite:
|
||||
raise Exception(ErrorCode.GAUSS_501["GAUSS_50102"] % (
|
||||
"parameter overwrite", dest))
|
||||
try:
|
||||
if overwrite:
|
||||
cmd = CmdUtil.getMoveFileCmd(src, dest)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + "The cmd is %s" % cmd)
|
||||
else:
|
||||
# move file
|
||||
shutil.move(src, dest)
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50232"] % (src, dest) +
|
||||
" Error:\n%s" % str(e))
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def readFile(filename, keyword="", rows=0):
|
||||
"""
|
||||
function: read the content of a file
|
||||
input:
|
||||
filename: the name and path of the file
|
||||
keyword: read line include keyword
|
||||
rows: the row number, which want to read
|
||||
offset: keep the parameter, but do nothing
|
||||
output:list
|
||||
"""
|
||||
list_key = []
|
||||
str_rows = ""
|
||||
all_lines = []
|
||||
# check if file exists.
|
||||
if not os.path.exists(filename):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % filename)
|
||||
try:
|
||||
with open(filename, 'rb') as fp:
|
||||
for line in fp:
|
||||
all_lines.append(line.decode("utf-8"))
|
||||
except Exception:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50204"] % filename)
|
||||
# get keyword lines
|
||||
if keyword != "":
|
||||
for line in all_lines:
|
||||
flag = line.find(keyword)
|
||||
if flag >= 0:
|
||||
list_key.append(line)
|
||||
# get content of row
|
||||
if rows:
|
||||
if not str(rows).isdigit():
|
||||
raise Exception
|
||||
if rows > 0:
|
||||
row_num = rows - 1
|
||||
else:
|
||||
row_num = rows
|
||||
try:
|
||||
if row_num < (len(all_lines)):
|
||||
str_rows = all_lines[row_num]
|
||||
except Exception:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50204"] % (
|
||||
"the %s line of the file [%s]" % (rows, filename)))
|
||||
# check which needs return
|
||||
if keyword != "" and rows != 0:
|
||||
return [str_rows]
|
||||
if keyword != "" and rows == 0:
|
||||
return list_key
|
||||
if keyword == "" and rows != 0:
|
||||
return [str_rows]
|
||||
if keyword == "" and rows == 0:
|
||||
return all_lines
|
||||
|
||||
@staticmethod
|
||||
def writeFile(path, context=None, mode="a+"):
|
||||
"""
|
||||
function: write content in a file
|
||||
input:
|
||||
path: the name and path of the file
|
||||
context: the content, which want to write
|
||||
mode: the write mode
|
||||
output:
|
||||
"""
|
||||
lock = _thread.allocate_lock()
|
||||
if context is None:
|
||||
context = []
|
||||
# check if not exists.
|
||||
if not os.path.exists(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % path)
|
||||
# check if is a file.
|
||||
if os.path.exists(path) and not os.path.isfile(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % path)
|
||||
# if no context, return
|
||||
if not context:
|
||||
return False
|
||||
FileUtil.createFileInSafeMode(path)
|
||||
with open(path, mode) as fp:
|
||||
fp.writelines(line + os.linesep for line in context)
|
||||
lock.acquire()
|
||||
try:
|
||||
# write context.
|
||||
fp.flush()
|
||||
except Exception as excep:
|
||||
lock.release()
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50205"] % path +
|
||||
"Error:\n%s" % str(excep))
|
||||
lock.release()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def withAsteriskPath(path):
|
||||
"""
|
||||
function: deal with the path with *
|
||||
input: the path to deal with
|
||||
output: cmd
|
||||
"""
|
||||
path_dir_list = os.path.realpath(path).split(os.path.sep)[1:]
|
||||
path = "'"
|
||||
for dir_name in path_dir_list:
|
||||
if "*" in dir_name:
|
||||
dir_path = "'" + os.path.sep + dir_name + "'"
|
||||
else:
|
||||
dir_path = os.path.sep + dir_name
|
||||
path += dir_path
|
||||
if path[-1] == "'":
|
||||
path = path[:-1]
|
||||
else:
|
||||
path += "'"
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def changeMode(mode, path, recursive=False, cmd_type="shell",
|
||||
retry_flag=False, retry_time=15, waite_time=1):
|
||||
"""
|
||||
function: change permission of file
|
||||
input:
|
||||
cmd_type: user shell or python
|
||||
mode:permission value, Type is int and start with 0. ex: 0700
|
||||
path:file path
|
||||
recursive: recursive or not
|
||||
output:
|
||||
"""
|
||||
# do with shell command.
|
||||
if cmd_type == "shell":
|
||||
if "*" in path:
|
||||
path = FileUtil.withAsteriskPath(path)
|
||||
else:
|
||||
path = "'" + path + "'"
|
||||
cmd = CmdUtil.getChmodCmd(str(mode), path, recursive)
|
||||
if retry_flag:
|
||||
CmdUtil.retryGetstatusoutput(cmd, retry_time, waite_time)
|
||||
else:
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_501[
|
||||
"GAUSS_50107"] % path +
|
||||
" Error:\n%s." % output +
|
||||
"The cmd is %s" % cmd)
|
||||
# do with python API. If the name has special characters.
|
||||
else:
|
||||
os.chmod(path, mode)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def changeOwner(user, path, recursive=False, cmd_type="shell",
|
||||
retry_flag=False, retry_time=15, waite_time=1, link=False):
|
||||
"""
|
||||
function: change the owner of file
|
||||
input: cmd_type, user, path, recursive
|
||||
output: return true
|
||||
"""
|
||||
try:
|
||||
# get uid and gid by username.
|
||||
try:
|
||||
userInfo = pwd.getpwnam(user)
|
||||
except KeyError:
|
||||
userInfo = pwd.getpwnam(user)
|
||||
group = grp.getgrgid(userInfo.pw_gid).gr_name
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_503["GAUSS_50308"] +
|
||||
" Error:\n%s" % str(e))
|
||||
try:
|
||||
# do with shell command.
|
||||
if cmd_type == "shell":
|
||||
if "*" in path:
|
||||
path = FileUtil.withAsteriskPath(path)
|
||||
else:
|
||||
path = "'" + path + "'"
|
||||
cmd = CmdUtil.getChownCmd(user, group, path, recursive)
|
||||
if link:
|
||||
cmd = cmd + " -h"
|
||||
if retry_flag:
|
||||
CmdUtil.retryGetstatusoutput(cmd, retry_time, waite_time)
|
||||
else:
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + " The cmd is %s" % cmd)
|
||||
# do with python API. If the name has special characters.
|
||||
else:
|
||||
os.chown(path, userInfo.pw_uid, userInfo.pw_gid)
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_501["GAUSS_50106"] % path +
|
||||
" Error:\n%s." % str(e))
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def modifyFileOwner(user, currentfile):
|
||||
"""
|
||||
function : Modify the file's owner
|
||||
input : String,String
|
||||
output : String
|
||||
"""
|
||||
# only root user can run this function
|
||||
if os.getuid() == 0:
|
||||
UserUtil.getGroupByUser(user)
|
||||
if os.path.exists(currentfile):
|
||||
FileUtil.changeOwner(user, currentfile)
|
||||
|
||||
@staticmethod
|
||||
def modifyFileOwnerFromGPHOME(currentfile):
|
||||
"""
|
||||
function : Modify the file's owner to the GPHOME's user
|
||||
input : String,String
|
||||
output : String
|
||||
"""
|
||||
gphome = EnvUtil.getEnv("GPHOME")
|
||||
if not gphome:
|
||||
raise Exception(ErrorCode.GAUSS_518["GAUSS_51802"] % "GPHOME")
|
||||
(user, group) = UserUtil.getPathOwner(gphome)
|
||||
if user == "" or group == "":
|
||||
raise Exception(ErrorCode.GAUSS_503["GAUSS_50308"])
|
||||
FileUtil.modifyFileOwner(user, currentfile)
|
||||
|
||||
@staticmethod
|
||||
def createDirectory(path, overwrite=True, mode=None):
|
||||
"""
|
||||
function: create a directory
|
||||
input: path, overwrite
|
||||
output: true
|
||||
"""
|
||||
try:
|
||||
if os.path.exists(path) and not overwrite:
|
||||
raise Exception(ErrorCode.GAUSS_501["GAUSS_50102"] % (
|
||||
"parameter overwrite", path))
|
||||
if overwrite:
|
||||
cmd = CmdUtil.getMakeDirCmd(path, overwrite)
|
||||
if mode:
|
||||
cmd += "; %s" % CmdUtil.getChmodCmd(str(mode), path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + " The cmd is %s" % cmd)
|
||||
if not overwrite:
|
||||
if mode:
|
||||
os.mkdir(path, mode)
|
||||
else:
|
||||
os.mkdir(path)
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50208"] % path +
|
||||
" Error:\n%s" % str(e))
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def cleanDirectoryContent(path):
|
||||
"""
|
||||
function: clean the content in a directory,
|
||||
but do not remove directory.
|
||||
input:path
|
||||
output:true
|
||||
"""
|
||||
prefix_cmd = "cd %s && ls | xargs -n 100000" % path
|
||||
if not os.path.exists(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % path)
|
||||
cmd = "%s %s && %s '%s'/.[^.]*" % (prefix_cmd, CmdUtil.getRemoveCmd(
|
||||
"directory"), CmdUtil.getRemoveCmd("directory"), path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"] % (
|
||||
"content in the directory %s " % path) +
|
||||
" Error:\n%s." % output + "The cmd is %s" % cmd)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def cleanFileContext(path):
|
||||
"""
|
||||
function: clean file context
|
||||
input: path, file path
|
||||
"""
|
||||
real_path = os.path.realpath(path)
|
||||
if os.path.exists(real_path) and os.path.getsize(real_path) > 0:
|
||||
with open(real_path, 'r+') as fp_file:
|
||||
fp_file.seek(0)
|
||||
fp_file.truncate()
|
||||
fp_file.flush()
|
||||
|
||||
@staticmethod
|
||||
def removeDirectory(path):
|
||||
"""
|
||||
function: remove the content in a directory
|
||||
input:path
|
||||
output:true
|
||||
"""
|
||||
if "*" in path:
|
||||
path = FileUtil.withAsteriskPath(path)
|
||||
cmd = "%s %s" % (CmdUtil.getRemoveCmd("directory"), path)
|
||||
else:
|
||||
cmd = "%s '%s'" % (CmdUtil.getRemoveCmd("directory"), path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50209"] % path +
|
||||
" Error:\n%s." % output + "The cmd is %s" % cmd)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def getDirectoryList(path, keywords="", recursive=False):
|
||||
"""
|
||||
function:give the list of file in the directory
|
||||
input:path, keywords, recursive
|
||||
output:list
|
||||
"""
|
||||
list_dir = []
|
||||
try:
|
||||
if keywords == "":
|
||||
if recursive:
|
||||
cmd = "%s -R '%s'" % (CmdUtil.getListCmd(), path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + "\nThe cmd is %s" % cmd)
|
||||
list_dir = output.split('\n')
|
||||
else:
|
||||
list_dir = os.listdir(path)
|
||||
else:
|
||||
if recursive:
|
||||
cmd = "%s -R '%s' |%s -E '%s'" % (
|
||||
CmdUtil.getListCmd(), path,
|
||||
CmdUtil.getGrepCmd(), keywords)
|
||||
else:
|
||||
cmd = "%s '%s' |%s -E '%s'" % (
|
||||
CmdUtil.getListCmd(), path,
|
||||
CmdUtil.getGrepCmd(), keywords)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0 and output != "":
|
||||
raise Exception(output + "\nThe cmd is %s" % cmd)
|
||||
else:
|
||||
list_dir = output.split('\n')
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] % (
|
||||
"the list of %s" % path) + " Error:\n%s" % str(e))
|
||||
while '' in list_dir:
|
||||
list_dir.remove('')
|
||||
return list_dir
|
||||
|
||||
@staticmethod
|
||||
def cpFile(src, dest, cmd_type="shell", skip_check=False):
|
||||
"""
|
||||
function: copy a file
|
||||
input:src, dest, cmd_type
|
||||
output:true
|
||||
"""
|
||||
if skip_check:
|
||||
if not os.path.exists(src):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % src)
|
||||
if not os.path.exists(os.path.dirname(dest)):
|
||||
raise Exception(ErrorCode.GAUSS_502[
|
||||
"GAUSS_50201"] % os.path.dirname(dest))
|
||||
try:
|
||||
if cmd_type != "shell":
|
||||
shutil.copy(src, dest)
|
||||
else:
|
||||
cmd = CmdUtil.getCopyCmd(src, dest, "directory")
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(output + "\nThe cmd is %s" % cmd)
|
||||
except Exception as e:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50214"] % src +
|
||||
" Error:\n%s" % str(e))
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def findFile(path, keyword, choice='name'):
|
||||
"""
|
||||
function:find a file by name or size or user
|
||||
input:path, keyword, choice, type
|
||||
output:NA
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % path)
|
||||
cmd = "%s '%s' -%s %s " % (CmdUtil.getFindCmd(), path,
|
||||
choice, keyword)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] % (
|
||||
"the files of path %s" % path) + " Error:\n%s" % output
|
||||
+ "\nThe cmd is %s" % cmd)
|
||||
list_file = output.split('\n')
|
||||
while '' in list_file:
|
||||
list_file.remove('')
|
||||
return list_file
|
||||
|
||||
@staticmethod
|
||||
def replaceFileLineContent(old_line, new_line, path):
|
||||
"""
|
||||
function: replace the line in a file to a new line
|
||||
input:
|
||||
old_line : Need to replace content
|
||||
new_line : Replaced content
|
||||
path
|
||||
output:NA
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % path)
|
||||
|
||||
cmd = CmdUtil.getReplaceFileLineContentCmd(old_line, new_line, path)
|
||||
proc = FastPopen(cmd, stdout=PIPE, stderr=PIPE, preexec_fn=os.setsid, close_fds=True)
|
||||
stdout, stderr = proc.communicate()
|
||||
output = stdout + stderr
|
||||
status = proc.returncode
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50223"] % path +
|
||||
" Error:\n%s" % SensitiveMask.mask_pwd(output) +
|
||||
"\nThe cmd is %s" % SensitiveMask.mask_pwd(cmd))
|
||||
|
||||
@staticmethod
|
||||
def checkIsInDirectory(file_name, directory_list):
|
||||
"""
|
||||
function : Check if the file is in directory_list.
|
||||
input : String,[]
|
||||
output : []
|
||||
"""
|
||||
is_exist = False
|
||||
for one_path in directory_list:
|
||||
dir_name = os.path.normpath(file_name)
|
||||
is_exist = False
|
||||
|
||||
while dir_name != "/":
|
||||
if dir_name == one_path:
|
||||
is_exist = True
|
||||
break
|
||||
dir_name = os.path.dirname(dir_name)
|
||||
|
||||
if is_exist:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50229"] % (
|
||||
file_name, one_path))
|
||||
return is_exist
|
||||
|
||||
@staticmethod
|
||||
def checkDirWriteable(dir_path):
|
||||
"""
|
||||
function : Check if target directory is writeable for execute user.
|
||||
input : String,String
|
||||
output : boolean
|
||||
"""
|
||||
# if we can touch a tmp file under the path, it is true;
|
||||
return os.access(dir_path, os.W_OK)
|
||||
|
||||
@staticmethod
|
||||
def getFileSHA256(filename):
|
||||
"""
|
||||
function : Get the ssh file by SHA256
|
||||
input : String
|
||||
output : String
|
||||
"""
|
||||
if not os.path.exists(filename):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % filename)
|
||||
if not os.path.isfile(filename):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % filename)
|
||||
|
||||
strSHA256 = ""
|
||||
cmd = CmdUtil.getFileSHA256Cmd(filename)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
return strSHA256
|
||||
strSHA256 = output.strip()
|
||||
|
||||
return strSHA256
|
||||
|
||||
@staticmethod
|
||||
def getDirSize(path, unit=""):
|
||||
"""
|
||||
function : Get the directory or file size
|
||||
input : String, String
|
||||
output : String
|
||||
"""
|
||||
sizeInfo = ""
|
||||
cmd = CmdUtil.getDirSizeCmd(path, unit)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
return sizeInfo
|
||||
return output.split()[0]
|
||||
|
||||
@staticmethod
|
||||
def getFilesType(given_path):
|
||||
"""
|
||||
function : get the file and subdirectory type of the given path
|
||||
input : String
|
||||
output : String
|
||||
"""
|
||||
if not os.path.exists(given_path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % given_path)
|
||||
# obtain the file type
|
||||
tmp_file = "/tmp/fileList_%d" % os.getpid()
|
||||
cmd = "%s '%s' ! -iname '.*' | %s file -F '::' > %s 2>/dev/null" % (
|
||||
CmdUtil.getFindCmd(), given_path,
|
||||
CmdUtil.getXargsCmd(), tmp_file)
|
||||
subprocess.getstatusoutput(cmd)
|
||||
# Return code is not equal to zero when file a non-existent
|
||||
# file in SLES SP4
|
||||
# But it is equal to zero in SLES SP1/SP2/SP3 and
|
||||
# RHEL 6.4/6.5/6.6 platform, skip check status and output
|
||||
res_dict = {}
|
||||
try:
|
||||
with open(tmp_file, 'r') as fp:
|
||||
file_name_type_list = fp.readlines()
|
||||
os.remove(tmp_file)
|
||||
for one_item in file_name_type_list:
|
||||
res = one_item.split("::")
|
||||
if len(res) != 2:
|
||||
continue
|
||||
else:
|
||||
res_dict[res[0]] = res[1]
|
||||
return res_dict
|
||||
except Exception as excep:
|
||||
if os.path.exists(tmp_file):
|
||||
FileUtil.removeFile(tmp_file)
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50221"] +
|
||||
" Error: \n%s" % str(excep))
|
||||
|
||||
@staticmethod
|
||||
def deleteLine(file_path, line_info):
|
||||
"""
|
||||
function : delete a line in file match with re
|
||||
input : file_path ,line_info
|
||||
output : NA
|
||||
"""
|
||||
cmd = CmdUtil.getSedCmd()
|
||||
cmd += " -i '/%s/d' %s" % (line_info, file_path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50205"] % file_path +
|
||||
" Error:\n%s" % output + "\nThe cmd is %s" % cmd)
|
||||
|
||||
@staticmethod
|
||||
def deleteLineByRowNum(file_path, line_num):
|
||||
"""
|
||||
function : delete line in a file by row num
|
||||
input : file_path ,line_info
|
||||
output : NA
|
||||
"""
|
||||
cmd = CmdUtil.getSedCmd()
|
||||
cmd += " -i '%sd' %s" % (line_num, file_path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50205"] % file_path +
|
||||
" Error:\n%s" % output + "\nThe cmd is %s" % cmd)
|
||||
|
||||
@staticmethod
|
||||
def rename(old_file_path, new_file_path):
|
||||
"""
|
||||
function : rename a file name to new name
|
||||
input : old_file_path, new_file_path
|
||||
output : NA
|
||||
"""
|
||||
cmd = CmdUtil.getMoveCmd(old_file_path, new_file_path)
|
||||
(status, output) = subprocess.getstatusoutput(cmd)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50218"] % old_file_path +
|
||||
" Error:\n%s" % output + "\nThe cmd is %s" % cmd)
|
||||
|
||||
@staticmethod
|
||||
def checkLink(file_path):
|
||||
"""
|
||||
function:check if file is a link
|
||||
input: file_path
|
||||
output:NA
|
||||
"""
|
||||
if os.path.exists(file_path):
|
||||
if os.path.islink(file_path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50210"] % file_path)
|
||||
|
||||
@staticmethod
|
||||
def getTopPathNotExist(top_dir_path):
|
||||
"""
|
||||
function : Get the top path if exist
|
||||
input : String
|
||||
output : String
|
||||
"""
|
||||
tmp_dir = top_dir_path
|
||||
while True:
|
||||
# find the top path to be created
|
||||
(tmp_dir, top_dir_name) = os.path.split(tmp_dir)
|
||||
if os.path.exists(tmp_dir) or top_dir_name == "":
|
||||
tmp_dir = os.path.join(tmp_dir, top_dir_name)
|
||||
break
|
||||
return tmp_dir
|
||||
|
||||
@staticmethod
|
||||
def getfileUser(path):
|
||||
"""
|
||||
function: get the info(username group) of a file
|
||||
input:path
|
||||
output:list of info
|
||||
"""
|
||||
if not os.path.exists(path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % path)
|
||||
|
||||
user = pwd.getpwuid(os.stat(path).st_uid).pw_name
|
||||
group = grp.getgrgid(os.stat(path).st_gid).gr_name
|
||||
return user, group
|
||||
|
||||
@staticmethod
|
||||
def judgePathUser(temp_path):
|
||||
"""
|
||||
function: judge the owner of path if exist
|
||||
input: temp_path
|
||||
output: True/False
|
||||
"""
|
||||
try:
|
||||
pwd.getpwuid(os.stat(temp_path).st_uid).pw_name
|
||||
return True
|
||||
except Exception as e:
|
||||
# if the user is not exist
|
||||
if str(e).find("uid not found") >= 0:
|
||||
return False
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50219"] %
|
||||
("the owner of %s" % temp_path) +
|
||||
" Error: \n%s" % str(e))
|
||||
|
||||
@staticmethod
|
||||
def checkPathandChangeOwner(one_path, user, permission):
|
||||
"""
|
||||
function: Get the owner of each layer path , if the user does not
|
||||
exist and change owner
|
||||
input: onePath---the specified path; user---the user of cluster;
|
||||
group---the group of cluster
|
||||
output: the owner of path
|
||||
precondiftion: the path exists
|
||||
"""
|
||||
pathlist = []
|
||||
if not os.path.exists(one_path):
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % one_path)
|
||||
|
||||
owner_path = one_path
|
||||
while True:
|
||||
# obtain the each top path
|
||||
(owner_path, dirName) = os.path.split(owner_path)
|
||||
if os.path.exists(owner_path) and dirName != "":
|
||||
pathlist.append(os.path.join(owner_path, dirName))
|
||||
else:
|
||||
break
|
||||
|
||||
for temp_path in pathlist:
|
||||
# the user does not exist
|
||||
if not FileUtil.judgePathUser(temp_path):
|
||||
FileUtil.changeMode(permission, temp_path)
|
||||
FileUtil.changeOwner(user, temp_path)
|
||||
|
||||
@staticmethod
|
||||
def getchangeDirModeCmd(user_dir):
|
||||
"""
|
||||
function : change directory permission
|
||||
input : user_dir
|
||||
output: NA
|
||||
"""
|
||||
# Use "find -exec" to mask special characters
|
||||
cmd_dir = "find '%s' -type d -exec chmod '%s' {} \;" % (
|
||||
user_dir, ConstantsBase.KEY_DIRECTORY_MODE)
|
||||
(status, diroutput) = subprocess.getstatusoutput(cmd_dir)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % user_dir +
|
||||
" Error: \n%s" % diroutput)
|
||||
|
||||
@staticmethod
|
||||
def getchangeFileModeCmd(user_dir):
|
||||
"""
|
||||
function : change log file permission
|
||||
input : user_dir
|
||||
output: NA
|
||||
"""
|
||||
# Use "find -exec" to mask special characters
|
||||
cmd_file = "find '%s' -type f -name '*.log' -exec chmod '%s' {} \;" \
|
||||
% (user_dir, ConstantsBase.KEY_FILE_MODE)
|
||||
(status, file_output) = subprocess.getstatusoutput(cmd_file)
|
||||
if status != 0:
|
||||
raise Exception(ErrorCode.GAUSS_502["GAUSS_50201"] % "log file" +
|
||||
" Directory:%s." % user_dir + " Error: \n%s" % file_output)
|
||||
Reference in New Issue
Block a user