[feature](tools)profile viewer (#14429)
It is a painful work to read profile, especially there are multi-parallel instances. This tool helps us to grasp the main information of profile in a graphical view. The profile is represented by a tree. Sql operation nodes contains operation type(join, scan...), its node id, its fragment id. The number on the arrow edge means how many rows output by child node. This tool will sum the output rows of the same node in multi-parallel instances, that is if there are 4 parallel instance, and each ScanNode on lineitem table output 10 rows, the label on the arrow beginning with ScanNode(lineitem) is 40. Here is a demo for tpch Q2 tpch q2 profile viewer Issue Number: close #xxx
This commit is contained in:
minghong
329
tools/profile_viewer.py
Normal file
329
tools/profile_viewer.py
Normal file
@ -0,0 +1,329 @@
|
||||
|
||||
#!/usr/bin/env python
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
#dot dig -T png -o dig.png
|
||||
|
||||
exchanges = {}
|
||||
|
||||
class Node:
|
||||
TERMINAL_NODE_TYPES={'VEXCHANGE_NODE', 'VNewOlapScanNode'}
|
||||
def __init__(self, id, type, frid):
|
||||
self.id = id
|
||||
self.frid = frid
|
||||
self.type = type
|
||||
self.parent = None
|
||||
self.children = []
|
||||
self.rows = 0
|
||||
m = re.search("VNewOlapScanNode\((.*)\)", type)
|
||||
if m != None:
|
||||
self.table = m.group(1)
|
||||
self.type = 'VNewOlapScanNode'
|
||||
else:
|
||||
self.table = ""
|
||||
|
||||
def set_data(self, data):
|
||||
self.data = data
|
||||
|
||||
def parse(self):
|
||||
for line in self.data:
|
||||
m = re.search(".*RowsReturned:.+\(([\d]+)\)", line)
|
||||
if m!= None:
|
||||
self.rows = int(m.group(1))
|
||||
break
|
||||
else:
|
||||
m = re.search(".*RowsReturned:.+([\d]+)", line)
|
||||
if m != None:
|
||||
self.rows = int(m.group(1))
|
||||
break
|
||||
|
||||
def build_tree(self, nodes):
|
||||
if self.type in Node.TERMINAL_NODE_TYPES:
|
||||
return 0
|
||||
self.children.append(nodes[0])
|
||||
consume = 1 + nodes[0].build_tree(nodes[1:])
|
||||
if self.type.find("JOIN") > 0:
|
||||
self.children.append(nodes[consume])
|
||||
consume = consume + 1 + nodes[consume].build_tree(nodes[consume+1:])
|
||||
return consume
|
||||
|
||||
def __repr__(self):
|
||||
if self.type == 'VNewOlapScanNode':
|
||||
return "{}_{}".format(self.table, self.id)
|
||||
return "{}_{}".format(self.type, self.id)
|
||||
|
||||
def to_label(self):
|
||||
name = self.type
|
||||
if self.type == 'VNewOlapScanNode':
|
||||
name = self.table
|
||||
return '"{}\\n#{}FR{}"'.format(name, self.id, self.frid)
|
||||
|
||||
def show(self, indent):
|
||||
print("{}{}".format(indent, self))
|
||||
for ch in self.children:
|
||||
ch.show(" " + indent)
|
||||
|
||||
def to_dot(self, dot):
|
||||
dot = dot + "\n{}[label={}]".format(self, self.to_label())
|
||||
for ch in reversed(self.children):
|
||||
dot = dot + "\n{}->{}[label={}]".format(ch, self, ch.rows)
|
||||
for ch in self.children:
|
||||
dot = ch.to_dot(dot)
|
||||
return dot
|
||||
class Instance:
|
||||
def __init__(self, id, frid):
|
||||
self.id = id
|
||||
self.frid = frid
|
||||
self.target = ""
|
||||
self.srcs = []
|
||||
self.nodes = []
|
||||
self.has_build_tree = False
|
||||
|
||||
def set_data(self, data):
|
||||
self.data = data
|
||||
|
||||
def build_tree(self):
|
||||
if self.has_build_tree:
|
||||
return
|
||||
self.has_build_tree = True
|
||||
self.root = self.nodes[0]
|
||||
self.root.build_tree(self.nodes[1:])
|
||||
|
||||
def parse(self):
|
||||
pos = 0
|
||||
for line in self.data:
|
||||
pos = pos + 1
|
||||
m = re.search("\s+VDataStreamSender \(dst_id=(\d+),.*", line)
|
||||
if m != None:
|
||||
self.target = m.group(1)
|
||||
break
|
||||
|
||||
for line in self.data:
|
||||
m = re.search("\s+VEXCHANGE_NODE \(id=(\d+)\):.*", line)
|
||||
if m != None:
|
||||
self.srcs.append(m.group(1))
|
||||
|
||||
prev = 0
|
||||
pos = 0
|
||||
for line in self.data:
|
||||
m = re.search("\s+([^\s]+N[o|O][d|D][e|E][^\s]*) \(id=(\d+)\.*", line)
|
||||
if m != None:
|
||||
type = m.group(1)
|
||||
id = m.group(2)
|
||||
node = Node(id, type, self.frid)
|
||||
if node.type == 'VEXCHANGE_NODE':
|
||||
exchanges[id] = node
|
||||
if prev != 0:
|
||||
#self.nodes
|
||||
self.nodes[-1].set_data(self.data[prev : pos])
|
||||
self.nodes.append(node)
|
||||
prev = pos
|
||||
pos = pos + 1
|
||||
self.nodes[-1].set_data(self.data[prev : pos])
|
||||
|
||||
for node in self.nodes:
|
||||
node.parse()
|
||||
self.build_tree()
|
||||
|
||||
|
||||
def show(self):
|
||||
self.root.show(" ")
|
||||
|
||||
class Fragment:
|
||||
def __init__(self, id):
|
||||
self.id = id
|
||||
self.target = ""
|
||||
self.srcs = []
|
||||
self.instances = []
|
||||
self.children = []
|
||||
self.parent = None
|
||||
|
||||
def set_data(self, data):
|
||||
self.data = data
|
||||
|
||||
def get_instance_id(self, line):
|
||||
m = re.search("\s+Instance (\w+-\w+)", line)
|
||||
is_inst = m != None
|
||||
return (is_inst, m.group(1) if is_inst else "")
|
||||
|
||||
def parse(self):
|
||||
pos = 1
|
||||
(is_inst, inst_id) = self.get_instance_id(self.data[pos])
|
||||
inst = Instance(inst_id, self.id)
|
||||
self.instances.append(inst)
|
||||
size = len(self.data)
|
||||
prev = pos
|
||||
while pos < size - 1:
|
||||
pos = pos + 1
|
||||
(is_inst, inst_id) = self.get_instance_id(self.data[pos])
|
||||
if is_inst:
|
||||
inst.set_data(self.data[prev : pos])
|
||||
inst = Instance(inst_id, self.id)
|
||||
self.instances.append(inst)
|
||||
prev = pos
|
||||
|
||||
inst.set_data(self.data[prev : pos])
|
||||
|
||||
for inst in self.instances:
|
||||
inst.parse()
|
||||
self.srcs = self.instances[0].srcs
|
||||
self.target = self.instances[0].target
|
||||
|
||||
|
||||
def show(self, indent):
|
||||
print("{}{}".format(indent, self.id))
|
||||
for fr in self.children:
|
||||
fr.show(indent+" ")
|
||||
|
||||
def sum_nodes(self, nodes):
|
||||
sum = 0
|
||||
for node in nodes:
|
||||
sum = sum + node.rows
|
||||
nodes[0].rows = sum
|
||||
for (i, ch) in enumerate(nodes[0].children):
|
||||
chs = [node.children[i] for node in nodes]
|
||||
self.sum_nodes(chs)
|
||||
|
||||
def zip_instances(self, instances):
|
||||
'''sum rows of same nodes in different instance'''
|
||||
roots = [inst.root for inst in instances]
|
||||
self.sum_nodes(roots)
|
||||
|
||||
def to_dot(self):
|
||||
inst = self.instances[0]
|
||||
dot = ""
|
||||
dot = inst.root.to_dot(dot)
|
||||
return dot
|
||||
|
||||
class Tree:
|
||||
def __init__(self, fragments):
|
||||
self.fragments = fragments
|
||||
|
||||
def build(self):
|
||||
src_fr_map = {}
|
||||
for fr in self.fragments:
|
||||
if fr.target == "":
|
||||
self.root = fr
|
||||
for src in fr.srcs:
|
||||
src_fr_map[src] = fr
|
||||
for fr in self.fragments:
|
||||
if fr.target != "":
|
||||
parent = src_fr_map.get(fr.target)
|
||||
|
||||
parent.children.append(fr)
|
||||
fr.parent = parent
|
||||
|
||||
def show(self):
|
||||
self.root.show(" ")
|
||||
|
||||
def load_profile(profile):
|
||||
with open(profile, 'r') as f:
|
||||
l = f.readlines()
|
||||
return l
|
||||
|
||||
def get_fragment_id(line):
|
||||
#line in format : Fragment /d+:
|
||||
#len("Fragment "):10
|
||||
line = line.strip()
|
||||
return line[10:-1]
|
||||
|
||||
def is_fragment_begin(line):
|
||||
line = line.strip()
|
||||
return line.startswith("Fragment")
|
||||
|
||||
def cut_fragment(data):
|
||||
fragments = []
|
||||
prev = 0
|
||||
pos = 0
|
||||
total_len = len(data)
|
||||
#skip some lines not in fragment
|
||||
for line in data:
|
||||
if is_fragment_begin(line):
|
||||
break
|
||||
prev = prev + 1
|
||||
pos = prev
|
||||
fr_id = get_fragment_id(data[pos])
|
||||
pos = pos + 1
|
||||
current_fr = Fragment(fr_id)
|
||||
fragments = []
|
||||
fragments.append(current_fr)
|
||||
while pos < total_len:
|
||||
line = data[pos]
|
||||
if is_fragment_begin(line):
|
||||
current_fr.set_data(data[prev : pos])
|
||||
fr_id = get_fragment_id(line)
|
||||
current_fr = Fragment(fr_id)
|
||||
fragments.append(current_fr)
|
||||
prev = pos
|
||||
|
||||
pos = pos + 1
|
||||
|
||||
current_fr.set_data(data[prev: pos - 1])
|
||||
for fr in fragments:
|
||||
fr.parse()
|
||||
return fragments
|
||||
|
||||
def print_usage():
|
||||
print("""
|
||||
USAGE:
|
||||
python profile_viewer.py [PATH_TO_PROFILE]
|
||||
|
||||
graphviz is required(https://graphviz.org/)
|
||||
on linux: apt install graphvize
|
||||
on mac: brew install graphvize
|
||||
""")
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print_usage()
|
||||
exit(0)
|
||||
prf_file = sys.argv[1]
|
||||
if not os.path.exists(prf_file):
|
||||
print("{} is not a file".format(prf_file))
|
||||
exit(0)
|
||||
|
||||
data = load_profile(prf_file)
|
||||
fragements = []
|
||||
fragments = cut_fragment(data)
|
||||
dot_header = """digraph BPMN {
|
||||
rankdir=BT;
|
||||
node[shape=rectanble style="rounded,filled" color="lightgoldenrodyellow" ]
|
||||
"""
|
||||
dot_tail = "}"
|
||||
with open(prf_file+".dot", "w") as f:
|
||||
f.write(dot_header)
|
||||
for fr in fragments:
|
||||
# print("fr{}:{} <-{}".format(fr.id, fr.target, fr.srcs))
|
||||
fr.zip_instances(fr.instances)
|
||||
f.write(fr.to_dot())
|
||||
if fr.target != "":
|
||||
f.write("\n{}->{}[label={}]".format(fr.instances[0].root, exchanges[fr.target], fr.instances[0].root.rows))
|
||||
f.write(dot_tail)
|
||||
cmd = "dot {} -T png -o {}".format(prf_file+".dot", prf_file+".png")
|
||||
status = os.system(cmd)
|
||||
if status != 0:
|
||||
print("convert failed")
|
||||
else:
|
||||
print("generate " + prf_file + ".png")
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user