036ef5bcb9
In order to be aware of the convert process from AlphaRowset to BetaRowset, we need a mechanism to know the process of convert.
111 lines
3.5 KiB
Python
111 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import json
|
|
import time
|
|
from urllib import urlopen
|
|
|
|
class BeTabletResolver:
|
|
def __init__(self, be_list, tablet_map):
|
|
self.tablet_map = tablet_map
|
|
self.tablet_infos = {}
|
|
|
|
self.be_map = {}
|
|
for be in be_list:
|
|
self.be_map[be['be_id']] = be
|
|
|
|
def debug_output(self):
|
|
print "tablet_infos:(%s), print up to ten here:" % len(self.tablet_infos)
|
|
self._print_list(self.tablet_infos.values()[0:10])
|
|
print
|
|
|
|
def _print_list(self, one_list):
|
|
for item in one_list:
|
|
print item
|
|
|
|
def init(self):
|
|
self.fetch_tablet_meta()
|
|
|
|
def fetch_tablet_meta(self):
|
|
print "fetching tablet metas from BEs..."
|
|
count = 0
|
|
for tablet in self.tablet_map.values():
|
|
be_id = tablet['be_id']
|
|
be = self.be_map[be_id]
|
|
url = self._make_url(be, tablet)
|
|
print url
|
|
tablet_meta = self._fetch_tablet_meta_by_id(url)
|
|
self._decode_rs_metas_of_tablet(tablet_meta)
|
|
# slow down, do not need too fast
|
|
count += 1
|
|
if count % 10 == 0:
|
|
time.sleep(0.005)
|
|
print "finished. \n"
|
|
return
|
|
|
|
def _make_url(self, be, tablet):
|
|
url_list = []
|
|
url_list.append("http://")
|
|
url_list.append(be["ip"])
|
|
url_list.append(":")
|
|
url_list.append(be["http_port"])
|
|
url_list.append("/api/meta/header/")
|
|
url_list.append(str(tablet["tablet_id"]))
|
|
url_list.append("/")
|
|
url_list.append(str(tablet["schema_hash"]))
|
|
return "".join(url_list)
|
|
|
|
def _fetch_tablet_meta_by_id(self, url):
|
|
tablet_meta = urlopen(url).read()
|
|
tablet_meta = json.loads(tablet_meta)
|
|
return tablet_meta
|
|
|
|
def _decode_rs_metas_of_tablet(self, tablet_meta):
|
|
# When something wrong, may do not have rs_metas attr, so use 'get()' instead of '[]'
|
|
rs_metas = tablet_meta.get('rs_metas')
|
|
if rs_metas is None:
|
|
return
|
|
size = len(rs_metas)
|
|
|
|
rowsets = []
|
|
for rs_meta in rs_metas:
|
|
rowset = {}
|
|
rowset['tablet_id'] = rs_meta['tablet_id']
|
|
rowset['num_rows'] = rs_meta['num_rows']
|
|
rowset['data_disk_size'] = rs_meta['data_disk_size']
|
|
if rs_meta['rowset_type'] == 'BETA_ROWSET':
|
|
rowset['is_beta'] = True
|
|
else:
|
|
rowset['is_beta'] = False
|
|
rowsets.append(rowset);
|
|
|
|
self.tablet_infos[rs_meta['tablet_id']] = rowsets
|
|
return
|
|
|
|
def get_rowsets_by_tablet(self, tablet_id):
|
|
return self.tablet_infos.get(tablet_id)
|
|
|
|
def get_all_rowsets(self):
|
|
return self.tablet_infos.values()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
|