feat: add logic for reading manifest from bolt file (#183)
This commit is contained in:
Daniel Moran
GitHub
parent
4452aac87d
commit
9b54bcb17f
349
clients/backup/bolt.go
Normal file
349
clients/backup/bolt.go
Normal file
@ -0,0 +1,349 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/influxdata/influx-cli/v2/api"
|
||||
"github.com/influxdata/influx-cli/v2/clients/backup/internal"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
//go:generate protoc --gogo_out=. internal/meta.proto
|
||||
|
||||
// NOTE: An unfortunate naming collision below. Bolt calls its databases "buckets".
|
||||
// These are the names that were used in the metadata DB for 2.0.x versions of influxdb.
|
||||
var (
|
||||
bucketsBoltBucket = []byte("bucketsv1")
|
||||
organizationsBoltBucket = []byte("organizationsv1")
|
||||
v1MetadataBoltBucket = []byte("v1_tsm1_metadata")
|
||||
v1MetadataBoltKey = []byte("meta.db")
|
||||
)
|
||||
|
||||
// influxdbBucketSchema models the JSON structure used by InfluxDB 2.0.x to serialize
|
||||
// bucket metadata in the embedded KV store.
|
||||
type influxdbBucketSchema struct {
|
||||
ID string `json:"id"`
|
||||
OrgID string `json:"orgID"`
|
||||
Type int `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Description *string `json:"description,omitempty"`
|
||||
RetentionPeriod time.Duration `json:"retentionPeriod"`
|
||||
ShardGroupDuration time.Duration `json:"ShardGroupDuration"`
|
||||
}
|
||||
|
||||
// influxdbOrganizationSchema models the JSON structure used by InfluxDB 2.0.x to serialize
|
||||
// organization metadata in the embedded KV store.
|
||||
type influxdbOrganizationSchema struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// influxdbV1DatabaseInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// database info in the embedded KV store.
|
||||
type influxdbV1DatabaseInfo struct {
|
||||
Name string
|
||||
DefaultRetentionPolicy string
|
||||
RetentionPolicies []influxdbV1RetentionPolicyInfo
|
||||
}
|
||||
|
||||
// influxdbV1RetentionPolicyInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// retention-policy info in the embedded KV store.
|
||||
type influxdbV1RetentionPolicyInfo struct {
|
||||
Name string
|
||||
ReplicaN int32
|
||||
Duration int64
|
||||
ShardGroupDuration int64
|
||||
ShardGroups []influxdbV1ShardGroupInfo
|
||||
Subscriptions []influxdbV1SubscriptionInfo
|
||||
}
|
||||
|
||||
// influxdbV1ShardGroupInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// shard-group info in the embedded KV store.
|
||||
type influxdbV1ShardGroupInfo struct {
|
||||
ID int64
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
DeletedAt time.Time
|
||||
Shards []influxdbV1ShardInfo
|
||||
TruncatedAt time.Time
|
||||
}
|
||||
|
||||
// influxdbV1ShardInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// shard info in the embedded KV store.
|
||||
type influxdbV1ShardInfo struct {
|
||||
ID int64
|
||||
Owners []influxdbV1ShardOwnerInfo
|
||||
}
|
||||
|
||||
// inflxudbV1ShardOwnerInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// shard-owner info in the embedded KV store.
|
||||
type influxdbV1ShardOwnerInfo struct {
|
||||
NodeID int64
|
||||
}
|
||||
|
||||
// influxdbV1SubscriptionInfo models the protobuf structure used by InfluxDB 2.0.x to serialize
|
||||
// subscription info in the embedded KV store.
|
||||
type influxdbV1SubscriptionInfo struct {
|
||||
Name string
|
||||
Mode string
|
||||
Destinations []string
|
||||
}
|
||||
|
||||
// extractBucketManifest reads a boltdb backed up from InfluxDB 2.0.x, converting a subset of the
|
||||
// metadata it contains into a set of 2.1.x bucket manifests.
|
||||
func extractBucketManifest(boltPath string) ([]api.BucketMetadataManifest, error) {
|
||||
db, err := bbolt.Open(boltPath, 0666, &bbolt.Options{ReadOnly: true, Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
// Hack to give a slightly nicer error message for a known failure mode when bolt calls
|
||||
// mmap on a file system that doesn't support the MAP_SHARED option.
|
||||
//
|
||||
// See: https://github.com/boltdb/bolt/issues/272
|
||||
// See: https://stackoverflow.com/a/18421071
|
||||
if err.Error() == "invalid argument" {
|
||||
return nil, fmt.Errorf("unable to open boltdb: mmap of %q may not support the MAP_SHARED option", boltPath)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unable to open boltdb: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
// Read raw metadata needed to construct a manifest.
|
||||
var buckets []influxdbBucketSchema
|
||||
orgNamesById := map[string]string{}
|
||||
dbInfoByBucketId := map[string]influxdbV1DatabaseInfo{}
|
||||
|
||||
if err := db.View(func(tx *bbolt.Tx) error {
|
||||
bucketDB := tx.Bucket(bucketsBoltBucket)
|
||||
if bucketDB == nil {
|
||||
return errors.New("bucket metadata not found in local KV store")
|
||||
}
|
||||
|
||||
if err := bucketDB.ForEach(func(k, v []byte) error {
|
||||
var b influxdbBucketSchema
|
||||
if err := json.Unmarshal(v, &b); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.Type != 1 { // 1 == "system"
|
||||
buckets = append(buckets, b)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to read bucket metadata from local KV store: %w", err)
|
||||
}
|
||||
|
||||
orgsDB := tx.Bucket(organizationsBoltBucket)
|
||||
if orgsDB == nil {
|
||||
return errors.New("organization metadata not found in local KV store")
|
||||
}
|
||||
|
||||
if err := orgsDB.ForEach(func(k, v []byte) error {
|
||||
var o influxdbOrganizationSchema
|
||||
if err := json.Unmarshal(v, &o); err != nil {
|
||||
return err
|
||||
}
|
||||
orgNamesById[o.ID] = o.Name
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to read organization metadata from local KV store: %w", err)
|
||||
}
|
||||
|
||||
v1DB := tx.Bucket(v1MetadataBoltBucket)
|
||||
if v1DB == nil {
|
||||
return errors.New("v1 database info not found in local KV store")
|
||||
}
|
||||
fullMeta := v1DB.Get(v1MetadataBoltKey)
|
||||
if fullMeta == nil {
|
||||
return errors.New("v1 database info not found in local KV store")
|
||||
}
|
||||
|
||||
var pb internal.Data
|
||||
if err := proto.Unmarshal(fullMeta, &pb); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal v1 database info: %w", err)
|
||||
}
|
||||
for _, rawDBI := range pb.GetDatabases() {
|
||||
if rawDBI == nil {
|
||||
continue
|
||||
}
|
||||
unmarshalled := unmarshalRawDBI(*rawDBI)
|
||||
dbInfoByBucketId[unmarshalled.Name] = unmarshalled
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifests := make([]api.BucketMetadataManifest, len(buckets))
|
||||
for i, b := range buckets {
|
||||
orgName, ok := orgNamesById[b.OrgID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("local KV store in inconsistent state: no organization found with ID %q", b.OrgID)
|
||||
}
|
||||
dbi, ok := dbInfoByBucketId[b.ID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("local KV store in inconsistent state: no V1 database info found for bucket %q", b.Name)
|
||||
}
|
||||
manifests[i] = combineMetadata(b, orgName, dbi)
|
||||
}
|
||||
|
||||
return manifests, nil
|
||||
}
|
||||
|
||||
func unmarshalRawDBI(rawDBI internal.DatabaseInfo) influxdbV1DatabaseInfo {
|
||||
dbi := influxdbV1DatabaseInfo{
|
||||
Name: rawDBI.GetName(),
|
||||
DefaultRetentionPolicy: rawDBI.GetDefaultRetentionPolicy(),
|
||||
RetentionPolicies: make([]influxdbV1RetentionPolicyInfo, 0, len(rawDBI.GetRetentionPolicies())),
|
||||
}
|
||||
for _, rp := range rawDBI.GetRetentionPolicies() {
|
||||
if rp == nil {
|
||||
continue
|
||||
}
|
||||
dbi.RetentionPolicies = append(dbi.RetentionPolicies, unmarshalRawRPI(*rp))
|
||||
}
|
||||
return dbi
|
||||
}
|
||||
|
||||
func unmarshalRawRPI(rawRPI internal.RetentionPolicyInfo) influxdbV1RetentionPolicyInfo {
|
||||
rpi := influxdbV1RetentionPolicyInfo{
|
||||
Name: rawRPI.GetName(),
|
||||
ReplicaN: int32(rawRPI.GetReplicaN()),
|
||||
Duration: rawRPI.GetDuration(),
|
||||
ShardGroupDuration: rawRPI.GetShardGroupDuration(),
|
||||
ShardGroups: make([]influxdbV1ShardGroupInfo, 0, len(rawRPI.GetShardGroups())),
|
||||
Subscriptions: make([]influxdbV1SubscriptionInfo, 0, len(rawRPI.GetSubscriptions())),
|
||||
}
|
||||
for _, sg := range rawRPI.GetShardGroups() {
|
||||
if sg == nil {
|
||||
continue
|
||||
}
|
||||
rpi.ShardGroups = append(rpi.ShardGroups, unmarshalRawSGI(*sg))
|
||||
}
|
||||
for _, s := range rawRPI.GetSubscriptions() {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
rpi.Subscriptions = append(rpi.Subscriptions, influxdbV1SubscriptionInfo{
|
||||
Name: s.GetName(),
|
||||
Mode: s.GetMode(),
|
||||
Destinations: s.GetDestinations(),
|
||||
})
|
||||
}
|
||||
return rpi
|
||||
}
|
||||
|
||||
func unmarshalRawSGI(rawSGI internal.ShardGroupInfo) influxdbV1ShardGroupInfo {
|
||||
sgi := influxdbV1ShardGroupInfo{
|
||||
ID: int64(rawSGI.GetID()),
|
||||
StartTime: time.Unix(0, rawSGI.GetStartTime()).UTC(),
|
||||
EndTime: time.Unix(0, rawSGI.GetEndTime()).UTC(),
|
||||
DeletedAt: time.Unix(0, rawSGI.GetDeletedAt()).UTC(),
|
||||
Shards: make([]influxdbV1ShardInfo, 0, len(rawSGI.GetShards())),
|
||||
TruncatedAt: time.Unix(0, rawSGI.GetTruncatedAt()).UTC(),
|
||||
}
|
||||
for _, s := range rawSGI.GetShards() {
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
sgi.Shards = append(sgi.Shards, unmarshalRawShard(*s))
|
||||
}
|
||||
return sgi
|
||||
}
|
||||
|
||||
func unmarshalRawShard(rawShard internal.ShardInfo) influxdbV1ShardInfo {
|
||||
si := influxdbV1ShardInfo{
|
||||
ID: int64(rawShard.GetID()),
|
||||
}
|
||||
// If deprecated "OwnerIDs" exists then convert it to "Owners" format.
|
||||
//lint:ignore SA1019 we need to check for the presence of the deprecated field so we can convert it
|
||||
oldStyleOwnerIds := rawShard.GetOwnerIDs()
|
||||
if len(oldStyleOwnerIds) > 0 {
|
||||
si.Owners = make([]influxdbV1ShardOwnerInfo, len(oldStyleOwnerIds))
|
||||
for i, oid := range oldStyleOwnerIds {
|
||||
si.Owners[i] = influxdbV1ShardOwnerInfo{NodeID: int64(oid)}
|
||||
}
|
||||
} else {
|
||||
si.Owners = make([]influxdbV1ShardOwnerInfo, 0, len(rawShard.GetOwners()))
|
||||
for _, o := range rawShard.GetOwners() {
|
||||
if o == nil {
|
||||
continue
|
||||
}
|
||||
si.Owners = append(si.Owners, influxdbV1ShardOwnerInfo{NodeID: int64(o.GetNodeID())})
|
||||
}
|
||||
}
|
||||
return si
|
||||
}
|
||||
|
||||
func combineMetadata(bucket influxdbBucketSchema, orgName string, dbi influxdbV1DatabaseInfo) api.BucketMetadataManifest {
|
||||
m := api.BucketMetadataManifest{
|
||||
OrganizationID: bucket.OrgID,
|
||||
OrganizationName: orgName,
|
||||
BucketID: bucket.ID,
|
||||
BucketName: bucket.Name,
|
||||
DefaultRetentionPolicy: dbi.DefaultRetentionPolicy,
|
||||
RetentionPolicies: make([]api.RetentionPolicyManifest, len(dbi.RetentionPolicies)),
|
||||
}
|
||||
if bucket.Description != nil && *bucket.Description != "" {
|
||||
m.Description = bucket.Description
|
||||
}
|
||||
for i, rp := range dbi.RetentionPolicies {
|
||||
m.RetentionPolicies[i] = convertRPI(rp)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func convertRPI(rpi influxdbV1RetentionPolicyInfo) api.RetentionPolicyManifest {
|
||||
m := api.RetentionPolicyManifest{
|
||||
Name: rpi.Name,
|
||||
ReplicaN: rpi.ReplicaN,
|
||||
Duration: rpi.Duration,
|
||||
ShardGroupDuration: rpi.ShardGroupDuration,
|
||||
ShardGroups: make([]api.ShardGroupManifest, len(rpi.ShardGroups)),
|
||||
Subscriptions: make([]api.SubscriptionManifest, len(rpi.Subscriptions)),
|
||||
}
|
||||
for i, sg := range rpi.ShardGroups {
|
||||
m.ShardGroups[i] = convertSGI(sg)
|
||||
}
|
||||
for i, s := range rpi.Subscriptions {
|
||||
m.Subscriptions[i] = api.SubscriptionManifest{
|
||||
Name: s.Name,
|
||||
Mode: s.Mode,
|
||||
Destinations: s.Destinations,
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func convertSGI(sgi influxdbV1ShardGroupInfo) api.ShardGroupManifest {
|
||||
m := api.ShardGroupManifest{
|
||||
Id: sgi.ID,
|
||||
StartTime: sgi.StartTime,
|
||||
EndTime: sgi.EndTime,
|
||||
Shards: make([]api.ShardManifest, len(sgi.Shards)),
|
||||
}
|
||||
if sgi.DeletedAt.Unix() != 0 {
|
||||
m.DeletedAt = &sgi.DeletedAt
|
||||
}
|
||||
if sgi.TruncatedAt.Unix() != 0 {
|
||||
m.TruncatedAt = &sgi.TruncatedAt
|
||||
}
|
||||
for i, s := range sgi.Shards {
|
||||
m.Shards[i] = convertShard(s)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func convertShard(shard influxdbV1ShardInfo) api.ShardManifest {
|
||||
m := api.ShardManifest{
|
||||
Id: shard.ID,
|
||||
ShardOwners: make([]api.ShardOwner, len(shard.Owners)),
|
||||
}
|
||||
for i, o := range shard.Owners {
|
||||
m.ShardOwners[i] = api.ShardOwner{NodeID: o.NodeID}
|
||||
}
|
||||
return m
|
||||
}
|
||||
106
clients/backup/bolt_internal_test.go
Normal file
106
clients/backup/bolt_internal_test.go
Normal file
@ -0,0 +1,106 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/influxdata/influx-cli/v2/api"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const testFile = "testdata/test.bolt.gz"
|
||||
|
||||
func TestExtractManifest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("skipping test on Windows: https://github.com/etcd-io/bbolt/issues/252")
|
||||
}
|
||||
|
||||
// Extract our example input into a format the bbolt client can use.
|
||||
boltIn, err := os.Open(testFile)
|
||||
require.NoError(t, err)
|
||||
defer boltIn.Close()
|
||||
gzipIn, err := gzip.NewReader(boltIn)
|
||||
require.NoError(t, err)
|
||||
defer gzipIn.Close()
|
||||
|
||||
tmp, err := os.MkdirTemp("", "")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
tmpBoltPath := filepath.Join(tmp, "test.bolt")
|
||||
tmpBolt, err := os.Create(tmpBoltPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = io.Copy(tmpBolt, gzipIn)
|
||||
require.NoError(t, tmpBolt.Close())
|
||||
require.NoError(t, err)
|
||||
|
||||
extracted, err := extractBucketManifest(tmpBoltPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := []api.BucketMetadataManifest{
|
||||
{
|
||||
OrganizationID: "80c29010030b3d83",
|
||||
OrganizationName: "test2",
|
||||
BucketID: "5379ef2d1655b0ab",
|
||||
BucketName: "test3",
|
||||
DefaultRetentionPolicy: "autogen",
|
||||
RetentionPolicies: []api.RetentionPolicyManifest{
|
||||
{
|
||||
Name: "autogen",
|
||||
ReplicaN: 1,
|
||||
Duration: 0,
|
||||
ShardGroupDuration: 604800000000000,
|
||||
ShardGroups: []api.ShardGroupManifest{},
|
||||
Subscriptions: []api.SubscriptionManifest{},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
OrganizationID: "375477729f9d7262",
|
||||
OrganizationName: "test",
|
||||
BucketID: "cce01ef3783e3678",
|
||||
BucketName: "test2",
|
||||
DefaultRetentionPolicy: "autogen",
|
||||
RetentionPolicies: []api.RetentionPolicyManifest{
|
||||
{
|
||||
Name: "autogen",
|
||||
ReplicaN: 1,
|
||||
Duration: 0,
|
||||
ShardGroupDuration: 3600000000000,
|
||||
ShardGroups: []api.ShardGroupManifest{},
|
||||
Subscriptions: []api.SubscriptionManifest{},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
OrganizationID: "375477729f9d7262",
|
||||
OrganizationName: "test",
|
||||
BucketID: "d66c5360b5aa91b4",
|
||||
BucketName: "test",
|
||||
DefaultRetentionPolicy: "autogen",
|
||||
RetentionPolicies: []api.RetentionPolicyManifest{
|
||||
{
|
||||
Name: "autogen",
|
||||
ReplicaN: 1,
|
||||
Duration: 259200000000000,
|
||||
ShardGroupDuration: 86400000000000,
|
||||
ShardGroups: []api.ShardGroupManifest{},
|
||||
Subscriptions: []api.SubscriptionManifest{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
require.Equal(t, len(expected), len(extracted))
|
||||
require.Equal(t, expected, extracted)
|
||||
for _, e := range expected {
|
||||
require.Contains(t, extracted, e)
|
||||
}
|
||||
}
|
||||
12
clients/backup/internal/README.md
Normal file
12
clients/backup/internal/README.md
Normal file
@ -0,0 +1,12 @@
|
||||
# V1 Meta Protobufs
|
||||
|
||||
For compatibility with backups made via the v2.0.x `influx` CLI, we include logic
|
||||
for opening & reading backed-up KV stores to derive bucket manifests. Part of that
|
||||
process requires reading & unmarshalling V1 database info, serialized as protobuf.
|
||||
To support that requirement, we've copied the `meta.proto` definition out of `influxdb`
|
||||
and into this repository. This file isn't intended to be modified.
|
||||
|
||||
If `meta.pb.go` ever needs to be re-generated, follow these steps:
|
||||
1. Install `protoc` (i.e. via `brew install protobuf`)
|
||||
2. Run `go install github.com/gogo/protobuf/protoc-gen-gogo` from within this repository
|
||||
3. Run `go generate <path to clients/backup>`
|
||||
2867
clients/backup/internal/meta.pb.go
Normal file
2867
clients/backup/internal/meta.pb.go
Normal file
File diff suppressed because it is too large
Load Diff
397
clients/backup/internal/meta.proto
Normal file
397
clients/backup/internal/meta.proto
Normal file
@ -0,0 +1,397 @@
|
||||
// NOTE: This is a snapshot of the schema used to serialize V1 database info
|
||||
// in the 2.0.x line of InfluxDB. The copy is here so we can support backing
|
||||
// up from older DB versions, it's not intended to be kept up-to-date.
|
||||
|
||||
package internal;
|
||||
|
||||
//========================================================================
|
||||
//
|
||||
// Metadata
|
||||
//
|
||||
//========================================================================
|
||||
|
||||
message Data {
|
||||
required uint64 Term = 1;
|
||||
required uint64 Index = 2;
|
||||
required uint64 ClusterID = 3;
|
||||
|
||||
repeated NodeInfo Nodes = 4;
|
||||
repeated DatabaseInfo Databases = 5;
|
||||
repeated UserInfo Users = 6;
|
||||
|
||||
required uint64 MaxNodeID = 7;
|
||||
required uint64 MaxShardGroupID = 8;
|
||||
required uint64 MaxShardID = 9;
|
||||
|
||||
// added for 0.10.0
|
||||
repeated NodeInfo DataNodes = 10;
|
||||
repeated NodeInfo MetaNodes = 11;
|
||||
}
|
||||
|
||||
message NodeInfo {
|
||||
required uint64 ID = 1;
|
||||
required string Host = 2;
|
||||
optional string TCPHost = 3;
|
||||
}
|
||||
|
||||
message DatabaseInfo {
|
||||
required string Name = 1;
|
||||
required string DefaultRetentionPolicy = 2;
|
||||
repeated RetentionPolicyInfo RetentionPolicies = 3;
|
||||
repeated ContinuousQueryInfo ContinuousQueries = 4;
|
||||
}
|
||||
|
||||
message RetentionPolicySpec {
|
||||
optional string Name = 1;
|
||||
optional int64 Duration = 2;
|
||||
optional int64 ShardGroupDuration = 3;
|
||||
optional uint32 ReplicaN = 4;
|
||||
}
|
||||
|
||||
message RetentionPolicyInfo {
|
||||
required string Name = 1;
|
||||
required int64 Duration = 2;
|
||||
required int64 ShardGroupDuration = 3;
|
||||
required uint32 ReplicaN = 4;
|
||||
repeated ShardGroupInfo ShardGroups = 5;
|
||||
repeated SubscriptionInfo Subscriptions = 6;
|
||||
}
|
||||
|
||||
message ShardGroupInfo {
|
||||
required uint64 ID = 1;
|
||||
required int64 StartTime = 2;
|
||||
required int64 EndTime = 3;
|
||||
required int64 DeletedAt = 4;
|
||||
repeated ShardInfo Shards = 5;
|
||||
optional int64 TruncatedAt = 6;
|
||||
}
|
||||
|
||||
message ShardInfo {
|
||||
required uint64 ID = 1;
|
||||
repeated uint64 OwnerIDs = 2 [deprecated=true];
|
||||
repeated ShardOwner Owners = 3;
|
||||
}
|
||||
|
||||
message SubscriptionInfo{
|
||||
required string Name = 1;
|
||||
required string Mode = 2;
|
||||
repeated string Destinations = 3;
|
||||
}
|
||||
|
||||
message ShardOwner {
|
||||
required uint64 NodeID = 1;
|
||||
}
|
||||
|
||||
message ContinuousQueryInfo {
|
||||
required string Name = 1;
|
||||
required string Query = 2;
|
||||
}
|
||||
|
||||
message UserInfo {
|
||||
required string Name = 1;
|
||||
required string Hash = 2;
|
||||
required bool Admin = 3;
|
||||
repeated UserPrivilege Privileges = 4;
|
||||
}
|
||||
|
||||
message UserPrivilege {
|
||||
required string Database = 1;
|
||||
required int32 Privilege = 2;
|
||||
}
|
||||
|
||||
|
||||
//========================================================================
|
||||
//
|
||||
// COMMANDS
|
||||
//
|
||||
//========================================================================
|
||||
|
||||
message Command {
|
||||
extensions 100 to max;
|
||||
|
||||
enum Type {
|
||||
CreateNodeCommand = 1;
|
||||
DeleteNodeCommand = 2;
|
||||
CreateDatabaseCommand = 3;
|
||||
DropDatabaseCommand = 4;
|
||||
CreateRetentionPolicyCommand = 5;
|
||||
DropRetentionPolicyCommand = 6;
|
||||
SetDefaultRetentionPolicyCommand = 7;
|
||||
UpdateRetentionPolicyCommand = 8;
|
||||
CreateShardGroupCommand = 9;
|
||||
DeleteShardGroupCommand = 10;
|
||||
CreateContinuousQueryCommand = 11;
|
||||
DropContinuousQueryCommand = 12;
|
||||
CreateUserCommand = 13;
|
||||
DropUserCommand = 14;
|
||||
UpdateUserCommand = 15;
|
||||
SetPrivilegeCommand = 16;
|
||||
SetDataCommand = 17;
|
||||
SetAdminPrivilegeCommand = 18;
|
||||
UpdateNodeCommand = 19;
|
||||
CreateSubscriptionCommand = 21;
|
||||
DropSubscriptionCommand = 22;
|
||||
RemovePeerCommand = 23;
|
||||
CreateMetaNodeCommand = 24;
|
||||
CreateDataNodeCommand = 25;
|
||||
UpdateDataNodeCommand = 26;
|
||||
DeleteMetaNodeCommand = 27;
|
||||
DeleteDataNodeCommand = 28;
|
||||
SetMetaNodeCommand = 29;
|
||||
DropShardCommand = 30;
|
||||
}
|
||||
|
||||
required Type type = 1;
|
||||
}
|
||||
|
||||
// This isn't used in >= 0.10.0. Kept around for upgrade purposes. Instead
|
||||
// look at CreateDataNodeCommand and CreateMetaNodeCommand
|
||||
message CreateNodeCommand {
|
||||
extend Command {
|
||||
optional CreateNodeCommand command = 101;
|
||||
}
|
||||
required string Host = 1;
|
||||
required uint64 Rand = 2;
|
||||
}
|
||||
|
||||
message DeleteNodeCommand {
|
||||
extend Command {
|
||||
optional DeleteNodeCommand command = 102;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
required bool Force = 2;
|
||||
}
|
||||
|
||||
message CreateDatabaseCommand {
|
||||
extend Command {
|
||||
optional CreateDatabaseCommand command = 103;
|
||||
}
|
||||
required string Name = 1;
|
||||
optional RetentionPolicyInfo RetentionPolicy = 2;
|
||||
}
|
||||
|
||||
message DropDatabaseCommand {
|
||||
extend Command {
|
||||
optional DropDatabaseCommand command = 104;
|
||||
}
|
||||
required string Name = 1;
|
||||
}
|
||||
|
||||
message CreateRetentionPolicyCommand {
|
||||
extend Command {
|
||||
optional CreateRetentionPolicyCommand command = 105;
|
||||
}
|
||||
required string Database = 1;
|
||||
required RetentionPolicyInfo RetentionPolicy = 2;
|
||||
}
|
||||
|
||||
message DropRetentionPolicyCommand {
|
||||
extend Command {
|
||||
optional DropRetentionPolicyCommand command = 106;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Name = 2;
|
||||
}
|
||||
|
||||
message SetDefaultRetentionPolicyCommand {
|
||||
extend Command {
|
||||
optional SetDefaultRetentionPolicyCommand command = 107;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Name = 2;
|
||||
}
|
||||
|
||||
message UpdateRetentionPolicyCommand {
|
||||
extend Command {
|
||||
optional UpdateRetentionPolicyCommand command = 108;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Name = 2;
|
||||
optional string NewName = 3;
|
||||
optional int64 Duration = 4;
|
||||
optional uint32 ReplicaN = 5;
|
||||
}
|
||||
|
||||
message CreateShardGroupCommand {
|
||||
extend Command {
|
||||
optional CreateShardGroupCommand command = 109;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Policy = 2;
|
||||
required int64 Timestamp = 3;
|
||||
}
|
||||
|
||||
message DeleteShardGroupCommand {
|
||||
extend Command {
|
||||
optional DeleteShardGroupCommand command = 110;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Policy = 2;
|
||||
required uint64 ShardGroupID = 3;
|
||||
}
|
||||
|
||||
message CreateContinuousQueryCommand {
|
||||
extend Command {
|
||||
optional CreateContinuousQueryCommand command = 111;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Name = 2;
|
||||
required string Query = 3;
|
||||
}
|
||||
|
||||
message DropContinuousQueryCommand {
|
||||
extend Command {
|
||||
optional DropContinuousQueryCommand command = 112;
|
||||
}
|
||||
required string Database = 1;
|
||||
required string Name = 2;
|
||||
}
|
||||
|
||||
message CreateUserCommand {
|
||||
extend Command {
|
||||
optional CreateUserCommand command = 113;
|
||||
}
|
||||
required string Name = 1;
|
||||
required string Hash = 2;
|
||||
required bool Admin = 3;
|
||||
}
|
||||
|
||||
message DropUserCommand {
|
||||
extend Command {
|
||||
optional DropUserCommand command = 114;
|
||||
}
|
||||
required string Name = 1;
|
||||
}
|
||||
|
||||
message UpdateUserCommand {
|
||||
extend Command {
|
||||
optional UpdateUserCommand command = 115;
|
||||
}
|
||||
required string Name = 1;
|
||||
required string Hash = 2;
|
||||
}
|
||||
|
||||
message SetPrivilegeCommand {
|
||||
extend Command {
|
||||
optional SetPrivilegeCommand command = 116;
|
||||
}
|
||||
required string Username = 1;
|
||||
required string Database = 2;
|
||||
required int32 Privilege = 3;
|
||||
}
|
||||
|
||||
message SetDataCommand {
|
||||
extend Command {
|
||||
optional SetDataCommand command = 117;
|
||||
}
|
||||
required Data Data = 1;
|
||||
}
|
||||
|
||||
message SetAdminPrivilegeCommand {
|
||||
extend Command {
|
||||
optional SetAdminPrivilegeCommand command = 118;
|
||||
}
|
||||
required string Username = 1;
|
||||
required bool Admin = 2;
|
||||
}
|
||||
|
||||
message UpdateNodeCommand {
|
||||
extend Command {
|
||||
optional UpdateNodeCommand command = 119;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
required string Host = 2;
|
||||
}
|
||||
|
||||
message CreateSubscriptionCommand {
|
||||
extend Command {
|
||||
optional CreateSubscriptionCommand command = 121;
|
||||
}
|
||||
required string Name = 1;
|
||||
required string Database = 2;
|
||||
required string RetentionPolicy = 3;
|
||||
required string Mode = 4;
|
||||
repeated string Destinations = 5;
|
||||
|
||||
}
|
||||
|
||||
message DropSubscriptionCommand {
|
||||
extend Command {
|
||||
optional DropSubscriptionCommand command = 122;
|
||||
}
|
||||
required string Name = 1;
|
||||
required string Database = 2;
|
||||
required string RetentionPolicy = 3;
|
||||
}
|
||||
|
||||
message RemovePeerCommand {
|
||||
extend Command {
|
||||
optional RemovePeerCommand command = 123;
|
||||
}
|
||||
optional uint64 ID = 1;
|
||||
required string Addr = 2;
|
||||
}
|
||||
|
||||
message CreateMetaNodeCommand {
|
||||
extend Command {
|
||||
optional CreateMetaNodeCommand command = 124;
|
||||
}
|
||||
required string HTTPAddr = 1;
|
||||
required string TCPAddr = 2;
|
||||
required uint64 Rand = 3;
|
||||
}
|
||||
|
||||
message CreateDataNodeCommand {
|
||||
extend Command {
|
||||
optional CreateDataNodeCommand command = 125;
|
||||
}
|
||||
required string HTTPAddr = 1;
|
||||
required string TCPAddr = 2;
|
||||
}
|
||||
|
||||
message UpdateDataNodeCommand {
|
||||
extend Command {
|
||||
optional UpdateDataNodeCommand command = 126;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
required string Host = 2;
|
||||
required string TCPHost = 3;
|
||||
}
|
||||
|
||||
message DeleteMetaNodeCommand {
|
||||
extend Command {
|
||||
optional DeleteMetaNodeCommand command = 127;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
}
|
||||
|
||||
message DeleteDataNodeCommand {
|
||||
extend Command {
|
||||
optional DeleteDataNodeCommand command = 128;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
}
|
||||
|
||||
message Response {
|
||||
required bool OK = 1;
|
||||
optional string Error = 2;
|
||||
optional uint64 Index = 3;
|
||||
}
|
||||
|
||||
// SetMetaNodeCommand is for the initial metanode in a cluster or
|
||||
// if the single host restarts and its hostname changes, this will update it
|
||||
message SetMetaNodeCommand {
|
||||
extend Command {
|
||||
optional SetMetaNodeCommand command = 129;
|
||||
}
|
||||
required string HTTPAddr = 1;
|
||||
required string TCPAddr = 2;
|
||||
required uint64 Rand = 3;
|
||||
}
|
||||
|
||||
message DropShardCommand {
|
||||
extend Command {
|
||||
optional DropShardCommand command = 130;
|
||||
}
|
||||
required uint64 ID = 1;
|
||||
}
|
||||
BIN
clients/backup/testdata/test.bolt.gz
vendored
Normal file
BIN
clients/backup/testdata/test.bolt.gz
vendored
Normal file
Binary file not shown.
2
go.mod
2
go.mod
@ -10,6 +10,7 @@ require (
|
||||
github.com/fatih/color v1.9.0
|
||||
github.com/fujiwara/shapeio v1.0.0
|
||||
github.com/gocarina/gocsv v0.0.0-20210408192840-02d7211d929d
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang/mock v1.5.0
|
||||
github.com/google/go-cmp v0.5.5
|
||||
github.com/google/go-jsonnet v0.17.0
|
||||
@ -17,6 +18,7 @@ require (
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/urfave/cli v1.22.5
|
||||
go.etcd.io/bbolt v1.3.6
|
||||
golang.org/x/text v0.3.3
|
||||
golang.org/x/tools v0.1.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
|
||||
|
||||
12
go.sum
12
go.sum
@ -21,6 +21,8 @@ github.com/fujiwara/shapeio v1.0.0 h1:xG5D9oNqCSUUbryZ/jQV3cqe1v2suEjwPIcEg1gKM8
|
||||
github.com/fujiwara/shapeio v1.0.0/go.mod h1:LmEmu6L/8jetyj1oewewFb7bZCNRwE7wLCUNzDLaLVA=
|
||||
github.com/gocarina/gocsv v0.0.0-20210408192840-02d7211d929d h1:r3mStZSyjKhEcgbJ5xtv7kT5PZw/tDiFBTMgQx2qsXE=
|
||||
github.com/gocarina/gocsv v0.0.0-20210408192840-02d7211d929d/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
@ -31,6 +33,7 @@ github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDG
|
||||
github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@ -68,18 +71,24 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
|
||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
|
||||
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -87,6 +96,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -97,7 +107,9 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiT
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20201118003311-bd56c0adb394/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user