hcq/influx-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement influx restore using new APIs (#121)

Browse Source
This commit is contained in:
Daniel Moran
2021-06-15 16:45:55 -04:00
committed by GitHub
parent abe521add0
commit 6757c2bcfa
10 changed files with 650 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
clients/backup/backup.go
View File

@ -15,6 +15,7 @@ import (
"github.com/influxdata/influx-cli/v2/api"
"github.com/influxdata/influx-cli/v2/clients"
br "github.com/influxdata/influx-cli/v2/internal/backup_restore"
)
type Client struct {
@ -24,7 +25,7 @@ type Client struct {
// Local state tracked across steps in the backup process.
baseName string
bucketMetadata []api.BucketMetadataManifest
manifest Manifest
manifest br.Manifest
}
type Params struct {
@ -42,7 +43,7 @@ type Params struct {
Path string
// Compression to use for local copies of snapshot files.
Compression FileCompression
Compression br.FileCompression
}
func (p *Params) matches(bkt api.BucketMetadataManifest) bool {
@ -104,9 +105,9 @@ func (c *Client) downloadMetadata(ctx context.Context, params *Params) error {
}
defer body.Close()
writeFile := func(from io.Reader, to string) (ManifestFileEntry, error) {
writeFile := func(from io.Reader, to string) (br.ManifestFileEntry, error) {
toPath := filepath.Join(params.Path, to)
if params.Compression == GzipCompression {
if params.Compression == br.GzipCompression {
toPath = toPath + ".gz"
}
@ -120,7 +121,7 @@ func (c *Client) downloadMetadata(ctx context.Context, params *Params) error {
defer out.Close()
var outW io.Writer = out
if params.Compression == GzipCompression {
if params.Compression == br.GzipCompression {
gw := gzip.NewWriter(out)
defer gw.Close()
outW = gw
@ -129,14 +130,14 @@ func (c *Client) downloadMetadata(ctx context.Context, params *Params) error {
_, err = io.Copy(outW, from)
return err
}(); err != nil {
return ManifestFileEntry{}, err
return br.ManifestFileEntry{}, err
}
fi, err := os.Stat(toPath)
if err != nil {
return ManifestFileEntry{}, err
return br.ManifestFileEntry{}, err
}
return ManifestFileEntry{
return br.ManifestFileEntry{
FileName: fi.Name(),
Size: fi.Size(),
Compression: params.Compression,
@ -185,12 +186,12 @@ func (c *Client) downloadMetadata(ctx context.Context, params *Params) error {
//
// Bucket metadata must be pre-seeded via downloadMetadata before this method is called.
func (c *Client) downloadBucketData(ctx context.Context, params *Params) error {
c.manifest.Buckets = make([]ManifestBucketEntry, 0, len(c.bucketMetadata))
c.manifest.Buckets = make([]br.ManifestBucketEntry, 0, len(c.bucketMetadata))
for _, b := range c.bucketMetadata {
if !params.matches(b) {
continue
}
bktManifest, err := ConvertBucketManifest(b, func(shardId int64) (*ManifestFileEntry, error) {
bktManifest, err := ConvertBucketManifest(b, func(shardId int64) (*br.ManifestFileEntry, error) {
return c.downloadShardData(ctx, params, shardId)
})
if err != nil {
@ -203,7 +204,7 @@ func (c *Client) downloadBucketData(ctx context.Context, params *Params) error {
// downloadShardData downloads the TSM snapshot for a single shard. The snapshot is written
// to a local file, and its metadata is returned for aggregation.
func (c Client) downloadShardData(ctx context.Context, params *Params, shardId int64) (*ManifestFileEntry, error) {
func (c Client) downloadShardData(ctx context.Context, params *Params, shardId int64) (*br.ManifestFileEntry, error) {
log.Printf("INFO: Backing up TSM for shard %d", shardId)
res, err := c.GetBackupShardId(ctx, shardId).AcceptEncoding("gzip").Execute()
if err != nil {
@ -218,7 +219,7 @@ func (c Client) downloadShardData(ctx context.Context, params *Params, shardId i
defer res.Body.Close()
fileName := fmt.Sprintf("%s.%d.tar", c.baseName, shardId)
if params.Compression == GzipCompression {
if params.Compression == br.GzipCompression {
fileName = fileName + ".gz"
}
path := filepath.Join(params.Path, fileName)
@ -236,12 +237,12 @@ func (c Client) downloadShardData(ctx context.Context, params *Params, shardId i
var outW io.Writer = f
// Make sure the locally-written data is compressed according to the user's request.
if params.Compression == GzipCompression && res.Header.Get("Content-Encoding") != "gzip" {
if params.Compression == br.GzipCompression && res.Header.Get("Content-Encoding") != "gzip" {
gzw := gzip.NewWriter(outW)
defer gzw.Close()
outW = gzw
}
if params.Compression == NoCompression && res.Header.Get("Content-Encoding") == "gzip" {
if params.Compression == br.NoCompression && res.Header.Get("Content-Encoding") == "gzip" {
gzr, err := gzip.NewReader(inR)
if err != nil {
return err
@ -260,7 +261,7 @@ func (c Client) downloadShardData(ctx context.Context, params *Params, shardId i
if err != nil {
return nil, err
}
return &ManifestFileEntry{
return &br.ManifestFileEntry{
FileName: fi.Name(),
Size: fi.Size(),
Compression: params.Compression,
@ -270,7 +271,7 @@ func (c Client) downloadShardData(ctx context.Context, params *Params, shardId i
// writeManifest writes a description of all files downloaded as part of the backup process
// to the backup folder, encoded as JSON.
func (c Client) writeManifest(params *Params) error {
manifestPath := filepath.Join(params.Path, fmt.Sprintf("%s.manifest", c.baseName))
manifestPath := filepath.Join(params.Path, fmt.Sprintf("%s.%s", c.baseName, br.ManifestExtension))
f, err := os.OpenFile(manifestPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err

55
clients/backup/backup_internal_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/golang/mock/gomock"
"github.com/influxdata/influx-cli/v2/api"
"github.com/influxdata/influx-cli/v2/clients"
br "github.com/influxdata/influx-cli/v2/internal/backup_restore"
"github.com/influxdata/influx-cli/v2/internal/mock"
"github.com/stretchr/testify/require"
)
@ -55,28 +56,28 @@ func TestBackup_DownloadMetadata(t *testing.T) {
testCases := []struct {
name string
compression FileCompression
responseCompression FileCompression
compression br.FileCompression
responseCompression br.FileCompression
}{
{
name: "no gzip",
compression: NoCompression,
responseCompression: NoCompression,
compression: br.NoCompression,
responseCompression: br.NoCompression,
},
{
name: "response gzip, no local gzip",
compression: NoCompression,
responseCompression: GzipCompression,
compression: br.NoCompression,
responseCompression: br.GzipCompression,
},
{
name: "no response gzip, local gzip",
compression: GzipCompression,
responseCompression: NoCompression,
compression: br.GzipCompression,
responseCompression: br.NoCompression,
},
{
name: "all gzip",
compression: GzipCompression,
responseCompression: GzipCompression,
compression: br.GzipCompression,
responseCompression: br.GzipCompression,
},
}
@ -94,7 +95,7 @@ func TestBackup_DownloadMetadata(t *testing.T) {
DoAndReturn(func(request api.ApiGetBackupMetadataRequest) (*http.Response, error) {
out := bytes.Buffer{}
var outW io.Writer = &out
if tc.responseCompression == GzipCompression {
if tc.responseCompression == br.GzipCompression {
gzw := gzip.NewWriter(outW)
defer gzw.Close()
outW = gzw
@ -144,7 +145,7 @@ func TestBackup_DownloadMetadata(t *testing.T) {
res := http.Response{Header: http.Header{}, Body: ioutil.NopCloser(&out)}
res.Header.Add("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", writer.Boundary()))
if tc.responseCompression == GzipCompression {
if tc.responseCompression == br.GzipCompression {
res.Header.Add("Content-Encoding", "gzip")
}
return &res, nil
@ -173,7 +174,7 @@ func TestBackup_DownloadMetadata(t *testing.T) {
defer localKv.Close()
var kvReader io.Reader = localKv
if tc.compression == GzipCompression {
if tc.compression == br.GzipCompression {
gzr, err := gzip.NewReader(kvReader)
require.NoError(t, err)
defer gzr.Close()
@ -188,7 +189,7 @@ func TestBackup_DownloadMetadata(t *testing.T) {
defer localSql.Close()
var sqlReader io.Reader = localSql
if tc.compression == GzipCompression {
if tc.compression == br.GzipCompression {
gzr, err := gzip.NewReader(sqlReader)
require.NoError(t, err)
defer gzr.Close()
@ -208,28 +209,28 @@ func TestBackup_DownloadShardData(t *testing.T) {
testCases := []struct {
name string
compression FileCompression
responseCompression FileCompression
compression br.FileCompression
responseCompression br.FileCompression
}{
{
name: "no gzip",
compression: NoCompression,
responseCompression: NoCompression,
compression: br.NoCompression,
responseCompression: br.NoCompression,
},
{
name: "response gzip, no local gzip",
compression: NoCompression,
responseCompression: GzipCompression,
compression: br.NoCompression,
responseCompression: br.GzipCompression,
},
{
name: "no response gzip, local gzip",
compression: GzipCompression,
responseCompression: NoCompression,
compression: br.GzipCompression,
responseCompression: br.NoCompression,
},
{
name: "all gzip",
compression: GzipCompression,
responseCompression: GzipCompression,
compression: br.GzipCompression,
responseCompression: br.GzipCompression,
},
}
@ -247,7 +248,7 @@ func TestBackup_DownloadShardData(t *testing.T) {
DoAndReturn(func(api.ApiGetBackupShardIdRequest) (*http.Response, error) {
out := bytes.Buffer{}
var outW io.Writer = &out
if tc.responseCompression == GzipCompression {
if tc.responseCompression == br.GzipCompression {
gzw := gzip.NewWriter(outW)
defer gzw.Close()
outW = gzw
@ -256,7 +257,7 @@ func TestBackup_DownloadShardData(t *testing.T) {
require.NoError(t, err)
res := http.Response{Header: http.Header{}, Body: ioutil.NopCloser(&out)}
res.Header.Add("Content-Type", "application/octet-stream")
if tc.responseCompression == GzipCompression {
if tc.responseCompression == br.GzipCompression {
res.Header.Add("Content-Encoding", "gzip")
}
return &res, nil
@ -285,7 +286,7 @@ func TestBackup_DownloadShardData(t *testing.T) {
defer localShard.Close()
var shardReader io.Reader = localShard
if tc.compression == GzipCompression {
if tc.compression == br.GzipCompression {
gzr, err := gzip.NewReader(shardReader)
require.NoError(t, err)
defer gzr.Close()

124
clients/backup/manifest.go
View File

@ -2,94 +2,53 @@ package backup
import (
"fmt"
"time"
"github.com/influxdata/influx-cli/v2/api"
br "github.com/influxdata/influx-cli/v2/internal/backup_restore"
)
type FileCompression int
const (
NoCompression FileCompression = iota
GzipCompression
)
func (c *FileCompression) Set(v string) error {
switch v {
case "none":
*c = NoCompression
case "gzip":
*c = GzipCompression
default:
return fmt.Errorf("unsupported format: %q", v)
}
return nil
}
func (c FileCompression) String() string {
switch c {
case NoCompression:
return "none"
case GzipCompression:
return "gzip"
default:
panic("Impossible!")
}
}
type Manifest struct {
KV ManifestFileEntry `json:"kv"`
SQL ManifestFileEntry `json:"sql"`
Buckets []ManifestBucketEntry `json:"buckets"`
}
type ManifestFileEntry struct {
FileName string `json:"fileName"`
Size int64 `json:"size"`
Compression FileCompression `json:"compression"`
}
func ConvertBucketManifest(manifest api.BucketMetadataManifest, getShard func(shardId int64) (*ManifestFileEntry, error)) (ManifestBucketEntry, error) {
m := ManifestBucketEntry{
func ConvertBucketManifest(manifest api.BucketMetadataManifest, getShard func(shardId int64) (*br.ManifestFileEntry, error)) (br.ManifestBucketEntry, error) {
m := br.ManifestBucketEntry{
OrganizationID: manifest.OrganizationID,
OrganizationName: manifest.OrganizationName,
BucketID: manifest.BucketID,
BucketName: manifest.BucketName,
Description: manifest.Description,
DefaultRetentionPolicy: manifest.DefaultRetentionPolicy,
RetentionPolicies: make([]ManifestRetentionPolicy, len(manifest.RetentionPolicies)),
RetentionPolicies: make([]br.ManifestRetentionPolicy, len(manifest.RetentionPolicies)),
}
for i, rp := range manifest.RetentionPolicies {
var err error
m.RetentionPolicies[i], err = ConvertRetentionPolicy(rp, getShard)
if err != nil {
return ManifestBucketEntry{}, err
return br.ManifestBucketEntry{}, err
}
}
return m, nil
}
func ConvertRetentionPolicy(manifest api.RetentionPolicyManifest, getShard func(shardId int64) (*ManifestFileEntry, error)) (ManifestRetentionPolicy, error) {
m := ManifestRetentionPolicy{
func ConvertRetentionPolicy(manifest api.RetentionPolicyManifest, getShard func(shardId int64) (*br.ManifestFileEntry, error)) (br.ManifestRetentionPolicy, error) {
m := br.ManifestRetentionPolicy{
Name: manifest.Name,
ReplicaN: manifest.ReplicaN,
Duration: manifest.Duration,
ShardGroupDuration: manifest.ShardGroupDuration,
ShardGroups: make([]ManifestShardGroup, len(manifest.ShardGroups)),
Subscriptions: make([]ManifestSubscription, len(manifest.Subscriptions)),
ShardGroups: make([]br.ManifestShardGroup, len(manifest.ShardGroups)),
Subscriptions: make([]br.ManifestSubscription, len(manifest.Subscriptions)),
}
for i, sg := range manifest.ShardGroups {
var err error
m.ShardGroups[i], err = ConvertShardGroup(sg, getShard)
if err != nil {
return ManifestRetentionPolicy{}, err
return br.ManifestRetentionPolicy{}, err
}
}
for i, s := range manifest.Subscriptions {
m.Subscriptions[i] = ManifestSubscription{
m.Subscriptions[i] = br.ManifestSubscription{
Name: s.Name,
Mode: s.Mode,
Destinations: s.Destinations,
@ -99,20 +58,20 @@ func ConvertRetentionPolicy(manifest api.RetentionPolicyManifest, getShard func(
return m, nil
}
func ConvertShardGroup(manifest api.ShardGroupManifest, getShard func(shardId int64) (*ManifestFileEntry, error)) (ManifestShardGroup, error) {
m := ManifestShardGroup{
func ConvertShardGroup(manifest api.ShardGroupManifest, getShard func(shardId int64) (*br.ManifestFileEntry, error)) (br.ManifestShardGroup, error) {
m := br.ManifestShardGroup{
ID: manifest.Id,
StartTime: manifest.StartTime,
EndTime: manifest.EndTime,
DeletedAt: manifest.DeletedAt,
TruncatedAt: manifest.TruncatedAt,
Shards: make([]ManifestShardEntry, 0, len(manifest.Shards)),
Shards: make([]br.ManifestShardEntry, 0, len(manifest.Shards)),
}
for _, sh := range manifest.Shards {
maybeShard, err := ConvertShard(sh, getShard)
if err != nil {
return ManifestShardGroup{}, err
return br.ManifestShardGroup{}, err
}
// Shard deleted mid-backup.
if maybeShard == nil {
@ -124,7 +83,7 @@ func ConvertShardGroup(manifest api.ShardGroupManifest, getShard func(shardId in
return m, nil
}
func ConvertShard(manifest api.ShardManifest, getShard func(shardId int64) (*ManifestFileEntry, error)) (*ManifestShardEntry, error) {
func ConvertShard(manifest api.ShardManifest, getShard func(shardId int64) (*br.ManifestFileEntry, error)) (*br.ManifestShardEntry, error) {
shardFileInfo, err := getShard(manifest.Id)
if err != nil {
return nil, fmt.Errorf("failed to download snapshot of shard %d: %w", manifest.Id, err)
@ -133,60 +92,17 @@ func ConvertShard(manifest api.ShardManifest, getShard func(shardId int64) (*Man
return nil, nil
}
m := ManifestShardEntry{
m := br.ManifestShardEntry{
ID: manifest.Id,
ShardOwners: make([]ShardOwner, len(manifest.ShardOwners)),
ShardOwners: make([]br.ShardOwner, len(manifest.ShardOwners)),
ManifestFileEntry: *shardFileInfo,
}
for i, o := range manifest.ShardOwners {
m.ShardOwners[i] = ShardOwner{
m.ShardOwners[i] = br.ShardOwner{
NodeID: o.NodeID,
}
}
return &m, nil
}
type ManifestBucketEntry struct {
OrganizationID string `json:"organizationID"`
OrganizationName string `json:"organizationName"`
BucketID string `json:"bucketID"`
BucketName string `json:"bucketName"`
DefaultRetentionPolicy string `json:"defaultRetentionPolicy"`
RetentionPolicies []ManifestRetentionPolicy `json:"retentionPolicies"`
}
type ManifestRetentionPolicy struct {
Name string `json:"name"`
ReplicaN int32 `json:"replicaN"`
Duration int64 `json:"duration"`
ShardGroupDuration int64 `json:"shardGroupDuration"`
ShardGroups []ManifestShardGroup `json:"shardGroups"`
Subscriptions []ManifestSubscription `json:"subscriptions"`
}
type ManifestShardGroup struct {
ID int64 `json:"id"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
DeletedAt *time.Time `json:"deletedAt,omitempty"`
TruncatedAt *time.Time `json:"truncatedAt,omitempty"`
Shards []ManifestShardEntry `json:"shards"`
}
type ManifestShardEntry struct {
ID int64 `json:"id"`
ShardOwners []ShardOwner `json:"shardOwners"`
ManifestFileEntry
}
type ShardOwner struct {
NodeID int64 `json:"nodeID"`
}
type ManifestSubscription struct {
Name string `json:"name"`
Mode string `json:"mode"`
Destinations []string `json:"destinations"`
}

37
clients/backup/manifest_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/influxdata/influx-cli/v2/api"
"github.com/influxdata/influx-cli/v2/clients/backup"
br "github.com/influxdata/influx-cli/v2/internal/backup_restore"
"github.com/stretchr/testify/require"
)
@ -87,46 +88,46 @@ func TestConvertBucketManifest(t *testing.T) {
},
}
fakeGetShard := func(id int64) (*backup.ManifestFileEntry, error) {
fakeGetShard := func(id int64) (*br.ManifestFileEntry, error) {
if id == 20 {
return nil, nil
}
return &backup.ManifestFileEntry{
return &br.ManifestFileEntry{
FileName: fmt.Sprintf("%d.gz", id),
Size: id * 100,
Compression: backup.GzipCompression,
Compression: br.GzipCompression,
}, nil
}
converted, err := backup.ConvertBucketManifest(manifest, fakeGetShard)
require.NoError(t, err)
expected := backup.ManifestBucketEntry{
expected := br.ManifestBucketEntry{
OrganizationID: "123",
OrganizationName: "org",
BucketID: "456",
BucketName: "bucket",
DefaultRetentionPolicy: "foo",
RetentionPolicies: []backup.ManifestRetentionPolicy{
RetentionPolicies: []br.ManifestRetentionPolicy{
{
Name: "foo",
ReplicaN: 1,
Duration: 100,
ShardGroupDuration: 10,
ShardGroups: []backup.ManifestShardGroup{
ShardGroups: []br.ManifestShardGroup{
{
ID: 1,
StartTime: now,
EndTime: now,
TruncatedAt: &now,
Shards: []backup.ManifestShardEntry{
Shards: []br.ManifestShardEntry{
{
ID: 10,
ShardOwners: []backup.ShardOwner{{NodeID: 1}},
ManifestFileEntry: backup.ManifestFileEntry{
ShardOwners: []br.ShardOwner{{NodeID: 1}},
ManifestFileEntry: br.ManifestFileEntry{
FileName: "10.gz",
Size: 1000,
Compression: backup.GzipCompression,
Compression: br.GzipCompression,
},
},
},
@ -136,35 +137,35 @@ func TestConvertBucketManifest(t *testing.T) {
StartTime: now,
EndTime: now,
DeletedAt: &now,
Shards: []backup.ManifestShardEntry{
Shards: []br.ManifestShardEntry{
{
ID: 30,
ShardOwners: []backup.ShardOwner{},
ManifestFileEntry: backup.ManifestFileEntry{
ShardOwners: []br.ShardOwner{},
ManifestFileEntry: br.ManifestFileEntry{
FileName: "30.gz",
Size: 3000,
Compression: backup.GzipCompression,
Compression: br.GzipCompression,
},
},
},
},
},
Subscriptions: []backup.ManifestSubscription{},
Subscriptions: []br.ManifestSubscription{},
},
{
Name: "bar",
ReplicaN: 3,
Duration: 9999,
ShardGroupDuration: 1,
ShardGroups: []backup.ManifestShardGroup{
ShardGroups: []br.ManifestShardGroup{
{
ID: 3,
StartTime: now,
EndTime: now,
Shards: []backup.ManifestShardEntry{},
Shards: []br.ManifestShardEntry{},
},
},
Subscriptions: []backup.ManifestSubscription{
Subscriptions: []br.ManifestSubscription{
{
Name: "test",
Mode: "on",