archive: --dry-run support was missing, added it

cmd/archive/archive.go

@@ -69,7 +69,7 @@ var listCommand = &cobra.Command{
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.70",
 	},
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		//
 		src, srcFile := cmd.NewFsFile(args[0])
@@ -77,7 +77,7 @@ var listCommand = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			return list.ArchiveList(context.Background(), src, srcFile, longList)
 		})
-
+		return nil
 	},
 }
 
@@ -89,7 +89,7 @@ var extractCommand = &cobra.Command{
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.70",
 	},
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
 		//
 		src, srcFile := cmd.NewFsFile(args[0])
@@ -98,7 +98,7 @@ var extractCommand = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			return extract.ArchiveExtract(context.Background(), src, srcFile, dst, dstFile)
 		})
-
+		return nil
 	},
 }
 
@@ -162,7 +162,7 @@ the contents of the archive will be:
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.70",
 	},
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		var src, dst fs.Fs
 		var dstFile string
 		if len(args) == 1 { // source only, archive to stdout
@@ -173,6 +173,7 @@ the contents of the archive will be:
 		} else {
 			cmd.CheckArgs(1, 2, command, args)
 		}
+		//
 		cmd.Run(false, false, command, func() error {
 			if prefix != "" {
 				return create.ArchiveCreate(context.Background(), src, dst, dstFile, format, prefix)
@@ -181,5 +182,6 @@ the contents of the archive will be:
 			}
 			return create.ArchiveCreate(context.Background(), src, dst, dstFile, format, "")
 		})
+		return nil
 	},
 }

cmd/archive/create/create.go

@@ -92,9 +92,10 @@ var (
 		// tar.sz
 		"*.tar.sz": "tar.sz",
 	}
-	filesAdded = 0
 )
 
+// sorted FileInfo list
+
 type archivesFileInfoList []archives.FileInfo
 
 func (a archivesFileInfoList) Len() int {
@@ -115,6 +116,8 @@ func (a archivesFileInfoList) Swap(i, j int) {
 	a[i], a[j] = a[j], a[i]
 }
 
+//
+
 func init() {
 }
 
@@ -200,13 +203,6 @@ func CheckValidDestination(ctx context.Context, dst fs.Fs, dstFile string) (fs.F
 	return dst, parentFile, fmt.Errorf("invalid parent dir %s: %w", parentDir, err)
 }
 
-func onProgress(snapshot accounting.TransferSnapshot, action int) {
-	if action == files.Closing {
-		fs.Printf(nil, "Add %s", snapshot.Name)
-		filesAdded++
-	}
-}
-
 func loadMetadata(ctx context.Context, o fs.DirEntry) fs.Metadata {
 	meta, err := fs.GetMetadata(ctx, o)
 	if err != nil {
@@ -220,6 +216,8 @@ func ArchiveCreate(ctx context.Context, src fs.Fs, dst fs.Fs, dstFile string, fo
 	var err error
 	var list archivesFileInfoList
 	var compArchive archives.CompressedArchive
+	var totalLength int64
+	var callback files.ProgressCallback
 	// check id dst is valid
 	if dst != nil {
 		dst, dstFile, err = CheckValidDestination(ctx, dst, dstFile)
@@ -228,26 +226,34 @@ func ArchiveCreate(ctx context.Context, src fs.Fs, dst fs.Fs, dstFile string, fo
 		}
 	}
 	//
+	ci := fs.GetConfig(ctx)
 	fi := filter.GetConfig(ctx)
 	// get archive format
 	compArchive, err = getCompressor(format, dstFile)
 	if err != nil {
 		return err
 	}
+	// set callback
+	callback = func(snapshot accounting.TransferSnapshot, action int) {
+		if action == files.Closing {
+			fs.Debugf(nil, "Add %s", snapshot.Name)
+		}
+	}
 	// get source files
 	err = walk.Walk(ctx, src, "", false, -1, func(path string, entries fs.DirEntries, err error) error {
 		// get directories
 		entries.ForDir(func(o fs.Directory) {
 			if fi.Include(o.Remote(), o.Size(), o.ModTime(ctx), loadMetadata(ctx, o)) {
-				info := files.NewArchiveFileInfo(ctx, o, prefix, onProgress)
+				info := files.NewArchiveFileInfo(ctx, o, prefix, callback)
 				list = append(list, info)
 			}
 		})
 		// get files
 		entries.ForObject(func(o fs.Object) {
 			if fi.Include(o.Remote(), o.Size(), o.ModTime(ctx), loadMetadata(ctx, o)) {
-				info := files.NewArchiveFileInfo(ctx, o, prefix, onProgress)
+				info := files.NewArchiveFileInfo(ctx, o, prefix, callback)
 				list = append(list, info)
+				totalLength += o.Size()
 			}
 		})
 		return nil
@@ -258,22 +264,42 @@ func ArchiveCreate(ctx context.Context, src fs.Fs, dst fs.Fs, dstFile string, fo
 		return fmt.Errorf("no files found in source")
 	}
 	sort.Stable(list)
-	// create destination
-	if dst != nil {
-		// create io.Pipe
-		pipeReader, pipeWriter := io.Pipe()
-		// write to pipewriter in background
-		go func() {
-			err := compArchive.Archive(ctx, pipeWriter, list)
-			pipeWriter.CloseWithError(err)
-		}()
-		// rcat to remote from pipereader
-		_, err = operations.Rcat(ctx, dst, dstFile, pipeReader, time.Now(), nil)
-		fs.Printf(nil, "Total files added %d", filesAdded)
+	// create archive
+	if ci.DryRun {
+		// write nowhere
+		counter := files.NewCountWriter(nil)
+		err = compArchive.Archive(ctx, counter, list)
+		// log totals
+		fs.Printf(nil, "Total files added %d", list.Len())
+		fs.Printf(nil, "Total bytes read %d", totalLength)
+		fs.Printf(nil, "Compressed file size %d", counter.Count())
+		//
+		return err
+	} else if dst == nil {
+		// write to stdout
+		counter := files.NewCountWriter(os.Stdout)
+		err = compArchive.Archive(ctx, counter, list)
+		// log totals
+		fs.Printf(nil, "Total files added %d", list.Len())
+		fs.Printf(nil, "Total bytes read %d", totalLength)
+		fs.Printf(nil, "Compressed file size %d", counter.Count())
+		//
 		return err
 	}
-	// write to stdout
-	err = compArchive.Archive(ctx, os.Stdout, list)
-	fs.Printf(nil, "Total files added %d", filesAdded)
+	// write to remote
+	pipeReader, pipeWriter := io.Pipe()
+	// write to pipewriter in background
+	counter := files.NewCountWriter(pipeWriter)
+	go func() {
+		err := compArchive.Archive(ctx, counter, list)
+		pipeWriter.CloseWithError(err)
+	}()
+	// rcat to remote from pipereader
+	_, err = operations.Rcat(ctx, dst, dstFile, pipeReader, time.Now(), nil)
+	// log totals
+	fs.Printf(nil, "Total files added %d", list.Len())
+	fs.Printf(nil, "Total bytes read %d", totalLength)
+	fs.Printf(nil, "Compressed file size %d", counter.Count())
+	//
 	return err
 }

cmd/archive/extract/extract.go

@@ -15,17 +15,17 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 )
 
-var filesExtracted = 0
-
 func init() {
 }
 
 // ArchiveExtract -- extracts files from source to destination
 func ArchiveExtract(ctx context.Context, src fs.Fs, srcFile string, dst fs.Fs, dstFile string) error {
 	var srcObj fs.Object
+	var filesExtracted = 0
 	var err error
 	//
 	fi := filter.GetConfig(ctx)
+	ci := fs.GetConfig(ctx)
 	// get source object
 	srcObj, err = src.NewObject(ctx, srcFile)
 	if errors.Is(err, fs.ErrorIsDir) {
@@ -83,31 +83,38 @@ func ArchiveExtract(ctx context.Context, src fs.Fs, srcFile string, dst fs.Fs, d
 		if !fi.Include(f.NameInArchive, f.Size(), f.ModTime(), fs.Metadata{}) {
 			return nil
 		}
-		//
+		// process directory
 		if f.IsDir() {
-			// directory, try and crerate it
-			err := operations.Mkdir(ctx, dst, f.NameInArchive)
-			if err == nil {
-				fs.Debugf(nil, "mkdir %s", f.NameInArchive)
-			}
-		} else {
-			// file, open it
-			fin, err := f.Open()
-			if err != nil {
-				return err
-			}
-			// extract the file to destination
-			_, err = operations.Rcat(ctx, dst, f.NameInArchive, fin, f.ModTime(), nil)
-			if err == nil {
-				fs.Printf(nil, "Extract %s", f.NameInArchive)
-				filesExtracted++
+			// directory
+			fs.Debugf(nil, "mkdir %s", f.NameInArchive)
+			// leave if --dry-run set
+			if ci.DryRun {
+				return nil
 			}
+			// create the directory
+			return operations.Mkdir(ctx, dst, f.NameInArchive)
+		}
+		// process file
+		fs.Debugf(nil, "Extract %s", f.NameInArchive)
+		// leave if --dry-run set
+		if ci.DryRun {
+			filesExtracted++
+			return nil
+		}
+		// open file
+		fin, err := f.Open()
+		if err != nil {
+			return err
+		}
+		// extract the file to destination
+		_, err = operations.Rcat(ctx, dst, f.NameInArchive, fin, f.ModTime(), nil)
+		if err == nil {
+			filesExtracted++
 		}
 		return err
 	})
 	//
 	fs.Printf(nil, "Total files extracted %d", filesExtracted)
-
 	//
 	return err
 }

cmd/archive/files/files.go

@@ -2,16 +2,16 @@
 package files
 
 import (
+	"archive/tar"
 	"context"
 	"fmt"
 	"io"
 	stdfs "io/fs"
 	"path"
 	"strings"
+	"sync/atomic"
 	"time"
 
-	"archive/tar"
-
 	"github.com/mholt/archives"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
@@ -241,3 +241,38 @@ func (a *fileImpl) Close() error {
 	//
 	return a.err
 }
+
+// CountWriter will counts bytes written
+type CountWriter struct {
+	count uint64
+	io.Writer
+}
+
+// NewCountWriter will create a writer that counts bytes written
+func NewCountWriter(w io.Writer) *CountWriter {
+	return &CountWriter{
+		Writer: w,
+	}
+}
+
+func (w *CountWriter) Write(buf []byte) (int, error) {
+	var n int
+	var err error
+	// if writer is null just count
+	if w.Writer == nil {
+		n = len(buf)
+		err = nil
+	} else {
+		n, err = w.Writer.Write(buf)
+	}
+	// add bytes written
+	if n >= 0 {
+		atomic.AddUint64(&w.count, uint64(n))
+	}
+	return n, err
+}
+
+// Count returns total bytes written to writer
+func (w *CountWriter) Count() uint64 {
+	return atomic.LoadUint64(&w.count)
+}