mirror of
https://github.com/rclone/rclone.git
synced 2025-04-19 10:08:56 +08:00
Merge ea4714564323949735acf0d43c010efe355b7902 into 0b9671313b14ffe839ecbd7dd2ae5ac7f6f05db8
This commit is contained in:
commit
e3678e4e77
@ -8,18 +8,23 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@ -90,19 +95,69 @@ for more info.
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
|
||||
**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
|
||||
|
||||
## Logger Flags
|
||||
|
||||
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--match| and |--error| flags write paths,
|
||||
one per line, to the file name (or stdout if it is |-|) supplied. What they write is described
|
||||
in the help below. For example |--differ| will write all paths which are present
|
||||
on both the source and destination but different.
|
||||
|
||||
The |--combined| flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- |= path| means path was found in source and destination and was identical
|
||||
- |- path| means path was missing on the source, so only in the destination
|
||||
- |+ path| means path was missing on the destination, so only in the source
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
|
||||
The |--dest-after| flag writes a list file using the same format flags
|
||||
as [|lsf|](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's |--itemize-changes|, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the copy.
|
||||
|
||||
When the |--no-traverse| flag is set, all logs involving files that exist only
|
||||
on the destination will be incomplete or completely missing.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- |--max-duration| / |CutoffModeHard|
|
||||
- |--compare-dest| / |--copy-dest|
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use |--retries 1| to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the copy, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
@ -6,12 +6,21 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@ -55,10 +64,21 @@ the destination.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, false)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, false)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
@ -8,6 +8,7 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@ -16,6 +17,8 @@ import (
|
||||
var (
|
||||
deleteEmptySrcDirs = false
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -23,6 +26,8 @@ func init() {
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@ -66,6 +71,46 @@ for more info.
|
||||
|--dry-run| or the |--interactive|/|-i| flag.
|
||||
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
|
||||
## Logger Flags
|
||||
|
||||
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--match| and |--error| flags write paths,
|
||||
one per line, to the file name (or stdout if it is |-|) supplied. What they write is described
|
||||
in the help below. For example |--differ| will write all paths which are present
|
||||
on both the source and destination but different.
|
||||
|
||||
The |--combined| flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- |= path| means path was found in source and destination and was identical
|
||||
- |- path| means path was missing on the source, so only in the destination
|
||||
- |+ path| means path was missing on the destination, so only in the source
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
|
||||
The |--dest-after| flag writes a list file using the same format flags
|
||||
as [|lsf|](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's |--itemize-changes|, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the move.
|
||||
|
||||
When the |--no-traverse| flag is set, all logs involving files that exist only
|
||||
on the destination will be incomplete or completely missing.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- |--max-duration| / |CutoffModeHard|
|
||||
- |--compare-dest| / |--copy-dest|
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use |--retries 1| to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the move, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.19",
|
||||
@ -75,10 +120,21 @@ for more info.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
@ -6,12 +6,21 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@ -57,10 +66,21 @@ successful transfer.
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, false, false)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, false, false)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
170
cmd/sync/sync.go
170
cmd/sync/sync.go
@ -3,13 +3,9 @@ package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
mutex "sync" // renamed as "sync" already in use
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
@ -19,7 +15,7 @@ import (
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
opt = operations.LoggerOpt{}
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
@ -27,119 +23,15 @@ func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
|
||||
// TODO: add same flags to move and copy
|
||||
}
|
||||
|
||||
var lock mutex.Mutex
|
||||
|
||||
func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(opt) != nil {
|
||||
operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSyncLoggerOpt gets the options corresponding to the logger flags
|
||||
func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
opt.LoggerFn = syncLoggerFn
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return opt, close, nil
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "sync source:path dest:path",
|
||||
Short: `Make source and dest identical, modifying destination only.`,
|
||||
Long: `Sync the source to the destination, changing the destination
|
||||
// Warning! "|" will be replaced by backticks below
|
||||
Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer files that are identical on source and
|
||||
destination, testing by size and modification time or MD5SUM.
|
||||
Destination is updated to match source, including deleting files
|
||||
@ -148,7 +40,7 @@ want to delete files from destination, use the
|
||||
[copy](/commands/rclone_copy/) command instead.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|--dry-run| or the |--interactive|/|i| flag.
|
||||
|
||||
rclone sync --interactive SOURCE remote:DESTINATION
|
||||
|
||||
@ -171,55 +63,54 @@ destination that is inside the source directory.
|
||||
|
||||
Rclone will sync the modification times of files and directories if
|
||||
the backend supports it. If metadata syncing is required then use the
|
||||
` + "`--metadata`" + ` flag.
|
||||
|--metadata| flag.
|
||||
|
||||
Note that the modification time and metadata for the root directory
|
||||
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
|
||||
for more info.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
|
||||
## Logger Flags
|
||||
|
||||
The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
|
||||
"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
|
||||
stdout if it is ` + "`-`" + `) supplied. What they write is described in the
|
||||
help below. For example ` + "`--differ`" + ` will write all paths which are
|
||||
present on both the source and destination but different.
|
||||
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--match| and |--error| flags write paths,
|
||||
one per line, to the file name (or stdout if it is |-|) supplied. What they write is described
|
||||
in the help below. For example |--differ| will write all paths which are present
|
||||
on both the source and destination but different.
|
||||
|
||||
The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
|
||||
The |--combined| flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- ` + "`= path`" + ` means path was found in source and destination and was identical
|
||||
- ` + "`- path`" + ` means path was missing on the source, so only in the destination
|
||||
- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
|
||||
- ` + "`* path`" + ` means path was present in source and destination but different.
|
||||
- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
|
||||
- |= path| means path was found in source and destination and was identical
|
||||
- |- path| means path was missing on the source, so only in the destination
|
||||
- |+ path| means path was missing on the destination, so only in the source
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
|
||||
The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
|
||||
as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
The |--dest-after| flag writes a list file using the same format flags
|
||||
as [|lsf|](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
|
||||
Conceptually it is similar to rsync's |--itemize-changes|, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the sync.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
|
||||
- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
|
||||
- |--max-duration| / |CutoffModeHard|
|
||||
- |--compare-dest| / |--copy-dest|
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
|
||||
- High-level retries, because there would be duplicates (use |--retries 1| to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the sync, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`,
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Sync,Copy,Filter,Listing,Important",
|
||||
},
|
||||
@ -228,15 +119,14 @@ is most useful as a predictor of what SHOULD happen to each file
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
|
||||
loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
|
||||
ctx = operations.WithSyncLogger(ctx, opt)
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
|
@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
mutex "sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
@ -106,6 +107,43 @@ type LoggerOpt struct {
|
||||
Absolute bool
|
||||
}
|
||||
|
||||
// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil
|
||||
func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn {
|
||||
var lock mutex.Mutex
|
||||
|
||||
return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(*opt) != nil {
|
||||
SyncFprintf(sigil.Writer(*opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
|
||||
func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
|
||||
return context.WithValue(ctx, loggerKey, logger)
|
||||
|
@ -3,9 +3,15 @@
|
||||
package operationsflags
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
@ -20,6 +26,20 @@ type AddLoggerFlagsOptions struct {
|
||||
DestAfter string // files that exist on the destination post-sync
|
||||
}
|
||||
|
||||
// AnySet checks if any of the logger flags have a non-blank value
|
||||
func (o AddLoggerFlagsOptions) AnySet() bool {
|
||||
return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter)
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AddLoggerFlags adds the logger flags to the cmdFlags command
|
||||
func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {
|
||||
flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")
|
||||
@ -43,3 +63,75 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
|
||||
flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
|
||||
// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
|
||||
}
|
||||
|
||||
// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags
|
||||
func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(flagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Match, &opt.Match); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.NoTraverse && opt.Combined != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.MissingOnSrc != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.DestAfter != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
|
||||
}
|
||||
|
||||
return close, nil
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user