diff --git a/docs/content/filtering.md b/docs/content/filtering.md index 8c7ce232f..0250bc92f 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -242,7 +242,9 @@ Any path/file included at that stage is processed by the rclone command. `--files-from` and `--files-from-raw` flags over-ride and cannot be -combined with other filter options. +combined with other filter options, with the exception of the flag +`--files-from--strict`, which makes the operation fail if at least +one of the files does not exist. To see the internal combined rule list, in regular expression form, for a command add the `--dump filters` flag. Running an rclone command diff --git a/docs/content/flags.md b/docs/content/flags.md index f9d32c490..d84273adb 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -187,6 +187,7 @@ Flags for filtering directory listings. --exclude-from stringArray Read file exclude patterns from file (use - to read from stdin) --exclude-if-present stringArray Exclude directories if filename is present --files-from stringArray Read list of source-file names from file (use - to read from stdin) + --files-from-strict Makes "files-from" fail if at least one of the files does not exist --files-from-raw stringArray Read list of source-file names from file without any processing of lines (use - to read from stdin) -f, --filter stringArray Add a file filtering rule --filter-from stringArray Read file filtering patterns from a file (use - to read from stdin) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index e978074d6..52369c23a 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -35,6 +35,11 @@ var OptionsInfo = fs.Options{{ Default: []string{}, Help: "Read list of source-file names from file (use - to read from stdin)", Groups: "Filter", +}, { + Name: "files_from_strict", + Default: false, + Help: "Fail if at least one of the files does not exist", + Groups: "Filter", }, { Name: "files_from_raw", Default: []string{}, @@ -130,17 +135,18 @@ var OptionsInfo = fs.Options{{ // Options configures the filter type Options struct { - DeleteExcluded bool `config:"delete_excluded"` - RulesOpt // embedded so we don't change the JSON API - ExcludeFile []string `config:"exclude_if_present"` - FilesFrom []string `config:"files_from"` - FilesFromRaw []string `config:"files_from_raw"` - MetaRules RulesOpt `config:"metadata"` - MinAge fs.Duration `config:"min_age"` - MaxAge fs.Duration `config:"max_age"` - MinSize fs.SizeSuffix `config:"min_size"` - MaxSize fs.SizeSuffix `config:"max_size"` - IgnoreCase bool `config:"ignore_case"` + DeleteExcluded bool `config:"delete_excluded"` + RulesOpt // embedded so we don't change the JSON API + ExcludeFile []string `config:"exclude_if_present"` + FilesFrom []string `config:"files_from"` + FilesFromStrict bool `config:"files_from_strict"` + FilesFromRaw []string `config:"files_from_raw"` + MetaRules RulesOpt `config:"metadata"` + MinAge fs.Duration `config:"min_age"` + MaxAge fs.Duration `config:"max_age"` + MinSize fs.SizeSuffix `config:"min_size"` + MaxSize fs.SizeSuffix `config:"max_size"` + IgnoreCase bool `config:"ignore_case"` } func init() { @@ -209,7 +215,7 @@ func NewFilter(opt *Options) (f *Filter, err error) { for _, rule := range f.Opt.FilesFrom { if !inActive { - return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw") + return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw, or optionally, --files-from-strict") } f.initAddFile() // init to show --files-from set even if no files within err := forEachLine(rule, false, func(line string) error { @@ -224,7 +230,7 @@ func NewFilter(opt *Options) (f *Filter, err error) { // --files-from-raw can be used with --files-from, hence we do // not need to get the value of f.InActive again if !inActive { - return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from") + return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from, or optionally, --files-from-strict") } f.initAddFile() // init to show --files-from set even if no files within err := forEachLine(rule, true, func(line string) error { @@ -570,7 +576,9 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte for remote := range remotes { entries[0], err = NewObject(gCtx, remote) if err == fs.ErrorObjectNotFound { - // Skip files that are not found + if f.Opt.FilesFromStrict { + return err + } } else if err != nil { return err } else { diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index ef7a1440b..2716ed80e 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -116,6 +116,40 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) { } } +func TestMakeListRFilesFromStrict(t *testing.T) { + f, err := NewFilter(nil) + require.NoError(t, err) + + // Add two files: one that will "exist" and one that will simulate a missing file. + err = f.AddFile("existing") + require.NoError(t, err) + err = f.AddFile("notfound") + require.NoError(t, err) + + // Enable strict mode so that missing files cause an error. + f.Opt.FilesFromStrict = true + + NewObject := func(ctx context.Context, remote string) (fs.Object, error) { + if remote == "notfound" { + return nil, fs.ErrorObjectNotFound + } + return mockobject.New(remote), nil + } + + // Define a callback which just ignores the entries. + callback := func(entries fs.DirEntries) error { + return nil + } + + listR := f.MakeListR(context.Background(), NewObject) + + // When running ListR, since "notfound" will trigger an error in strict mode, + // we expect ListR to return fs.ErrorObjectNotFound, if it doesn't, the test isn't working. + err = listR(context.Background(), "", callback) + require.Error(t, err) + require.Equal(t, fs.ErrorObjectNotFound, err) +} + func TestNewFilterWithFilesFromRaw(t *testing.T) { Opt := Opt