Merge 0c9014db9914bf166951124b542de688a81592b6 into 63c4fef27aa1c9ae1e36e2642499f9199c11f720

docs/content/filtering.md

@@ -242,7 +242,9 @@ Any path/file included at that stage is processed by the rclone
 command.
 
 `--files-from` and `--files-from-raw` flags over-ride and cannot be
-combined with other filter options.
+combined with other filter options, with the exception of the flag
+`--files-from--strict`, which makes the operation fail if at least
+one of the files does not exist.
 
 To see the internal combined rule list, in regular expression form,
 for a command add the `--dump filters` flag. Running an rclone command

docs/content/flags.md

@@ -187,6 +187,7 @@ Flags for filtering directory listings.
       --exclude-from stringArray            Read file exclude patterns from file (use - to read from stdin)
       --exclude-if-present stringArray      Exclude directories if filename is present
       --files-from stringArray              Read list of source-file names from file (use - to read from stdin)
+      --files-from-strict                   Makes "files-from" fail if at least one of the files does not exist
       --files-from-raw stringArray          Read list of source-file names from file without any processing of lines (use - to read from stdin)
   -f, --filter stringArray                  Add a file filtering rule
       --filter-from stringArray             Read file filtering patterns from a file (use - to read from stdin)

fs/filter/filter.go

@@ -35,6 +35,11 @@ var OptionsInfo = fs.Options{{
 	Default: []string{},
 	Help:    "Read list of source-file names from file (use - to read from stdin)",
 	Groups:  "Filter",
+}, {
+	Name:    "files_from_strict",
+	Default: false,
+	Help:    "Fail if at least one of the files does not exist",
+	Groups:  "Filter",
 }, {
 	Name:    "files_from_raw",
 	Default: []string{},
@@ -130,17 +135,18 @@ var OptionsInfo = fs.Options{{
 
 // Options configures the filter
 type Options struct {
-	DeleteExcluded bool          `config:"delete_excluded"`
-	RulesOpt                     // embedded so we don't change the JSON API
-	ExcludeFile    []string      `config:"exclude_if_present"`
-	FilesFrom      []string      `config:"files_from"`
-	FilesFromRaw   []string      `config:"files_from_raw"`
-	MetaRules      RulesOpt      `config:"metadata"`
-	MinAge         fs.Duration   `config:"min_age"`
-	MaxAge         fs.Duration   `config:"max_age"`
-	MinSize        fs.SizeSuffix `config:"min_size"`
-	MaxSize        fs.SizeSuffix `config:"max_size"`
-	IgnoreCase     bool          `config:"ignore_case"`
+	DeleteExcluded  bool          `config:"delete_excluded"`
+	RulesOpt                      // embedded so we don't change the JSON API
+	ExcludeFile     []string      `config:"exclude_if_present"`
+	FilesFrom       []string      `config:"files_from"`
+	FilesFromStrict bool          `config:"files_from_strict"`
+	FilesFromRaw    []string      `config:"files_from_raw"`
+	MetaRules       RulesOpt      `config:"metadata"`
+	MinAge          fs.Duration   `config:"min_age"`
+	MaxAge          fs.Duration   `config:"max_age"`
+	MinSize         fs.SizeSuffix `config:"min_size"`
+	MaxSize         fs.SizeSuffix `config:"max_size"`
+	IgnoreCase      bool          `config:"ignore_case"`
 }
 
 func init() {
@@ -209,7 +215,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
 
 	for _, rule := range f.Opt.FilesFrom {
 		if !inActive {
-			return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw")
+			return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw, or optionally, --files-from-strict")
 		}
 		f.initAddFile() // init to show --files-from set even if no files within
 		err := forEachLine(rule, false, func(line string) error {
@@ -224,7 +230,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
 		// --files-from-raw can be used with --files-from, hence we do
 		// not need to get the value of f.InActive again
 		if !inActive {
-			return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from")
+			return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from, or optionally, --files-from-strict")
 		}
 		f.initAddFile() // init to show --files-from set even if no files within
 		err := forEachLine(rule, true, func(line string) error {
@@ -570,7 +576,9 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
 				for remote := range remotes {
 					entries[0], err = NewObject(gCtx, remote)
 					if err == fs.ErrorObjectNotFound {
-						// Skip files that are not found
+						if f.Opt.FilesFromStrict {
+							return err
+						}
 					} else if err != nil {
 						return err
 					} else {

fs/filter/filter_test.go

@@ -116,6 +116,40 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) {
 	}
 }
 
+func TestMakeListRFilesFromStrict(t *testing.T) {
+	f, err := NewFilter(nil)
+	require.NoError(t, err)
+
+	// Add two files: one that will "exist" and one that will simulate a missing file.
+	err = f.AddFile("existing")
+	require.NoError(t, err)
+	err = f.AddFile("notfound")
+	require.NoError(t, err)
+
+	// Enable strict mode so that missing files cause an error.
+	f.Opt.FilesFromStrict = true
+
+	NewObject := func(ctx context.Context, remote string) (fs.Object, error) {
+		if remote == "notfound" {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return mockobject.New(remote), nil
+	}
+
+	// Define a callback which just ignores the entries.
+	callback := func(entries fs.DirEntries) error {
+		return nil
+	}
+
+	listR := f.MakeListR(context.Background(), NewObject)
+
+	// When running ListR, since "notfound" will trigger an error in strict mode,
+	// we expect ListR to return fs.ErrorObjectNotFound, if it doesn't, the test isn't working.
+	err = listR(context.Background(), "", callback)
+	require.Error(t, err)
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+}
+
 func TestNewFilterWithFilesFromRaw(t *testing.T) {
 	Opt := Opt