Merge 8348bdd802d924022b53329a077306d7c4261de2 into 0b9671313b14ffe839ecbd7dd2ae5ac7f6f05db8

docs/content/docs.md

@@ -2505,6 +2505,13 @@ Using this flag on a sync operation without also using `--update` would cause
 all files modified at any time other than the last upload time to be uploaded
 again, which is probably not what you want.
 
+### --use-ssh-config ###
+
+When set, then settings from ~/.ssh/config will also be considered for the
+sftp backend. Any setting in ~/.ssh/config will have precedence over the
+rclone config file.
+
+
 ### -v, -vv, --verbose ###
 
 With `-v` rclone will tell you about each file that is transferred and

fs/config/configflags/configflags.go

@@ -13,6 +13,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/lib/sshconfig"
 	"github.com/spf13/pflag"
 )
 
@@ -35,6 +36,7 @@ var (
 	downloadHeaders []string
 	headers         []string
 	metadataSet     []string
+	useSSHConfig    bool
 )
 
 // AddFlags adds the non filing system specific flags to the command
@@ -59,6 +61,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking")
 	flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata")
 	flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking")
+	flags.BoolVarP(flagSet, &useSSHConfig, "use-ssh-config", "", false, "Use ~/.ssh/config file for sftp/ssh connections", "Config")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions
@@ -199,6 +202,14 @@ func SetFlags(ci *fs.ConfigInfo) {
 		fs.Fatalf(nil, "--temp-dir: Failed to set %q as temp dir: %v", tempDir, err)
 	}
 
+	// Process --use-ssh-config
+	if useSSHConfig {
+		err := sshconfig.LoadSSHConfigIntoEnv()
+		if err != nil {
+			fs.Fatalf(nil, "--use-ssh-config: Failed loading ssh config: %v", err)
+		}
+	}
+
 	// Process --multi-thread-streams - set whether multi-thread-streams was set
 	multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
 	ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3
 	github.com/josephspurrier/goversioninfo v1.4.1
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
+	github.com/kevinburke/ssh_config v1.2.0
 	github.com/klauspost/compress v1.18.0
 	github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6

go.sum

@@ -423,6 +423,8 @@ github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 h1:JcltaO1HXM5S2K
 github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7/go.mod h1:MEkhEPFwP3yudWO0lj6vfYpLIB+3eIcuIW+e0AZzUQk=
 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg=
 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c=
+github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
+github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -430,7 +432,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV10kKGuY=
 github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=

lib/sshconfig/use_sshconfig.go

@@ -0,0 +1,106 @@
+// Package sshconfig functions to convert ssh config file to rclone config and to add to temp env vars
+package sshconfig
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/kevinburke/ssh_config"
+)
+
+// keyMapping defines the mapping between SSH configuration keys and Rclone configuration keys.
+var keyMapping = map[string]string{
+	"identityfile": "key_file",
+	"pubkeyfile":   "pubkey_file",
+	"user":         "user",
+	"hostname":     "host",
+	"port":         "port",
+	"password":     "pass",
+}
+
+// sshConfig represents the SSH configuration structure
+type sshConfig map[string]map[string]string
+
+// LoadSSHConfigIntoEnv loads SSH configuration into environment variables by mapping SSH settings to
+// rclone configuration and setting the environment accordingly.
+// Returns an error if any step fails during the process.
+// Note: type=sftp is added for each host (section), also key_use_agent=true is set, when if key_file was given.
+func LoadSSHConfigIntoEnv() error {
+	path := filepath.Join(os.Getenv("HOME"), ".ssh", "config")
+	f, err := os.Open(path) // returning file handler, so caller needs close
+
+	if err != nil {
+		return fmt.Errorf("error opening ssh config file: %w", err)
+	}
+
+	c, err := mapSSHToRcloneConfig(f)
+	if err != nil {
+		return fmt.Errorf("error mapping ssh config to rclone config: %w", err)
+	}
+
+	if err := EnvLoadSSHConfig(c); err != nil {
+		return fmt.Errorf("error setting Env with ssh config: %v", err)
+	}
+
+	return nil
+}
+
+// EnvLoadSSHConfig sets the environment variables based on the Ssh configuration.
+func EnvLoadSSHConfig(sshCfg sshConfig) error {
+	for sectionName, section := range sshCfg {
+
+		for key, value := range section {
+			s := fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(sectionName), strings.ToUpper(convertToIniKey(key)))
+			if err := os.Setenv(s, value); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// convertToIniKey converts custom SSH configuration keys to Rclone configuration keys using keyMapping.
+func convertToIniKey(customKey string) string {
+	if iniKey, found := keyMapping[strings.ToLower(customKey)]; found {
+		return iniKey
+	}
+
+	return customKey
+}
+
+// mapSSHToRcloneConfig maps Ssh configuration to rclone configuration.
+func mapSSHToRcloneConfig(r io.Reader) (sshConfig, error) {
+	cfg, err := ssh_config.Decode(r)
+	if err != nil {
+		return nil, fmt.Errorf("error deocing ssh config file: %w", err)
+	}
+	sections := sshConfig(map[string]map[string]string{})
+
+	for _, host := range cfg.Hosts {
+		pattern := host.Patterns[0].String()
+		sections[pattern] = make(map[string]string)
+
+		// ssh configs are always type sftp
+		sections[pattern]["type"] = "sftp"
+
+		for _, node := range host.Nodes {
+			keyval := strings.Fields(strings.TrimSpace(node.String()))
+			if len(keyval) > 1 {
+				key := strings.ToLower(strings.TrimSpace(keyval[0]))
+				value := strings.TrimSpace(strings.Join(keyval[1:], " "))
+
+				sections[pattern][convertToIniKey(key)] = value
+			}
+		}
+
+		// add missing key_use_agent if there is identityfile or here mapped key_file
+		_, ok := sections[pattern]["key_file"]
+		if ok {
+			sections[pattern]["key_use_agent"] = "true"
+		}
+	}
+	return sections, nil
+}

lib/sshconfig/use_sshconfig_test.go

@@ -0,0 +1,56 @@
+package sshconfig
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var sshConfigData = `Host one
+  hostname 127.1.1.20
+  user onerclone
+  preferredauthentications publickey
+  identityfile ~/.ssh/id_ed123
+
+Host tworclone 
+  hostname 127.1.1.40
+  User vik
+  LocalForward localhost:8090 127.0.0.1:8190
+
+`
+
+// Converts known custom keys to their corresponding INI keys
+func TestConvertToIniKey(t *testing.T) {
+	tests := map[string]string{
+		"identityfile": "key_file",
+		"pubkeyfile":   "pubkey_file",
+		"user":         "user",
+		"hostname":     "host",
+		"port":         "port",
+		"password":     "pass",
+	}
+
+	for customKey, expectedIniKey := range tests {
+		result := convertToIniKey(customKey)
+		if result != expectedIniKey {
+			t.Errorf("expected: %s, got: %s", expectedIniKey, result)
+		}
+	}
+}
+
+func TestMapSshToRcloneConfig(t *testing.T) {
+	r := strings.NewReader(sshConfigData)
+	c, err := mapSSHToRcloneConfig(r)
+
+	require.NoError(t, err)
+
+	assert.Equal(t, "sftp", c["one"]["type"])
+	assert.Equal(t, "sftp", c["tworclone"]["type"])
+	assert.Equal(t, "127.1.1.20", c["one"]["host"])
+	assert.Equal(t, "~/.ssh/id_ed123", c["one"]["key_file"])
+	assert.Equal(t, "true", c["one"]["key_use_agent"])
+	assert.Equal(t, "localhost:8090 127.0.0.1:8190", c["tworclone"]["localforward"])
+	assert.Empty(t, c["towrclone"]["key_use_agent"])
+}