feat: ftp server support (#7634 close #1898)

* feat: ftp server support

* fix(ftp): incorrect mode for dirs in LIST returns
This commit is contained in:
KirCute_ECT
2024-12-10 20:17:46 +08:00
committed by GitHub
parent 7341846499
commit 650b03aeb1
13 changed files with 835 additions and 2 deletions

285
server/ftp.go Normal file
View File

@ -0,0 +1,285 @@
package server
import (
"context"
"crypto/tls"
"errors"
"fmt"
ftpserver "github.com/KirCute/ftpserverlib-pasvportmap"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/ftp"
"math/rand"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
)
type FtpMainDriver struct {
settings *ftpserver.Settings
proxyHeader *http.Header
clients map[uint32]ftpserver.ClientContext
shutdownLock sync.RWMutex
isShutdown bool
tlsConfig *tls.Config
}
func NewMainDriver() (*FtpMainDriver, error) {
header := &http.Header{}
header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent))
transferType := ftpserver.TransferTypeASCII
if conf.Conf.FTP.DefaultTransferBinary {
transferType = ftpserver.TransferTypeBinary
}
activeConnCheck := ftpserver.IPMatchDisabled
if conf.Conf.FTP.EnableActiveConnIPCheck {
activeConnCheck = ftpserver.IPMatchRequired
}
pasvConnCheck := ftpserver.IPMatchDisabled
if conf.Conf.FTP.EnablePasvConnIPCheck {
pasvConnCheck = ftpserver.IPMatchRequired
}
tlsRequired := ftpserver.ClearOrEncrypted
if setting.GetBool(conf.FTPImplicitTLS) {
tlsRequired = ftpserver.ImplicitEncryption
} else if setting.GetBool(conf.FTPMandatoryTLS) {
tlsRequired = ftpserver.MandatoryEncryption
}
tlsConf, err := getTlsConf(setting.GetStr(conf.FTPTLSPrivateKeyPath), setting.GetStr(conf.FTPTLSPublicCertPath))
if err != nil && tlsRequired != ftpserver.ClearOrEncrypted {
return nil, fmt.Errorf("FTP mandatory TLS has been enabled, but the certificate failed to load: %w", err)
}
return &FtpMainDriver{
settings: &ftpserver.Settings{
ListenAddr: conf.Conf.FTP.Listen,
PublicHost: lookupIP(setting.GetStr(conf.FTPPublicHost)),
PassiveTransferPortGetter: newPortMapper(setting.GetStr(conf.FTPPasvPortMap)),
FindPasvPortAttempts: conf.Conf.FTP.FindPasvPortAttempts,
ActiveTransferPortNon20: conf.Conf.FTP.ActiveTransferPortNon20,
IdleTimeout: conf.Conf.FTP.IdleTimeout,
ConnectionTimeout: conf.Conf.FTP.ConnectionTimeout,
DisableMLSD: false,
DisableMLST: false,
DisableMFMT: true,
Banner: setting.GetStr(conf.Announcement),
TLSRequired: tlsRequired,
DisableLISTArgs: false,
DisableSite: true,
DisableActiveMode: conf.Conf.FTP.DisableActiveMode,
EnableHASH: false,
DisableSTAT: false,
DisableSYST: false,
EnableCOMB: false,
DefaultTransferType: transferType,
ActiveConnectionsCheck: activeConnCheck,
PasvConnectionsCheck: pasvConnCheck,
},
proxyHeader: header,
clients: make(map[uint32]ftpserver.ClientContext),
shutdownLock: sync.RWMutex{},
isShutdown: false,
tlsConfig: tlsConf,
}, nil
}
func (d *FtpMainDriver) GetSettings() (*ftpserver.Settings, error) {
return d.settings, nil
}
func (d *FtpMainDriver) ClientConnected(cc ftpserver.ClientContext) (string, error) {
if d.isShutdown || !d.shutdownLock.TryRLock() {
return "", errors.New("server has shutdown")
}
defer d.shutdownLock.RUnlock()
d.clients[cc.ID()] = cc
return "AList FTP Endpoint", nil
}
func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) {
err := cc.Close()
if err != nil {
utils.Log.Errorf("failed to close client: %v", err)
}
delete(d.clients, cc.ID())
}
func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) {
var userObj *model.User
var err error
if user == "anonymous" || user == "guest" {
userObj, err = op.GetGuest()
if err != nil {
return nil, err
}
} else {
userObj, err = op.GetUserByName(user)
if err != nil {
return nil, err
}
passHash := model.StaticHash(pass)
if err = userObj.ValidatePwdStaticHash(passHash); err != nil {
return nil, err
}
}
if userObj.Disabled || !userObj.CanFTPAccess() {
return nil, errors.New("user not allowed to access FTP")
}
ctx := context.Background()
ctx = context.WithValue(ctx, "user", userObj)
if user == "anonymous" || user == "guest" {
ctx = context.WithValue(ctx, "meta_pass", pass)
} else {
ctx = context.WithValue(ctx, "meta_pass", "")
}
ctx = context.WithValue(ctx, "client_ip", cc.RemoteAddr().String())
ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader)
return ftp.NewAferoAdapter(ctx), nil
}
func (d *FtpMainDriver) GetTLSConfig() (*tls.Config, error) {
if d.tlsConfig == nil {
return nil, errors.New("TLS config not provided")
}
return d.tlsConfig, nil
}
func (d *FtpMainDriver) Stop() {
d.isShutdown = true
d.shutdownLock.Lock()
defer d.shutdownLock.Unlock()
for _, value := range d.clients {
_ = value.Close()
}
}
func lookupIP(host string) string {
if host == "" || net.ParseIP(host) != nil {
return host
}
ips, err := net.LookupIP(host)
if err != nil || len(ips) == 0 {
utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err)
return ""
}
for _, ip := range ips {
if ip.To4() != nil {
return ip.String()
}
}
v6 := ips[0].String()
utils.Log.Warnf("no IPv4 record looked up, %s will be used as public host, and it might do not work.", v6)
return v6
}
func newPortMapper(str string) ftpserver.PasvPortGetter {
if str == "" {
return nil
}
pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",")
type group struct {
ExposedStart int
ListenedStart int
Length int
}
groups := make([]group, len(pasvPortMappers))
totalLength := 0
convertToPorts := func(str string) (int, int, error) {
start, end, multi := strings.Cut(str, "-")
if multi {
si, err := strconv.Atoi(start)
if err != nil {
return 0, 0, err
}
ei, err := strconv.Atoi(end)
if err != nil {
return 0, 0, err
}
if ei < si || ei < 1024 || si < 1024 || ei > 65535 || si > 65535 {
return 0, 0, errors.New("invalid port")
}
return si, ei - si + 1, nil
} else {
ret, err := strconv.Atoi(str)
if err != nil {
return 0, 0, err
} else {
return ret, 1, nil
}
}
}
for i, mapper := range pasvPortMappers {
var err error
exposed, listened, mapped := strings.Cut(mapper, ":")
for {
if mapped {
var es, ls, el, ll int
es, el, err = convertToPorts(exposed)
if err != nil {
break
}
ls, ll, err = convertToPorts(listened)
if err != nil {
break
}
if el != ll {
err = errors.New("the number of exposed ports and listened ports does not match")
break
}
groups[i].ExposedStart = es
groups[i].ListenedStart = ls
groups[i].Length = el
totalLength += el
} else {
var start, length int
start, length, err = convertToPorts(mapper)
groups[i].ExposedStart = start
groups[i].ListenedStart = start
groups[i].Length = length
totalLength += length
}
break
}
if err != nil {
utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err)
return nil
}
}
return func() (int, int, bool) {
idxPort := rand.Intn(totalLength)
for _, g := range groups {
if idxPort >= g.Length {
idxPort -= g.Length
} else {
return g.ExposedStart + idxPort, g.ListenedStart + idxPort, true
}
}
// unreachable
return 0, 0, false
}
}
func getTlsConf(keyPath, certPath string) (*tls.Config, error) {
if keyPath == "" || certPath == "" {
return nil, errors.New("private key or certificate is not provided")
}
cert, err := os.ReadFile(certPath)
if err != nil {
return nil, err
}
key, err := os.ReadFile(keyPath)
if err != nil {
return nil, err
}
tlsCert, err := tls.X509KeyPair(cert, key)
if err != nil {
return nil, err
}
return &tls.Config{Certificates: []tls.Certificate{tlsCert}}, nil
}

91
server/ftp/afero.go Normal file
View File

@ -0,0 +1,91 @@
package ftp
import (
"context"
"errors"
ftpserver "github.com/KirCute/ftpserverlib-pasvportmap"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/spf13/afero"
"os"
"time"
)
type AferoAdapter struct {
ctx context.Context
}
func NewAferoAdapter(ctx context.Context) *AferoAdapter {
return &AferoAdapter{ctx: ctx}
}
func (a *AferoAdapter) Create(_ string) (afero.File, error) {
// See also GetHandle
return nil, errs.NotImplement
}
func (a *AferoAdapter) Mkdir(name string, _ os.FileMode) error {
return Mkdir(a.ctx, name)
}
func (a *AferoAdapter) MkdirAll(path string, perm os.FileMode) error {
return a.Mkdir(path, perm)
}
func (a *AferoAdapter) Open(_ string) (afero.File, error) {
// See also GetHandle and ReadDir
return nil, errs.NotImplement
}
func (a *AferoAdapter) OpenFile(_ string, _ int, _ os.FileMode) (afero.File, error) {
// See also GetHandle
return nil, errs.NotImplement
}
func (a *AferoAdapter) Remove(name string) error {
return Remove(a.ctx, name)
}
func (a *AferoAdapter) RemoveAll(path string) error {
return a.Remove(path)
}
func (a *AferoAdapter) Rename(oldName, newName string) error {
return Rename(a.ctx, oldName, newName)
}
func (a *AferoAdapter) Stat(name string) (os.FileInfo, error) {
return Stat(a.ctx, name)
}
func (a *AferoAdapter) Name() string {
return "AList FTP Endpoint"
}
func (a *AferoAdapter) Chmod(_ string, _ os.FileMode) error {
return errs.NotSupport
}
func (a *AferoAdapter) Chown(_ string, _, _ int) error {
return errs.NotSupport
}
func (a *AferoAdapter) Chtimes(_ string, _ time.Time, _ time.Time) error {
return errs.NotSupport
}
func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) {
return List(a.ctx, name)
}
func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) {
if offset != 0 {
return nil, errors.New("offset")
}
if (flags & os.O_APPEND) > 0 {
return nil, errors.New("append")
}
if (flags & os.O_WRONLY) > 0 {
return OpenUpload(a.ctx, name)
}
return OpenDownload(a.ctx, name)
}

75
server/ftp/fsmanage.go Normal file
View File

@ -0,0 +1,75 @@
package ftp
import (
"context"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/server/common"
"github.com/pkg/errors"
stdpath "path"
)
func Mkdir(ctx context.Context, path string) error {
user := ctx.Value("user").(*model.User)
reqPath, err := user.JoinPath(path)
if err != nil {
return err
}
if !user.CanWrite() || !user.CanFTPManage() {
meta, err := op.GetNearestMeta(stdpath.Dir(reqPath))
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return err
}
}
if !common.CanWrite(meta, reqPath) {
return errs.PermissionDenied
}
}
return fs.MakeDir(ctx, reqPath)
}
func Remove(ctx context.Context, path string) error {
user := ctx.Value("user").(*model.User)
if !user.CanRemove() || !user.CanFTPManage() {
return errs.PermissionDenied
}
reqPath, err := user.JoinPath(path)
if err != nil {
return err
}
return fs.Remove(ctx, reqPath)
}
func Rename(ctx context.Context, oldPath, newPath string) error {
user := ctx.Value("user").(*model.User)
srcPath, err := user.JoinPath(oldPath)
if err != nil {
return err
}
dstPath, err := user.JoinPath(newPath)
if err != nil {
return err
}
srcDir, srcBase := stdpath.Split(srcPath)
dstDir, dstBase := stdpath.Split(dstPath)
if srcDir == dstDir {
if !user.CanRename() || !user.CanFTPManage() {
return errs.PermissionDenied
}
return fs.Rename(ctx, srcPath, dstBase)
} else {
if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) {
return errs.PermissionDenied
}
if err := fs.Move(ctx, srcPath, dstDir); err != nil {
return err
}
if srcBase != dstBase {
return fs.Rename(ctx, stdpath.Join(dstDir, srcBase), dstBase)
}
return nil
}
}

188
server/ftp/fsread.go Normal file
View File

@ -0,0 +1,188 @@
package ftp
import (
"context"
ftpserver "github.com/KirCute/ftpserverlib-pasvportmap"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/net"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
"github.com/pkg/errors"
"io"
fs2 "io/fs"
"net/http"
"os"
"time"
)
type FileDownloadProxy struct {
ftpserver.FileTransfer
reader io.ReadCloser
closers *utils.Closers
}
func OpenDownload(ctx context.Context, path string) (*FileDownloadProxy, error) {
user := ctx.Value("user").(*model.User)
reqPath, err := user.JoinPath(path)
if err != nil {
return nil, err
}
meta, err := op.GetNearestMeta(reqPath)
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return nil, err
}
}
ctx = context.WithValue(ctx, "meta", meta)
if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) {
return nil, errs.PermissionDenied
}
// directly use proxy
header := *(ctx.Value("proxy_header").(*http.Header))
link, obj, err := fs.Link(ctx, reqPath, model.LinkArgs{
IP: ctx.Value("client_ip").(string),
Header: header,
})
if err != nil {
return nil, err
}
storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
if err != nil {
return nil, err
}
if storage.GetStorage().ProxyRange {
common.ProxyRange(link, obj.GetSize())
}
reader, closers, err := proxy(link)
if err != nil {
return nil, err
}
return &FileDownloadProxy{reader: reader, closers: closers}, nil
}
func proxy(link *model.Link) (io.ReadCloser, *utils.Closers, error) {
if link.MFile != nil {
return link.MFile, nil, nil
} else if link.RangeReadCloser != nil {
rc, err := link.RangeReadCloser.RangeRead(context.Background(), http_range.Range{Length: -1})
if err != nil {
return nil, nil, err
}
closers := link.RangeReadCloser.GetClosers()
return rc, &closers, nil
} else {
res, err := net.RequestHttp(context.Background(), http.MethodGet, link.Header, link.URL)
if err != nil {
return nil, nil, err
}
return res.Body, nil, nil
}
}
func (f *FileDownloadProxy) Read(p []byte) (n int, err error) {
return f.reader.Read(p)
}
func (f *FileDownloadProxy) Write(p []byte) (n int, err error) {
return 0, errs.NotSupport
}
func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) {
return 0, errs.NotSupport
}
func (f *FileDownloadProxy) Close() error {
defer func() {
if f.closers != nil {
_ = f.closers.Close()
}
}()
return f.reader.Close()
}
type OsFileInfoAdapter struct {
obj model.Obj
}
func (o *OsFileInfoAdapter) Name() string {
return o.obj.GetName()
}
func (o *OsFileInfoAdapter) Size() int64 {
return o.obj.GetSize()
}
func (o *OsFileInfoAdapter) Mode() fs2.FileMode {
var mode fs2.FileMode = 0755
if o.IsDir() {
mode |= fs2.ModeDir
}
return mode
}
func (o *OsFileInfoAdapter) ModTime() time.Time {
return o.obj.ModTime()
}
func (o *OsFileInfoAdapter) IsDir() bool {
return o.obj.IsDir()
}
func (o *OsFileInfoAdapter) Sys() any {
return o.obj
}
func Stat(ctx context.Context, path string) (os.FileInfo, error) {
user := ctx.Value("user").(*model.User)
reqPath, err := user.JoinPath(path)
if err != nil {
return nil, err
}
meta, err := op.GetNearestMeta(reqPath)
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return nil, err
}
}
ctx = context.WithValue(ctx, "meta", meta)
if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) {
return nil, errs.PermissionDenied
}
obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
if err != nil {
return nil, err
}
return &OsFileInfoAdapter{obj: obj}, nil
}
func List(ctx context.Context, path string) ([]os.FileInfo, error) {
user := ctx.Value("user").(*model.User)
reqPath, err := user.JoinPath(path)
if err != nil {
return nil, err
}
meta, err := op.GetNearestMeta(reqPath)
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return nil, err
}
}
ctx = context.WithValue(ctx, "meta", meta)
if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) {
return nil, errs.PermissionDenied
}
objs, err := fs.List(ctx, reqPath, &fs.ListArgs{})
if err != nil {
return nil, err
}
ret := make([]os.FileInfo, len(objs))
for i, obj := range objs {
ret[i] = &OsFileInfoAdapter{obj: obj}
}
return ret, nil
}

91
server/ftp/fsup.go Normal file
View File

@ -0,0 +1,91 @@
package ftp
import (
"context"
ftpserver "github.com/KirCute/ftpserverlib-pasvportmap"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/server/common"
"github.com/pkg/errors"
"io"
"net/http"
"os"
stdpath "path"
"time"
)
type FileUploadProxy struct {
ftpserver.FileTransfer
buffer *os.File
path string
ctx context.Context
}
func OpenUpload(ctx context.Context, path string) (*FileUploadProxy, error) {
user := ctx.Value("user").(*model.User)
path, err := user.JoinPath(path)
if err != nil {
return nil, err
}
meta, err := op.GetNearestMeta(stdpath.Dir(path))
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
return nil, err
}
}
if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) &&
((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) {
return nil, errs.PermissionDenied
}
tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*")
if err != nil {
return nil, err
}
return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx}, nil
}
func (f *FileUploadProxy) Read(p []byte) (n int, err error) {
return 0, errs.NotSupport
}
func (f *FileUploadProxy) Write(p []byte) (n int, err error) {
return f.buffer.Write(p)
}
func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) {
return 0, errs.NotSupport
}
func (f *FileUploadProxy) Close() error {
dir, name := stdpath.Split(f.path)
size, err := f.buffer.Seek(0, io.SeekCurrent)
if err != nil {
return err
}
if _, err := f.buffer.Seek(0, io.SeekStart); err != nil {
return err
}
arr := make([]byte, 512)
if _, err := f.buffer.Read(arr); err != nil {
return err
}
contentType := http.DetectContentType(arr)
if _, err := f.buffer.Seek(0, io.SeekStart); err != nil {
return err
}
s := &stream.FileStream{
Obj: &model.Object{
Name: name,
Size: size,
Modified: time.Now(),
},
Mimetype: contentType,
WebPutAsTask: false,
}
s.SetTmpFile(f.buffer)
return fs.PutDirectly(f.ctx, dir, s, true)
}