diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index f3cb384f..d5110afd 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -143,7 +143,6 @@ func InitialSettings() []model.SettingItem { // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX}, - {Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, } diff --git a/internal/op/hook.go b/internal/op/hook.go index 1dd4a3c1..e37e52df 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -90,12 +91,17 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) { return false, nil } -//func HandleSettingItemsHook(items []model.SettingItem) (err error) { -// for i := 0; i < len(items); i++ { -// _, err = HandleSettingItemHook(&items[i]) -// if err != nil { -// return err -// } -// } -// return nil -//} +// Storage +type StorageHook func(typ string, storage driver.Driver) + +var storageHooks = make([]StorageHook, 0) + +func callStorageHooks(typ string, storage driver.Driver) { + for _, hook := range storageHooks { + hook(typ, storage) + } +} + +func RegisterStorageHook(hook StorageHook) { + storageHooks = append(storageHooks, hook) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 3acdc38e..75c755ee 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -29,11 +29,11 @@ func HasStorage(mountPath string) bool { return storagesMap.Has(utils.FixAndCleanPath(mountPath)) } -func GetStorageByMountPath(virtualPath string) (driver.Driver, error) { - virtualPath = utils.FixAndCleanPath(virtualPath) - storageDriver, ok := storagesMap.Load(virtualPath) +func GetStorageByMountPath(mountPath string) (driver.Driver, error) { + mountPath = utils.FixAndCleanPath(mountPath) + storageDriver, ok := storagesMap.Load(mountPath) if !ok { - return nil, errors.Errorf("no mount path for an storage is: %s", virtualPath) + return nil, errors.Errorf("no mount path for an storage is: %s", mountPath) } return storageDriver, nil } @@ -58,6 +58,7 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { } // already has an id err = initStorage(ctx, storage, storageDriver) + go callStorageHooks("add", storageDriver) if err != nil { return storage.ID, errors.Wrap(err, "failed init storage but storage is already created") } @@ -77,6 +78,7 @@ func LoadStorage(ctx context.Context, storage model.Storage) error { storageDriver := driverNew() err = initStorage(ctx, storage, storageDriver) + go callStorageHooks("add", storageDriver) log.Debugf("storage %+v is created", storageDriver) return err } @@ -145,6 +147,7 @@ func DisableStorage(ctx context.Context, id uint) error { return errors.WithMessage(err, "failed update storage in db") } storagesMap.Delete(storage.MountPath) + go callStorageHooks("del", storageDriver) return nil } @@ -182,6 +185,7 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } err = initStorage(ctx, storage, storageDriver) + go callStorageHooks("update", storageDriver) log.Debugf("storage %+v is update", storageDriver) return err } @@ -200,13 +204,14 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err := storageDriver.Drop(ctx); err != nil { return errors.Wrapf(err, "failed drop storage") } + // delete the storage in the memory + storagesMap.Delete(storage.MountPath) + go callStorageHooks("del", storageDriver) } // delete the storage in the database if err := db.DeleteStorageById(id); err != nil { return errors.WithMessage(err, "failed delete storage in database") } - // delete the storage in the memory - storagesMap.Delete(storage.MountPath) return nil } diff --git a/internal/search/build.go b/internal/search/build.go index 53b944f7..25af1a56 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -8,9 +8,11 @@ import ( "sync/atomic" "time" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/search/searcher" "github.com/alist-org/alist/v3/pkg/mq" "github.com/alist-org/alist/v3/pkg/utils" mapset "github.com/deckarep/golang-set/v2" @@ -157,20 +159,15 @@ func Clear(ctx context.Context) error { return instance.Clear(ctx) } +func Config(ctx context.Context) searcher.Config { + return instance.Config() +} + func Update(parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || Running.Load() { return } - indexPaths := GetIndexPaths() - if !isIndexPath(parent, indexPaths) { - return - } - ignorePaths, err := GetIgnorePaths() - if err != nil { - log.Errorf("update search index error while get ignore paths: %+v", err) - return - } - if isIgnorePath(parent, ignorePaths) { + if isIgnorePath(parent) { return } ctx := context.Background() @@ -219,7 +216,10 @@ func Update(parent string, objs []model.Obj) { } // build index if it's a folder if objs[i].IsDir() { - err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false) + err = BuildIndex(ctx, + []string{path.Join(parent, objs[i].GetName())}, + conf.SlicesMap[conf.IgnorePaths], + -1, false) if err != nil { log.Errorf("update search index error while build index: %+v", err) return diff --git a/internal/search/util.go b/internal/search/util.go index cfb98c45..531be619 100644 --- a/internal/search/util.go +++ b/internal/search/util.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/drivers/alist_v3" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" @@ -37,25 +38,7 @@ func WriteProgress(progress *model.IndexProgress) { } } -func GetIndexPaths() []string { - indexPaths := make([]string, 0) - customIndexPaths := setting.GetStr(conf.IndexPaths) - if customIndexPaths != "" { - indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...) - } - return indexPaths -} - -func isIndexPath(path string, indexPaths []string) bool { - for _, indexPaths := range indexPaths { - if strings.HasPrefix(path, indexPaths) { - return true - } - } - return false -} - -func GetIgnorePaths() ([]string, error) { +func updateIgnorePaths() { storages := op.GetAllStorages() ignorePaths := make([]string, 0) var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} @@ -84,14 +67,27 @@ func GetIgnorePaths() ([]string, error) { if customIgnorePaths != "" { ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...) } - return ignorePaths, nil + conf.SlicesMap[conf.IgnorePaths] = ignorePaths } -func isIgnorePath(path string, ignorePaths []string) bool { - for _, ignorePath := range ignorePaths { +func isIgnorePath(path string) bool { + for _, ignorePath := range conf.SlicesMap[conf.IgnorePaths] { if strings.HasPrefix(path, ignorePath) { return true } } return false } + +func init() { + op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error { + updateIgnorePaths() + return nil + }) + op.RegisterStorageHook(func(typ string, storage driver.Driver) { + var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} + if utils.SliceContains(skipDrivers, storage.Config().Name) { + updateIgnorePaths() + } + }) +} diff --git a/server/handles/index.go b/server/handles/index.go index 0c0793a2..754e1fea 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -3,23 +3,22 @@ package handles import ( "context" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/search" "github.com/alist-org/alist/v3/server/common" - mapset "github.com/deckarep/golang-set/v2" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) -type BuildIndexReq struct { - Paths []string `json:"paths"` - MaxDepth int `json:"max_depth"` - IgnorePaths []string `json:"ignore_paths"` - Clear bool `json:"clear"` +type BuildOrUpdateIndexReq struct { + Paths []string `json:"paths"` + MaxDepth int `json:"max_depth"` + //IgnorePaths []string `json:"ignore_paths"` } func BuildIndex(c *gin.Context) { - var req BuildIndexReq + var req BuildOrUpdateIndexReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return @@ -28,38 +27,15 @@ func BuildIndex(c *gin.Context) { common.ErrorStrResp(c, "index is running", 400) return } - indexPaths := search.GetIndexPaths() - indexPaths = append(indexPaths, req.Paths...) - indexPathsSet := mapset.NewSet[string]() - for _, indexPath := range indexPaths { - indexPathsSet.Add(indexPath) - } - indexPaths = indexPathsSet.ToSlice() - ignorePaths, err := search.GetIgnorePaths() - if err != nil { - common.ErrorResp(c, err, 500) - return - } - ignorePaths = append(ignorePaths, req.IgnorePaths...) go func() { ctx := context.Background() - var err error - if req.Clear { - err = search.Clear(ctx) - if err != nil { - log.Errorf("clear index error: %+v", err) - return - } - } else { - for _, path := range req.Paths { - err = search.Del(ctx, path) - if err != nil { - log.Errorf("delete index on %s error: %+v", path, err) - return - } - } + err := search.Clear(ctx) + if err != nil { + log.Errorf("clear index error: %+v", err) + return } - err = search.BuildIndex(context.Background(), indexPaths, ignorePaths, req.MaxDepth, true) + err = search.BuildIndex(context.Background(), req.Paths, + conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, true) if err != nil { log.Errorf("build index error: %+v", err) } @@ -67,6 +43,37 @@ func BuildIndex(c *gin.Context) { common.SuccessResp(c) } +func UpdateIndex(c *gin.Context) { + var req BuildOrUpdateIndexReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if search.Running.Load() { + common.ErrorStrResp(c, "index is running", 400) + return + } + if !search.Config(c).AutoUpdate { + common.ErrorStrResp(c, "update is not supported for current index", 400) + } + go func() { + ctx := context.Background() + for _, path := range req.Paths { + err := search.Del(ctx, path) + if err != nil { + log.Errorf("delete index on %s error: %+v", path, err) + return + } + } + err := search.BuildIndex(context.Background(), req.Paths, + conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, false) + if err != nil { + log.Errorf("update index error: %+v", err) + } + }() + common.SuccessResp(c) +} + func StopIndex(c *gin.Context) { if !search.Running.Load() { common.ErrorStrResp(c, "index is not running", 400) diff --git a/server/router.go b/server/router.go index 80d05a67..e29755c0 100644 --- a/server/router.go +++ b/server/router.go @@ -114,6 +114,7 @@ func admin(g *gin.RouterGroup) { index := g.Group("/index") index.POST("/build", middlewares.SearchIndex, handles.BuildIndex) + index.POST("/update", middlewares.SearchIndex, handles.UpdateIndex) index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)