feat: update index by req.Paths

internal/bootstrap/data/setting.go

@@ -143,7 +143,6 @@ func InitialSettings() []model.SettingItem {
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
 		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
-		{Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
 	}

internal/op/hook.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/pkg/errors"
@@ -90,12 +91,17 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) {
 	return false, nil
 }
 
-//func HandleSettingItemsHook(items []model.SettingItem) (err error) {
-//	for i := 0; i < len(items); i++ {
-//		_, err = HandleSettingItemHook(&items[i])
-//		if err != nil {
-//			return err
-//		}
-//	}
-//	return nil
-//}
+// Storage
+type StorageHook func(typ string, storage driver.Driver)
+
+var storageHooks = make([]StorageHook, 0)
+
+func callStorageHooks(typ string, storage driver.Driver) {
+	for _, hook := range storageHooks {
+		hook(typ, storage)
+	}
+}
+
+func RegisterStorageHook(hook StorageHook) {
+	storageHooks = append(storageHooks, hook)
+}

internal/op/storage.go

@@ -29,11 +29,11 @@ func HasStorage(mountPath string) bool {
 	return storagesMap.Has(utils.FixAndCleanPath(mountPath))
 }
 
-func GetStorageByMountPath(virtualPath string) (driver.Driver, error) {
-	virtualPath = utils.FixAndCleanPath(virtualPath)
-	storageDriver, ok := storagesMap.Load(virtualPath)
+func GetStorageByMountPath(mountPath string) (driver.Driver, error) {
+	mountPath = utils.FixAndCleanPath(mountPath)
+	storageDriver, ok := storagesMap.Load(mountPath)
 	if !ok {
-		return nil, errors.Errorf("no mount path for an storage is: %s", virtualPath)
+		return nil, errors.Errorf("no mount path for an storage is: %s", mountPath)
 	}
 	return storageDriver, nil
 }
@@ -58,6 +58,7 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) {
 	}
 	// already has an id
 	err = initStorage(ctx, storage, storageDriver)
+	go callStorageHooks("add", storageDriver)
 	if err != nil {
 		return storage.ID, errors.Wrap(err, "failed init storage but storage is already created")
 	}
@@ -77,6 +78,7 @@ func LoadStorage(ctx context.Context, storage model.Storage) error {
 	storageDriver := driverNew()
 
 	err = initStorage(ctx, storage, storageDriver)
+	go callStorageHooks("add", storageDriver)
 	log.Debugf("storage %+v is created", storageDriver)
 	return err
 }
@@ -145,6 +147,7 @@ func DisableStorage(ctx context.Context, id uint) error {
 		return errors.WithMessage(err, "failed update storage in db")
 	}
 	storagesMap.Delete(storage.MountPath)
+	go callStorageHooks("del", storageDriver)
 	return nil
 }
 
@@ -182,6 +185,7 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
 	}
 
 	err = initStorage(ctx, storage, storageDriver)
+	go callStorageHooks("update", storageDriver)
 	log.Debugf("storage %+v is update", storageDriver)
 	return err
 }
@@ -200,13 +204,14 @@ func DeleteStorageById(ctx context.Context, id uint) error {
 		if err := storageDriver.Drop(ctx); err != nil {
 			return errors.Wrapf(err, "failed drop storage")
 		}
+		// delete the storage in the memory
+		storagesMap.Delete(storage.MountPath)
+		go callStorageHooks("del", storageDriver)
 	}
 	// delete the storage in the database
 	if err := db.DeleteStorageById(id); err != nil {
 		return errors.WithMessage(err, "failed delete storage in database")
 	}
-	// delete the storage in the memory
-	storagesMap.Delete(storage.MountPath)
 	return nil
 }
 

internal/search/build.go

@@ -8,9 +8,11 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
 	"github.com/alist-org/alist/v3/pkg/mq"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	mapset "github.com/deckarep/golang-set/v2"
@@ -157,20 +159,15 @@ func Clear(ctx context.Context) error {
 	return instance.Clear(ctx)
 }
 
+func Config(ctx context.Context) searcher.Config {
+	return instance.Config()
+}
+
 func Update(parent string, objs []model.Obj) {
 	if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
 		return
 	}
-	indexPaths := GetIndexPaths()
-	if !isIndexPath(parent, indexPaths) {
-		return
-	}
-	ignorePaths, err := GetIgnorePaths()
-	if err != nil {
-		log.Errorf("update search index error while get ignore paths: %+v", err)
-		return
-	}
-	if isIgnorePath(parent, ignorePaths) {
+	if isIgnorePath(parent) {
 		return
 	}
 	ctx := context.Background()
@@ -219,7 +216,10 @@ func Update(parent string, objs []model.Obj) {
 			}
 			// build index if it's a folder
 			if objs[i].IsDir() {
-				err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false)
+				err = BuildIndex(ctx,
+					[]string{path.Join(parent, objs[i].GetName())},
+					conf.SlicesMap[conf.IgnorePaths],
+					-1, false)
 				if err != nil {
 					log.Errorf("update search index error while build index: %+v", err)
 					return

internal/search/util.go

@@ -6,6 +6,7 @@ import (
 	"github.com/alist-org/alist/v3/drivers/alist_v3"
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/driver"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/setting"
@@ -37,25 +38,7 @@ func WriteProgress(progress *model.IndexProgress) {
 	}
 }
 
-func GetIndexPaths() []string {
-	indexPaths := make([]string, 0)
-	customIndexPaths := setting.GetStr(conf.IndexPaths)
-	if customIndexPaths != "" {
-		indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...)
-	}
-	return indexPaths
-}
-
-func isIndexPath(path string, indexPaths []string) bool {
-	for _, indexPaths := range indexPaths {
-		if strings.HasPrefix(path, indexPaths) {
-			return true
-		}
-	}
-	return false
-}
-
-func GetIgnorePaths() ([]string, error) {
+func updateIgnorePaths() {
 	storages := op.GetAllStorages()
 	ignorePaths := make([]string, 0)
 	var skipDrivers = []string{"AList V2", "AList V3", "Virtual"}
@@ -84,14 +67,27 @@ func GetIgnorePaths() ([]string, error) {
 	if customIgnorePaths != "" {
 		ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...)
 	}
-	return ignorePaths, nil
+	conf.SlicesMap[conf.IgnorePaths] = ignorePaths
 }
 
-func isIgnorePath(path string, ignorePaths []string) bool {
-	for _, ignorePath := range ignorePaths {
+func isIgnorePath(path string) bool {
+	for _, ignorePath := range conf.SlicesMap[conf.IgnorePaths] {
 		if strings.HasPrefix(path, ignorePath) {
 			return true
 		}
 	}
 	return false
 }
+
+func init() {
+	op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error {
+		updateIgnorePaths()
+		return nil
+	})
+	op.RegisterStorageHook(func(typ string, storage driver.Driver) {
+		var skipDrivers = []string{"AList V2", "AList V3", "Virtual"}
+		if utils.SliceContains(skipDrivers, storage.Config().Name) {
+			updateIgnorePaths()
+		}
+	})
+}

server/handles/index.go

@@ -3,23 +3,22 @@ package handles
 import (
 	"context"
 
+	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/search"
 	"github.com/alist-org/alist/v3/server/common"
-	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
 )
 
-type BuildIndexReq struct {
-	Paths       []string `json:"paths"`
-	MaxDepth    int      `json:"max_depth"`
-	IgnorePaths []string `json:"ignore_paths"`
-	Clear       bool     `json:"clear"`
+type BuildOrUpdateIndexReq struct {
+	Paths    []string `json:"paths"`
+	MaxDepth int      `json:"max_depth"`
+	//IgnorePaths []string `json:"ignore_paths"`
 }
 
 func BuildIndex(c *gin.Context) {
-	var req BuildIndexReq
+	var req BuildOrUpdateIndexReq
 	if err := c.ShouldBind(&req); err != nil {
 		common.ErrorResp(c, err, 400)
 		return
@@ -28,38 +27,15 @@ func BuildIndex(c *gin.Context) {
 		common.ErrorStrResp(c, "index is running", 400)
 		return
 	}
-	indexPaths := search.GetIndexPaths()
-	indexPaths = append(indexPaths, req.Paths...)
-	indexPathsSet := mapset.NewSet[string]()
-	for _, indexPath := range indexPaths {
-		indexPathsSet.Add(indexPath)
-	}
-	indexPaths = indexPathsSet.ToSlice()
-	ignorePaths, err := search.GetIgnorePaths()
-	if err != nil {
-		common.ErrorResp(c, err, 500)
-		return
-	}
-	ignorePaths = append(ignorePaths, req.IgnorePaths...)
 	go func() {
 		ctx := context.Background()
-		var err error
-		if req.Clear {
-			err = search.Clear(ctx)
-			if err != nil {
-				log.Errorf("clear index error: %+v", err)
-				return
-			}
-		} else {
-			for _, path := range req.Paths {
-				err = search.Del(ctx, path)
-				if err != nil {
-					log.Errorf("delete index on %s error: %+v", path, err)
-					return
-				}
-			}
+		err := search.Clear(ctx)
+		if err != nil {
+			log.Errorf("clear index error: %+v", err)
+			return
 		}
-		err = search.BuildIndex(context.Background(), indexPaths, ignorePaths, req.MaxDepth, true)
+		err = search.BuildIndex(context.Background(), req.Paths,
+			conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, true)
 		if err != nil {
 			log.Errorf("build index error: %+v", err)
 		}
@@ -67,6 +43,37 @@ func BuildIndex(c *gin.Context) {
 	common.SuccessResp(c)
 }
 
+func UpdateIndex(c *gin.Context) {
+	var req BuildOrUpdateIndexReq
+	if err := c.ShouldBind(&req); err != nil {
+		common.ErrorResp(c, err, 400)
+		return
+	}
+	if search.Running.Load() {
+		common.ErrorStrResp(c, "index is running", 400)
+		return
+	}
+	if !search.Config(c).AutoUpdate {
+		common.ErrorStrResp(c, "update is not supported for current index", 400)
+	}
+	go func() {
+		ctx := context.Background()
+		for _, path := range req.Paths {
+			err := search.Del(ctx, path)
+			if err != nil {
+				log.Errorf("delete index on %s error: %+v", path, err)
+				return
+			}
+		}
+		err := search.BuildIndex(context.Background(), req.Paths,
+			conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, false)
+		if err != nil {
+			log.Errorf("update index error: %+v", err)
+		}
+	}()
+	common.SuccessResp(c)
+}
+
 func StopIndex(c *gin.Context) {
 	if !search.Running.Load() {
 		common.ErrorStrResp(c, "index is not running", 400)

server/router.go

@@ -114,6 +114,7 @@ func admin(g *gin.RouterGroup) {
 
 	index := g.Group("/index")
 	index.POST("/build", middlewares.SearchIndex, handles.BuildIndex)
+	index.POST("/update", middlewares.SearchIndex, handles.UpdateIndex)
 	index.POST("/stop", middlewares.SearchIndex, handles.StopIndex)
 	index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex)
 	index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)