github-mirror/alist
github-mirror/alist
2
0
Fork 0
mirror of https://github.com/AlistGo/alist.git synced 2025-04-24 22:34:04 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: customize ignore paths of indexes

Browse Source
This commit is contained in:
Noah Hsu 2022-12-05 16:45:11 +08:00
parent bd33c200dc
commit bf8b6f4c2c
7 changed files with 120 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/bootstrap/data/setting.go
View File

@ -131,7 +131,8 @@ func InitialSettings() []model.SettingItem {
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.SINGLE},
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
}
if flags.Dev {

5
internal/conf/const.go
View File

@ -41,7 +41,10 @@ const (
PrivacyRegs = "privacy_regs"
OcrApi = "ocr_api"
FilenameCharMapping = "filename_char_mapping"
SearchIndex = "search_index"
// index
SearchIndex = "search_index"
IgnorePaths = "ignore_paths"
// aria2
Aria2Uri = "aria2_uri"

5
internal/model/setting.go
View File

@ -1,12 +1,13 @@
package model
const (
SITE = iota
SINGLE = iota
SITE
STYLE
PREVIEW
GLOBAL
SINGLE
ARIA2
INDEX
)
const (

84
internal/search/build.go
View File

@ -11,8 +11,10 @@ import (
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/mq"
"github.com/alist-org/alist/v3/pkg/utils"
mapset "github.com/deckarep/golang-set/v2"
log "github.com/sirupsen/logrus"
)
@ -22,18 +24,8 @@ var (
)
func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error {
storages, err := db.GetEnabledStorages()
if err != nil {
return err
}
var skipDrivers = []string{"AList V2", "AList V3"}
for _, storage := range storages {
if utils.SliceContains(skipDrivers, storage.Driver) {
// TODO: request for indexing permission
ignorePaths = append(ignorePaths, storage.MountPath)
}
}
var (
err error
objCount uint64 = 0
fi model.Obj
)
@ -154,3 +146,73 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
func Clear(ctx context.Context) error {
return instance.Clear(ctx)
}
func Update(parent string, objs []model.Obj) {
if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
return
}
ignorePaths, err := GetIgnorePaths()
if err != nil {
log.Errorf("update search index error while get ignore paths: %+v", err)
return
}
if isIgnorePath(parent, ignorePaths) {
return
}
ctx := context.Background()
// only update when index have built
progress, err := Progress()
if err != nil {
log.Errorf("update search index error while get progress: %+v", err)
return
}
if !progress.IsDone {
return
}
nodes, err := instance.Get(ctx, parent)
if err != nil {
log.Errorf("update search index error while get nodes: %+v", err)
return
}
now := mapset.NewSet[string]()
for i := range objs {
now.Add(objs[i].GetName())
}
old := mapset.NewSet[string]()
for i := range nodes {
old.Add(nodes[i].Name)
}
// delete data that no longer exists
toDelete := old.Difference(now)
toAdd := now.Difference(old)
for i := range nodes {
if toDelete.Contains(nodes[i].Name) {
err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
if err != nil {
log.Errorf("update search index error while del old node: %+v", err)
return
}
}
}
for i := range objs {
if toAdd.Contains(objs[i].GetName()) {
err = Index(ctx, parent, objs[i])
if err != nil {
log.Errorf("update search index error while index new node: %+v", err)
return
}
// build index if it's a folder
if objs[i].IsDir() {
err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false)
if err != nil {
log.Errorf("update search index error while build index: %+v", err)
return
}
}
}
}
}
func init() {
op.RegisterObjsUpdateHook(Update)
}

73
internal/search/update.go
View File

@ -1,73 +0,0 @@
package search
import (
"context"
"path"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
mapset "github.com/deckarep/golang-set/v2"
log "github.com/sirupsen/logrus"
)
func Update(parent string, objs []model.Obj) {
if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
return
}
ctx := context.Background()
// only update when index have built
progress, err := Progress()
if err != nil {
log.Errorf("update search index error while get progress: %+v", err)
return
}
if !progress.IsDone {
return
}
nodes, err := instance.Get(ctx, parent)
if err != nil {
log.Errorf("update search index error while get nodes: %+v", err)
return
}
now := mapset.NewSet[string]()
for i := range objs {
now.Add(objs[i].GetName())
}
old := mapset.NewSet[string]()
for i := range nodes {
old.Add(nodes[i].Name)
}
// delete data that no longer exists
toDelete := old.Difference(now)
toAdd := now.Difference(old)
for i := range nodes {
if toDelete.Contains(nodes[i].Name) {
err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
if err != nil {
log.Errorf("update search index error while del old node: %+v", err)
return
}
}
}
for i := range objs {
if toAdd.Contains(objs[i].GetName()) {
err = Index(ctx, parent, objs[i])
if err != nil {
log.Errorf("update search index error while index new node: %+v", err)
return
}
// build index if it's a folder
if objs[i].IsDir() {
err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false)
if err != nil {
log.Errorf("update search index error while build index: %+v", err)
return
}
}
}
}
}
func init() {
op.RegisterObjsUpdateHook(Update)
}

31
internal/search/progress.go → internal/search/util.go
View File

@ -1,6 +1,8 @@
package search
import (
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
@ -32,3 +34,32 @@ func WriteProgress(progress *model.IndexProgress) {
log.Errorf("save progress error: %+v", err)
}
}
func GetIgnorePaths() ([]string, error) {
storages, err := db.GetEnabledStorages()
if err != nil {
return nil, err
}
ignorePaths := make([]string, 0)
var skipDrivers = []string{"AList V2", "AList V3"}
for _, storage := range storages {
if utils.SliceContains(skipDrivers, storage.Driver) {
// TODO: request for indexing permission
ignorePaths = append(ignorePaths, storage.MountPath)
}
}
customIgnorePaths := setting.GetStr(conf.IgnorePaths)
if customIgnorePaths != "" {
ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...)
}
return ignorePaths, nil
}
func isIgnorePath(path string, ignorePaths []string) bool {
for _, ignorePath := range ignorePaths {
if strings.HasPrefix(path, ignorePath) {
return true
}
}
return false
}

8
server/handles/index.go
View File

@ -25,6 +25,12 @@ func BuildIndex(c *gin.Context) {
common.ErrorStrResp(c, "index is running", 400)
return
}
ignorePaths, err := search.GetIgnorePaths()
if err != nil {
common.ErrorResp(c, err, 500)
return
}
ignorePaths = append(ignorePaths, req.IgnorePaths...)
go func() {
ctx := context.Background()
err := search.Clear(ctx)
@ -32,7 +38,7 @@ func BuildIndex(c *gin.Context) {
log.Errorf("clear index error: %+v", err)
return
}
err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true)
err = search.BuildIndex(context.Background(), req.Paths, ignorePaths, req.MaxDepth, true)
if err != nil {
log.Errorf("build index error: %+v", err)
}