backend/kvfs: a new backend using lib/kv

backend/all/all.go

@@ -32,6 +32,7 @@ import (
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
+	_ "github.com/rclone/rclone/backend/kvfs"
 	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/mailru"

backend/kvfs/kvfs.go

@@ -0,0 +1,328 @@
+// Package kv provides an interface to the kv backend.
+package kvfs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Register Fs with rclone
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "kvfs",
+		Description: "kv based filesystem",
+		NewFs:       NewFs,
+		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+			return nil, nil
+		},
+		Options: []fs.Option{
+			{
+				Name:      "config_dir",
+				Help:      "Where you would like to store the kvfs kv db (e.g. /tmp/my/kv) ?",
+				Required:  true,
+				Sensitive: true,
+				Default:   "~/.config/rclone/kvfs",
+			},
+		},
+	})
+}
+
+// NewFs constructs a new filesystem given a root path and rclone configuration options
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
+	fs.Debugf(nil, "[NewFs] name: %q root: %q", name, root)
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	f := &Fs{
+		name: name,
+		root: root,
+		opt:  *opt,
+		features: &fs.Features{
+			CanHaveEmptyDirectories: true,
+			FilterAware:             true,
+		},
+	}
+
+	f.db, err = f.getDb()
+	if err != nil {
+		return nil, err
+	}
+	err = f.db.Do(true, &opPut{
+		key:   "NewFs",
+		value: []byte(time.Now().UTC().Format(time.RFC3339)),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := f.findFile(root)
+	if err != nil {
+		return nil, err
+	}
+
+	// check if file or directory
+	if file != nil && file.Type != "dir" {
+		root = path.Dir(file.Filename)
+		err = fs.ErrorIsFile
+		f.root = root
+	}
+
+	return f, err
+}
+
+// List returns a list of items in a directory
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	fullPath := f.fullPath(dir)
+	fs.Debugf(nil, "[List] dir: %q fullPath: %q", dir, fullPath)
+	files, err := f.getFiles(fullPath)
+	if err != nil {
+		return nil, err
+	}
+
+	entries = make([]fs.DirEntry, len(*files))
+	for i, file := range *files {
+		// remote is the fullpath of the file.Filename relative to the root
+		remote := strings.TrimPrefix(strings.TrimPrefix(file.Filename, f.root), "/")
+
+		if file.Type == "dir" {
+			entries[i] = fs.NewDir(remote, time.UnixMilli(file.ModTime))
+		} else {
+			obj := &Object{
+				fs:      f,
+				info:    file,
+				remote:  remote,
+				size:    file.Size,
+				modTime: time.UnixMilli(file.ModTime),
+				sha1:    file.SHA1,
+			}
+			entries[i] = obj
+		}
+	}
+
+	return entries, nil
+}
+
+// NewObject creates a new remote Object for a given remote path
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	fs.Debugf(nil, "[NewObject] remote: %q", remote)
+	// Find the file with matching remote path
+	file, err := f.findFile(path.Join(f.root, remote))
+	if err != nil {
+		return nil, err
+	}
+	if file != nil && file.Type != "dir" {
+		return &Object{
+			fs:      f,
+			remote:  remote,
+			info:    *file,
+			size:    file.Size,
+			modTime: time.UnixMilli(file.ModTime),
+			sha1:    file.SHA1,
+		}, nil
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put updates a remote Object
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
+	modTime := src.ModTime(ctx)
+	remote := src.Remote()
+	fullPath := f.fullPath(src.Remote())
+	dirPath := path.Dir(fullPath)
+
+	fs.Debugf(nil, "[Put] saving file: %q", fullPath)
+	err = f.mkDir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := f.putFile(in, fullPath, modTime)
+	if err != nil {
+		return nil, err
+	}
+	return &Object{
+		fs:      f,
+		info:    *file,
+		remote:  remote,
+		modTime: time.UnixMilli(file.ModTime),
+		size:    file.Size,
+		sha1:    file.SHA1,
+	}, nil
+}
+
+// Mkdir makes the directory (container, bucket)
+// Creates ancestors if necessary
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return f.mkDir(f.fullPath(dir))
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	fullPath := f.fullPath(dir)
+	fs.Debugf(nil, "[Rmdir] attempting removing dir: %q", fullPath)
+	return f.rmDir(fullPath)
+}
+
+// Object Methods
+// ------------------
+
+// SetModTime is not supported
+func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
+	return fs.ErrorCantSetModTimeWithoutDelete
+}
+
+// Open opens the Object for reading
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	fs.Debugf(nil, "[Open] opening object: %q options: %+v", o.info.Filename, options)
+
+	sOff, eOff := 0, len(o.info.Content)
+
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			sOff = int(x.Offset)
+			if sOff < 0 {
+				sOff = eOff - (1 * sOff)
+			}
+		case *fs.RangeOption:
+			sOff = int(x.Start)
+			if sOff < 0 {
+				sOff = eOff - (1 * sOff)
+			}
+			eOff = int(x.End) + 1
+			if eOff <= 0 {
+				eOff = len(o.info.Content)
+			}
+		default:
+			if option.Mandatory() {
+				fs.Debugf(o, "[Open] Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	content := o.info.Content[sOff:eOff]
+	reader := io.NopCloser(strings.NewReader(content))
+	return reader, nil
+}
+
+// Update updates the Object contents
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	modTime := src.ModTime(ctx)
+	fs.Debugf(nil, "[Update] updating object: %q", o.info.Filename)
+	file, err := o.fs.putFile(in, o.info.Filename, modTime)
+	if err != nil {
+		return err
+	}
+	o.info = *file
+	o.size = file.Size
+	o.sha1 = file.SHA1
+	o.modTime = modTime
+	return nil
+}
+
+// Remove deletes the remote Object
+func (o *Object) Remove(ctx context.Context) error {
+	fs.Debugf(nil, "[Remove] removing object: %q", o.info.Filename)
+	err := o.fs.remove(o.info.Filename)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ObjectInfo Methods
+// ------------------
+
+// Hash returns an SHA1 hash of the Object
+func (o *Object) Hash(ctx context.Context, typ hash.Type) (string, error) {
+	if typ == hash.SHA1 {
+		return o.sha1, nil
+	}
+	return "", nil
+}
+
+// Storable returns true if the Object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Info Methods
+// ------------------
+
+// Name returns the name of the Fs
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root path of the Fs
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a string representation of the Fs
+func (f *Fs) String() string {
+	return fmt.Sprintf("['%s']", f.root)
+}
+
+// Precision denotes that setting modification times is not supported
+func (f *Fs) Precision() time.Duration {
+	return time.Millisecond
+}
+
+// Hashes returns a set of hashes are Provided by the Fs
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.SHA1)
+}
+
+// Features returns the optional features supported by this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// DirEntry Methods
+// ------------------
+
+// Fs returns a reference to the Stub Fs containing the Object
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// String returns a string representation of the remote Object
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path of the Object, relative to Fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the Object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// Size return the size of the Object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}

backend/kvfs/kvfs_test.go

@@ -0,0 +1,19 @@
+//go:build !plan9
+
+// Test Telegram filesystem interface
+package kvfs_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/kvfs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKvfs:",
+		NilObject:  (*kvfs.Object)(nil),
+	})
+}

backend/kvfs/kvfs_utils.go

@@ -0,0 +1,313 @@
+package kvfs
+
+import (
+	"context"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/kv"
+)
+
+// An Fs is a representation of a remote KVFS Fs
+type Fs struct {
+	name     string
+	root     string
+	opt      Options
+	features *fs.Features
+	db       *kv.DB
+}
+
+// An File is a representation of an actual file in the KVFS store
+type File struct {
+	Filename string
+	Type     string
+	Content  string
+	Size     int64
+	ModTime  int64
+	SHA1     string
+}
+
+// An Object on the remote KVFS Fs
+type Object struct {
+	fs      *Fs       // what this object is part of
+	info    File      // info about the object
+	remote  string    // The remote path
+	size    int64     // size of the object
+	modTime time.Time // modification time of the object
+	sha1    string    // SHA-1 of the object content
+}
+
+// Options represent the configuration of the KVFS backend
+type Options struct {
+	ConfigDir string
+}
+
+func (f *Fs) getDb() (*kv.DB, error) {
+	var err error
+	if f.db == nil {
+		f.db, err = kv.Start(context.Background(), "kvfs", filepath.Join(f.opt.ConfigDir, "db"), f)
+		if err != nil {
+			return nil, fmt.Errorf("failed to insert file: %w", err)
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return f.db, nil
+}
+
+func (f *Fs) findFile(fullPath string) (*File, error) {
+	fs.Debugf(nil, "[findFile] fullPath: %q", fullPath)
+	var file File
+	err := f.db.Do(false, &opGet{
+		key:   fullPath,
+		value: &file,
+	})
+	if err == kv.ErrEmpty {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &file, nil
+}
+
+func (f *Fs) fileExists(fullPath string) bool {
+	file, err := f.findFile(fullPath)
+	if err != nil {
+		return false
+	}
+	return file != nil
+}
+
+func (f *Fs) getFiles(fullPath string) (*[]File, error) {
+	dirExists := fullPath == "/"
+
+	var files []File
+	err := f.db.Do(false, &opList{
+		prefix: fullPath,
+		fn: func(key string, value []byte) error {
+			var file File
+			if key == "NewFs" {
+				return nil
+			}
+			if err := json.Unmarshal(value, &file); err != nil {
+				return err
+			}
+			if file.Filename == fullPath {
+				dirExists = true
+				return nil
+			}
+			dir := path.Dir(file.Filename)
+			if dir == fullPath {
+				files = append(files, file)
+			}
+			return nil
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list files: %w", err)
+	}
+
+	if !dirExists {
+		return nil, fs.ErrorDirNotFound
+	}
+
+	return &files, nil
+}
+
+func (f *Fs) hasFiles(fullPath string) (bool, error) {
+	files, err := f.getFiles(fullPath)
+	if err != nil {
+		return false, err
+	}
+	return len(*files) > 0, nil
+}
+
+func (f *Fs) mkDir(fullPath string) error {
+	parts := strings.Split(fullPath, "/")
+
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		dir := strings.Join(parts[:i+1], "/")
+		if f.fileExists(dir) {
+			continue
+		}
+		fs.Debugf(nil, "[mkdirTree] creating directory: %q part: %q", dir, part)
+
+		file := &File{
+			Filename: dir,
+			Type:     "dir",
+			ModTime:  time.Now().UnixMilli(),
+		}
+
+		data, err := json.Marshal(file)
+		if err != nil {
+			return fmt.Errorf("failed to marshal directory: %w", err)
+		}
+
+		err = f.db.Do(true, &opPut{
+			key:   dir,
+			value: data,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to insert directory: %w", err)
+		}
+	}
+	return nil
+}
+
+func (f *Fs) rmDir(fullPath string) error {
+	fs.Debugf(nil, "[rmdir] fullPath: %q", fullPath)
+
+	// Check if directory is empty
+	result, err := f.hasFiles(fullPath)
+	if err != nil {
+		return err
+	}
+	if result {
+		return fs.ErrorDirectoryNotEmpty
+	}
+
+	// Check if directory exists
+	file, err := f.findFile(fullPath)
+	if err != nil {
+		return err
+	}
+	if file == nil {
+		return fs.ErrorDirNotFound
+	}
+
+	err = f.db.Do(true, &opDelete{
+		key: fullPath,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete directory: %w", err)
+	}
+	return nil
+}
+
+func (f *Fs) putFile(in io.Reader, fullPath string, modTime time.Time) (*File, error) {
+	fs.Debugf(nil, "[putFile] fullPath: %q", fullPath)
+
+	content, err := func() (string, error) {
+		data, err := io.ReadAll(in)
+		if err != nil {
+			return "", err
+		}
+		return string(data), nil
+	}()
+	if err != nil {
+		return nil, err
+	}
+
+	file := &File{
+		Filename: fullPath,
+		Type:     "file",
+		ModTime:  modTime.UnixMilli(),
+		Content:  content,
+	}
+
+	file.calculateMetadata()
+
+	data, err := json.Marshal(file)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal file: %w", err)
+	}
+
+	err = f.db.Do(true, &opPut{
+		key:   fullPath,
+		value: data,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to insert file: %w", err)
+	}
+
+	return file, nil
+}
+
+func (f *Fs) remove(fullPath string) error {
+	fs.Debugf(nil, "[remove] fullPath: %q", fullPath)
+	err := f.db.Do(true, &opDelete{
+		key: fullPath,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete file: %w", err)
+	}
+	return nil
+}
+
+// Calculate size and SHA1 hash for a file
+func (f *File) calculateMetadata() {
+	// Calculate size from content in bytes
+	f.Size = int64(len(f.Content))
+	// f.Size = int64(len([]byte(f.Content)))
+
+	// Create a new SHA-1 hash object
+	hasher := sha1.New()
+	// Write the input string to the hasher
+	hasher.Write([]byte(f.Content))
+	f.SHA1 = hex.EncodeToString(hasher.Sum(nil))
+}
+
+// fullPath constructs a full, absolute path from an Fs root relative path,
+func (f *Fs) fullPath(part string) string {
+	return path.Join(f.root, part)
+}
+
+// KVFS store operations
+type opGet struct {
+	key   string
+	value interface{}
+}
+
+func (op *opGet) Do(ctx context.Context, b kv.Bucket) error {
+	data := b.Get([]byte(op.key))
+	if data == nil {
+		return kv.ErrEmpty
+	}
+	return json.Unmarshal(data, op.value)
+}
+
+type opPut struct {
+	key   string
+	value []byte
+}
+
+func (op *opPut) Do(ctx context.Context, b kv.Bucket) error {
+	return b.Put([]byte(op.key), op.value)
+}
+
+type opDelete struct {
+	key string
+}
+
+func (op *opDelete) Do(ctx context.Context, b kv.Bucket) error {
+	return b.Delete([]byte(op.key))
+}
+
+type opList struct {
+	prefix string
+	fn     func(key string, value []byte) error
+}
+
+func (op *opList) Do(ctx context.Context, b kv.Bucket) error {
+	c := b.Cursor()
+	for k, v := c.Seek([]byte(op.prefix)); k != nil && strings.HasPrefix(string(k), op.prefix); k, v = c.Next() {
+		if err := op.fn(string(k), v); err != nil {
+			return err
+		}
+	}
+	return nil
+}

bin/make_manual.py

@@ -58,6 +58,7 @@ docs = [
     "internetarchive.md",
     "jottacloud.md",
     "koofr.md",
+    "kvfs.md",
     "linkbox.md",
     "mailru.md",
     "mega.md",

docs/content/kvfs.md

@@ -0,0 +1,58 @@
+---
+title: "KVFS"
+description: "Rclone docs for KVFS"
+versionIntroduced: "v1.70"
+status: Beta
+---
+
+# {{< icon "<i class="fa-regular fa-key"></i>"}} KVFS
+
+
+## Configuration
+
+The initial setup for an KVFS backend. 
+
+`rclone config` walks you through the backend configuration.  
+**<ins>Notice:</ins>** the configuration process will create the kvfs folder with the database file if it doesn't exist.
+
+Here is an example of how to make a remote called `kvfs`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> kvfs1
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / kv based filesystem
+   \ (kvfs)
+[snip]
+Storage> kvfs
+
+Option config_dir.
+Where you would like to store the kvfs kv db (e.g. /tmp/my/kv) ?
+Enter a value of type string. Press Enter for the default (~/.config/rclone/kvfs).
+config_dir> /tmp/my/kv
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: kvfs
+- config_dir: /tmp/my/kv
+Keep this "kvfs1" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+```

fstest/test_all/config.yaml

@@ -528,3 +528,6 @@ backends:
  - backend:  "iclouddrive"
    remote:   "TestICloudDrive:"
    fastlist: false
+ - backend:  "kvfs"
+   remote:   "TestKvfs:"
+   fastlist: false