sqlite: add sqlite backend

backend/all/all.go

@@ -55,6 +55,7 @@ import (
 	_ "github.com/rclone/rclone/backend/sharefile"
 	_ "github.com/rclone/rclone/backend/sia"
 	_ "github.com/rclone/rclone/backend/smb"
+	_ "github.com/rclone/rclone/backend/sqlite"
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"

backend/sqlite/sqlite.go

@@ -0,0 +1,339 @@
+package sqlite
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3" // SQLite driver
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Register Fs with rclone
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "sqlite",
+		Description: "sqlite",
+		NewFs:       NewFs,
+		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+			sqliteFile, ok := m.Get("sqlite_file")
+			var err error
+			if !ok || sqliteFile == "" {
+				return nil, fmt.Errorf("sqlite_file is required")
+			}
+
+			db, err := getConnection(sqliteFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get connection: %w", err)
+			}
+			defer db.Close()
+
+			if err := initSqlite(db); err != nil {
+				return nil, fmt.Errorf("failed to initialize tables: %w", err)
+			}
+
+			return nil, nil
+		},
+		Options: []fs.Option{
+			{
+				Name:      "sqlite_file",
+				Help:      "Enter the full path to the SQLite database file. e.g. /tmp/rclone_sqlite.db",
+				Required:  true,
+				Sensitive: true,
+				Default:   "~/.config/rclone/sqlite.db",
+			},
+		},
+	})
+}
+
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
+	fs.Debugf(nil, "[NewFs] name: %q root: %q", name, root)
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := getConnection(opt.SqliteFile)
+	if err != nil {
+		return nil, err
+	}
+	err = initSqlite(db)
+	if err != nil {
+		return nil, err
+	}
+
+	f := &Fs{
+		name: name,
+		root: root,
+		opt:  *opt,
+		features: &fs.Features{
+			CanHaveEmptyDirectories: true,
+			FilterAware:             true,
+			NoMultiThreading:        true,
+		}, // Initialize features
+		db: db,
+	}
+
+	file, err := f.findFile(root)
+	if err != nil {
+		return nil, err
+	}
+
+	// check if file or directory
+	if file != nil && file.Type != "dir" {
+		root = path.Dir(file.Filename)
+		err = fs.ErrorIsFile
+		f.root = root
+	}
+
+	return f, err
+}
+
+// List returns a list of items in a directory
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	fullPath := f.fullPath(dir)
+	fs.Debugf(nil, "[List] dir: %q fullPath: %q", dir, fullPath)
+	files, err := f.getFiles(fullPath)
+	if err != nil {
+		return nil, err
+	}
+
+	entries = make([]fs.DirEntry, 0, len(*files))
+	for _, file := range *files {
+
+		// remote is the fullpath of the file.Filename relative to the root
+		remote := strings.TrimPrefix(strings.TrimPrefix(file.Filename, f.root), "/")
+
+		if file.Type == "dir" {
+			entries = append(entries, fs.NewDir(remote, time.UnixMilli(file.ModTime)))
+		} else {
+			obj := &Object{
+				fs:      f,
+				info:    file,
+				remote:  remote,
+				size:    file.Size,
+				modTime: time.UnixMilli(file.ModTime),
+				sha1:    file.SHA1,
+			}
+			entries = append(entries, obj)
+		}
+	}
+
+	return entries, nil
+}
+
+// NewObject creates a new remote Object for a given remote path
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	fs.Debugf(nil, "[NewObject] remote: %q", remote)
+	// Find the file with matching remote path
+	file, err := f.findFile(path.Join(f.root, remote))
+	if err != nil {
+		return nil, err
+	}
+	if file != nil && file.Type != "dir" {
+		return &Object{
+			fs:      f,
+			remote:  remote,
+			info:    *file,
+			size:    file.Size,
+			modTime: time.UnixMilli(file.ModTime),
+			sha1:    file.SHA1,
+		}, nil
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put updates a remote Object
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (obj fs.Object, err error) {
+	modTime := src.ModTime(ctx)
+	remote := src.Remote()
+	fullPath := f.fullPath(src.Remote())
+	dirPath := path.Dir(fullPath)
+
+	fs.Debugf(nil, "[Put] saving file: %q", fullPath)
+	err = f.mkDir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := f.putFile(in, fullPath, modTime)
+	if err != nil {
+		return nil, err
+	}
+	return &Object{
+		fs:      f,
+		info:    *file,
+		remote:  remote,
+		modTime: time.UnixMilli(file.ModTime),
+		size:    file.Size,
+		sha1:    file.SHA1,
+	}, nil
+}
+
+// Mkdir makes the directory (container, bucket)
+// Creates ancestors if necessary
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return f.mkDir(f.fullPath(dir))
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	fullPath := f.fullPath(dir)
+	fs.Debugf(nil, "[Rmdir] attemting removing dir: %q", fullPath)
+	return f.rmDir(fullPath)
+}
+
+// Object Methods
+// ------------------
+
+// SetModTime is not supported
+func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
+	return fs.ErrorCantSetModTimeWithoutDelete
+}
+
+// Open opens the Object for reading
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	fs.Debugf(nil, "[Open] opening object: %q options: %+v", o.info.Filename, options)
+
+	sOff, eOff := 0, len(o.info.Content)
+
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.SeekOption:
+			sOff = int(x.Offset)
+			if sOff < 0 {
+				sOff = eOff - (1 * sOff)
+			}
+		case *fs.RangeOption:
+			sOff = int(x.Start)
+			if sOff < 0 {
+				sOff = eOff - (1 * sOff)
+			}
+			eOff = int(x.End) + 1
+			if eOff <= 0 {
+				eOff = len(o.info.Content)
+			}
+		default:
+			if option.Mandatory() {
+				fs.Debugf(o, "[Open] Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	content := o.info.Content[sOff:eOff]
+	reader := io.NopCloser(strings.NewReader(content))
+	return reader, nil
+}
+
+// Update updates the Object contents
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	modTime := src.ModTime(ctx)
+	fs.Debugf(nil, "[Update] updating object: %q", o.info.Filename)
+	file, err := o.fs.putFile(in, o.info.Filename, modTime)
+	if err != nil {
+		return err
+	}
+	o.info = *file
+	o.size = file.Size
+	o.sha1 = file.SHA1
+	o.modTime = modTime
+	return nil
+}
+
+// Remove deletes the remote Object
+func (o *Object) Remove(ctx context.Context) error {
+	fs.Debugf(nil, "[Remove] removing object: %q", o.info.Filename)
+	o.fs.remove(o.info.Filename)
+	return nil
+}
+
+// ObjectInfo Methods
+// ------------------
+
+// Hash returns an SHA1 hash of the Object
+func (o *Object) Hash(ctx context.Context, typ hash.Type) (string, error) {
+	if typ == hash.SHA1 {
+		return o.sha1, nil
+	}
+	return "", nil
+}
+
+// Storable returns true if the Object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Info Methods
+// ------------------
+
+// Name returns the name of the Fs
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root path of the Fs
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a string representation of the Fs
+func (f *Fs) String() string {
+	return fmt.Sprintf("['%s']", f.root)
+}
+
+func (f *Fs) Precision() time.Duration {
+	return time.Millisecond
+}
+
+// Hashes returns a set of hashes are Provided by the Fs
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.SHA1)
+}
+
+// Features returns the optional features supported by this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// DirEntry Methods
+// ------------------
+
+// Fs returns a reference to the Stub Fs containing the Object
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// String returns a string representation of the remote Object
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path of the Object, relative to Fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification time of the Object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// Size return the size of the Object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}

backend/sqlite/sqlite_test.go

@@ -0,0 +1,19 @@
+//go:build !plan9
+
+// Test Storj filesystem interface
+package sqlite_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/sqlite"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSqlite:",
+		NilObject:  (*sqlite.Object)(nil),
+	})
+}

backend/sqlite/sqlite_utils.go

@@ -0,0 +1,278 @@
+package sqlite
+
+import (
+	"crypto/sha1"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/rclone/rclone/fs"
+)
+
+type Fs struct {
+	name     string
+	root     string
+	opt      Options
+	features *fs.Features
+	db       *sql.DB
+}
+
+type File struct {
+	Filename string
+	Type     string
+	Content  string
+	Size     int64
+	ModTime  int64
+	SHA1     string
+}
+
+type Object struct {
+	fs      *Fs       // what this object is part of
+	info    File      // info about the object
+	remote  string    // The remote path
+	size    int64     // size of the object
+	modTime time.Time // modification time of the object
+	sha1    string    // SHA-1 of the object content
+}
+
+type Options struct {
+	SqliteFile string
+}
+
+// Schema for the files table
+const schema = `
+CREATE TABLE IF NOT EXISTS files (
+    filename TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    content BLOB NOT NULL DEFAULT '',  -- Using BLOB for large file contents
+    size INTEGER NOT NULL DEFAULT 0,
+    mod_time INTEGER NOT NULL, -- Store as Unix timestamp
+    sha1 TEXT NOT NULL DEFAULT '',
+    CONSTRAINT type_check CHECK (type IN ('file', 'dir'))
+);
+
+-- Index to improve directory listing performance
+CREATE INDEX IF NOT EXISTS idx_filename_prefix ON files(filename);
+`
+
+func getConnection(dbPath string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite3", dbPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open database: %w", err)
+	}
+	return db, nil
+}
+
+// initSqlite initializes the SQLite database with the required schema
+func initSqlite(db *sql.DB) error {
+	// Check if table exists first
+	var tableName string
+	err := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='files'").Scan(&tableName)
+	if err != nil && err != sql.ErrNoRows {
+		return fmt.Errorf("failed to check if table exists: %w", err)
+	}
+
+	// Only create schema if table doesn't exist
+	if err == sql.ErrNoRows {
+		_, err = db.Exec(schema)
+		if err != nil {
+			return fmt.Errorf("failed to initialize database schema: %w", err)
+		}
+	}
+	return nil
+}
+
+func (f *Fs) findFile(fullPath string) (*File, error) {
+	fs.Debugf(nil, "[findFile] fullPath: %q", fullPath)
+	var err error = nil
+	if f.db == nil {
+		f.db, err = getConnection(f.opt.SqliteFile)
+		if err != nil {
+			return nil, err
+		}
+	}
+	row := f.db.QueryRow("SELECT filename, type, content, size, mod_time, sha1 FROM files WHERE filename = ?", fullPath)
+	err = row.Err()
+	if row == nil || err == sql.ErrNoRows {
+		return nil, nil
+	}
+
+	file := &File{}
+	err = row.Scan(&file.Filename, &file.Type, &file.Content, &file.Size, &file.ModTime, &file.SHA1)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}
+
+func (f *Fs) fileExists(fullPath string) bool {
+	file, err := f.findFile(fullPath)
+	if err != nil {
+		return false
+	}
+	return file != nil
+}
+
+func (f *Fs) getFiles(fullPath string) (*[]File, error) {
+	var err error = nil
+	if f.db == nil {
+		f.db, err = getConnection(f.opt.SqliteFile)
+		if err != nil {
+			return nil, err
+		}
+	}
+	rows, err := f.db.Query("SELECT filename, type, content, size, mod_time, sha1 FROM files WHERE filename LIKE ?", fullPath+"%")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query files: %w", err)
+	}
+	defer rows.Close()
+
+	var files []File
+	fileOrDirExists := false
+	for rows.Next() {
+		var file File
+		err := rows.Scan(&file.Filename, &file.Type, &file.Content, &file.Size, &file.ModTime, &file.SHA1)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan row: %w", err)
+		}
+		dir := path.Dir(file.Filename) // Extract directory path and filename
+		if dir == fullPath {
+			files = append(files, file)
+			fileOrDirExists = true
+		} else if file.Filename == fullPath {
+			fileOrDirExists = true
+		}
+	}
+	if err = rows.Err(); err != nil {
+		return nil, fmt.Errorf("error iterating rows: %w", err)
+	}
+	if !fileOrDirExists {
+		return nil, fs.ErrorDirNotFound
+	}
+
+	return &files, nil
+}
+
+func (f *Fs) hasFiles(fullPath string) (bool, error) {
+	files, err := f.getFiles(fullPath)
+	if err != nil {
+		return false, err
+	}
+	return len(*files) > 0, nil
+}
+
+func (f *Fs) mkDir(fullPath string) error {
+	parts := strings.Split(fullPath, "/")
+
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		dir := strings.Join(parts[:i+1], "/")
+		if f.fileExists(dir) {
+			continue
+		}
+		fs.Debugf(nil, "[mkdirTree] creating directory: %q part: %q", dir, part)
+
+		_, err := f.db.Exec("INSERT OR REPLACE INTO files (filename, type, mod_time) VALUES (?, ?, ?)", dir, "dir", time.Now().UnixMilli())
+		if err != nil {
+			return fmt.Errorf("failed to insert directory: %w", err)
+		}
+	}
+	return nil
+}
+
+func (f *Fs) rmDir(fullPath string) error {
+	fs.Debugf(nil, "[rmdir] fullPath: %q", fullPath)
+	var err error = nil
+	// Check if directory is empty
+	result, err := f.hasFiles(fullPath)
+	if err != nil {
+		return err
+	}
+	if result {
+		return fs.ErrorDirectoryNotEmpty
+	}
+
+	// Check if directory exists
+	file, err := f.findFile(fullPath)
+	if err != nil {
+		return err
+	}
+	if file == nil {
+		return fs.ErrorDirNotFound
+	}
+
+	_, err = f.db.Exec("DELETE FROM files WHERE filename = ?", fullPath)
+	if err != nil {
+		return fmt.Errorf("failed to delete directory: %w", err)
+	}
+	return nil
+}
+
+func (f *Fs) putFile(in io.Reader, fullPath string, modTime time.Time) (*File, error) {
+	fs.Debugf(nil, "[putFile] fullPath: %q", fullPath)
+
+	content, err := func() (string, error) {
+		data, err := io.ReadAll(in)
+		if err != nil {
+			return "", err
+		}
+		return string(data), nil
+	}()
+	if err != nil {
+		return nil, err
+	}
+
+	file := &File{
+		Filename: fullPath,
+		Type:     "file",
+		ModTime:  modTime.UnixMilli(),
+		Content:  content,
+	}
+
+	file.calculateMetadata()
+
+	_, err = f.db.Exec("INSERT OR REPLACE INTO files (filename, type, content, size, mod_time, sha1) VALUES (?, ?, ?, ?, ?, ?)", file.Filename, file.Type, file.Content, file.Size, file.ModTime, file.SHA1)
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}
+
+func (f *Fs) remove(fullPath string) error {
+	fs.Debugf(nil, "[remove] fullPath: %q", fullPath)
+	_, err := f.db.Exec("DELETE FROM files WHERE filename = ?", fullPath)
+	if err != nil {
+		return fmt.Errorf("failed to delete file: %w", err)
+	}
+	return nil
+}
+
+// Calculate size and SHA1 hash for a file
+func (f *File) calculateMetadata() {
+	// Calculate size from content in bytes
+	f.Size = int64(len(f.Content))
+	// f.Size = int64(len([]byte(f.Content)))
+
+	// Create a new SHA-1 hash object
+	hasher := sha1.New()
+	// Write the input string to the hasher
+	hasher.Write([]byte(f.Content))
+	f.SHA1 = hex.EncodeToString(hasher.Sum(nil))
+}
+
+// fullPath constructs a full, absolute path from an Fs root relative path,
+func (f *Fs) fullPath(part string) string {
+	return path.Join(f.root, part)
+}

fstest/test_all/config.yaml

@@ -525,3 +525,6 @@ backends:
  - backend:  "iclouddrive"
    remote:   "TestICloudDrive:"
    fastlist: false
+ - backend:  "sqlite"
+   remote:   "TestSqlite:"
+   fastlist: false

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
 	github.com/mattn/go-colorable v0.1.14
 	github.com/mattn/go-runewidth v0.0.16
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/minio/minio-go/v7 v7.0.83
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/sys/mountinfo v0.7.2
@@ -91,7 +92,6 @@ require (
 	gopkg.in/validator.v2 v2.0.1
 	gopkg.in/yaml.v3 v3.0.1
 	storj.io/uplink v1.13.1
-
 )
 
 require (

go.sum

@@ -460,6 +460,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=