rclone/backend/doi/dataverse.go

// Implementation for Dataverse

package doi

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/rclone/rclone/backend/doi/api"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/lib/rest"
)

// Returns true if resolvedURL is likely a DOI hosted on a Dataverse intallation
func activateDataverse(resolvedURL *url.URL) (isActive bool) {
	queryValues := resolvedURL.Query()
	persistentID := queryValues.Get("persistentId")
	return persistentID != ""
}

// Resolve the main API endpoint for a DOI hosted on a Dataverse installation
func resolveDataverseEndpoint(resolvedURL *url.URL) (provider Provider, endpoint *url.URL, err error) {
	queryValues := resolvedURL.Query()
	persistentID := queryValues.Get("persistentId")

	query := url.Values{}
	query.Add("persistentId", persistentID)
	endpointURL := resolvedURL.ResolveReference(&url.URL{Path: "/api/datasets/:persistentId/", RawQuery: query.Encode()})

	return Dataverse, endpointURL, nil
}

// dataverseProvider implements the doiProvider interface for Dataverse installations
type dataverseProvider struct {
	f *Fs
}

// ListEntries returns the full list of entries found at the remote, regardless of root
func (dp *dataverseProvider) ListEntries(ctx context.Context) (entries []*Object, err error) {
	// Use the cache if populated
	cachedEntries, found := dp.f.cache.GetMaybe("files")
	if found {
		parsedEntries, ok := cachedEntries.([]Object)
		if ok {
			for _, entry := range parsedEntries {
				newEntry := entry
				entries = append(entries, &newEntry)
			}
			return entries, nil
		}
	}

	filesURL := dp.f.endpoint
	var res *http.Response
	var result api.DataverseDatasetResponse
	opts := rest.Opts{
		Method:     "GET",
		Path:       strings.TrimLeft(filesURL.EscapedPath(), "/"),
		Parameters: filesURL.Query(),
	}
	err = dp.f.pacer.Call(func() (bool, error) {
		res, err = dp.f.srv.CallJSON(ctx, &opts, nil, &result)
		return shouldRetry(ctx, res, err)
	})
	if err != nil {
		return nil, fmt.Errorf("readDir failed: %w", err)
	}
	modTime, modTimeErr := time.Parse(time.RFC3339, result.Data.LatestVersion.LastUpdateTime)
	if modTimeErr != nil {
		fs.Logf(dp.f, "error: could not parse last update time %v", modTimeErr)
		modTime = timeUnset
	}
	for _, file := range result.Data.LatestVersion.Files {
		contentURLPath := fmt.Sprintf("/api/access/datafile/%d", file.DataFile.ID)
		query := url.Values{}
		query.Add("format", "original")
		contentURL := dp.f.endpoint.ResolveReference(&url.URL{Path: contentURLPath, RawQuery: query.Encode()})
		entry := &Object{
			fs:          dp.f,
			remote:      path.Join(file.DirectoryLabel, file.DataFile.Filename),
			contentURL:  contentURL.String(),
			size:        file.DataFile.FileSize,
			modTime:     modTime,
			md5:         file.DataFile.MD5,
			contentType: file.DataFile.ContentType,
		}
		if file.DataFile.OriginalFileName != "" {
			entry.remote = path.Join(file.DirectoryLabel, file.DataFile.OriginalFileName)
			entry.size = file.DataFile.OriginalFileSize
			entry.contentType = file.DataFile.OriginalFileFormat
		}
		entries = append(entries, entry)
	}
	// Populate the cache
	cacheEntries := []Object{}
	for _, entry := range entries {
		cacheEntries = append(cacheEntries, *entry)
	}
	dp.f.cache.Put("files", cacheEntries)
	return entries, nil
}

func newDataverseProvider(f *Fs) doiProvider {
	return &dataverseProvider{
		f: f,
	}
}