feat: port template-parsing logic from influxdb (#146)

clients/apply/source.go

@@ -0,0 +1,217 @@
+package apply
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/influxdata/influx-cli/v2/api"
+	"github.com/influxdata/influx-cli/v2/pkg/github"
+	"github.com/influxdata/influx-cli/v2/pkg/jsonnet"
+	"gopkg.in/yaml.v3"
+)
+
+type TemplateEncoding int
+
+const (
+	TemplateEncodingUnknown TemplateEncoding = iota
+	TemplateEncodingJson
+	TemplateEncodingJsonnet
+	TemplateEncodingYaml
+)
+
+func (e *TemplateEncoding) Set(v string) error {
+	switch v {
+	case "jsonnet":
+		*e = TemplateEncodingJsonnet
+	case "json":
+		*e = TemplateEncodingJson
+	case "yml", "yaml":
+		*e = TemplateEncodingYaml
+	default:
+		return fmt.Errorf("unknown inEncoding %q", v)
+	}
+	return nil
+}
+
+func (e TemplateEncoding) String() string {
+	switch e {
+	case TemplateEncodingJsonnet:
+		return "jsonnet"
+	case TemplateEncodingJson:
+		return "json"
+	case TemplateEncodingYaml:
+		return "yaml"
+	case TemplateEncodingUnknown:
+		fallthrough
+	default:
+		return "unknown"
+	}
+}
+
+type TemplateSource struct {
+	Name     string
+	Encoding TemplateEncoding
+	Open     func(context.Context) (io.ReadCloser, error)
+}
+
+func SourcesFromPath(path string, recur bool, encoding TemplateEncoding) ([]TemplateSource, error) {
+	paths, err := findPaths(path, recur)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find inputs at path %q: %w", path, err)
+	}
+
+	sources := make([]TemplateSource, len(paths))
+	for i := range paths {
+		path := paths[i] // Local var for the `Open` closure to capture.
+		encoding := encoding
+		if encoding == TemplateEncodingUnknown {
+			switch filepath.Ext(path) {
+			case ".jsonnet":
+				encoding = TemplateEncodingJsonnet
+			case ".json":
+				encoding = TemplateEncodingJson
+			case ".yml":
+				fallthrough
+			case ".yaml":
+				encoding = TemplateEncodingYaml
+			default:
+			}
+		}
+
+		sources[i] = TemplateSource{
+			Name:     path,
+			Encoding: encoding,
+			Open: func(context.Context) (io.ReadCloser, error) {
+				return os.Open(path)
+			},
+		}
+	}
+	return sources, nil
+}
+
+func findPaths(path string, recur bool) ([]string, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+	if !fi.IsDir() {
+		return []string{path}, nil
+	}
+
+	dirFiles, err := os.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var paths []string
+	for _, df := range dirFiles {
+		fullPath := filepath.Join(path, df.Name())
+		if df.IsDir() && recur {
+			subPaths, err := findPaths(fullPath, recur)
+			if err != nil {
+				return nil, err
+			}
+			paths = append(paths, subPaths...)
+		} else if !df.IsDir() {
+			paths = append(paths, fullPath)
+		}
+	}
+	return paths, nil
+}
+
+func SourceFromURL(u *url.URL, encoding TemplateEncoding) TemplateSource {
+	if encoding == TemplateEncodingUnknown {
+		switch path.Ext(u.Path) {
+		case ".jsonnet":
+			encoding = TemplateEncodingJsonnet
+		case ".json":
+			encoding = TemplateEncodingJson
+		case ".yml":
+			fallthrough
+		case ".yaml":
+			encoding = TemplateEncodingYaml
+		default:
+		}
+	}
+
+	normalized := github.NormalizeURLToContent(u, "yaml", "yml", "jsonnet", "json").String()
+
+	return TemplateSource{
+		Name:     normalized,
+		Encoding: encoding,
+		Open: func(ctx context.Context) (io.ReadCloser, error) {
+			req, err := http.NewRequestWithContext(ctx, "GET", normalized, nil)
+			if err != nil {
+				return nil, err
+			}
+			res, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return nil, err
+			}
+			if res.StatusCode/100 != 2 {
+				body, err := io.ReadAll(res.Body)
+				res.Body.Close()
+				if err != nil {
+					return nil, err
+				}
+				return nil, fmt.Errorf("bad response: address=%s status_code=%d body=%q",
+					normalized, res.StatusCode, strings.TrimSpace(string(body)))
+			}
+			return res.Body, nil
+		},
+	}
+}
+
+func SourceFromReader(r io.Reader, encoding TemplateEncoding) TemplateSource {
+	return TemplateSource{
+		Name:     "byte stream",
+		Encoding: encoding,
+		Open: func(context.Context) (io.ReadCloser, error) {
+			return io.NopCloser(r), nil
+		},
+	}
+}
+
+func (s TemplateSource) Read(ctx context.Context) ([]api.TemplateEntry, error) {
+	var entries []api.TemplateEntry
+	if err := func() error {
+		in, err := s.Open(ctx)
+		if err != nil {
+			return err
+		}
+		defer in.Close()
+
+		switch s.Encoding {
+		case TemplateEncodingJsonnet:
+			err = jsonnet.NewDecoder(in).Decode(&entries)
+		case TemplateEncodingJson:
+			err = json.NewDecoder(in).Decode(&entries)
+		case TemplateEncodingUnknown:
+			fallthrough // Assume YAML if we can't make a better guess
+		case TemplateEncodingYaml:
+			dec := yaml.NewDecoder(in)
+			for {
+				var e api.TemplateEntry
+				if err := dec.Decode(&e); err == io.EOF {
+					break
+				} else if err != nil {
+					return err
+				}
+				entries = append(entries, e)
+			}
+		}
+		return err
+	}(); err != nil {
+		return nil, fmt.Errorf("failed to read template(s) from %q: %w", s.Name, err)
+	}
+
+	return entries, nil
+}

clients/apply/source_test.go

@@ -0,0 +1,433 @@
+package apply_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/influxdata/influx-cli/v2/api"
+	"github.com/influxdata/influx-cli/v2/clients/apply"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSourcesFromPath(t *testing.T) {
+	t.Parallel()
+
+	type contents struct {
+		name     string
+		encoding apply.TemplateEncoding
+		contents string
+	}
+	testCases := []struct {
+		name       string
+		setup      func(t *testing.T, rootDir string)
+		inPath     func(rootDir string) string
+		inEncoding apply.TemplateEncoding
+		recursive  bool
+		expected   func(rootDir string) []contents
+	}{
+		{
+			name: "JSON file",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return filepath.Join(rootDir, "foo.json")
+			},
+			expected: func(string) []contents {
+				return []contents{{
+					name:     "foo.json",
+					encoding: apply.TemplateEncodingJson,
+					contents: "foo",
+				}}
+			},
+		},
+		{
+			name: "YAML file",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yaml"), []byte("foo"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return filepath.Join(rootDir, "foo.yaml")
+			},
+			expected: func(string) []contents {
+				return []contents{{
+					name:     "foo.yaml",
+					encoding: apply.TemplateEncodingYaml,
+					contents: "foo",
+				}}
+			},
+		},
+		{
+			name: "YML file",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return filepath.Join(rootDir, "foo.yml")
+			},
+			expected: func(string) []contents {
+				return []contents{{
+					name:     "foo.yml",
+					encoding: apply.TemplateEncodingYaml,
+					contents: "foo",
+				}}
+			},
+		},
+		{
+			name: "JSONNET file",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.jsonnet"), []byte("foo"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return filepath.Join(rootDir, "foo.jsonnet")
+			},
+			expected: func(string) []contents {
+				return []contents{{
+					name:     "foo.jsonnet",
+					encoding: apply.TemplateEncodingJsonnet,
+					contents: "foo",
+				}}
+			},
+		},
+		{
+			name: "explicit inEncoding",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo"), []byte("foo"), os.ModePerm))
+			},
+			inEncoding: apply.TemplateEncodingJson,
+			inPath: func(rootDir string) string {
+				return filepath.Join(rootDir, "foo")
+			},
+			expected: func(string) []contents {
+				return []contents{{
+					name:     "foo",
+					encoding: apply.TemplateEncodingJson,
+					contents: "foo",
+				}}
+			},
+		},
+		{
+			name: "directory - non-recursive",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.Mkdir(filepath.Join(rootDir, "bar"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo.json"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo.yml"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.jsonnet"), []byte("foo.jsonnet"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.yaml"), []byte("foo.yaml"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return rootDir
+			},
+			expected: func(rootDir string) []contents {
+				return []contents{
+					{
+						name:     filepath.Join(rootDir, "foo.json"),
+						contents: "foo.json",
+						encoding: apply.TemplateEncodingJson,
+					},
+					{
+						name:     filepath.Join(rootDir, "foo.yml"),
+						contents: "foo.yml",
+						encoding: apply.TemplateEncodingYaml,
+					},
+				}
+			},
+		},
+		{
+			name: "directory - recursive",
+			setup: func(t *testing.T, rootDir string) {
+				require.NoError(t, os.Mkdir(filepath.Join(rootDir, "bar"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo.json"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo.yml"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.jsonnet"), []byte("foo.jsonnet"), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.yaml"), []byte("foo.yaml"), os.ModePerm))
+			},
+			inPath: func(rootDir string) string {
+				return rootDir
+			},
+			recursive: true,
+			expected: func(rootDir string) []contents {
+				return []contents{
+					{
+						name:     filepath.Join(rootDir, "foo.json"),
+						contents: "foo.json",
+						encoding: apply.TemplateEncodingJson,
+					},
+					{
+						name:     filepath.Join(rootDir, "foo.yml"),
+						contents: "foo.yml",
+						encoding: apply.TemplateEncodingYaml,
+					},
+					{
+						name:     filepath.Join(rootDir, "bar", "foo.yaml"),
+						contents: "foo.yaml",
+						encoding: apply.TemplateEncodingYaml,
+					},
+					{
+						name:     filepath.Join(rootDir, "bar", "foo.jsonnet"),
+						contents: "foo.jsonnet",
+						encoding: apply.TemplateEncodingJsonnet,
+					},
+				}
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			tmp, err := os.MkdirTemp("", "")
+			require.NoError(t, err)
+			defer os.RemoveAll(tmp)
+			tc.setup(t, tmp)
+
+			sources, err := apply.SourcesFromPath(tc.inPath(tmp), tc.recursive, tc.inEncoding)
+			require.NoError(t, err)
+			expected := tc.expected(tmp)
+			require.Len(t, sources, len(expected))
+
+			sort.Slice(sources, func(i, j int) bool {
+				return sources[i].Name < sources[j].Name
+			})
+			sort.Slice(expected, func(i, j int) bool {
+				return expected[i].name < expected[j].name
+			})
+
+			for i := range expected {
+				source := sources[i]
+				contents := expected[i]
+
+				require.Equal(t, contents.encoding, source.Encoding)
+				sourceIn, err := source.Open(context.Background())
+				require.NoError(t, err)
+				bytes, err := io.ReadAll(sourceIn)
+				sourceIn.Close()
+				require.NoError(t, err)
+				require.Equal(t, contents.contents, string(bytes))
+			}
+		})
+	}
+}
+
+func TestSourceFromURL(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name           string
+		filename       string
+		inEncoding     apply.TemplateEncoding
+		expectEncoding apply.TemplateEncoding
+		resStatus      int
+		resBody        string
+		expectErr      bool
+	}{
+		{
+			name:           "JSON file",
+			filename:       "foo.json",
+			expectEncoding: apply.TemplateEncodingJson,
+			resStatus:      200,
+			resBody:        "Foo bar",
+		},
+		{
+			name:           "YAML file",
+			filename:       "foo.yaml",
+			expectEncoding: apply.TemplateEncodingYaml,
+			resStatus:      200,
+			resBody:        "Foo bar",
+		},
+		{
+			name:           "YML file",
+			filename:       "foo.yml",
+			expectEncoding: apply.TemplateEncodingYaml,
+			resStatus:      200,
+			resBody:        "Foo bar",
+		},
+		{
+			name:           "JSONNET file",
+			filename:       "foo.jsonnet",
+			expectEncoding: apply.TemplateEncodingJsonnet,
+			resStatus:      200,
+			resBody:        "Foo bar",
+		},
+		{
+			name:           "explicit encoding",
+			filename:       "foo",
+			inEncoding:     apply.TemplateEncodingJson,
+			expectEncoding: apply.TemplateEncodingJson,
+			resStatus:      200,
+			resBody:        "Foo bar",
+		},
+		{
+			name:           "err response",
+			filename:       "foo.json",
+			expectEncoding: apply.TemplateEncodingJson,
+			resStatus:      403,
+			resBody:        "OH NO",
+			expectErr:      true,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+				rw.WriteHeader(tc.resStatus)
+				rw.Write([]byte(tc.resBody))
+			}))
+			defer server.Close()
+
+			u, err := url.Parse(server.URL)
+			require.NoError(t, err)
+			u.Path = tc.filename
+
+			source := apply.SourceFromURL(u, tc.inEncoding)
+			in, err := source.Open(context.Background())
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "bad response")
+				require.Contains(t, err.Error(), tc.resBody)
+				return
+			}
+			require.NoError(t, err)
+			defer in.Close()
+			bytes, err := io.ReadAll(in)
+			require.NoError(t, err)
+			require.Equal(t, tc.resBody, string(bytes))
+		})
+	}
+}
+
+func TestTemplateSource_Read(t *testing.T) {
+	t.Parallel()
+
+	yamlTemplate := `---
+apiversion: influxdata.com/v2alpha1
+kind: Bucket
+meta: null
+spec:
+    name: test
+    retentionRules:
+      - type: expire
+---
+apiversion: influxdata.com/v2alpha1
+kind: Bucket
+meta: null
+spec:
+    name: test2
+    retentionRules:
+      - type: expire
+`
+	jsonTemplate := `[
+	{
+		"apiVersion": "influxdata.com/v2alpha1",
+		"kind": "Bucket",
+		"spec": {
+			"name": "test",
+			"retentionRules": [
+				{
+					"type": "expire"
+				}
+			]
+		}
+	},
+	{
+		"apiVersion": "influxdata.com/v2alpha1",
+		"kind": "Bucket",
+		"spec": {
+			"name": "test2",
+			"retentionRules": [
+				{
+					"type": "expire"
+				}
+			]
+		}
+	}
+]
+`
+	jsonnetTemplate := `local Bucket(name) = {
+	apiVersion: "influxdata.com/v2alpha1",
+	kind: "Bucket",
+	spec: {
+		name: name,
+		retentionRules: [{
+			type: "expire",
+		}],
+	},
+};
+[Bucket("test"), Bucket("test2")]
+`
+
+	parsed := []api.TemplateEntry{
+		{
+			ApiVersion: "influxdata.com/v2alpha1",
+			Kind:       "Bucket",
+			Spec: map[string]interface{}{
+				"name": "test",
+				"retentionRules": []interface{}{
+					map[string]interface{}{
+						"type":         "expire",
+					},
+				},
+			},
+		},
+		{
+			ApiVersion: "influxdata.com/v2alpha1",
+			Kind:       "Bucket",
+			Spec: map[string]interface{}{
+				"name": "test2",
+				"retentionRules": []interface{}{
+					map[string]interface{}{
+						"type":         "expire",
+					},
+				},
+			},
+		},
+	}
+
+	testCases := []struct {
+		name     string
+		encoding apply.TemplateEncoding
+		data     string
+	}{
+		{
+			name:     "JSON",
+			encoding: apply.TemplateEncodingJson,
+			data:     jsonTemplate,
+		},
+		{
+			name:     "YAML",
+			encoding: apply.TemplateEncodingYaml,
+			data:     yamlTemplate,
+		},
+		{
+			name:     "JSONNET",
+			encoding: apply.TemplateEncodingJsonnet,
+			data:     jsonnetTemplate,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			source := apply.SourceFromReader(strings.NewReader(tc.data), tc.encoding)
+			tmpls, err := source.Read(context.Background())
+			require.NoError(t, err)
+
+			require.Equal(t, parsed, tmpls)
+		})
+	}
+}

pkg/github/normalize.go

@@ -0,0 +1,45 @@
+package github
+
+import (
+	"net/url"
+	"path"
+	"strings"
+)
+
+const (
+	githubRawContentHost = "raw.githubusercontent.com"
+	githubHost           = "github.com"
+)
+
+func NormalizeURLToContent(u *url.URL, extensions ...string) *url.URL {
+	if u.Host != githubHost {
+		return u
+	}
+	if len(extensions) > 0 && !extensionMatches(u, extensions) {
+		return u
+	}
+
+	p := u.Path
+	if !strings.HasPrefix(p, "/") {
+		p = "/" + p
+	}
+	parts := strings.Split(p, "/")
+	if len(parts) < 4 {
+		return u
+	}
+
+	normalized := *u
+	normalized.Host = githubRawContentHost
+	normalized.Path = "/" + path.Join(append(parts[:3], parts[4:]...)...)
+	return &normalized
+}
+
+func extensionMatches(u *url.URL, extensions []string) bool {
+	ext := path.Ext(u.Path)
+	for _, e := range extensions {
+		if strings.EqualFold(e, "."+ext) {
+			return true
+		}
+	}
+	return false
+}

pkg/github/normalize_test.go

@@ -0,0 +1,47 @@
+package github_test
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/influxdata/influx-cli/v2/pkg/github"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNormalize(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name string
+		in   url.URL
+		exts []string
+		out  url.URL
+	}{
+		{
+			name: "github URL",
+			in:   url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"},
+			out:  url.URL{Host: "raw.githubusercontent.com", Path: "/influxdata/influxdb/master/flags.yml"},
+		},
+		{
+			name: "other URL",
+			in:   url.URL{Host: "google.com", Path: "/fake.yml"},
+			out:  url.URL{Host: "google.com", Path: "/fake.yml"},
+		},
+		{
+			name: "github URL - wrong extension",
+			in:   url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"},
+			exts: []string{"json"},
+			out:  url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"},
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			normalized := github.NormalizeURLToContent(&tc.in, tc.exts...)
+			require.Equal(t, tc.out, *normalized)
+		})
+	}
+}