diff --git a/clients/apply/source.go b/clients/apply/source.go new file mode 100644 index 0000000..3846c44 --- /dev/null +++ b/clients/apply/source.go @@ -0,0 +1,217 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/influxdata/influx-cli/v2/api" + "github.com/influxdata/influx-cli/v2/pkg/github" + "github.com/influxdata/influx-cli/v2/pkg/jsonnet" + "gopkg.in/yaml.v3" +) + +type TemplateEncoding int + +const ( + TemplateEncodingUnknown TemplateEncoding = iota + TemplateEncodingJson + TemplateEncodingJsonnet + TemplateEncodingYaml +) + +func (e *TemplateEncoding) Set(v string) error { + switch v { + case "jsonnet": + *e = TemplateEncodingJsonnet + case "json": + *e = TemplateEncodingJson + case "yml", "yaml": + *e = TemplateEncodingYaml + default: + return fmt.Errorf("unknown inEncoding %q", v) + } + return nil +} + +func (e TemplateEncoding) String() string { + switch e { + case TemplateEncodingJsonnet: + return "jsonnet" + case TemplateEncodingJson: + return "json" + case TemplateEncodingYaml: + return "yaml" + case TemplateEncodingUnknown: + fallthrough + default: + return "unknown" + } +} + +type TemplateSource struct { + Name string + Encoding TemplateEncoding + Open func(context.Context) (io.ReadCloser, error) +} + +func SourcesFromPath(path string, recur bool, encoding TemplateEncoding) ([]TemplateSource, error) { + paths, err := findPaths(path, recur) + if err != nil { + return nil, fmt.Errorf("failed to find inputs at path %q: %w", path, err) + } + + sources := make([]TemplateSource, len(paths)) + for i := range paths { + path := paths[i] // Local var for the `Open` closure to capture. + encoding := encoding + if encoding == TemplateEncodingUnknown { + switch filepath.Ext(path) { + case ".jsonnet": + encoding = TemplateEncodingJsonnet + case ".json": + encoding = TemplateEncodingJson + case ".yml": + fallthrough + case ".yaml": + encoding = TemplateEncodingYaml + default: + } + } + + sources[i] = TemplateSource{ + Name: path, + Encoding: encoding, + Open: func(context.Context) (io.ReadCloser, error) { + return os.Open(path) + }, + } + } + return sources, nil +} + +func findPaths(path string, recur bool) ([]string, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return []string{path}, nil + } + + dirFiles, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + var paths []string + for _, df := range dirFiles { + fullPath := filepath.Join(path, df.Name()) + if df.IsDir() && recur { + subPaths, err := findPaths(fullPath, recur) + if err != nil { + return nil, err + } + paths = append(paths, subPaths...) + } else if !df.IsDir() { + paths = append(paths, fullPath) + } + } + return paths, nil +} + +func SourceFromURL(u *url.URL, encoding TemplateEncoding) TemplateSource { + if encoding == TemplateEncodingUnknown { + switch path.Ext(u.Path) { + case ".jsonnet": + encoding = TemplateEncodingJsonnet + case ".json": + encoding = TemplateEncodingJson + case ".yml": + fallthrough + case ".yaml": + encoding = TemplateEncodingYaml + default: + } + } + + normalized := github.NormalizeURLToContent(u, "yaml", "yml", "jsonnet", "json").String() + + return TemplateSource{ + Name: normalized, + Encoding: encoding, + Open: func(ctx context.Context) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, "GET", normalized, nil) + if err != nil { + return nil, err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode/100 != 2 { + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, err + } + return nil, fmt.Errorf("bad response: address=%s status_code=%d body=%q", + normalized, res.StatusCode, strings.TrimSpace(string(body))) + } + return res.Body, nil + }, + } +} + +func SourceFromReader(r io.Reader, encoding TemplateEncoding) TemplateSource { + return TemplateSource{ + Name: "byte stream", + Encoding: encoding, + Open: func(context.Context) (io.ReadCloser, error) { + return io.NopCloser(r), nil + }, + } +} + +func (s TemplateSource) Read(ctx context.Context) ([]api.TemplateEntry, error) { + var entries []api.TemplateEntry + if err := func() error { + in, err := s.Open(ctx) + if err != nil { + return err + } + defer in.Close() + + switch s.Encoding { + case TemplateEncodingJsonnet: + err = jsonnet.NewDecoder(in).Decode(&entries) + case TemplateEncodingJson: + err = json.NewDecoder(in).Decode(&entries) + case TemplateEncodingUnknown: + fallthrough // Assume YAML if we can't make a better guess + case TemplateEncodingYaml: + dec := yaml.NewDecoder(in) + for { + var e api.TemplateEntry + if err := dec.Decode(&e); err == io.EOF { + break + } else if err != nil { + return err + } + entries = append(entries, e) + } + } + return err + }(); err != nil { + return nil, fmt.Errorf("failed to read template(s) from %q: %w", s.Name, err) + } + + return entries, nil +} diff --git a/clients/apply/source_test.go b/clients/apply/source_test.go new file mode 100644 index 0000000..e66b204 --- /dev/null +++ b/clients/apply/source_test.go @@ -0,0 +1,433 @@ +package apply_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/influxdata/influx-cli/v2/api" + "github.com/influxdata/influx-cli/v2/clients/apply" + "github.com/stretchr/testify/require" +) + +func TestSourcesFromPath(t *testing.T) { + t.Parallel() + + type contents struct { + name string + encoding apply.TemplateEncoding + contents string + } + testCases := []struct { + name string + setup func(t *testing.T, rootDir string) + inPath func(rootDir string) string + inEncoding apply.TemplateEncoding + recursive bool + expected func(rootDir string) []contents + }{ + { + name: "JSON file", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return filepath.Join(rootDir, "foo.json") + }, + expected: func(string) []contents { + return []contents{{ + name: "foo.json", + encoding: apply.TemplateEncodingJson, + contents: "foo", + }} + }, + }, + { + name: "YAML file", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yaml"), []byte("foo"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return filepath.Join(rootDir, "foo.yaml") + }, + expected: func(string) []contents { + return []contents{{ + name: "foo.yaml", + encoding: apply.TemplateEncodingYaml, + contents: "foo", + }} + }, + }, + { + name: "YML file", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return filepath.Join(rootDir, "foo.yml") + }, + expected: func(string) []contents { + return []contents{{ + name: "foo.yml", + encoding: apply.TemplateEncodingYaml, + contents: "foo", + }} + }, + }, + { + name: "JSONNET file", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.jsonnet"), []byte("foo"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return filepath.Join(rootDir, "foo.jsonnet") + }, + expected: func(string) []contents { + return []contents{{ + name: "foo.jsonnet", + encoding: apply.TemplateEncodingJsonnet, + contents: "foo", + }} + }, + }, + { + name: "explicit inEncoding", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo"), []byte("foo"), os.ModePerm)) + }, + inEncoding: apply.TemplateEncodingJson, + inPath: func(rootDir string) string { + return filepath.Join(rootDir, "foo") + }, + expected: func(string) []contents { + return []contents{{ + name: "foo", + encoding: apply.TemplateEncodingJson, + contents: "foo", + }} + }, + }, + { + name: "directory - non-recursive", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.Mkdir(filepath.Join(rootDir, "bar"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo.json"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo.yml"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.jsonnet"), []byte("foo.jsonnet"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.yaml"), []byte("foo.yaml"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return rootDir + }, + expected: func(rootDir string) []contents { + return []contents{ + { + name: filepath.Join(rootDir, "foo.json"), + contents: "foo.json", + encoding: apply.TemplateEncodingJson, + }, + { + name: filepath.Join(rootDir, "foo.yml"), + contents: "foo.yml", + encoding: apply.TemplateEncodingYaml, + }, + } + }, + }, + { + name: "directory - recursive", + setup: func(t *testing.T, rootDir string) { + require.NoError(t, os.Mkdir(filepath.Join(rootDir, "bar"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.json"), []byte("foo.json"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "foo.yml"), []byte("foo.yml"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.jsonnet"), []byte("foo.jsonnet"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bar", "foo.yaml"), []byte("foo.yaml"), os.ModePerm)) + }, + inPath: func(rootDir string) string { + return rootDir + }, + recursive: true, + expected: func(rootDir string) []contents { + return []contents{ + { + name: filepath.Join(rootDir, "foo.json"), + contents: "foo.json", + encoding: apply.TemplateEncodingJson, + }, + { + name: filepath.Join(rootDir, "foo.yml"), + contents: "foo.yml", + encoding: apply.TemplateEncodingYaml, + }, + { + name: filepath.Join(rootDir, "bar", "foo.yaml"), + contents: "foo.yaml", + encoding: apply.TemplateEncodingYaml, + }, + { + name: filepath.Join(rootDir, "bar", "foo.jsonnet"), + contents: "foo.jsonnet", + encoding: apply.TemplateEncodingJsonnet, + }, + } + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer os.RemoveAll(tmp) + tc.setup(t, tmp) + + sources, err := apply.SourcesFromPath(tc.inPath(tmp), tc.recursive, tc.inEncoding) + require.NoError(t, err) + expected := tc.expected(tmp) + require.Len(t, sources, len(expected)) + + sort.Slice(sources, func(i, j int) bool { + return sources[i].Name < sources[j].Name + }) + sort.Slice(expected, func(i, j int) bool { + return expected[i].name < expected[j].name + }) + + for i := range expected { + source := sources[i] + contents := expected[i] + + require.Equal(t, contents.encoding, source.Encoding) + sourceIn, err := source.Open(context.Background()) + require.NoError(t, err) + bytes, err := io.ReadAll(sourceIn) + sourceIn.Close() + require.NoError(t, err) + require.Equal(t, contents.contents, string(bytes)) + } + }) + } +} + +func TestSourceFromURL(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + filename string + inEncoding apply.TemplateEncoding + expectEncoding apply.TemplateEncoding + resStatus int + resBody string + expectErr bool + }{ + { + name: "JSON file", + filename: "foo.json", + expectEncoding: apply.TemplateEncodingJson, + resStatus: 200, + resBody: "Foo bar", + }, + { + name: "YAML file", + filename: "foo.yaml", + expectEncoding: apply.TemplateEncodingYaml, + resStatus: 200, + resBody: "Foo bar", + }, + { + name: "YML file", + filename: "foo.yml", + expectEncoding: apply.TemplateEncodingYaml, + resStatus: 200, + resBody: "Foo bar", + }, + { + name: "JSONNET file", + filename: "foo.jsonnet", + expectEncoding: apply.TemplateEncodingJsonnet, + resStatus: 200, + resBody: "Foo bar", + }, + { + name: "explicit encoding", + filename: "foo", + inEncoding: apply.TemplateEncodingJson, + expectEncoding: apply.TemplateEncodingJson, + resStatus: 200, + resBody: "Foo bar", + }, + { + name: "err response", + filename: "foo.json", + expectEncoding: apply.TemplateEncodingJson, + resStatus: 403, + resBody: "OH NO", + expectErr: true, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + rw.WriteHeader(tc.resStatus) + rw.Write([]byte(tc.resBody)) + })) + defer server.Close() + + u, err := url.Parse(server.URL) + require.NoError(t, err) + u.Path = tc.filename + + source := apply.SourceFromURL(u, tc.inEncoding) + in, err := source.Open(context.Background()) + if tc.expectErr { + require.Error(t, err) + require.Contains(t, err.Error(), "bad response") + require.Contains(t, err.Error(), tc.resBody) + return + } + require.NoError(t, err) + defer in.Close() + bytes, err := io.ReadAll(in) + require.NoError(t, err) + require.Equal(t, tc.resBody, string(bytes)) + }) + } +} + +func TestTemplateSource_Read(t *testing.T) { + t.Parallel() + + yamlTemplate := `--- +apiversion: influxdata.com/v2alpha1 +kind: Bucket +meta: null +spec: + name: test + retentionRules: + - type: expire +--- +apiversion: influxdata.com/v2alpha1 +kind: Bucket +meta: null +spec: + name: test2 + retentionRules: + - type: expire +` + jsonTemplate := `[ + { + "apiVersion": "influxdata.com/v2alpha1", + "kind": "Bucket", + "spec": { + "name": "test", + "retentionRules": [ + { + "type": "expire" + } + ] + } + }, + { + "apiVersion": "influxdata.com/v2alpha1", + "kind": "Bucket", + "spec": { + "name": "test2", + "retentionRules": [ + { + "type": "expire" + } + ] + } + } +] +` + jsonnetTemplate := `local Bucket(name) = { + apiVersion: "influxdata.com/v2alpha1", + kind: "Bucket", + spec: { + name: name, + retentionRules: [{ + type: "expire", + }], + }, +}; +[Bucket("test"), Bucket("test2")] +` + + parsed := []api.TemplateEntry{ + { + ApiVersion: "influxdata.com/v2alpha1", + Kind: "Bucket", + Spec: map[string]interface{}{ + "name": "test", + "retentionRules": []interface{}{ + map[string]interface{}{ + "type": "expire", + }, + }, + }, + }, + { + ApiVersion: "influxdata.com/v2alpha1", + Kind: "Bucket", + Spec: map[string]interface{}{ + "name": "test2", + "retentionRules": []interface{}{ + map[string]interface{}{ + "type": "expire", + }, + }, + }, + }, + } + + testCases := []struct { + name string + encoding apply.TemplateEncoding + data string + }{ + { + name: "JSON", + encoding: apply.TemplateEncodingJson, + data: jsonTemplate, + }, + { + name: "YAML", + encoding: apply.TemplateEncodingYaml, + data: yamlTemplate, + }, + { + name: "JSONNET", + encoding: apply.TemplateEncodingJsonnet, + data: jsonnetTemplate, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + source := apply.SourceFromReader(strings.NewReader(tc.data), tc.encoding) + tmpls, err := source.Read(context.Background()) + require.NoError(t, err) + + require.Equal(t, parsed, tmpls) + }) + } +} diff --git a/pkg/github/normalize.go b/pkg/github/normalize.go new file mode 100644 index 0000000..6732007 --- /dev/null +++ b/pkg/github/normalize.go @@ -0,0 +1,45 @@ +package github + +import ( + "net/url" + "path" + "strings" +) + +const ( + githubRawContentHost = "raw.githubusercontent.com" + githubHost = "github.com" +) + +func NormalizeURLToContent(u *url.URL, extensions ...string) *url.URL { + if u.Host != githubHost { + return u + } + if len(extensions) > 0 && !extensionMatches(u, extensions) { + return u + } + + p := u.Path + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + parts := strings.Split(p, "/") + if len(parts) < 4 { + return u + } + + normalized := *u + normalized.Host = githubRawContentHost + normalized.Path = "/" + path.Join(append(parts[:3], parts[4:]...)...) + return &normalized +} + +func extensionMatches(u *url.URL, extensions []string) bool { + ext := path.Ext(u.Path) + for _, e := range extensions { + if strings.EqualFold(e, "."+ext) { + return true + } + } + return false +} diff --git a/pkg/github/normalize_test.go b/pkg/github/normalize_test.go new file mode 100644 index 0000000..5ca5263 --- /dev/null +++ b/pkg/github/normalize_test.go @@ -0,0 +1,47 @@ +package github_test + +import ( + "net/url" + "testing" + + "github.com/influxdata/influx-cli/v2/pkg/github" + "github.com/stretchr/testify/require" +) + +func TestNormalize(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + in url.URL + exts []string + out url.URL + }{ + { + name: "github URL", + in: url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"}, + out: url.URL{Host: "raw.githubusercontent.com", Path: "/influxdata/influxdb/master/flags.yml"}, + }, + { + name: "other URL", + in: url.URL{Host: "google.com", Path: "/fake.yml"}, + out: url.URL{Host: "google.com", Path: "/fake.yml"}, + }, + { + name: "github URL - wrong extension", + in: url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"}, + exts: []string{"json"}, + out: url.URL{Host: "github.com", Path: "/influxdata/influxdb/blob/master/flags.yml"}, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + normalized := github.NormalizeURLToContent(&tc.in, tc.exts...) + require.Equal(t, tc.out, *normalized) + }) + } +}