From b1851eb8190f064a988b7c6e4b62c88ef3e8b0c6 Mon Sep 17 00:00:00 2001 From: Daniel Moran Date: Mon, 17 May 2021 14:53:55 -0400 Subject: [PATCH] refactor(api): move CLI-specific API contract into this repo, add openapi as submodule (#95) --- .circleci/config.yml | 3 + .gitmodules | 3 + etc/generate-openapi.sh | 14 +- internal/api/README.md | 5 +- internal/api/contract/README.md | 53 +++++++ internal/api/contract/cli.yml | 132 ++++++++++++++++++ internal/api/contract/openapi | 1 + .../api/contract/overrides/paths/query.yml | 81 +++++++++++ .../api/contract/overrides/schemas/Extern.yml | 12 ++ .../api/contract/overrides/schemas/Query.yml | 22 +++ internal/api/templates/README.md | 3 + internal/cmd/config/config_test.go | 14 +- 12 files changed, 332 insertions(+), 11 deletions(-) create mode 100644 .gitmodules create mode 100644 internal/api/contract/README.md create mode 100644 internal/api/contract/cli.yml create mode 160000 internal/api/contract/openapi create mode 100644 internal/api/contract/overrides/paths/query.yml create mode 100644 internal/api/contract/overrides/schemas/Extern.yml create mode 100644 internal/api/contract/overrides/schemas/Query.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index e7ab8e1..65be891 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,9 @@ jobs: working_directory: /home/circleci/go/src/github.com/influxdata/influx-cli steps: - checkout + - run: + name: Init openapi submodule + command: git submodule update --init --recursive - run: name: Upgrade Go command: | diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..26cfb54 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "openapi"] + path = internal/api/contract/openapi + url = ../openapi diff --git a/etc/generate-openapi.sh b/etc/generate-openapi.sh index 7b275f1..f55724a 100755 --- a/etc/generate-openapi.sh +++ b/etc/generate-openapi.sh @@ -1,20 +1,24 @@ #!/usr/bin/env bash +set -euo pipefail declare -r ETC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" declare -r ROOT_DIR="$(dirname ${ETC_DIR})" declare -r API_DIR="${ROOT_DIR}/internal/api" declare -r GENERATED_PATTERN='^// Code generated .* DO NOT EDIT\.$' +declare -r MERGE_DOCKER_IMG=quay.io/influxdb/swagger-cli declare -r GENERATOR_DOCKER_IMG=openapitools/openapi-generator-cli:v5.1.0 -declare -r OPENAPI_COMMIT=dd675843404dbb3881f4d245581b916df319091f # Clean up all the generated files in the target directory. rm $(grep -Elr "${GENERATED_PATTERN}" "${API_DIR}") -# Download our target API spec. -# NOTE: openapi-generator supports HTTP references to API docs, but using that feature -# causes the host of the URL to be injected into the base paths of generated code. -curl -o "${API_DIR}/cli.yml" https://raw.githubusercontent.com/influxdata/openapi/${OPENAPI_COMMIT}/contracts/cli.yml +# Merge all API contracts into a single file to drive codegen. +docker run --rm -it -u "$(id -u):$(id -g)" \ + -v "${API_DIR}":/api \ + ${MERGE_DOCKER_IMG} \ + swagger-cli bundle /api/contract/cli.yml \ + --outfile /api/cli.yml \ + --type yaml # Run the generator - This produces many more files than we want to track in git. docker run --rm -it -u "$(id -u):$(id -g)" \ diff --git a/internal/api/README.md b/internal/api/README.md index 160ab26..f90ecfe 100644 --- a/internal/api/README.md +++ b/internal/api/README.md @@ -1,4 +1,7 @@ # Influx CLI - HTTP Client The `.go` files in this module are generated using [`OpenAPITools/openapi-generator`](https://github.com/OpenAPITools/openapi-generator), -based off of our public API documentation. Run `etc/generate-openapi.sh` to regenerate files as needed. +based off of our public API documentation. + +Run `make openapi` from the project root to regenerate files as needed. See [`contract/README.md`](./contract/README.md) +and [`templates/README.md`](./templates/README.md) for more detailed information and use-cases. diff --git a/internal/api/contract/README.md b/internal/api/contract/README.md new file mode 100644 index 0000000..3b590b5 --- /dev/null +++ b/internal/api/contract/README.md @@ -0,0 +1,53 @@ +# API Contract + +This directory contains the source YMLs used to drive code generation of HTTP clients in [`internal/api`](../). + +## YML Structure + +Most YMLs used here are pulled from the source-of-truth [`openapi`](https://github.com/influxdata/openapi) repo via a git +submodule. In rare cases, the full description of an API is too complex for our codegen tooling to handle; +the [`overrides/`](./overrides) directory contains alternate definitions for paths/schemas that work around these cases. +[`cli.yml`](./cli.yml) ties together all the pieces by linking all routes and schemas used by the CLI. + +## Updating the API contract + +To extend/modify the API contract used by the CLI, first make sure the `openapi` submodule is cloned and up-to-date: +```shell +# Run from the project root. +git submodule update --init --recursive +``` + +Then create a new branch to track your work: +```shell +git checkout +``` + +Next, decide if any modifications are needed in the source-of-truth `openapi` repo. If so, create a branch in the +submodule to track changes there: +```shell +cd internal/api/contract/openapi && git checkout -b +``` + +Edit/add to the files under `api-contract/` to describe the new API contract. Run the following from the project +root test your changes and see the outputs in Go code: +```shell +make openapi +# Use `git status` to see new/modified files under `internal/api` +``` + +Once you're happy with the new API contract, submit your changes for review & merge. +If you added/edited files within `openapi`, you'll first need to: +1. Push your submodule branch to GitHub + ```shell + cd internal/api/contract/openapi && git push + ``` +2. Create a PR in `openapi`, eventually merge to `master` there +3. Update your submodule to point at the merge result: + ```shell + cd internal/api/contract/openapi && git fetch && git checkout master && git pull origin master + ``` +4. Update the submodule reference from the main repo: + ```shell + git add internal/api/contract/openapi + git commit + ``` diff --git a/internal/api/contract/cli.yml b/internal/api/contract/cli.yml new file mode 100644 index 0000000..ca45bc5 --- /dev/null +++ b/internal/api/contract/cli.yml @@ -0,0 +1,132 @@ +openapi: "3.0.0" +info: + title: Subset of Influx API covered by Influx CLI + version: 2.0.0 +servers: + - url: /api/v2 +paths: + /health: + servers: + - url: '' + $ref: "./openapi/src/oss/paths/health.yml" + /setup: + $ref: "./openapi/src/common/paths/setup.yml" + /write: + $ref: "./openapi/src/common/paths/write.yml" + /buckets: + $ref: "./openapi/src/common/paths/buckets.yml" + /buckets/{bucketID}: + $ref: "./openapi/src/common/paths/buckets_bucketID.yml" + /orgs: + $ref: "./openapi/src/common/paths/orgs.yml" + /orgs/{orgID}: + $ref: "./openapi/src/common/paths/orgs_orgID.yml" + /orgs/{orgID}/members: + $ref: "./openapi/src/common/paths/orgs_orgID_members.yml" + /orgs/{orgID}/members/{userID}: + $ref: "./openapi/src/common/paths/orgs_orgID_members_userID.yml" + /buckets/{bucketID}/schema/measurements: + $ref: "./openapi/src/cloud/paths/measurements.yml" + /buckets/{bucketID}/schema/measurements/{measurementID}: + $ref: "./openapi/src/cloud/paths/measurements_measurementID.yml" + /query: + $ref: "./overrides/paths/query.yml" +components: + parameters: + TraceSpan: + $ref: "./openapi/src/common/parameters/TraceSpan.yml" + Offset: + $ref: "./openapi/src/common/parameters/Offset.yml" + Limit: + $ref: "./openapi/src/common/parameters/Limit.yml" + After: + $ref: "./openapi/src/common/parameters/After.yml" + Descending: + $ref: "./openapi/src/common/parameters/Descending.yml" + schemas: + Error: + $ref: "./openapi/src/common/schemas/Error.yml" + ErrorCode: + $ref: "./openapi/src/common/schemas/ErrorCode.yml" + HealthCheck: + $ref: "./openapi/src/common/schemas/HealthCheck.yml" + HealthCheckStatus: + $ref: "./openapi/src/common/schemas/HealthCheckStatus.yml" + OnboardingRequest: + $ref: "./openapi/src/common/schemas/OnboardingRequest.yml" + OnboardingResponse: + $ref: "./openapi/src/common/schemas/OnboardingResponse.yml" + UserResponse: + $ref: "./openapi/src/common/schemas/UserResponse.yml" + Links: + $ref: "./openapi/src/common/schemas/Links.yml" + Link: + $ref: "./openapi/src/common/schemas/Link.yml" + Organizations: + $ref: "./openapi/src/common/schemas/Organizations.yml" + Organization: + $ref: "./openapi/src/common/schemas/Organization.yml" + PostOrganizationRequest: + $ref: "./openapi/src/common/schemas/PostOrganizationRequest.yml" + PatchOrganizationRequest: + $ref: "./openapi/src/common/schemas/PatchOrganizationRequest.yml" + Buckets: + $ref: "./openapi/src/common/schemas/Buckets.yml" + Bucket: + $ref: "./openapi/src/common/schemas/Bucket.yml" + PostBucketRequest: + $ref: "./openapi/src/common/schemas/PostBucketRequest.yml" + RetentionRules: + $ref: "./openapi/src/common/schemas/RetentionRules.yml" + RetentionRule: + $ref: "./openapi/src/common/schemas/RetentionRule.yml" + PatchBucketRequest: + $ref: "./openapi/src/common/schemas/PatchBucketRequest.yml" + PatchRetentionRules: + $ref: "./openapi/src/common/schemas/PatchRetentionRules.yml" + PatchRetentionRule: + $ref: "./openapi/src/common/schemas/PatchRetentionRule.yml" + Labels: + $ref: "./openapi/src/common/schemas/Labels.yml" + Label: + $ref: "./openapi/src/common/schemas/Label.yml" + Authorization: + $ref: "./openapi/src/common/schemas/Authorization.yml" + AuthorizationUpdateRequest: + $ref: "./openapi/src/common/schemas/AuthorizationUpdateRequest.yml" + Permission: + $ref: "./openapi/src/common/schemas/Permission.yml" + ResourceMembers: + $ref: "./openapi/src/common/schemas/ResourceMembers.yml" + ResourceMember: + $ref: "./openapi/src/common/schemas/ResourceMember.yml" + AddResourceMemberRequestBody: + $ref: "./openapi/src/common/schemas/AddResourceMemberRequestBody.yml" + WritePrecision: + $ref: "./openapi/src/common/schemas/WritePrecision.yml" + LineProtocolError: + $ref: "./openapi/src/common/schemas/LineProtocolError.yml" + LineProtocolLengthError: + $ref: "./openapi/src/common/schemas/LineProtocolLengthError.yml" + SchemaType: + $ref: "./openapi/src/common/schemas/SchemaType.yml" + ColumnDataType: + $ref: "./openapi/src/cloud/schemas/ColumnDataType.yml" + ColumnSemanticType: + $ref: "./openapi/src/cloud/schemas/ColumnSemanticType.yml" + MeasurementSchema: + $ref: "./openapi/src/cloud/schemas/MeasurementSchema.yml" + MeasurementSchemaColumn: + $ref: "./openapi/src/cloud/schemas/MeasurementSchemaColumn.yml" + MeasurementSchemaCreateRequest: + $ref: "./openapi/src/cloud/schemas/MeasurementSchemaCreateRequest.yml" + MeasurementSchemaList: + $ref: "./openapi/src/cloud/schemas/MeasurementSchemaList.yml" + MeasurementSchemaUpdateRequest: + $ref: "./openapi/src/cloud/schemas/MeasurementSchemaUpdateRequest.yml" + Query: + $ref: "./overrides/schemas/Query.yml" + Dialect: + $ref: "./openapi/src/common/schemas/Dialect.yml" + Extern: + $ref: "./overrides/schemas/Extern.yml" diff --git a/internal/api/contract/openapi b/internal/api/contract/openapi new file mode 160000 index 0000000..ea92cf2 --- /dev/null +++ b/internal/api/contract/openapi @@ -0,0 +1 @@ +Subproject commit ea92cf2265f9bfa7d9f70a50c4bade78a263e0e7 diff --git a/internal/api/contract/overrides/paths/query.yml b/internal/api/contract/overrides/paths/query.yml new file mode 100644 index 0000000..5d1225e --- /dev/null +++ b/internal/api/contract/overrides/paths/query.yml @@ -0,0 +1,81 @@ +post: + operationId: PostQuery + tags: + - Query + summary: Query InfluxDB + parameters: + - $ref: "../../openapi/src/common/parameters/TraceSpan.yml" + - in: header + name: Accept-Encoding + description: The Accept-Encoding request HTTP header advertises which content encoding, usually a compression algorithm, the client is able to understand. + schema: + type: string + description: Specifies that the query response in the body should be encoded with gzip or not encoded with identity. + default: identity + enum: + - gzip + - identity + - in: header + name: Content-Type + schema: + type: string + enum: + - application/json + - in: query + name: org + description: Specifies the name of the organization executing the query. Takes either the ID or Name interchangeably. If both `orgID` and `org` are specified, `org` takes precedence. + schema: + type: string + - in: query + name: orgID + description: Specifies the ID of the organization executing the query. If both `orgID` and `org` are specified, `org` takes precedence. + schema: + type: string + requestBody: + description: Flux query or specification to execute + content: + application/json: + schema: + $ref: "../schemas/Query.yml" + responses: + "200": + description: Query results + headers: + Content-Encoding: + description: The Content-Encoding entity header is used to compress the media-type. When present, its value indicates which encodings were applied to the entity-body + schema: + type: string + description: Specifies that the response in the body is encoded with gzip or not encoded with identity. + default: identity + enum: + - gzip + - identity + Trace-Id: + description: The Trace-Id header reports the request's trace ID, if one was generated. + schema: + type: string + description: Specifies the request's trace ID. + content: + text/csv: + schema: + type: string + format: binary + example: > + result,table,_start,_stop,_time,region,host,_value + mean,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:00Z,east,A,15.43 + mean,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:20Z,east,B,59.25 + mean,0,2018-05-08T20:50:00Z,2018-05-08T20:51:00Z,2018-05-08T20:50:40Z,east,C,52.62 + "429": + description: Token is temporarily over quota. The Retry-After header describes when to try the read again. + headers: + Retry-After: + description: A non-negative decimal integer indicating the seconds to delay after the response is received. + schema: + type: integer + format: int32 + default: + description: Error processing query + content: + application/json: + schema: + $ref: "../../openapi/src/common/schemas/Error.yml" diff --git a/internal/api/contract/overrides/schemas/Extern.yml b/internal/api/contract/overrides/schemas/Extern.yml new file mode 100644 index 0000000..f2931d1 --- /dev/null +++ b/internal/api/contract/overrides/schemas/Extern.yml @@ -0,0 +1,12 @@ +description: Free-form Flux AST to prepend to query requests +type: object +properties: + type: + type: string + enum: [File] + default: File +# NOTE: Intentionally type-less here because the boilerplate produced +# by codegen off the Flux AST spec is unmangeable. The CLI only needs +# to use a small subset of the AST and rarely changes what it sends, +# so we can live with a generic map in the codegen request body. +additionalProperties: true diff --git a/internal/api/contract/overrides/schemas/Query.yml b/internal/api/contract/overrides/schemas/Query.yml new file mode 100644 index 0000000..8e9fd60 --- /dev/null +++ b/internal/api/contract/overrides/schemas/Query.yml @@ -0,0 +1,22 @@ +description: Query influx using the Flux language +type: object +required: + - query +properties: + extern: + $ref: "./Extern.yml" + query: + description: Query script to execute. + type: string + type: + description: The type of query. Must be "flux". + type: string + enum: + - flux + default: flux + dialect: + $ref: "../../openapi/src/common/schemas/Dialect.yml" + now: + description: Specifies the time that should be reported as "now" in the query. Default is the server's now time. + type: string + format: date-time diff --git a/internal/api/templates/README.md b/internal/api/templates/README.md index 4e11027..ae8200d 100644 --- a/internal/api/templates/README.md +++ b/internal/api/templates/README.md @@ -30,3 +30,6 @@ multiple locations. `configuration.mustache` * Deleted `ContextOAuth2` key to match modification in client * Fixed error strings to be idiomatic according to staticcheck (lowercase, no punctuation) + +`model_oneof.mustache` +* Fixed error strings to be idiomatic according to staticcheck (lowercase, no punctuation) diff --git a/internal/cmd/config/config_test.go b/internal/cmd/config/config_test.go index 4bbe293..2f675a8 100644 --- a/internal/cmd/config/config_test.go +++ b/internal/cmd/config/config_test.go @@ -114,7 +114,7 @@ func TestClient_Delete(t *testing.T) { svc.EXPECT().DeleteConfig(gomock.Eq("foo")). Return(iconfig.Config{Name: "foo", Host: "bar", Org: "baz"}, nil) }, - out: []string{`^\s+foo\s+bar\s+baz\s+true`}, + out: []string{`\s+foo\s+bar\s+baz\s+true`}, }, { name: "many", @@ -128,7 +128,7 @@ func TestClient_Delete(t *testing.T) { Return(iconfig.Config{Name: "wibble", Host: "bar", Active: true}, nil) }, out: []string{ - `^\s+foo\s+bar\s+baz\s+true`, + `\s+foo\s+bar\s+baz\s+true`, `\*\s+wibble\s+bar\s+true`, }, }, @@ -151,9 +151,13 @@ func TestClient_Delete(t *testing.T) { cli := config.Client{CLI: cmd.CLI{ConfigService: svc, StdIO: stdio}} require.NoError(t, cli.Delete(tc.in)) - testutils.MatchLines(t, - append([]string{`Active\s+Name\s+URL\s+Org\s+Deleted`}, tc.out...), - strings.Split(writtenBytes.String(), "\n")) + + // Can't use our usual 'MatchLines' because list output depends on map iteration, + // so the order isn't well-defined. + out := writtenBytes.String() + for _, l := range append([]string{`Active\s+Name\s+URL\s+Org\s+Deleted`}, tc.out...) { + require.Regexp(t, l, out) + } }) } }