98 lines
3.0 KiB
Go
98 lines
3.0 KiB
Go
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"github.com/spf13/cast"
|
|
)
|
|
|
|
var (
|
|
dorisStreamLoadURL string
|
|
dorisUsername string
|
|
dorisPassword string
|
|
)
|
|
|
|
func init() {
|
|
dorisStreamLoadURL = "http://{fe_host}:{fe_port}/api/{db_name}/{table_name}/_stream_load"
|
|
dorisUsername = "{doris_username}"
|
|
dorisPassword = "{doris_password}"
|
|
}
|
|
|
|
func SubmitStreamLoadData(ctx context.Context, crowd, logID, num int64, body, partition string) error {
|
|
client := &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
|
req.SetBasicAuth(dorisUsername, dorisPassword)
|
|
return nil, nil
|
|
},
|
|
},
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, "PUT", dorisStreamLoadURL, strings.NewReader(body))
|
|
if err != nil {
|
|
log.Fatalf("[stream load failed err=%+v]\n", err)
|
|
return err
|
|
}
|
|
req.Header.Add("columns", fmt.Sprintf("member_id,confidence=100,bucket=floor(member_id/1000000),tag_range=%d,tag_group=%d,tag_value_id='-1',members=to_bitmap(member_id),partition_sign='%s'", logID, crowd, partition))
|
|
req.Header.Add("format", "json")
|
|
req.Header.Add("Label", fmt.Sprintf("crowd_%d_%d_%d", crowd, logID, num))
|
|
req.Header.Add("jsonpaths", "[\"$.member_id\"]")
|
|
req.Header.Add("Expect", "100-continue")
|
|
req.Header.Set("Content-Type", "text/plain")
|
|
req.Header.Set("strip_outer_array", "true")
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Read Response Body
|
|
respBody, _ := ioutil.ReadAll(resp.Body)
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
return fmt.Errorf("stream load：code=%d, body=%s", resp.StatusCode, string(respBody))
|
|
}
|
|
|
|
res := make(map[string]interface{})
|
|
err = json.Unmarshal(respBody, &res)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Display Results
|
|
log.Printf("[response Status : %+v]", resp.Status)
|
|
log.Printf("[response Headers : %+v]", resp.Header)
|
|
log.Printf("[response Body : %s]", string(respBody))
|
|
if cast.ToString(res["Status"]) != "Success" {
|
|
return fmt.Errorf("mas=%s, Label = %s", cast.ToString(res["Message"]), cast.ToString(res["Label"]))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func main() {
|
|
err := SubmitStreamLoadData(context.Background(), 1, 1, 1, `{"member_id": 1}`, "2020-01-01")
|
|
if err != nil {
|
|
log.Fatalf("[stream load failed err=%+v]\n", err)
|
|
}
|
|
}
|