diff --git a/docs/documentation/cn/sql-reference/sql-functions/cast.md b/docs/documentation/cn/sql-reference/sql-functions/cast.md new file mode 100644 index 0000000000..d4bc514f7a --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/cast.md @@ -0,0 +1,56 @@ +# cast + +## Syntax + +``` +cast (input as type) +``` + +## Description + +将 input 转成 指定的 type + +## BIGINT type + +### Syntax + +``` cast (input as BIGINT) ``` + +### Description + +将当前列 input 转换为 BIGINT 类型 + +### Examples + +1. 转常量，或表中某列 + +``` +mysql> select cast (1 as BIGINT); ++-------------------+ +| CAST(1 AS BIGINT) | ++-------------------+ +| 1 | ++-------------------+ +``` + +2. 转导入的原始数据 + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(tmp_k1 as BIGINT)" http://host:port/api/test/bigint/_stream_load +``` + +*注：在导入中，由于原始类型均为String，将值为浮点的元数数据做 cast的时候数据会被转换成 NULL ，比如 12.0 。Doris目前不会对原始数据做截断。* + +如果想强制将这种类型的原始数据 cast to int 的话。请看下面写法： + +``` +curl --location-trusted -u root: -T ~/user_data/bigint -H "columns: tmp_k1, k1=cast(cast(tmp_k1 as DOUBLE) as BIGINT)" http://host:port/api/test/bigint/_stream_load + +mysql> select cast(cast ("11.2" as double) as bigint); ++----------------------------------------+ +| CAST(CAST('11.2' AS DOUBLE) AS BIGINT) | ++----------------------------------------+ +| 11 | ++----------------------------------------+ +1 row in set (0.00 sec) +``` diff --git a/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index bc1e8d9e59..10256ebc49 100644 --- a/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -32,6 +32,8 @@ import org.apache.doris.load.Load; import org.apache.doris.load.PullLoadSourceInfo; import org.apache.doris.load.Source; import org.apache.doris.task.MasterTaskExecutor; +import org.apache.doris.transaction.TransactionState; +import org.apache.doris.transaction.TxnCommitAttachment; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -366,4 +368,55 @@ public class BrokerLoadJobTest { int progress = Deencapsulation.getField(brokerLoadJob, "progress"); Assert.assertEquals(99, progress); } + + @Test + public void testExecuteReplayOnAborted(@Injectable TransactionState txnState, + @Injectable LoadJobFinalOperation attachment, + @Injectable EtlStatus etlStatus) { + BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); + new Expectations() { + { + txnState.getTxnCommitAttachment(); + result = attachment; + attachment.getLoadingStatus(); + result = etlStatus; + attachment.getProgress(); + result = 99; + attachment.getFinishTimestamp(); + result = 1; + attachment.getJobState(); + result = JobState.CANCELLED; + } + }; + brokerLoadJob.executeReplayOnAborted(txnState); + Assert.assertEquals(99, (int) Deencapsulation.getField(brokerLoadJob, "progress")); + Assert.assertEquals(1, brokerLoadJob.getFinishTimestamp()); + Assert.assertEquals(JobState.CANCELLED, brokerLoadJob.getState()); + } + + + @Test + public void testExecuteReplayOnVisible(@Injectable TransactionState txnState, + @Injectable LoadJobFinalOperation attachment, + @Injectable EtlStatus etlStatus) { + BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); + new Expectations() { + { + txnState.getTxnCommitAttachment(); + result = attachment; + attachment.getLoadingStatus(); + result = etlStatus; + attachment.getProgress(); + result = 99; + attachment.getFinishTimestamp(); + result = 1; + attachment.getJobState(); + result = JobState.LOADING; + } + }; + brokerLoadJob.executeReplayOnAborted(txnState); + Assert.assertEquals(99, (int) Deencapsulation.getField(brokerLoadJob, "progress")); + Assert.assertEquals(1, brokerLoadJob.getFinishTimestamp()); + Assert.assertEquals(JobState.LOADING, brokerLoadJob.getState()); + } } diff --git a/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java b/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java new file mode 100644 index 0000000000..57105e23db --- /dev/null +++ b/fe/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.doris.load.loadv2; + +import org.apache.doris.analysis.LoadStmt; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.LabelAlreadyUsedException; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.load.Load; +import org.apache.doris.metric.LongCounterMetric; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.persist.EditLog; +import org.apache.doris.transaction.BeginTransactionException; +import org.apache.doris.transaction.GlobalTransactionMgr; +import org.apache.doris.transaction.TransactionState; + +import com.google.common.collect.Maps; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +import mockit.Deencapsulation; +import mockit.Expectations; +import mockit.Mocked; + +public class LoadJobTest { + + @Test + public void testGetDbNotExists(@Mocked Catalog catalog) { + LoadJob loadJob = new BrokerLoadJob(); + Deencapsulation.setField(loadJob, "dbId", 1L); + new Expectations() { + { + catalog.getDb(1L); + result = null; + } + }; + + try { + loadJob.getDb(); + Assert.fail(); + } catch (MetaNotFoundException e) { + } + } + + + @Test + public void testSetJobPropertiesWithErrorTimeout() { + Map jobProperties = Maps.newHashMap(); + jobProperties.put(LoadStmt.TIMEOUT_PROPERTY, "abc"); + LoadJob loadJob = new BrokerLoadJob(); + try { + loadJob.setJobProperties(jobProperties); + Assert.fail(); + } catch (DdlException e) { + } + } + + @Test + public void testSetJobProperties() { + Map jobProperties = Maps.newHashMap(); + jobProperties.put(LoadStmt.TIMEOUT_PROPERTY, "1000"); + jobProperties.put(LoadStmt.MAX_FILTER_RATIO_PROPERTY, "0.1"); + jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, "1024"); + jobProperties.put(LoadStmt.STRICT_MODE, "True"); + + LoadJob loadJob = new BrokerLoadJob(); + try { + loadJob.setJobProperties(jobProperties); + Assert.assertEquals(1000, (long) Deencapsulation.getField(loadJob, "timeoutSecond")); + Assert.assertEquals(0.1, Deencapsulation.getField(loadJob, "maxFilterRatio"), 0); + Assert.assertEquals(1024, (long) Deencapsulation.getField(loadJob, "execMemLimit")); + Assert.assertTrue(Deencapsulation.getField(loadJob, "strictMode")); + } catch (DdlException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testExecute(@Mocked GlobalTransactionMgr globalTransactionMgr) throws LabelAlreadyUsedException, BeginTransactionException, AnalysisException { + LoadJob loadJob = new BrokerLoadJob(); + new Expectations() { + { + globalTransactionMgr.beginTransaction(anyLong, anyString, anyLong, anyString, (TransactionState.LoadJobSourceType) any, anyLong, anyLong); + result = 1; + } + }; + + loadJob.execute(); + Assert.assertEquals(JobState.LOADING, loadJob.getState()); + Assert.assertEquals(1, loadJob.getTransactionId()); + + } + + @Test + public void testProcessTimeoutWithCompleted() { + LoadJob loadJob = new BrokerLoadJob(); + Deencapsulation.setField(loadJob, "state", JobState.FINISHED); + + loadJob.processTimeout(); + Assert.assertEquals(JobState.FINISHED, loadJob.getState()); + } + + @Test + public void testProcessTimeoutWithIsCommitting() { + LoadJob loadJob = new BrokerLoadJob(); + Deencapsulation.setField(loadJob, "isCommitting", true); + Deencapsulation.setField(loadJob, "state", JobState.LOADING); + + loadJob.processTimeout(); + Assert.assertEquals(JobState.LOADING, loadJob.getState()); + } + + @Test + public void testProcessTimeoutWithLongTimeoutSecond() { + LoadJob loadJob = new BrokerLoadJob(); + Deencapsulation.setField(loadJob, "timeoutSecond", 1000L); + + loadJob.processTimeout(); + Assert.assertEquals(JobState.PENDING, loadJob.getState()); + } + + @Test + public void testProcessTimeout(@Mocked Catalog catalog, @Mocked EditLog editLog) { + LoadJob loadJob = new BrokerLoadJob(); + Deencapsulation.setField(loadJob, "timeoutSecond", 0); + new Expectations() { + { + catalog.getEditLog(); + result = editLog; + } + }; + + loadJob.processTimeout(); + Assert.assertEquals(JobState.CANCELLED, loadJob.getState()); + } + + @Test + public void testUpdateStateToLoading() { + LoadJob loadJob = new BrokerLoadJob(); + loadJob.updateState(JobState.LOADING); + Assert.assertEquals(JobState.LOADING, loadJob.getState()); + Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "loadStartTimestamp")); + } + + @Test + public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo, + @Mocked LongCounterMetric longCounterMetric) { + metricRepo.COUNTER_LOAD_FINISHED = longCounterMetric; + LoadJob loadJob = new BrokerLoadJob(); + loadJob.updateState(JobState.FINISHED); + Assert.assertEquals(JobState.FINISHED, loadJob.getState()); + Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp")); + Assert.assertEquals(100, (int)Deencapsulation.getField(loadJob, "progress")); + } +}