database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](backup) backup throw NPE when no partition in table (#17546)

Browse Source 
If table has no partition, backup will report error:

2023-03-06 17:35:32,971 ERROR (backupHandler|24) [Daemon.run():118] daemon thread got exception. name: backupHandler
java.util.NoSuchElementException: No value present
        at java.util.Optional.get(Optional.java:135) ~[?:1.8.0_152]
        at org.apache.doris.catalog.OlapTable.selectiveCopy(OlapTable.java:1259) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.backup.BackupJob.prepareBackupMeta(BackupJob.java:505) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.backup.BackupJob.prepareAndSendSnapshotTask(BackupJob.java:398) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.backup.BackupJob.run(BackupJob.java:301) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.backup.BackupHandler.runAfterCatalogReady(BackupHandler.java:188) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.common.util.MasterDaemon.runOneCycle(MasterDaemon.java:58) ~[doris-fe.jar:1.0-SNAPSHOT]
        at org.apache.doris.common.util.Daemon.run(Daemon.java:116) ~[doris-fe.jar:1.0-SNAPSHOT]
This commit is contained in:
Mingyu Chen
2023-03-10 10:19:37 +08:00
committed by GitHub
parent 4ba93efc98
commit c7aa3f9717
6 changed files with 72 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/BACKUP.md
View File

@ -88,68 +88,9 @@ TO example_repo
EXCLUDE (example_tbl);
```
4. Create a warehouse named hdfs_repo, rely on Baidu hdfs broker "hdfs_broker", the data root directory is: hdfs://hadoop-name-node:54310/path/to/repo/
```
CREATE REPOSITORY `hdfs_repo`
WITH BROKER `hdfs_broker`
ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/"
PROPERTIES
(
"username" = "user",
"password" = "password"
);
```
5. Create a repository named s3_repo to link cloud storage directly without going through the broker.
```
CREATE REPOSITORY `s3_repo`
WITH S3
ON LOCATION "s3://s3-repo"
PROPERTIES
(
"AWS_ENDPOINT" = "http://s3-REGION.amazonaws.com",
"AWS_ACCESS_KEY" = "AWS_ACCESS_KEY",
"AWS_SECRET_KEY"="AWS_SECRET_KEY",
"AWS_REGION" = "REGION"
);
```
6. Create a repository named hdfs_repo to link HDFS directly without going through the broker.
```
CREATE REPOSITORY `hdfs_repo`
WITH hdfs
ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/"
PROPERTIES
(
"fs.defaultFS"="hdfs://hadoop-name-node:54310",
"hadoop.username"="user"
);
```
7. Create a repository named minio_repo to link minio storage directly through the s3 protocol.
```
CREATE REPOSITORY `minio_repo`
WITH S3
ON LOCATION "s3://minio_repo"
PROPERTIES
(
"AWS_ENDPOINT" = "http://minio.com",
"AWS_ACCESS_KEY" = "MINIO_USER",
"AWS_SECRET_KEY"="MINIO_PASSWORD",
"AWS_REGION" = "REGION",
"use_path_style" = "true"
);
```
### Keywords
```text
BACKUP
```
### Best Practice

18
docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
View File

@ -139,10 +139,11 @@ PROPERTIES
);
```
<version since="1.2">
7. Create a repository named minio_repo via temporary security credentials.
<version since="1.2"></version>
```
CREATE REPOSITORY `minio_repo`
WITH S3
@ -157,7 +158,20 @@ PROPERTIES
)
```
</version>
8. Create repository using Tencent COS
```
CREATE REPOSITORY `cos_repo`
WITH S3
ON LOCATION "s3://backet1/"
PROPERTIES
(
"AWS_ACCESS_KEY" = "ak",
"AWS_SECRET_KEY" = "sk",
"AWS_ENDPOINT" = "http://cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing"
);
```
### Keywords

59
docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/BACKUP.md
View File

@ -90,65 +90,6 @@ TO example_repo
EXCLUDE (example_tbl);
```
4. 创建名为 hdfs_repo 的仓库，依赖 Baidu hdfs broker "hdfs_broker"，数据根目录为：hdfs://hadoop-name-node:54310/path/to/repo/
```
CREATE REPOSITORY `hdfs_repo`
WITH BROKER `hdfs_broker`
ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/"
PROPERTIES
(
"username" = "user",
"password" = "password"
);
```
5. 创建名为 s3_repo 的仓库，直接链接云存储，而不通过broker.
```
CREATE REPOSITORY `s3_repo`
WITH S3
ON LOCATION "s3://s3-repo"
PROPERTIES
(
"AWS_ENDPOINT" = "http://s3-REGION.amazonaws.com",
"AWS_ACCESS_KEY" = "AWS_ACCESS_KEY",
"AWS_SECRET_KEY"="AWS_SECRET_KEY",
"AWS_REGION" = "REGION"
);
```
6. 创建名为 hdfs_repo 的仓库，直接链接HDFS，而不通过broker.
```
CREATE REPOSITORY `hdfs_repo`
WITH hdfs
ON LOCATION "hdfs://hadoop-name-node:54310/path/to/repo/"
PROPERTIES
(
"fs.defaultFS"="hdfs://hadoop-name-node:54310",
"hadoop.username"="user"
);
```
7. 创建名为 minio_repo 的仓库，直接通过 s3 协议链接 minio.
```
CREATE REPOSITORY `minio_repo`
WITH S3
ON LOCATION "s3://minio_repo"
PROPERTIES
(
"AWS_ENDPOINT" = "http://minio.com",
"AWS_ACCESS_KEY" = "MINIO_USER",
"AWS_SECRET_KEY"="MINIO_PASSWORD",
"AWS_REGION" = "REGION",
"use_path_style" = "true"
);
```
### Keywords
```text

18
docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/CREATE-REPOSITORY.md
View File

@ -136,10 +136,11 @@ PROPERTIES
"use_path_style" = "true"
);
```
<version since="1.2">
7. 使用临时秘钥创建名为 minio_repo 的仓库
<version since="1.2"></version>
```
CREATE REPOSITORY `minio_repo`
WITH S3
@ -154,7 +155,20 @@ PROPERTIES
)
```
</version>
8. 使用腾讯云 COS 创建仓库
```
CREATE REPOSITORY `cos_repo`
WITH S3
ON LOCATION "s3://backet1/"
PROPERTIES
(
"AWS_ACCESS_KEY" = "ak",
"AWS_SECRET_KEY" = "sk",
"AWS_ENDPOINT" = "http://cos.ap-beijing.myqcloud.com",
"AWS_REGION" = "ap-beijing"
);
```
### Keywords

9
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -1320,8 +1320,13 @@ public class OlapTable extends Table {
}
// remove shadow index from copied table
List<MaterializedIndex> shadowIndex = copied.getPartitions().stream().findFirst()
.get().getMaterializedIndices(IndexExtState.SHADOW);
// NOTICE that there maybe not partition in table.
List<MaterializedIndex> shadowIndex = Lists.newArrayList();
Optional<Partition> firstPartition = copied.getPartitions().stream().findFirst();
if (firstPartition.isPresent()) {
shadowIndex = firstPartition.get().getMaterializedIndices(IndexExtState.SHADOW);
}
for (MaterializedIndex deleteIndex : shadowIndex) {
LOG.debug("copied table delete shadow index : {}", deleteIndex.getId());
copied.deleteIndexInfo(copied.getIndexNameById(deleteIndex.getId()));

37
fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@ -40,6 +41,7 @@ import org.junit.rules.ExpectedException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
@ -1537,10 +1539,37 @@ public class DynamicPartitionTableTest {
+ ");";
ExceptionChecker.expectThrowsWithMsg(DdlException.class,
"errCode = 2, detailMessage = Invalid \" dynamic_partition.reserved_history_periods \""
+ " value [2020-01-01,2020-03-01]. "
+ "It must be like "
+ "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH "
+ "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.",
+ " value [2020-01-01,2020-03-01]. "
+ "It must be like "
+ "\"[yyyy-MM-dd,yyyy-MM-dd],[...,...]\" while time_unit is DAY/WEEK/MONTH "
+ "or \"[yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss],[...,...]\" while time_unit is HOUR.",
() -> createTable(createOlapTblStmt4));
}
@Test
public void testNoPartition() throws AnalysisException {
String createOlapTblStmt = "CREATE TABLE test.`no_partition` (\n"
+ " `k1` datetime NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\",\n"
+ " `k3` smallint NULL COMMENT \"\",\n"
+ " `v1` varchar(2048) NULL COMMENT \"\",\n"
+ " `v2` datetime NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`k1`, `k2`, `k3`)\n"
+ "COMMENT \"OLAP\"\n"
+ "PARTITION BY RANGE (k1)()\n"
+ "DISTRIBUTED BY HASH(`k1`) BUCKETS 32\n"
+ "PROPERTIES (\n"
+ "\"replication_num\" = \"1\"\n"
+ ");";
ExceptionChecker.expectThrowsNoException(() -> createTable(createOlapTblStmt));
OlapTable table = (OlapTable) Env.getCurrentInternalCatalog()
.getDbOrAnalysisException("default_cluster:test")
.getTableOrAnalysisException("no_partition");
Collection<Partition> partitions = table.getPartitions();
Assert.assertTrue(partitions.isEmpty());
OlapTable copiedTable = table.selectiveCopy(Collections.emptyList(), IndexExtState.VISIBLE, true);
partitions = copiedTable.getPartitions();
Assert.assertTrue(partitions.isEmpty());
}
}