database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](backup) add property to remove snapshot before creating repo (#25847)

Browse Source 
Doris is not responsible for managing snapshots, but it needs to clear all
snapshots before doing backup/restore regression testing, so a property is
added to indicate that existing snapshots need to be cleared when creating a
repo.

In addition, a regression test case for backup/restore has been added.
This commit is contained in:
walter
2023-10-27 21:03:26 +08:00
committed by GitHub
parent c715facafa
commit 365fdd2f4d
11 changed files with 440 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3ObjStorageTest.java
View File

@ -21,8 +21,8 @@ import org.apache.doris.backup.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.S3URI;
import org.apache.commons.lang3.StringUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import software.amazon.awssdk.core.sync.RequestBody;
@ -36,26 +36,85 @@ import java.util.Map;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class S3ObjStorageTest {
private S3ObjStorage storage;
@Test
public void testS3BaseOp() throws UserException {
String ak = System.getenv("S3_ACCESS_KEY");
String sk = System.getenv("S3_SECRET_KEY");
String endpoint = System.getenv("S3_ENDPOINT");
String region = System.getenv("S3_REGION");
String bucket = System.getenv("S3_BUCKET");
String prefix = System.getenv("S3_PREFIX");
private MockedS3Client mockedClient;
// Skip this test if ENV variables are not set.
if (StringUtils.isEmpty(endpoint) || StringUtils.isEmpty(ak)
|| StringUtils.isEmpty(sk) || StringUtils.isEmpty(region)
|| StringUtils.isEmpty(bucket) || StringUtils.isEmpty(prefix)) {
return;
}
@BeforeAll
public void beforeAll() throws Exception {
Map<String, String> properties = new HashMap<>();
properties.put("s3.endpoint", endpoint);
properties.put("s3.access_key", ak);
properties.put("s3.secret_key", sk);
properties.put("s3.region", region);
S3ObjStorage storage = new S3ObjStorage(properties);
String baseUrl = "s3://" + bucket + "/" + prefix + "/";
for (int i = 0; i < 5; ++i) {
Status st = storage.putObject(baseUrl + "key" + i, RequestBody.fromString("mocked"));
Assertions.assertEquals(Status.OK, st);
}
RemoteObjects remoteObjects = storage.listObjects(baseUrl, null);
Assertions.assertEquals(5, remoteObjects.getObjectList().size());
Assertions.assertFalse(remoteObjects.isTruncated());
Assertions.assertEquals(null, remoteObjects.getContinuationToken());
List<RemoteObject> objectList = remoteObjects.getObjectList();
for (int i = 0; i < objectList.size(); i++) {
RemoteObject remoteObject = objectList.get(i);
Assertions.assertEquals("key" + i, remoteObject.getRelativePath());
}
Status st = storage.headObject(baseUrl + "key" + 0);
Assertions.assertEquals(Status.OK, st);
File file = new File("test-file.txt");
file.delete();
st = storage.getObject(baseUrl + "key" + 0, file);
Assertions.assertEquals(Status.OK, st);
st = storage.deleteObject(baseUrl + "key" + 0);
Assertions.assertEquals(Status.OK, st);
file.delete();
st = storage.getObject(baseUrl + "key" + 0, file);
Assertions.assertEquals(Status.ErrCode.COMMON_ERROR, st.getErrCode());
Assertions.assertTrue(st.getErrMsg().contains("The specified key does not exist"));
file.delete();
st = storage.deleteObjects(baseUrl);
Assertions.assertEquals(Status.OK, st);
remoteObjects = storage.listObjects(baseUrl, null);
Assertions.assertEquals(0, remoteObjects.getObjectList().size());
Assertions.assertFalse(remoteObjects.isTruncated());
Assertions.assertEquals(null, remoteObjects.getContinuationToken());
}
@Test
public void testBaseOp() throws Exception {
Map<String, String> properties = new HashMap<>();
properties.put("s3.endpoint", "s3.e.c");
properties.put("s3.access_key", "abc");
properties.put("s3.secret_key", "123");
storage = new S3ObjStorage(properties);
S3ObjStorage storage = new S3ObjStorage(properties);
Field client = storage.getClass().getDeclaredField("client");
client.setAccessible(true);
mockedClient = new MockedS3Client();
MockedS3Client mockedClient = new MockedS3Client();
client.set(storage, mockedClient);
Assertions.assertTrue(storage.getClient("mocked") instanceof MockedS3Client);
}
@Test
public void testBaseOp() throws UserException {
S3URI vUri = S3URI.create("s3://bucket/key", true);
S3URI uri = S3URI.create("s3://bucket/key", false);
Assertions.assertEquals(vUri.getVirtualBucket(), "bucket");
@ -98,7 +157,16 @@ class S3ObjStorageTest {
List<RemoteObject> list = remoteObjectsVBucket.getObjectList();
for (int i = 0; i < list.size(); i++) {
RemoteObject remoteObject = list.get(i);
Assertions.assertTrue(remoteObject.getRelativePath().startsWith("keys/key" + i));
Assertions.assertTrue(remoteObject.getRelativePath().startsWith("key" + i));
}
storage.properties.put("use_path_style", "true");
storage.setProperties(storage.properties);
remoteObjectsVBucket = storage.listObjects("oss://bucket/keys", null);
list = remoteObjectsVBucket.getObjectList();
for (int i = 0; i < list.size(); i++) {
RemoteObject remoteObject = list.get(i);
Assertions.assertTrue(remoteObject.getRelativePath().startsWith("key" + i));
}
}
}