From a2d418cff12283cc344d4aff730f7fb21b65faf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=94=A6=E9=BE=99?= <15822173373@163.com> Date: Wed, 26 Oct 2022 09:21:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8E=8B=E7=BC=A9=E7=9B=B8=E5=85=B3=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_basebackup/pg_basebackup.cpp | 2 +- src/bin/pg_ctl/backup.cpp | 2 +- src/bin/pg_rewind/fetch.cpp | 2 + src/bin/pg_rewind/filemap.cpp | 2 +- src/common/backend/utils/adt/genfile.cpp | 62 +++-- .../optimizer/commands/indexcmds.cpp | 5 +- .../storage/replication/basebackup.cpp | 5 +- .../storage/smgr/cfs/cfs_tools.cpp | 4 +- src/include/storage/cfs/cfs_tools.h | 2 +- src/include/storage/page_compression_impl.h | 243 +++++++++--------- src/lib/page_compression/PageCompression.cpp | 59 ++++- src/lib/page_compression/PageCompression.h | 1 + 12 files changed, 228 insertions(+), 161 deletions(-) diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 1e5661d7e..6095ae35a 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -1042,7 +1042,7 @@ static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) /* * regular file */ - file = fopen(filename, IsCompressedFile(filename, strlen(filename)) ? "w+r+b" : "wb"); + file = fopen(filename, IsCompressedFile(filename, strlen(filename)) ? "wb+" : "wb"); if (NULL == file) { pg_log(stderr, _("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno)); disconnect_and_exit(1); diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index e7928eb8f..f32954906 100755 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -962,7 +962,7 @@ static bool ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) if (forbid_write) { file = fopen(filename, "ab"); } else { - file = fopen(filename, "wb"); + file = fopen(filename, IsCompressedFile(filename, strlen(filename)) ? "wb+" : "wb"); } if (NULL == file) { pg_log(PG_WARNING, _("could not create file \"%s\": %s\n"), filename, strerror(errno)); diff --git a/src/bin/pg_rewind/fetch.cpp b/src/bin/pg_rewind/fetch.cpp index 8313fa5db..b52fc8f0e 100755 --- a/src/bin/pg_rewind/fetch.cpp +++ b/src/bin/pg_rewind/fetch.cpp @@ -477,6 +477,8 @@ static BuildErrorCode receiveFileChunks(const char* sql, FILE* file) chunkSize = (int32)ntohl((uint32)chunkSize); bool rebuild = *PQgetvalue(res, 0, 6) != 0; CompressedFileInit(filename, rebuild); + pg_free(filename); + filename = NULL; /* fetch result */ FetchCompressedFile(chunk, (uint32)chunkoff, (int32)chunkspace, (uint16)chunkSize, (uint8)algorithm); } diff --git a/src/bin/pg_rewind/filemap.cpp b/src/bin/pg_rewind/filemap.cpp index 47dab3fc9..692ea179c 100755 --- a/src/bin/pg_rewind/filemap.cpp +++ b/src/bin/pg_rewind/filemap.cpp @@ -1022,7 +1022,7 @@ static char* datasegpath(RelFileNode rnode, ForkNumber forknum, BlockNumber segn path = relpathbackend_t((rnode), InvalidBackendId, (forknum)); if (segno > 0 || forknum > MAX_FORKNUM) { - segpath = compress ? format_text("%s.%u", path, segno) : format_text("%s.%u" COMPRESS_STR, path, segno); + segpath = compress ? format_text("%s.%u" COMPRESS_STR, path, segno) : format_text("%s.%u", path, segno); pg_free(path); path = NULL; return segpath; diff --git a/src/common/backend/utils/adt/genfile.cpp b/src/common/backend/utils/adt/genfile.cpp index 484270630..737351308 100644 --- a/src/common/backend/utils/adt/genfile.cpp +++ b/src/common/backend/utils/adt/genfile.cpp @@ -100,6 +100,16 @@ static char* convert_and_check_filename(text* arg) return filename; } +static bool IsCheckFileBlank(FILE* file, const char* filename) +{ + int rc = fseeko(file, 0L, SEEK_END); + if (rc != 0) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not seek in file \"%s\": %m", filename))); + } + + return (ftell(file) == 0); +} + /* * Read a section of a file, returning it as bytea * @@ -119,6 +129,7 @@ bytea* read_binary_file(const char* filename, int64 seek_offset, int64 bytes_to_ int retryCnt = 0; errno_t rc = 0; UndoFileType undoFileType = UNDO_INVALID; + bool isCompressed = IsCompressedFile(filename, strlen(filename)); if (bytes_to_read < 0) { if (seek_offset < 0) @@ -149,11 +160,18 @@ bytea* read_binary_file(const char* filename, int64 seek_offset, int64 bytes_to_ } /* do not need to check if table is compressed table file */ - isNeedCheck = - is_row_data_file(filename, &segNo, &undoFileType) && !IsCompressedFile(filename, strlen(filename)); + isNeedCheck = is_row_data_file(filename, &segNo, &undoFileType) && !isCompressed; ereport(DEBUG1, (errmsg("read_binary_file, filename is %s, isNeedCheck is %d", filename, isNeedCheck))); buf = (bytea*)palloc((Size)bytes_to_read + VARHDRSZ); + if (isCompressed && IsCheckFileBlank(file, filename)) { + rc = memset_s(VARDATA(buf), bytes_to_read, 0, bytes_to_read); + securec_check_c(rc, "", ""); + + SET_VARSIZE(buf, bytes_to_read + VARHDRSZ); + FreeFile(file); + return buf; + } recheck: if (fseeko(file, (off_t)seek_offset, (seek_offset >= 0) ? SEEK_SET : SEEK_END) != 0) @@ -339,6 +357,7 @@ Datum pg_read_binary_file_all(PG_FUNCTION_ARGS) struct CompressAddressItemState { uint32 blkno; int segmentNo; + off_t fileLen; CfsHeaderMap rbStruct; FILE *compressedFd; }; @@ -362,9 +381,9 @@ static void ReadBinaryFileBlocksFirstCall(PG_FUNCTION_ARGS, int32 startBlockNum, CompressAddressItemState* itemState = (CompressAddressItemState*)palloc(sizeof(CompressAddressItemState)); /* save mmap to inter_call_data->pcMap */ - + itemState->fileLen = 0; FILE *compressedFd = AllocateFile((const char *)path, "rb"); - auto blockNumber = ReadBlockNumberOfCFile(compressedFd); + auto blockNumber = ReadBlockNumberOfCFile(compressedFd, &itemState->fileLen); if (blockNumber >= MIN_COMPRESS_ERROR_RT) { ereport(ERROR, (ERRCODE_INVALID_PARAMETER_VALUE, errmsg("can not read actual block from %s, error code: %lu,", path, blockNumber))); @@ -970,7 +989,7 @@ Datum pg_read_binary_file_blocks(PG_FUNCTION_ARGS) int32 startBlockNum = PG_GETARG_INT32(1); int32 blockCount = PG_GETARG_INT32(2); - if (startBlockNum < 0 || blockCount <= 0 || startBlockNum + blockCount > RELSEG_SIZE) { + if (startBlockNum < 0 || blockCount < 0 || startBlockNum + blockCount > RELSEG_SIZE) { ereport(ERROR, (ERRCODE_INVALID_PARAMETER_VALUE, errmsg("invalid blocknum \"%d\" or block count \"%d\"", startBlockNum, blockCount))); } @@ -986,28 +1005,31 @@ Datum pg_read_binary_file_blocks(PG_FUNCTION_ARGS) if (fctx->call_cntr < fctx->max_calls) { bytea *buf = (bytea *)palloc(BLCKSZ + VARHDRSZ); - BlockNumber extentCount = itemState->blkno / CFS_LOGIC_BLOCKS_PER_EXTENT; - if (itemState->rbStruct.extentCount != extentCount || itemState->rbStruct.header == NULL) { - MmapFree(&itemState->rbStruct); - auto curHeader = MMapHeader(itemState->compressedFd, extentCount, true); + size_t len = 0; + if (itemState->fileLen > 0) { + BlockNumber extentCount = itemState->blkno / CFS_LOGIC_BLOCKS_PER_EXTENT; + if (itemState->rbStruct.extentCount != extentCount || itemState->rbStruct.header == NULL) { + MmapFree(&itemState->rbStruct); + auto curHeader = MMapHeader(itemState->compressedFd, extentCount, true); - itemState->rbStruct.header = curHeader.header; - itemState->rbStruct.extentCount = curHeader.extentCount; - itemState->rbStruct.pointer = curHeader.pointer; - itemState->rbStruct.mmapLen = curHeader.mmapLen; + + itemState->rbStruct.header = curHeader.header; + itemState->rbStruct.extentCount = curHeader.extentCount; + itemState->rbStruct.pointer = curHeader.pointer; + itemState->rbStruct.mmapLen = curHeader.mmapLen; + } + + CfsReadStruct cfsReadStruct{itemState->compressedFd, itemState->rbStruct.header, extentCount}; + len = CfsReadCompressedPage(VARDATA(buf), BLCKSZ, itemState->blkno % CFS_LOGIC_BLOCKS_PER_EXTENT, + &cfsReadStruct, CFS_LOGIC_BLOCKS_PER_FILE * itemState->segmentNo + itemState->blkno); } - - CfsReadStruct cfsReadStruct{itemState->compressedFd, itemState->rbStruct.header, extentCount}; - size_t len = CfsReadCompressedPage(VARDATA(buf), BLCKSZ, - itemState->blkno % CFS_LOGIC_BLOCKS_PER_EXTENT, &cfsReadStruct, - CFS_LOGIC_BLOCKS_PER_FILE * itemState->segmentNo + itemState->blkno); SET_VARSIZE(buf, len + VARHDRSZ); Datum values[6]; values[0] = PG_GETARG_DATUM(0); values[1] = Int32GetDatum(itemState->blkno); values[2] = Int32GetDatum(len); - values[3] = Int32GetDatum(itemState->rbStruct.header->algorithm); - values[4] = Int32GetDatum(itemState->rbStruct.header->chunk_size); + values[3] = Int32GetDatum(itemState->fileLen == 0 ? 0 : itemState->rbStruct.header->algorithm); + values[4] = Int32GetDatum(itemState->fileLen == 0 ? 0 : itemState->rbStruct.header->chunk_size); values[5] = PointerGetDatum(buf); /* Build and return the result tuple. */ diff --git a/src/gausskernel/optimizer/commands/indexcmds.cpp b/src/gausskernel/optimizer/commands/indexcmds.cpp index 64848cf77..3ef18d56b 100755 --- a/src/gausskernel/optimizer/commands/indexcmds.cpp +++ b/src/gausskernel/optimizer/commands/indexcmds.cpp @@ -682,9 +682,10 @@ Oid DefineIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId, bool is_al CheckCompressOption(&indexCreateSupport); /* do not suppport to create compressed index for temp table. */ if ((indexCreateSupport.compressType != (int)COMPRESS_TYPE_NONE) && - (relPersistence == RELPERSISTENCE_TEMP || relPersistence == RELPERSISTENCE_GLOBAL_TEMP)) { + (relPersistence == RELPERSISTENCE_TEMP || relPersistence == RELPERSISTENCE_GLOBAL_TEMP || + relPersistence == RELPERSISTENCE_UNLOGGED)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("compressed index \"%s\" is not supported for temporary table," + errmsg("compressed index \"%s\" is not supported for temporary table and unlogged table," " please use uncompressed one instead", stmt->idxname))); } diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index 59c22a46f..e7a43998d 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -1293,7 +1293,7 @@ static bool IsDCFPath(const char *pathname) * @param basepathlen subfix of path * @param statbuf path stat */ -static void SendRealFile(bool sizeOnly, char* pathbuf, int basepathlen, struct stat* statbuf) +static int64 SendRealFile(bool sizeOnly, char* pathbuf, int basepathlen, struct stat* statbuf) { int64 size = 0; // we must ensure the page integrity when in IncrementalCheckpoint @@ -1310,6 +1310,7 @@ static void SendRealFile(bool sizeOnly, char* pathbuf, int basepathlen, struct s SEND_DIR_ADD_SIZE(size, (*statbuf)); } } + return size; } /* @@ -1610,7 +1611,7 @@ static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tab if (!skip_this_dir) size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks); } else if (S_ISREG(statbuf.st_mode)) { - SendRealFile(sizeonly, pathbuf, basepathlen, &statbuf); + size += SendRealFile(sizeonly, pathbuf, basepathlen, &statbuf); } else ereport(WARNING, (errmsg("skipping special file \"%s\"", pathbuf))); } diff --git a/src/gausskernel/storage/smgr/cfs/cfs_tools.cpp b/src/gausskernel/storage/smgr/cfs/cfs_tools.cpp index 13eb987e9..5e0f7d739 100644 --- a/src/gausskernel/storage/smgr/cfs/cfs_tools.cpp +++ b/src/gausskernel/storage/smgr/cfs/cfs_tools.cpp @@ -144,6 +144,7 @@ void MmapFree(CfsHeaderMap *cfsHeaderMap) { if (cfsHeaderMap->pointer != nullptr) { (void)munmap(cfsHeaderMap->pointer, cfsHeaderMap->mmapLen); + cfsHeaderMap->pointer = nullptr; } } @@ -183,13 +184,14 @@ bool ReadCompressedInfo(T &t, off_t offset, FILE *file) return true; } -size_t ReadBlockNumberOfCFile(FILE* compressFd) +size_t ReadBlockNumberOfCFile(FILE* compressFd, off_t *curFileLen) { if (fseek(compressFd, 0L, SEEK_END) < 0) { return COMPRESS_FSEEK_ERROR; } /* read file size of toFullPath */ off_t fileLen = ftell(compressFd); + *curFileLen = fileLen; if (fileLen == 0) { return 0; } else { diff --git a/src/include/storage/cfs/cfs_tools.h b/src/include/storage/cfs/cfs_tools.h index feba2e687..5c6f4764f 100644 --- a/src/include/storage/cfs/cfs_tools.h +++ b/src/include/storage/cfs/cfs_tools.h @@ -47,6 +47,6 @@ CfsHeaderMap MMapHeader(FILE* fd, BlockNumber extentIndex, bool readOnly = false void MmapFree(CfsHeaderMap *cfsHeaderMap); -size_t ReadBlockNumberOfCFile(FILE* compressFd); +size_t ReadBlockNumberOfCFile(FILE* compressFd, off_t *curFileLen); #endif //OPENGAUSS_CFS_TOOLS_H diff --git a/src/include/storage/page_compression_impl.h b/src/include/storage/page_compression_impl.h index 9517d1a10..c077fff93 100644 --- a/src/include/storage/page_compression_impl.h +++ b/src/include/storage/page_compression_impl.h @@ -96,152 +96,151 @@ static inline void FreePointer(void* pointer) #define COMMON "" #if defined(_M_AMD64) || defined(_M_X64) || defined(__amd64) || defined(__x86_64__) -void Transpose8x8_8U(uint8 **src, uint8 **dst) +void Transpose8x8U(uint8 **src, uint8 **dst) { /* load and crossed-store 0~3 lines */ - __m128i S0 = _mm_loadl_epi64((__m128i *)(src[0])); // 0 0 0 0 0 0 0 0 A7 A6 A5 A4 A3 A2 A1 A0 - __m128i S1 = _mm_loadl_epi64((__m128i *)(src[1])); // 0 0 0 0 0 0 0 0 B7 B6 B5 B4 B3 B2 B1 B0 - __m128i S2 = _mm_loadl_epi64((__m128i *)(src[2])); // 0 0 0 0 0 0 0 0 C7 C6 C5 C4 C3 C2 C1 C0 - __m128i S3 = _mm_loadl_epi64((__m128i *)(src[3])); // 0 0 0 0 0 0 0 0 D7 D6 D5 D4 D3 D2 D1 D0 - __m128i S01 = _mm_unpacklo_epi8(S0, S1); // B7 A7 B6 A6 B5 A5 B4 A4 B3 A3 B2 A2 B1 A1 B0 A0 - __m128i S23 = _mm_unpacklo_epi8(S2, S3); // D7 C7 D6 C6 D5 C5 D4 C4 D3 C3 D2 C2 D1 C1 D0 C0 - __m128i S0123L = _mm_unpacklo_epi16(S01, S23); // D3 C3 B3 A3 D2 C2 B2 A2 D1 C1 B1 A1 D0 C0 B0 A0 - __m128i S0123H = _mm_unpackhi_epi16(S01, S23); // D7 C7 B7 A7 D6 C6 B6 A6 D5 C5 B5 A5 D4 C4 B4 A4 + __m128i str0 = _mm_loadl_epi64((__m128i *)(src[0])); + __m128i str1 = _mm_loadl_epi64((__m128i *)(src[1])); + __m128i str2 = _mm_loadl_epi64((__m128i *)(src[2])); + __m128i str3 = _mm_loadl_epi64((__m128i *)(src[3])); + __m128i str2x1 = _mm_unpacklo_epi8(str0, str1); + __m128i str2x2 = _mm_unpacklo_epi8(str2, str3); + __m128i str4x1L = _mm_unpacklo_epi16(str2x1, str2x2); + __m128i str4x1H = _mm_unpackhi_epi16(str2x1, str2x2); /* load and crossed-store 4~7 lines */ - __m128i S4 = _mm_loadl_epi64((__m128i *)(src[4])); // 0 0 0 0 0 0 0 0 E7 E6 E5 E4 E3 E2 E1 E0 - __m128i S5 = _mm_loadl_epi64((__m128i *)(src[5])); // 0 0 0 0 0 0 0 0 F7 F6 F5 F4 F3 F2 F1 F0 - __m128i S6 = _mm_loadl_epi64((__m128i *)(src[6])); // 0 0 0 0 0 0 0 0 G7 G6 G5 G4 G3 G2 G1 G0 - __m128i S7 = _mm_loadl_epi64((__m128i *)(src[7])); // 0 0 0 0 0 0 0 0 H7 H6 H5 H4 H3 H2 H1 H0 - __m128i S45 = _mm_unpacklo_epi8(S4, S5); // F7 E7 F6 E6 F5 E5 F4 E4 F3 E3 F2 E2 F1 E1 F0 E0 - __m128i S67 = _mm_unpacklo_epi8(S6, S7); // H7 G7 H6 G6 H5 G5 H4 G4 H3 G3 H2 G2 H1 G1 H0 G0 - __m128i S4567L = _mm_unpacklo_epi16(S45, S67); // H3 G3 F3 E3 H2 G2 F2 E2 H1 G1 F1 E1 H0 G0 F0 E0 - __m128i S4567H = _mm_unpackhi_epi16(S45, S67); // H7 G7 F7 E7 H6 G6 F6 E6 H5 G5 F5 E5 H4 G4 F4 E4 + __m128i str4 = _mm_loadl_epi64((__m128i *)(src[4])); + __m128i str5 = _mm_loadl_epi64((__m128i *)(src[5])); + __m128i str6 = _mm_loadl_epi64((__m128i *)(src[6])); + __m128i str7 = _mm_loadl_epi64((__m128i *)(src[7])); + __m128i str2x3 = _mm_unpacklo_epi8(str4, str5); + __m128i str2x4 = _mm_unpacklo_epi8(str6, str7); + __m128i str4x2L = _mm_unpacklo_epi16(str2x3, str2x4); + __m128i str4x2H = _mm_unpackhi_epi16(str2x3, str2x4); /* store into dst */ - __m128i T0 = _mm_unpacklo_epi32(S0123L, S4567L); - _mm_storel_epi64((__m128i *)(dst[0]), T0); - _mm_storel_epi64((__m128i *)(dst[1]), _mm_srli_si128(T0, 8)); + __m128i dst0 = _mm_unpacklo_epi32(str4x1L, str4x2L); + _mm_storel_epi64((__m128i *)(dst[0]), dst0); + _mm_storel_epi64((__m128i *)(dst[1]), _mm_srli_si128(dst0, 8)); - __m128i T1 = _mm_unpackhi_epi32(S0123L, S4567L); - _mm_storel_epi64((__m128i *)(dst[2]), T1); - _mm_storel_epi64((__m128i *)(dst[3]), _mm_srli_si128(T1, 8)); + __m128i dst1 = _mm_unpackhi_epi32(str4x1L, str4x2L); + _mm_storel_epi64((__m128i *)(dst[2]), dst1); + _mm_storel_epi64((__m128i *)(dst[3]), _mm_srli_si128(dst1, 8)); - __m128i T2 = _mm_unpacklo_epi32(S0123H, S4567H); - _mm_storel_epi64((__m128i *)(dst[4]), T2); - _mm_storel_epi64((__m128i *)(dst[5]), _mm_srli_si128(T2, 8)); + __m128i dst2 = _mm_unpacklo_epi32(str4x1H, str4x2H); + _mm_storel_epi64((__m128i *)(dst[4]), dst2); + _mm_storel_epi64((__m128i *)(dst[5]), _mm_srli_si128(dst2, 8)); - __m128i T3 = _mm_unpackhi_epi32(S0123H, S4567H); - _mm_storel_epi64((__m128i *)(dst[6]), T3); - _mm_storel_epi64((__m128i *)(dst[7]), _mm_srli_si128(T3, 8)); + __m128i dst3 = _mm_unpackhi_epi32(str4x1H, str4x2H); + _mm_storel_epi64((__m128i *)(dst[6]), dst3); + _mm_storel_epi64((__m128i *)(dst[7]), _mm_srli_si128(dst3, 8)); } #elif defined(_M_ARM64) || defined(__aarch64__) -void Transpose8x16_8U(uint8 **src, uint8 **dst) +void Transpose8x16U(uint8 **src, uint8 **dst) { /* load and crossed-store 0~3 lines */ - uint8x16x4_t mat0123_line; - uint8x16x4_t mat4567_line; - uint8 S0123_ptr[64]; - uint8 S4567_ptr[64]; + uint8x16x4_t matLine4x1; + uint8x16x4_t matLine4x2; + uint8 sptr4x1[64]; + uint8 sptr4x2[64]; - mat0123_line.val[0] = vld1q_u8(src[0]); // AF AE AD AC AB AA A9 A8 A7 A6 A5 A4 A3 A2 A1 A0 - mat0123_line.val[1] = vld1q_u8(src[1]); // BF BE BD BC BB BA B9 B8 B7 B6 B5 B4 B3 B2 B1 B0 - mat0123_line.val[2] = vld1q_u8(src[2]); // CF CE CD CC CB CA C9 C8 C7 C6 C5 C4 C3 C2 C1 C0 - mat0123_line.val[3] = vld1q_u8(src[3]); // DF DE DD DC DB DA D9 D8 D7 D6 D5 D4 D3 D2 D1 D0 - vst4q_u8(S0123_ptr, mat0123_line); // AF BF CF DF AE BE CE DE ... A0 B0 C0 D0 + matLine4x1.val[0] = vld1q_u8(src[0]); + matLine4x1.val[1] = vld1q_u8(src[1]); + matLine4x1.val[2] = vld1q_u8(src[2]); + matLine4x1.val[3] = vld1q_u8(src[3]); + vst4q_u8(sptr4x1, matLine4x1); /* load and crossed-store 4~7 lines */ - mat4567_line.val[0] = vld1q_u8(src[4]); // EF EE ED EC EB EA E9 E8 E7 E6 E5 E4 E3 E2 E1 E0 - mat4567_line.val[1] = vld1q_u8(src[5]); // FF FE FD FC FB FA F9 F8 F7 F6 F5 F4 F3 F2 F1 F0 - mat4567_line.val[2] = vld1q_u8(src[6]); // GF GE GD GC GB GA G9 G8 G7 G6 G5 G4 G3 G2 G1 G0 - mat4567_line.val[3] = vld1q_u8(src[7]); // HF HE HD HC HB HA H9 H8 H7 H6 H5 H4 H3 H2 H1 H0 - vst4q_u8(S4567_ptr, mat4567_line); // EF FF GF HF EE FE GE HE ... E0 F0 G0 H0 + matLine4x2.val[0] = vld1q_u8(src[4]); + matLine4x2.val[1] = vld1q_u8(src[5]); + matLine4x2.val[2] = vld1q_u8(src[6]); + matLine4x2.val[3] = vld1q_u8(src[7]); + vst4q_u8(sptr4x2, matLine4x2); - uint32x4x2_t dst04_line; - uint32x4x2_t dst15_line; - uint32x4x2_t dst26_line; - uint32x4x2_t dst37_line; - uint8 dst_ptr[128]; + matLine4x2.val[0] = vld1q_u8(src[4]); + matLine4x2.val[1] = vld1q_u8(src[5]); + matLine4x2.val[2] = vld1q_u8(src[6]); + matLine4x2.val[3] = vld1q_u8(src[7]); + vst4q_u8(sptr4x2, matLine4x2); - dst04_line.val[0] = vld1q_u32((uint32_t *)S0123_ptr); // (AF~DF)(AE~DE)(AD~DD)(AC~DC) - dst04_line.val[1] = vld1q_u32((uint32_t *)S4567_ptr); // (EF~EF)(FE~FE)(GD~GD)(HC~HC) - dst15_line.val[0] = vld1q_u32((uint32_t *)(S0123_ptr + 16)); // (AB~DB)(AA~DA)(A9~D9)(A8~D8) - dst15_line.val[1] = vld1q_u32((uint32_t *)(S4567_ptr + 16)); // (EB~EB)(FA~FA)(G9~G9)(H8~H8) - dst26_line.val[0] = vld1q_u32((uint32_t *)(S0123_ptr + 32)); // (A7~D7)(A6~D6)(A5~D5)(A4~D4) - dst26_line.val[1] = vld1q_u32((uint32_t *)(S4567_ptr + 32)); // (E7~E7)(F6~F6)(G5~G5)(H4~H4) - dst37_line.val[0] = vld1q_u32((uint32_t *)(S0123_ptr + 48)); // (A3~D3)(A2~D2)(A1~D1)(A0~D0) - dst37_line.val[1] = vld1q_u32((uint32_t *)(S4567_ptr + 48)); // (E3~E3)(F2~F2)(G1~G1)(H0~H0) + dstLine04.val[0] = vld1q_u32((uint32_t *)sptr4x1); + dstLine04.val[1] = vld1q_u32((uint32_t *)sptr4x2); + dstLine15.val[0] = vld1q_u32((uint32_t *)(sptr4x1 + 16)); + dstLine15.val[1] = vld1q_u32((uint32_t *)(sptr4x2 + 16)); + dstLine26.val[0] = vld1q_u32((uint32_t *)(sptr4x1 + 32)); + dstLine26.val[1] = vld1q_u32((uint32_t *)(sptr4x2 + 32)); + dstLine37.val[0] = vld1q_u32((uint32_t *)(sptr4x1 + 48)); + dstLine37.val[1] = vld1q_u32((uint32_t *)(sptr4x2 + 48)); - vst2q_u32((uint32_t *)dst_ptr, dst04_line); // (AF~HF)(AE~HE)(AD~HD)(AC~HC) - vst2q_u32((uint32_t *)(dst_ptr + 32), dst15_line); // (AB~HB)(AA~HA)(A9~H8)(A0~H8) - vst2q_u32((uint32_t *)(dst_ptr + 64), dst26_line); // (A7~H7)(A6~H6)(A5~H5)(A4~H4) - vst2q_u32((uint32_t *)(dst_ptr + 96), dst37_line); // (A3~H3)(A2~H2)(A1~H1)(A0~H0) + vst2q_u32((uint32_t *)dstPtr, dstLine04); + vst2q_u32((uint32_t *)(dstPtr + 32), dstLine15); + vst2q_u32((uint32_t *)(dstPtr + 64), dstLine26); + vst2q_u32((uint32_t *)(dstPtr + 96), dstLine37); /* store into dst */ - *((uint64 *)(dst[0])) = *((uint64 *)(dst_ptr + 0 * 8)); - *((uint64 *)(dst[1])) = *((uint64 *)(dst_ptr + 1 * 8)); - *((uint64 *)(dst[2])) = *((uint64 *)(dst_ptr + 2 * 8)); - *((uint64 *)(dst[3])) = *((uint64 *)(dst_ptr + 3 * 8)); - *((uint64 *)(dst[4])) = *((uint64 *)(dst_ptr + 4 * 8)); - *((uint64 *)(dst[5])) = *((uint64 *)(dst_ptr + 5 * 8)); - *((uint64 *)(dst[6])) = *((uint64 *)(dst_ptr + 6 * 8)); - *((uint64 *)(dst[7])) = *((uint64 *)(dst_ptr + 7 * 8)); - *((uint64 *)(dst[8])) = *((uint64 *)(dst_ptr + 8 * 8)); - *((uint64 *)(dst[9])) = *((uint64 *)(dst_ptr + 9 * 8)); - *((uint64 *)(dst[10])) = *((uint64 *)(dst_ptr + 10 * 8)); - *((uint64 *)(dst[11])) = *((uint64 *)(dst_ptr + 11 * 8)); - *((uint64 *)(dst[12])) = *((uint64 *)(dst_ptr + 12 * 8)); - *((uint64 *)(dst[13])) = *((uint64 *)(dst_ptr + 13 * 8)); - *((uint64 *)(dst[14])) = *((uint64 *)(dst_ptr + 14 * 8)); - *((uint64 *)(dst[15])) = *((uint64 *)(dst_ptr + 15 * 8)); + *((uint64 *)(dst[0])) = *((uint64 *)(dstPtr + 0 * 8)); + *((uint64 *)(dst[1])) = *((uint64 *)(dstPtr + 1 * 8)); + *((uint64 *)(dst[2])) = *((uint64 *)(dstPtr + 2 * 8)); + *((uint64 *)(dst[3])) = *((uint64 *)(dstPtr + 3 * 8)); + *((uint64 *)(dst[4])) = *((uint64 *)(dstPtr + 4 * 8)); + *((uint64 *)(dst[5])) = *((uint64 *)(dstPtr + 5 * 8)); + *((uint64 *)(dst[6])) = *((uint64 *)(dstPtr + 6 * 8)); + *((uint64 *)(dst[7])) = *((uint64 *)(dstPtr + 7 * 8)); + *((uint64 *)(dst[8])) = *((uint64 *)(dstPtr + 8 * 8)); + *((uint64 *)(dst[9])) = *((uint64 *)(dstPtr + 9 * 8)); + *((uint64 *)(dst[10])) = *((uint64 *)(dstPtr + 10 * 8)); + *((uint64 *)(dst[11])) = *((uint64 *)(dstPtr + 11 * 8)); + *((uint64 *)(dst[12])) = *((uint64 *)(dstPtr + 12 * 8)); + *((uint64 *)(dst[13])) = *((uint64 *)(dstPtr + 13 * 8)); + *((uint64 *)(dst[14])) = *((uint64 *)(dstPtr + 14 * 8)); + *((uint64 *)(dst[15])) = *((uint64 *)(dstPtr + 15 * 8)); return ; } -void Transpose8x8_8U(uint8 **src, uint8 **dst) +void Transpose8x8U(uint8 **src, uint8 **dst) { /* load and crossed-store 0~3 lines */ - uint8x8x4_t mat0123_line; - uint8x8x4_t mat4567_line; - uint8 S0123_ptr[32]; - uint8 S4567_ptr[32]; + uint8x8x4_t matLine4x1; + uint8x8x4_t matLine4x2; + uint8 sptr4x1[32]; + uint8 sptr4x2[32]; - mat0123_line.val[0] = vld1_u8(src[0]); // A7 A6 A5 A4 A3 A2 A1 A0 - mat0123_line.val[1] = vld1_u8(src[1]); // B7 B6 B5 B4 B3 B2 B1 B0 - mat0123_line.val[2] = vld1_u8(src[2]); // C7 C6 C5 C4 C3 C2 C1 C0 - mat0123_line.val[3] = vld1_u8(src[3]); // D7 D6 D5 D4 D3 D2 D1 D0 - vst4_u8(S0123_ptr, mat0123_line); // A7 B7 C7 D7 A6 B6 C6 D6 ... A0 B0 C0 D0 + matLine4x1.val[0] = vld1_u8(src[0]); + matLine4x1.val[1] = vld1_u8(src[1]); + matLine4x1.val[2] = vld1_u8(src[2]); + matLine4x1.val[3] = vld1_u8(src[3]); + vst4_u8(sptr4x1, matLine4x1); /* load and crossed-store 4~7 lines */ - mat4567_line.val[0] = vld1_u8(src[4]); // E7 E6 E5 E4 E3 E2 E1 E0 - mat4567_line.val[1] = vld1_u8(src[5]); // F7 F6 F5 F4 F3 F2 F1 F0 - mat4567_line.val[2] = vld1_u8(src[6]); // G7 G6 G5 G4 G3 G2 G1 G0 - mat4567_line.val[3] = vld1_u8(src[7]); // H7 H6 H5 H4 H3 H2 H1 H0 - vst4_u8(S4567_ptr, mat4567_line); // E7 F7 G7 H7 E6 F6 G6 H6 ... E0 F0 G0 H0 + matLine4x2.val[0] = vld1_u8(src[4]); + matLine4x2.val[1] = vld1_u8(src[5]); + matLine4x2.val[2] = vld1_u8(src[6]); + matLine4x2.val[3] = vld1_u8(src[7]); + vst4_u8(sptr4x2, matLine4x2); /* store into dst */ - uint32x4x2_t dst0145_line; - uint32x4x2_t dst2367_line; - uint8 dst_ptr[64]; + uint32x4x2_t dstLine4x1; + uint32x4x2_t dstLine4x2; + uint8 dstPtr[64]; - dst0145_line.val[0] = vld1q_u32((uint32_t *)S0123_ptr); // (A7~D7)(A6~D6)(A5~D5)(A4~D4) - dst0145_line.val[1] = vld1q_u32((uint32_t *)S4567_ptr); // (E7~H7)(E6~H6)(E5~H5)(E4~H4) - dst2367_line.val[0] = vld1q_u32((uint32_t *)(S0123_ptr + 16)); // (A3~D3)(A2~D2)(A1~D1)(A0~D0) - dst2367_line.val[1] = vld1q_u32((uint32_t *)(S4567_ptr + 16)); // (E3~H3)(E2~H2)(E1~H1)(E0~H0) + dstLine4x1.val[0] = vld1q_u32((uint32_t *)sptr4x1); + dstLine4x1.val[1] = vld1q_u32((uint32_t *)sptr4x2); + dstLine4x2.val[0] = vld1q_u32((uint32_t *)(sptr4x1 + 16)); + dstLine4x2.val[1] = vld1q_u32((uint32_t *)(sptr4x2 + 16)); - vst2q_u32((uint32_t *)dst_ptr, dst0145_line); // (A7~H7)(A6~H6)(A5~H5)(A4~H4) - vst2q_u32((uint32_t *)(dst_ptr + 32), dst2367_line); // (A3~H3)(A2~H2)(A1~H1)(A0~H0) + vst2q_u32((uint32_t *)dstPtr, dstLine4x1); + vst2q_u32((uint32_t *)(dstPtr + 32), dstLine4x2); - //==================== - *((uint64 *)(dst[0])) = *((uint64 *)(dst_ptr + 0 * 8)); - *((uint64 *)(dst[1])) = *((uint64 *)(dst_ptr + 1 * 8)); - *((uint64 *)(dst[2])) = *((uint64 *)(dst_ptr + 2 * 8)); - *((uint64 *)(dst[3])) = *((uint64 *)(dst_ptr + 3 * 8)); - *((uint64 *)(dst[4])) = *((uint64 *)(dst_ptr + 4 * 8)); - *((uint64 *)(dst[5])) = *((uint64 *)(dst_ptr + 5 * 8)); - *((uint64 *)(dst[6])) = *((uint64 *)(dst_ptr + 6 * 8)); - *((uint64 *)(dst[7])) = *((uint64 *)(dst_ptr + 7 * 8)); + *((uint64 *)(dst[0])) = *((uint64 *)(dstPtr + 0 * 8)); + *((uint64 *)(dst[1])) = *((uint64 *)(dstPtr + 1 * 8)); + *((uint64 *)(dst[2])) = *((uint64 *)(dstPtr + 2 * 8)); + *((uint64 *)(dst[3])) = *((uint64 *)(dstPtr + 3 * 8)); + *((uint64 *)(dst[4])) = *((uint64 *)(dstPtr + 4 * 8)); + *((uint64 *)(dst[5])) = *((uint64 *)(dstPtr + 5 * 8)); + *((uint64 *)(dst[6])) = *((uint64 *)(dstPtr + 6 * 8)); + *((uint64 *)(dst[7])) = *((uint64 *)(dstPtr + 7 * 8)); return; } @@ -455,7 +454,7 @@ static void CompressConvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 *rea aux_buf + (x + 2) * h + y, aux_buf + (x + 3) * h + y, aux_buf + (x + 4) * h + y, aux_buf + (x + 5) * h + y, aux_buf + (x + 6) * h + y, aux_buf + (x + 7) * h + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(y = sh; y < h; y++) { @@ -498,7 +497,7 @@ static void CompressConvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 *rea aux_buf + (x + 10) * h + y, aux_buf + (x +11) * h + y, aux_buf + (x + 12) * h + y, aux_buf + (x +13) * h + y, aux_buf + (x + 14) * h + y, aux_buf + (x +15) * h + y}; - Transpose8x16_8U(src, dst); + Transpose8x16U(src, dst); } if ((x + 8) <= sw) { uint8 *src[8] = {buf + line0_off + x, buf + line1_off + x, buf + line2_off + x, buf + line3_off + x, @@ -506,7 +505,7 @@ static void CompressConvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 *rea uint8 *dst[8] = {aux_buf + x * h + y, aux_buf + (x + 1) * h + y, aux_buf + (x + 2) * h + y, aux_buf + (x + 3) * h + y, aux_buf + (x + 4) * h + y, aux_buf + (x + 5) * h + y, aux_buf + (x + 6) * h + y, aux_buf + (x + 7) * h + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } @@ -607,7 +606,7 @@ static void CompressConvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 *real aux_buf + (x + 2) * h + y, aux_buf + (x + 3) * h + y, aux_buf + (x + 4) * h + y, aux_buf + (x + 5) * h + y, aux_buf + (x + 6) * h + y, aux_buf + (x + 7) * h + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(y = sh; y < h; y++) { @@ -650,7 +649,7 @@ static void CompressConvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 *real aux_buf + (x + 10) * h + y, aux_buf + (x + 11) * h + y, aux_buf + (x + 12) * h + y, aux_buf + (x + 13) * h + y, aux_buf + (x + 14) * h + y, aux_buf + (x + 15) * h + y}; - Transpose8x16_8U(src, dst); + Transpose8x16U(src, dst); } if ((x + 8) <= sw) { @@ -660,7 +659,7 @@ static void CompressConvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 *real aux_buf + (x + 2) * h + y, aux_buf + (x + 3) * h + y, aux_buf + (x + 4) * h + y, aux_buf + (x + 5) * h + y, aux_buf + (x + 6) * h + y, aux_buf + (x + 7) * h + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } @@ -1615,7 +1614,7 @@ static void DecompressDeconvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 aux_buf + line2_off + y, aux_buf + line3_off + y, aux_buf + line4_off + y, aux_buf + line5_off + y, aux_buf + line6_off + y, aux_buf + line7_off + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(x = 0;x < sw; x++) { @@ -1666,7 +1665,7 @@ static void DecompressDeconvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 aux_buf + line10_off + y, aux_buf + line11_off + y, aux_buf + line12_off + y, aux_buf + line13_off + y, aux_buf + line14_off + y, aux_buf + line15_off + y}; - Transpose8x16_8U(src, dst); + Transpose8x16U(src, dst); } } if ((x + 8) <= sw) { @@ -1687,7 +1686,7 @@ static void DecompressDeconvertIndexKeysPart1(uint8 *buf, uint8 *aux_buf, int16 aux_buf + line2_off + y, aux_buf + line3_off + y, aux_buf + line4_off + y, aux_buf + line5_off + y, aux_buf + line6_off + y, aux_buf + line7_off + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(x = 0; x < sw; x++) { @@ -1793,7 +1792,7 @@ static void DecompressDeconvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 * aux_buf + line2_off + y, aux_buf + line3_off + y, aux_buf + line4_off + y, aux_buf + line5_off + y, aux_buf + line6_off + y, aux_buf + line7_off + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(x = 0; x < sw; x++) { @@ -1842,7 +1841,7 @@ static void DecompressDeconvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 * aux_buf + line7_off + y, aux_buf + line8_off + y, aux_buf + line9_off + y, aux_buf + line10_off + y, aux_buf + line11_off + y, aux_buf + line12_off + y, aux_buf + line13_off + y, aux_buf + line14_off + y, aux_buf + line15_off + y}; - Transpose8x16_8U(src, dst); + Transpose8x16U(src, dst); } } if ((x + 8) <= sw) { @@ -1863,7 +1862,7 @@ static void DecompressDeconvertHeapRowsPart1(uint8 *buf, uint8 *aux_buf, int16 * uint8 *dst[8] = {aux_buf + line0_off + y, aux_buf + line1_off + y, aux_buf + line2_off + y, aux_buf + line3_off + y, aux_buf + line4_off + y, aux_buf + line5_off + y, aux_buf + line6_off + y, aux_buf + line7_off + y}; - Transpose8x8_8U(src, dst); + Transpose8x8U(src, dst); } } for(x = 0; x < sw; x++) { diff --git a/src/lib/page_compression/PageCompression.cpp b/src/lib/page_compression/PageCompression.cpp index 4e2135f6d..ec02fbb7b 100644 --- a/src/lib/page_compression/PageCompression.cpp +++ b/src/lib/page_compression/PageCompression.cpp @@ -19,7 +19,8 @@ */ static bool ReadRewindCompressedInfo(FILE *file, RewindCompressInfo *rewindCompressInfo) { - auto result = ReadBlockNumberOfCFile(file); + off_t curFileLen = 0; + auto result = ReadBlockNumberOfCFile(file, &curFileLen); if (result >= MIN_COMPRESS_ERROR_RT) { return false; } @@ -32,14 +33,22 @@ bool FetchSourcePca(unsigned char *header, size_t len, RewindCompressInfo *rewin { CfsExtentHeader *ptr = (CfsExtentHeader *)(void *)header; rewindCompressInfo->compressed = false; - if (len == sizeof(CfsExtentHeader)) { - rewindCompressInfo->compressed = true; + if (len != sizeof(CfsExtentHeader)) { + return rewindCompressInfo->compressed; + } + + rewindCompressInfo->compressed = true; + if (fileSize == 0) { + rewindCompressInfo->newBlockNumber = 0; + rewindCompressInfo->oldBlockNumber = 0; + } else { BlockNumber fileBlockNum = (BlockNumber) fileSize / BLCKSZ; BlockNumber extentCount = fileBlockNum / CFS_EXTENT_SIZE; BlockNumber result = (extentCount - 1) * (CFS_EXTENT_SIZE - 1); rewindCompressInfo->newBlockNumber = result + ptr->nblocks; rewindCompressInfo->oldBlockNumber = 0; } + return rewindCompressInfo->compressed; } @@ -99,9 +108,15 @@ size_t PageCompression::ReadCompressedBuffer(BlockNumber blockNum, char *buffer, CfsExtentHeader* PageCompression::GetHeaderByExtentNumber(BlockNumber extentCount, CfsCompressOption *option) { - if (this->cfsHeaderMap.extentCount != extentCount || this->cfsHeaderMap.header == nullptr) { + if (this->cfsHeaderMap.extentCount != extentCount || + this->cfsHeaderMap.header == nullptr) { MmapFree(&(cfsHeaderMap)); - if (option) { + bool needExtend = (this->extentIdx == InvalidBlockNumber || extentCount > this->extentIdx); + /* need extend one extent */ + if (needExtend) { + if (!option) { + return nullptr; + } auto extentOffset = ((extentCount + 1) * CFS_EXTENT_SIZE - 1) * BLCKSZ; if (fallocate(fileno(fd), 0, extentOffset, BLCKSZ) < 0) { return nullptr; @@ -111,9 +126,13 @@ CfsExtentHeader* PageCompression::GetHeaderByExtentNumber(BlockNumber extentCoun if (fallocate(fileno(fd), FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, extentStart, allocateSize) < 0) { return nullptr; } + this->extentIdx = extentCount; } auto curHeader = MMapHeader(this->fd, extentCount); - if (option) { + if (curHeader.header == MAP_FAILED) { + return nullptr; + } + if (needExtend) { curHeader.header->chunk_size = option->chunk_size; curHeader.header->algorithm = option->algorithm; } @@ -128,7 +147,7 @@ CfsExtentHeader* PageCompression::GetHeaderByExtentNumber(BlockNumber extentCoun CfsExtentHeader* PageCompression::GetStruct(BlockNumber blockNum, CfsCompressOption *option) { - return PageCompression::GetHeaderByExtentNumber(blockNum / CFS_LOGIC_BLOCKS_PER_EXTENT, option); + return GetHeaderByExtentNumber(blockNum / CFS_LOGIC_BLOCKS_PER_EXTENT, option); } PageCompression::~PageCompression() @@ -136,6 +155,11 @@ PageCompression::~PageCompression() if (this->fd) { (void)fclose(this->fd); } + + if (this->cfsHeaderMap.pointer != nullptr) { + (void)munmap(this->cfsHeaderMap.pointer, this->cfsHeaderMap.mmapLen); + this->cfsHeaderMap.pointer = nullptr; + } } const char *PageCompression::GetInitPath() const @@ -159,10 +183,14 @@ COMPRESS_ERROR_STATE PageCompression::Init(const char *filePath, BlockNumber inS this->blockNumber = 0; this->algorithm = 0; this->chunkSize = 0; + this->extentIdx = InvalidBlockNumber; errno_t rc = memset_s(&this->cfsHeaderMap, sizeof(CfsHeaderMap), 0, sizeof(CfsHeaderMap)); securec_check_ss_c(rc, "\0", "\0"); + rc = snprintf_s(initPath, MAXPGPATH, MAXPGPATH - 1, "%s", filePath); + securec_check_ss_c(rc, "\0", "\0"); + if (create) { return SUCCESS; } @@ -173,11 +201,17 @@ COMPRESS_ERROR_STATE PageCompression::Init(const char *filePath, BlockNumber inS return PCA_SEEK_ERROR; } off_t fileLen = ftell(this->fd); + if (fileLen % CFS_EXTENT_SIZE != 0) { + (void)fclose(file); + return NORMAL_READ_ERROR; + } if (fileLen > 0) { BlockNumber fileBlockNum = (BlockNumber) fileLen / BLCKSZ; BlockNumber extentCount = fileBlockNum / CFS_EXTENT_SIZE; BlockNumber result = (extentCount - 1) * (CFS_EXTENT_SIZE - 1); + this->extentIdx = extentCount - 1; + /* read header of last extent */ auto header = this->GetHeaderByExtentNumber(extentCount - 1); if (header == nullptr) { @@ -225,10 +259,14 @@ bool PageCompression::WriteBufferToCurrentBlock(char *buf, BlockNumber blkNumber securec_check(rc, "", ""); } size = realSize; - BlockNumber extentOffset = blkNumber % CFS_LOGIC_BLOCKS_PER_EXTENT; + BlockNumber logicBlockNumber = blkNumber % CFS_LOGIC_BLOCKS_PER_EXTENT; + BlockNumber extentOffset = (blkNumber / CFS_LOGIC_BLOCKS_PER_EXTENT) % CFS_EXTENT_COUNT_PER_FILE; int needChunks = size / (int32)chkSize; + if (logicBlockNumber >= cfsExtentHeader->nblocks) { + cfsExtentHeader->nblocks = logicBlockNumber + 1; + } - CfsExtentAddress *cfsExtentAddress = GetExtentAddress(cfsExtentHeader, (uint16)extentOffset); + CfsExtentAddress *cfsExtentAddress = GetExtentAddress(cfsExtentHeader, (uint16)logicBlockNumber); /* allocate chunks */ if (cfsExtentAddress->allocated_chunks < needChunks) { @@ -244,7 +282,8 @@ bool PageCompression::WriteBufferToCurrentBlock(char *buf, BlockNumber blkNumber for (int32 i = 0; i < needChunks; ++i) { char *buffer_pos = buf + (long)chkSize * i; - off_t seekPos = OffsetOfPageCompressChunk(chkSize, cfsExtentAddress->chunknos[i]); + off_t seekPos = OffsetOfPageCompressChunk(chkSize, cfsExtentAddress->chunknos[i]) + + extentOffset * CFS_EXTENT_SIZE * BLCKSZ; int32 start = i; /* merge continuous write */ while (i < needChunks - 1 && cfsExtentAddress->chunknos[i + 1] == cfsExtentAddress->chunknos[i] + 1) { diff --git a/src/lib/page_compression/PageCompression.h b/src/lib/page_compression/PageCompression.h index a8ede0839..475b356f4 100644 --- a/src/lib/page_compression/PageCompression.h +++ b/src/lib/page_compression/PageCompression.h @@ -79,6 +79,7 @@ public: private: char initPath[MAXPGPATH]; BlockNumber blockNumber; + BlockNumber extentIdx; CfsHeaderMap cfsHeaderMap; BlockNumber segmentNo; CfsExtentHeader* GetStruct(BlockNumber blockNum, CfsCompressOption *option = nullptr);