[fix](compress) snappy does not work right (#20934)
This commit is contained in:
@ -194,6 +194,12 @@ public:
|
||||
_num_elems = decode_fixed32_le((const uint8_t*)&_data[_data.get_size() - sizeof(uint32_t)]);
|
||||
_offsets_pos = _data.get_size() - (_num_elems + 1) * sizeof(uint32_t);
|
||||
|
||||
if (_offsets_pos > _data.get_size() - sizeof(uint32_t)) {
|
||||
return Status::Corruption(
|
||||
"file corruption: offsets pos beyonds data_size: {}, num_element: {}"
|
||||
", offset_pos: {}",
|
||||
_data.size, _num_elems, _offsets_pos);
|
||||
}
|
||||
_parsed = true;
|
||||
|
||||
return Status::OK();
|
||||
|
||||
@ -444,7 +444,7 @@ public:
|
||||
// we should assure that *len is not 0
|
||||
*len = _slices[_cur_slice].size - _slice_off;
|
||||
DCHECK(*len != 0);
|
||||
return _slices[_cur_slice].data;
|
||||
return _slices[_cur_slice].data + _slice_off;
|
||||
}
|
||||
|
||||
// Skip the next n bytes. Invalidates any buffer returned by
|
||||
|
||||
82845
regression-test/data/compress_p1/snappy/ipv4.csv
Normal file
82845
regression-test/data/compress_p1/snappy/ipv4.csv
Normal file
File diff suppressed because it is too large
Load Diff
45
regression-test/suites/compress_p1/snappy/load.groovy
Normal file
45
regression-test/suites/compress_p1/snappy/load.groovy
Normal file
@ -0,0 +1,45 @@
|
||||
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
suite("test_snappy", "p1") {
|
||||
def tableName = "test_snappy"
|
||||
|
||||
// create table
|
||||
sql """ DROP TABLE IF EXISTS ${tableName} """
|
||||
sql """
|
||||
CREATE TABLE IF NOT EXISTS ${tableName} (
|
||||
`k1` varchar(40) NULL
|
||||
) ENGINE=OLAP
|
||||
DUPLICATE KEY(`k1`)
|
||||
COMMENT 'OLAP'
|
||||
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
|
||||
PROPERTIES ("replication_allocation" = "tag.location.default: 1",
|
||||
"compression" = "snappy");
|
||||
"""
|
||||
|
||||
// skip 3 lines and file have 4 lines
|
||||
streamLoad {
|
||||
table "${tableName}"
|
||||
|
||||
file 'ipv4.csv'
|
||||
}
|
||||
|
||||
sql "sync"
|
||||
def count = sql "select count(*) from ${tableName} limit 10"
|
||||
assertEquals(82845, count[0][0])
|
||||
}
|
||||
Reference in New Issue
Block a user