[fix](compress) snappy does not work right (#20934)

This commit is contained in:
Yongqiang YANG
2023-06-19 14:11:10 +08:00
committed by GitHub
parent 26cca5e00a
commit dd5ecea36a
4 changed files with 82897 additions and 1 deletions

View File

@ -194,6 +194,12 @@ public:
_num_elems = decode_fixed32_le((const uint8_t*)&_data[_data.get_size() - sizeof(uint32_t)]);
_offsets_pos = _data.get_size() - (_num_elems + 1) * sizeof(uint32_t);
if (_offsets_pos > _data.get_size() - sizeof(uint32_t)) {
return Status::Corruption(
"file corruption: offsets pos beyonds data_size: {}, num_element: {}"
", offset_pos: {}",
_data.size, _num_elems, _offsets_pos);
}
_parsed = true;
return Status::OK();

View File

@ -444,7 +444,7 @@ public:
// we should assure that *len is not 0
*len = _slices[_cur_slice].size - _slice_off;
DCHECK(*len != 0);
return _slices[_cur_slice].data;
return _slices[_cur_slice].data + _slice_off;
}
// Skip the next n bytes. Invalidates any buffer returned by

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_snappy", "p1") {
def tableName = "test_snappy"
// create table
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName} (
`k1` varchar(40) NULL
) ENGINE=OLAP
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 1
PROPERTIES ("replication_allocation" = "tag.location.default: 1",
"compression" = "snappy");
"""
// skip 3 lines and file have 4 lines
streamLoad {
table "${tableName}"
file 'ipv4.csv'
}
sql "sync"
def count = sql "select count(*) from ${tableName} limit 10"
assertEquals(82845, count[0][0])
}