database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[SQL][Function] Fix from/to_base64 may return incorrect value (#4183)

Browse Source 
from/to_base64 may return incorrect value when the value is null #4130 
remove the duplicated base64 code
fix the base64 encoded string length is wrong， and this will cause the memory error
This commit is contained in:
Zhengguo Yang
2020-07-27 22:55:05 +08:00
committed by GitHub
parent 9e5ca697f3
commit 50e6a2c8a0
11 changed files with 100 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
be/src/exprs/CMakeLists.txt
View File

@ -22,7 +22,6 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exprs")
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exprs")
add_library(Exprs
base64.cpp
encryption_functions.cpp
aggregate_functions.cpp
agg_fn_evaluator.cpp

169
be/src/exprs/base64.cpp
View File

@ -1,169 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "base64.h"
#include <math.h>
#include <stdint.h>
#include <string>
static char s_encoding_table[] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '0', '1', '2', '3',
'4', '5', '6', '7', '8', '9', '+', '/'
};
static const char base64_pad = '=';
static short s_decoding_table[256] = {
-2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -2, -2, -1, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, 62, -2, -2, -2, 63,
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -2, -2, -2, -2, -2, -2,
-2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -2, -2, -2, -2, -2,
-2, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2,
-2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2
};
static int s_mod_table[] = {0, 2, 1};
namespace doris {
size_t base64_encode2(const unsigned char *data,
size_t length,
unsigned char *encoded_data) {
size_t output_length = (size_t) (4.0 * ceil((double) length / 3.0));
if (encoded_data == NULL) {
return 0;
}
for (uint32_t i = 0, j = 0; i < length;) {
uint32_t octet_a = i < length ? data[i++] : 0;
uint32_t octet_b = i < length ? data[i++] : 0;
uint32_t octet_c = i < length ? data[i++] : 0;
uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c;
encoded_data[j++] = s_encoding_table[(triple >> 3 * 6) & 0x3F];
encoded_data[j++] = s_encoding_table[(triple >> 2 * 6) & 0x3F];
encoded_data[j++] = s_encoding_table[(triple >> 1 * 6) & 0x3F];
encoded_data[j++] = s_encoding_table[(triple >> 0 * 6) & 0x3F];
}
for (int i = 0; i < s_mod_table[length % 3]; i++) {
encoded_data[output_length - 1 - i] = '=';
}
return output_length;
}
int64_t base64_decode2(
const char *data,
size_t length,
char *decoded_data) {
const char *current = data;
int ch = 0;
int i = 0;
int j = 0;
int k = 0;
// run through the whole string, converting as we go
while ((ch = *current++) != '\0' && length-- > 0) {
if (ch == base64_pad) {
if (*current != '=' && (i % 4) == 1) {
return -1;
}
continue;
}
ch = s_decoding_table[ch];
// a space or some other separator character, we simply skip over
if (ch == -1) {
continue;
} else if (ch == -2) {
return -1;
}
switch (i % 4) {
case 0:
decoded_data[j] = ch << 2;
break;
case 1:
decoded_data[j++] |= ch >> 4;
decoded_data[j] = (ch & 0x0f) << 4;
break;
case 2:
decoded_data[j++] |= ch >>2;
decoded_data[j] = (ch & 0x03) << 6;
break;
case 3:
decoded_data[j++] |= ch;
break;
default:
break;
}
i++;
}
k = j;
/* mop things up if we ended on a boundary */
if (ch == base64_pad) {
switch (i % 4) {
case 1:
return 0;
case 2:
k++;
case 3:
decoded_data[k] = 0;
default:
break;
}
}
decoded_data[j] = '\0';
return j;
}
/*bool base64_decode2(const std::string& in, std::string* out) {
char* tmp = new char[in.length()];
int64_t len = base64_decode(in.c_str(), in.length(), tmp);
if (len < 0) {
delete[] tmp;
return false;
}
out->assign(tmp, len);
delete[] tmp;
return true;
}
*/
}

36
be/src/exprs/base64.h
View File

@ -1,36 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#ifndef DORIS_BE_EXPRS_BASE64_H
#define DORIS_BE_EXPRS_BASE64_H
#include <stdio.h>
#include <stdint.h>
namespace doris {
int64_t base64_decode2(
const char *data,
size_t length,
char *decoded_data);
size_t base64_encode2(const unsigned char *data,
size_t length,
unsigned char *encoded_data);
}
#endif

12
be/src/exprs/encryption_functions.cpp
View File

@ -23,7 +23,7 @@
#include "exprs/expr.h"
#include "util/debug_util.h"
#include "runtime/tuple_row.h"
#include "exprs/base64.h"
#include "util/url_coding.h"
#include <boost/smart_ptr.hpp>
#include "runtime/string_value.h"
@ -69,7 +69,7 @@ StringVal EncryptionFunctions::aes_decrypt(FunctionContext* ctx,
}
StringVal EncryptionFunctions::from_base64(FunctionContext* ctx, const StringVal &src) {
if (src.len == 0) {
if (src.len == 0 || src.is_null) {
return StringVal::null();
}
@ -77,7 +77,7 @@ StringVal EncryptionFunctions::from_base64(FunctionContext* ctx, const StringVal
boost::scoped_array<char> p;
p.reset(new char[cipher_len]);
int ret_code = base64_decode2((const char *)src.ptr, src.len, p.get());
int ret_code = base64_decode((const char *)src.ptr, src.len, p.get());
if (ret_code < 0) {
return StringVal::null();
}
@ -85,15 +85,15 @@ StringVal EncryptionFunctions::from_base64(FunctionContext* ctx, const StringVal
}
StringVal EncryptionFunctions::to_base64(FunctionContext* ctx, const StringVal &src) {
if (src.len == 0) {
if (src.len == 0 || src.is_null) {
return StringVal::null();
}
int cipher_len = src.len * 4 / 3 + 1;
int cipher_len = (size_t) (4.0 * ceil((double) src.len / 3.0));
boost::scoped_array<char> p;
p.reset(new char[cipher_len]);
int ret_code = base64_encode2((unsigned char *)src.ptr, src.len, (unsigned char *)p.get());
int ret_code = base64_encode((unsigned char *)src.ptr, src.len, (unsigned char *)p.get());
if (ret_code < 0) {
return StringVal::null();
}