From 4f9562650da561918dbf3308a5fa950cc6ae34e1 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Sat, 18 Jan 2025 09:12:39 +0800 Subject: [PATCH] Branch-2.1 [Fix](Variant) fix variant serialize to string (#47121) (#47147) cherry-pick from #47121 --- be/src/vec/columns/column_object.cpp | 34 ++++++++++++++++--- .../load.out | 23 +++++++++++++ regression-test/data/variant_p0/agg.out | 4 +-- .../variant_p0/compaction/test_compaction.out | 8 ++--- .../suites/variant_log_data_p2/load.groovy | 6 ++-- regression-test/suites/variant_p0/agg.groovy | 8 ++--- .../suites/variant_p0/delete_update.groovy | 2 +- regression-test/suites/variant_p0/desc.groovy | 2 +- regression-test/suites/variant_p0/load.groovy | 4 +-- regression-test/suites/variant_p2/load.groovy | 2 +- 10 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 regression-test/data/variant_github_events_nonConcurrent_p2/load.out diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index 47dbb14d37..bad93600bb 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -1270,20 +1270,46 @@ Status ColumnObject::serialize_one_row_to_json_format(int row, rapidjson::String root.CopyFrom(*doc_structure, doc_structure->GetAllocator()); } Arena mem_pool; + + bool serialize_root = true; // Assume all subcolumns are null by default + for (const auto& subcolumn : subcolumns) { + if (subcolumn->data.is_root) { + continue; // Skip the root column + } + + // If any non-root subcolumn is NOT null, set serialize_root to false and exit early + if (!assert_cast(*subcolumn->data.get_finalized_column_ptr()) + .is_null_at(row)) { + serialize_root = false; + break; + } + } #ifndef NDEBUG VLOG_DEBUG << "dump structure " << JsonFunctions::print_json_value(*doc_structure); #endif + if (serialize_root && !is_null_root()) { + // only serialize root when all other subcolumns is null + RETURN_IF_ERROR( + subcolumns.get_root()->data.get_least_common_type_serde()->write_one_cell_to_json( + subcolumns.get_root()->data.get_finalized_column(), root, + doc_structure->GetAllocator(), mem_pool, row)); + output->Clear(); + compact_null_values(root, doc_structure->GetAllocator()); + rapidjson::Writer writer(*output); + root.Accept(writer); + return Status::OK(); + } + // handle subcolumns exclude root node for (const auto& subcolumn : subcolumns) { + if (subcolumn->data.is_root) { + continue; + } RETURN_IF_ERROR(find_and_set_leave_value( subcolumn->data.get_finalized_column_ptr(), subcolumn->path, subcolumn->data.get_least_common_type_serde(), subcolumn->data.get_least_common_type(), subcolumn->data.least_common_type.get_base_type_id(), root, doc_structure->GetAllocator(), mem_pool, row)); - if (subcolumn->path.empty() && !root.IsObject()) { - // root was modified, only handle root node - break; - } } compact_null_values(root, doc_structure->GetAllocator()); if (root.IsNull() && is_null != nullptr) { diff --git a/regression-test/data/variant_github_events_nonConcurrent_p2/load.out b/regression-test/data/variant_github_events_nonConcurrent_p2/load.out new file mode 100644 index 0000000000..4bee99a71a --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_p2/load.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +\N +\N +\N +4748 + +-- !sql -- +864 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/4381756?","gravatar_id":"","id":4381756,"login":"qw5414","url":"https://api.github.com/users/qw5414"},"created_at":"2015-01-01T02:06:41Z","id":"2489420466","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"created","comment":{"body":"乃\\r\\n","created_at":"2015-01-01T02:06:40Z","html_url":"https://github.com/xpressengine/xe-core/pull/1120#issuecomment-68478485","id":68478485,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120","updated_at":"2015-01-01T02:06:40Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/68478485","user":{"avatar_url":"https://avatars.githubusercontent.com/u/4381756?v=3","events_url":"https://api.github.com/users/qw5414/events{/privacy}","followers_url":"https://api.github.com/users/qw5414/followers","following_url":"https://api.github.com/users/qw5414/following{/other_user}","gists_url":"https://api.github.com/users/qw5414/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/qw5414","id":4381756,"login":"qw5414","organizations_url":"https://api.github.com/users/qw5414/orgs","received_events_url":"https://api.github.com/users/qw5414/received_events","repos_url":"https://api.github.com/users/qw5414/repos","site_admin":0,"starred_url":"https://api.github.com/users/qw5414/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qw5414/subscriptions","type":"User","url":"https://api.github.com/users/qw5414"}},"issue":{"body":"https://github.com/xpressengine/xe-core/issues/634\\r\\n\\r\\n# Internet Explorer send punycode URL(ASCII) URL and non-alphabet\\r\\nUnicode URL URL as a referer. 인터넷 익스플로러는 리퍼러 주소로 퓨니코드 주소와 유니코드 URL을 섞어\\r\\n쓰고 있습니다. AJAX 통신에는 리퍼러로 Unicode를 사용하고 요청 호스트로는 퓨니코드 URL을 사용(이건 다국어 주소\\r\\n형식으로 접속하려면 이렇게 했어야 할 것)합니다.\\r\\n- XE strictly compare referer and server host for denying CSRF, but\\r\\npunycode URL and Unicode URL should be dealt as a same one. 그런데 XE는 리퍼러의\\r\\n호스트와 서버 호스트를 비교합니다. punycode로 쓰인 주소와 Unicode로 쓰인 주소는 같은 주소를 지시하더라도 문자열이\\r\\n다릅니다. 같은 주소를 지칭하는 다른 문자열을 punycode로 변환해서 같은 주소라고 인식할 수 있게 수정했습니다.\\r\\n- Fix checkCSRF function to deal both form as a same one.\\r\\n- Convert Unicode URL input to punycode URL on the Admin Default URL\\r\\nSettings. 관리자가 유니코드 형식으로 기본 주소를 입력하더라도, 퓨니코드로 변환해 저장하도록 했습니다. 퓨니코드로 저장하는\\r\\n것이 여러모로 유용하기 때문입니다.\\r\\n- For converting punycode URL, include IDNA coverting class. 퓨니코드와 유니코드\\r\\n간 변환을 위해서 IDNA 변환 클래스(LGPL사용권)를 포함시켰습니다.\\r\\n\\r\\n**이 수정을 하면 *한글 도메인에서 글 작성이 가능*합니다. 하지만, *파일 업로드의 경우는 SWF Uploader 이슈로 파일 업로드가 불가능*합니다. 이 문제는, HTML5를 지원하는 파일 업로더를 이용하면 해결됩니다. (워드 프레스 등의 해결법) HTML5를 지원하는 파일 업로더는 AXISJ 업로더 등을 포함해서 XE 공식 홈페이지 자료실에서 다운받아 사용할 수 있습니다.(에디터 변경)**","comments":1,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments","created_at":"2015-01-01T01:47:43Z","events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/events","html_url":"https://github.com/xpressengine/xe-core/pull/1120","id":53211000,"labels_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/labels{/name}","locked":0,"number":1120,"pull_request":{"diff_url":"https://github.com/xpressengine/xe-core/pull/1120.diff","html_url":"https://github.com/xpressengine/xe-core/pull/1120","patch_url":"https://github.com/xpressengine/xe-core/pull/1120.patch","url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120"},"state":"open","title":"fix for Not-Alphabet URL document writing (#634)","updated_at":"2015-01-01T02:06:40Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"IssueCommentEvent"} +4842 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?","gravatar_id":"","id":53764,"login":"bnu","url":"https://api.github.com/users/bnu"},"created_at":"2015-01-01T03:47:41Z","id":"2489448854","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"created","comment":{"body":"@misol 고르기가 어렵네요.\\r\\nplup가 좋은데 GPL이네요^^;","created_at":"2015-01-01T03:47:41Z","html_url":"https://github.com/xpressengine/xe-core/issues/1086#issuecomment-68479786","id":68479786,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","updated_at":"2015-01-01T03:47:41Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/68479786","user":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?v=3","events_url":"https://api.github.com/users/bnu/events{/privacy}","followers_url":"https://api.github.com/users/bnu/followers","following_url":"https://api.github.com/users/bnu/following{/other_user}","gists_url":"https://api.github.com/users/bnu/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/bnu","id":53764,"login":"bnu","organizations_url":"https://api.github.com/users/bnu/orgs","received_events_url":"https://api.github.com/users/bnu/received_events","repos_url":"https://api.github.com/users/bnu/repos","site_admin":0,"starred_url":"https://api.github.com/users/bnu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bnu/subscriptions","type":"User","url":"https://api.github.com/users/bnu"}},"issue":{"comments":5,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/comments","created_at":"2014-12-12T11:48:03Z","events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/events","html_url":"https://github.com/xpressengine/xe-core/issues/1086","id":51797879,"labels_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/labels{/name}","locked":0,"number":1086,"state":"open","title":"파일 업로더 교체","updated_at":"2015-01-01T03:47:41Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","user":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?v=3","events_url":"https://api.github.com/users/bnu/events{/privacy}","followers_url":"https://api.github.com/users/bnu/followers","following_url":"https://api.github.com/users/bnu/following{/other_user}","gists_url":"https://api.github.com/users/bnu/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/bnu","id":53764,"login":"bnu","organizations_url":"https://api.github.com/users/bnu/orgs","received_events_url":"https://api.github.com/users/bnu/received_events","repos_url":"https://api.github.com/users/bnu/repos","site_admin":0,"starred_url":"https://api.github.com/users/bnu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bnu/subscriptions","type":"User","url":"https://api.github.com/users/bnu"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"IssueCommentEvent"} +5451 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T02:48:28Z","id":"2489433218","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"created","comment":{"body":"Html5 도 같이 지원하는 업로더였으면 좋겠어요! 구글링 해보면 꽤 나와요 :)","created_at":"2015-01-01T02:48:27Z","html_url":"https://github.com/xpressengine/xe-core/issues/1086#issuecomment-68479093","id":68479093,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/68479093","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"issue":{"comments":4,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/comments","created_at":"2014-12-12T11:48:03Z","events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/events","html_url":"https://github.com/xpressengine/xe-core/issues/1086","id":51797879,"labels_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086/labels{/name}","locked":0,"number":1086,"state":"open","title":"파일 업로더 교체","updated_at":"2015-01-01T02:48:27Z","url":"https://api.github.com/repos/xpressengine/xe-core/issues/1086","user":{"avatar_url":"https://avatars.githubusercontent.com/u/53764?v=3","events_url":"https://api.github.com/users/bnu/events{/privacy}","followers_url":"https://api.github.com/users/bnu/followers","following_url":"https://api.github.com/users/bnu/following{/other_user}","gists_url":"https://api.github.com/users/bnu/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/bnu","id":53764,"login":"bnu","organizations_url":"https://api.github.com/users/bnu/orgs","received_events_url":"https://api.github.com/users/bnu/received_events","repos_url":"https://api.github.com/users/bnu/repos","site_admin":0,"starred_url":"https://api.github.com/users/bnu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bnu/subscriptions","type":"User","url":"https://api.github.com/users/bnu"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"IssueCommentEvent"} +5995 {"actor":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?","gravatar_id":"","id":3437916,"login":"misol","url":"https://api.github.com/users/misol"},"created_at":"2015-01-01T01:47:44Z","id":"2489414108","org":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?","gravatar_id":"","id":1429259,"login":"xpressengine","url":"https://api.github.com/orgs/xpressengine"},"payload":{"action":"opened","number":1120,"pull_request":{"_links":{"comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments"},"commits":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits"},"html":{"href":"https://github.com/xpressengine/xe-core/pull/1120"},"issue":{"href":"https://api.github.com/repos/xpressengine/xe-core/issues/1120"},"review_comment":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}"},"review_comments":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments"},"self":{"href":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120"},"statuses":{"href":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654"}},"additions":4748,"base":{"label":"xpressengine:develop","ref":"develop","repo":{"archive_url":"https://api.github.com/repos/xpressengine/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/xpressengine/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/xpressengine/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/xpressengine/xe-core/branches{/branch}","clone_url":"https://github.com/xpressengine/xe-core.git","collaborators_url":"https://api.github.com/repos/xpressengine/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/xpressengine/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/xpressengine/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/xpressengine/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/xpressengine/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/xpressengine/xe-core/contributors","created_at":"2013-02-01T07:16:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/xpressengine/xe-core/downloads","events_url":"https://api.github.com/repos/xpressengine/xe-core/events","fork":0,"forks":143,"forks_count":143,"forks_url":"https://api.github.com/repos/xpressengine/xe-core/forks","full_name":"xpressengine/xe-core","git_commits_url":"https://api.github.com/repos/xpressengine/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/xpressengine/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/xpressengine/xe-core/git/tags{/sha}","git_url":"git://github.com/xpressengine/xe-core.git","has_downloads":1,"has_issues":1,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/xpressengine/xe-core/hooks","html_url":"https://github.com/xpressengine/xe-core","id":7953576,"issue_comment_url":"https://api.github.com/repos/xpressengine/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/xpressengine/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/xpressengine/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/xpressengine/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/xpressengine/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/xpressengine/xe-core/languages","merges_url":"https://api.github.com/repos/xpressengine/xe-core/merges","milestones_url":"https://api.github.com/repos/xpressengine/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/xpressengine/xe-core/notifications{?since,all,participating}","open_issues":156,"open_issues_count":156,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"},"private":0,"pulls_url":"https://api.github.com/repos/xpressengine/xe-core/pulls{/number}","pushed_at":"2014-12-31T08:52:46Z","releases_url":"https://api.github.com/repos/xpressengine/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:xpressengine/xe-core.git","stargazers_count":149,"stargazers_url":"https://api.github.com/repos/xpressengine/xe-core/stargazers","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/xpressengine/xe-core/subscribers","subscription_url":"https://api.github.com/repos/xpressengine/xe-core/subscription","svn_url":"https://github.com/xpressengine/xe-core","tags_url":"https://api.github.com/repos/xpressengine/xe-core/tags","teams_url":"https://api.github.com/repos/xpressengine/xe-core/teams","trees_url":"https://api.github.com/repos/xpressengine/xe-core/git/trees{/sha}","updated_at":"2014-12-30T00:05:52Z","url":"https://api.github.com/repos/xpressengine/xe-core","watchers":149,"watchers_count":149},"sha":"c3430d1c724f42154ca5dd648637c4df796d1708","user":{"avatar_url":"https://avatars.githubusercontent.com/u/1429259?v=3","events_url":"https://api.github.com/users/xpressengine/events{/privacy}","followers_url":"https://api.github.com/users/xpressengine/followers","following_url":"https://api.github.com/users/xpressengine/following{/other_user}","gists_url":"https://api.github.com/users/xpressengine/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/xpressengine","id":1429259,"login":"xpressengine","organizations_url":"https://api.github.com/users/xpressengine/orgs","received_events_url":"https://api.github.com/users/xpressengine/received_events","repos_url":"https://api.github.com/users/xpressengine/repos","site_admin":0,"starred_url":"https://api.github.com/users/xpressengine/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xpressengine/subscriptions","type":"Organization","url":"https://api.github.com/users/xpressengine"}},"body":"https://github.com/xpressengine/xe-core/issues/634\\r\\n\\r\\n# Internet Explorer send punycode URL(ASCII) URL and non-alphabet\\r\\nUnicode URL URL as a referer. 인터넷 익스플로러는 리퍼러 주소로 퓨니코드 주소와 유니코드 URL을 섞어\\r\\n쓰고 있습니다. AJAX 통신에는 리퍼러로 Unicode를 사용하고 요청 호스트로는 퓨니코드 URL을 사용(이건 다국어 주소\\r\\n형식으로 접속하려면 이렇게 했어야 할 것)합니다.\\r\\n- XE strictly compare referer and server host for denying CSRF, but\\r\\npunycode URL and Unicode URL should be dealt as a same one. 그런데 XE는 리퍼러의\\r\\n호스트와 서버 호스트를 비교합니다. punycode로 쓰인 주소와 Unicode로 쓰인 주소는 같은 주소를 지시하더라도 문자열이\\r\\n다릅니다. 같은 주소를 지칭하는 다른 문자열을 punycode로 변환해서 같은 주소라고 인식할 수 있게 수정했습니다.\\r\\n- Fix checkCSRF function to deal both form as a same one.\\r\\n- Convert Unicode URL input to punycode URL on the Admin Default URL\\r\\nSettings. 관리자가 유니코드 형식으로 기본 주소를 입력하더라도, 퓨니코드로 변환해 저장하도록 했습니다. 퓨니코드로 저장하는\\r\\n것이 여러모로 유용하기 때문입니다.\\r\\n- For converting punycode URL, include IDNA coverting class. 퓨니코드와 유니코드\\r\\n간 변환을 위해서 IDNA 변환 클래스(LGPL사용권)를 포함시켰습니다.\\r\\n\\r\\n**이 수정을 하면 *한글 도메인에서 글 작성이 가능*합니다. 하지만, *파일 업로드의 경우는 SWF Uploader 이슈로 파일 업로드가 불가능*합니다. 이 문제는, HTML5를 지원하는 파일 업로더를 이용하면 해결됩니다. (워드 프레스 등의 해결법) HTML5를 지원하는 파일 업로더는 AXISJ 업로더 등을 포함해서 XE 공식 홈페이지 자료실에서 다운받아 사용할 수 있습니다.(에디터 변경)**","changed_files":8,"comments":0,"comments_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120/comments","commits":1,"commits_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/commits","created_at":"2015-01-01T01:47:43Z","deletions":1,"diff_url":"https://github.com/xpressengine/xe-core/pull/1120.diff","head":{"label":"misol:support_punycode_domain","ref":"support_punycode_domain","repo":{"archive_url":"https://api.github.com/repos/misol/xe-core/{archive_format}{/ref}","assignees_url":"https://api.github.com/repos/misol/xe-core/assignees{/user}","blobs_url":"https://api.github.com/repos/misol/xe-core/git/blobs{/sha}","branches_url":"https://api.github.com/repos/misol/xe-core/branches{/branch}","clone_url":"https://github.com/misol/xe-core.git","collaborators_url":"https://api.github.com/repos/misol/xe-core/collaborators{/collaborator}","comments_url":"https://api.github.com/repos/misol/xe-core/comments{/number}","commits_url":"https://api.github.com/repos/misol/xe-core/commits{/sha}","compare_url":"https://api.github.com/repos/misol/xe-core/compare/{base}...{head}","contents_url":"https://api.github.com/repos/misol/xe-core/contents/{+path}","contributors_url":"https://api.github.com/repos/misol/xe-core/contributors","created_at":"2014-12-31T14:41:05Z","default_branch":"master","description":"PHP Open Source CMS","downloads_url":"https://api.github.com/repos/misol/xe-core/downloads","events_url":"https://api.github.com/repos/misol/xe-core/events","fork":1,"forks":0,"forks_count":0,"forks_url":"https://api.github.com/repos/misol/xe-core/forks","full_name":"misol/xe-core","git_commits_url":"https://api.github.com/repos/misol/xe-core/git/commits{/sha}","git_refs_url":"https://api.github.com/repos/misol/xe-core/git/refs{/sha}","git_tags_url":"https://api.github.com/repos/misol/xe-core/git/tags{/sha}","git_url":"git://github.com/misol/xe-core.git","has_downloads":1,"has_issues":0,"has_pages":0,"has_wiki":1,"homepage":"http://www.xpressengine.com","hooks_url":"https://api.github.com/repos/misol/xe-core/hooks","html_url":"https://github.com/misol/xe-core","id":28667946,"issue_comment_url":"https://api.github.com/repos/misol/xe-core/issues/comments/{number}","issue_events_url":"https://api.github.com/repos/misol/xe-core/issues/events{/number}","issues_url":"https://api.github.com/repos/misol/xe-core/issues{/number}","keys_url":"https://api.github.com/repos/misol/xe-core/keys{/key_id}","labels_url":"https://api.github.com/repos/misol/xe-core/labels{/name}","language":"PHP","languages_url":"https://api.github.com/repos/misol/xe-core/languages","merges_url":"https://api.github.com/repos/misol/xe-core/merges","milestones_url":"https://api.github.com/repos/misol/xe-core/milestones{/number}","name":"xe-core","notifications_url":"https://api.github.com/repos/misol/xe-core/notifications{?since,all,participating}","open_issues":0,"open_issues_count":0,"owner":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"},"private":0,"pulls_url":"https://api.github.com/repos/misol/xe-core/pulls{/number}","pushed_at":"2015-01-01T01:36:28Z","releases_url":"https://api.github.com/repos/misol/xe-core/releases{/id}","size":90250,"ssh_url":"git@github.com:misol/xe-core.git","stargazers_count":0,"stargazers_url":"https://api.github.com/repos/misol/xe-core/stargazers","statuses_url":"https://api.github.com/repos/misol/xe-core/statuses/{sha}","subscribers_url":"https://api.github.com/repos/misol/xe-core/subscribers","subscription_url":"https://api.github.com/repos/misol/xe-core/subscription","svn_url":"https://github.com/misol/xe-core","tags_url":"https://api.github.com/repos/misol/xe-core/tags","teams_url":"https://api.github.com/repos/misol/xe-core/teams","trees_url":"https://api.github.com/repos/misol/xe-core/git/trees{/sha}","updated_at":"2014-12-31T14:41:10Z","url":"https://api.github.com/repos/misol/xe-core","watchers":0,"watchers_count":0},"sha":"d2b05732abfd85020335ce272abd37c0ad1c6654","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}},"html_url":"https://github.com/xpressengine/xe-core/pull/1120","id":26739793,"issue_url":"https://api.github.com/repos/xpressengine/xe-core/issues/1120","locked":0,"mergeable_state":"unknown","merged":0,"number":1120,"patch_url":"https://github.com/xpressengine/xe-core/pull/1120.patch","review_comment_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/comments/{number}","review_comments":0,"review_comments_url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120/comments","state":"open","statuses_url":"https://api.github.com/repos/xpressengine/xe-core/statuses/d2b05732abfd85020335ce272abd37c0ad1c6654","title":"fix for Not-Alphabet URL document writing (#634)","updated_at":"2015-01-01T01:47:43Z","url":"https://api.github.com/repos/xpressengine/xe-core/pulls/1120","user":{"avatar_url":"https://avatars.githubusercontent.com/u/3437916?v=3","events_url":"https://api.github.com/users/misol/events{/privacy}","followers_url":"https://api.github.com/users/misol/followers","following_url":"https://api.github.com/users/misol/following{/other_user}","gists_url":"https://api.github.com/users/misol/gists{/gist_id}","gravatar_id":"","html_url":"https://github.com/misol","id":3437916,"login":"misol","organizations_url":"https://api.github.com/users/misol/orgs","received_events_url":"https://api.github.com/users/misol/received_events","repos_url":"https://api.github.com/users/misol/repos","site_admin":0,"starred_url":"https://api.github.com/users/misol/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/misol/subscriptions","type":"User","url":"https://api.github.com/users/misol"}}},"public":1,"repo":{"id":7953576,"name":"xpressengine/xe-core","url":"https://api.github.com/repos/xpressengine/xe-core"},"type":"PullRequestEvent"} + +-- !sql -- +\N +\N +\N +\N +4748 + +-- !sql -- +135 + diff --git a/regression-test/data/variant_p0/agg.out b/regression-test/data/variant_p0/agg.out index 958e3d41a7..3b62eec463 100644 --- a/regression-test/data/variant_p0/agg.out +++ b/regression-test/data/variant_p0/agg.out @@ -31,7 +31,7 @@ -- !sql7 -- 1 {"a":1,"b":{"c":[{"a":1}]}} 59 1022 {"a":1,"b":{"f":17034,"g":1.111}} 12 -1029 \N 12 +1029 {"a":1,"b":{"c":1}} 12 1999 {"a":1,"b":{"c":1}} 11 -- !sql8 -- @@ -48,7 +48,7 @@ 11 [123] 11 12 [123.2] 12 1022 {"a":1,"b":{"f":17034,"g":1.111}} 12 -1029 \N 12 +1029 {"a":1,"b":{"c":1}} 12 1999 {"a":1,"b":{"c":1}} 11 19921 {"a":1,"d":10} 11 diff --git a/regression-test/data/variant_p0/compaction/test_compaction.out b/regression-test/data/variant_p0/compaction/test_compaction.out index 0b905e3930..1d5a21a349 100644 --- a/regression-test/data/variant_p0/compaction/test_compaction.out +++ b/regression-test/data/variant_p0/compaction/test_compaction.out @@ -225,8 +225,8 @@ {"c":[{"a":1}]} [{"a":1}] {"c":[{"a":1}]} [{"a":1}] {"c":1} 1 -{} \N -{} \N +null \N +null \N -- !sql_11 -- 1 {"x":[1]} @@ -329,8 +329,8 @@ {"c":[{"a":1}]} [{"a":1}] {"c":[{"a":1}]} [{"a":1}] {"c":1} 1 -{} \N -{} \N +null \N +null \N -- !sql_11 -- 1 {"x":[1]} diff --git a/regression-test/suites/variant_log_data_p2/load.groovy b/regression-test/suites/variant_log_data_p2/load.groovy index 5c0370fd44..df79e60165 100644 --- a/regression-test/suites/variant_log_data_p2/load.groovy +++ b/regression-test/suites/variant_log_data_p2/load.groovy @@ -79,14 +79,14 @@ suite("regression_test_variant_logdata", "nonConcurrent,p2"){ sql "truncate table ${table_name}" // 0.95 default ratio - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") load_json_data.call(table_name, """${getS3Url() + '/load/logdata.json'}""") qt_sql_33 """ select json_extract(v,"\$.json.parseFailed") from logdata where json_extract(v,"\$.json.parseFailed") != 'null' order by k limit 1;""" qt_sql_33_1 """select cast(v['json']['parseFailed'] as string) from logdata where cast(v['json']['parseFailed'] as string) is not null and k = 162 limit 1;""" sql "truncate table ${table_name}" // always sparse column - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") load_json_data.call(table_name, """${getS3Url() + '/load/logdata.json'}""") qt_sql_34 """ select json_extract(v, "\$.json.parseFailed") from logdata where json_extract(v,"\$.json.parseFailed") != 'null' order by k limit 1;""" sql "truncate table ${table_name}" @@ -95,4 +95,4 @@ suite("regression_test_variant_logdata", "nonConcurrent,p2"){ // TODO add test case that some certain columns are materialized in some file while others are not materilized(sparse) // unique table set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") -} \ No newline at end of file +} diff --git a/regression-test/suites/variant_p0/agg.groovy b/regression-test/suites/variant_p0/agg.groovy index b5010ee7f7..5788d05396 100644 --- a/regression-test/suites/variant_p0/agg.groovy +++ b/regression-test/suites/variant_p0/agg.groovy @@ -25,7 +25,7 @@ suite("regression_test_variant_agg"){ ) AGGREGATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "disable_auto_compaction" = "false"); + properties("replication_num" = "1", "disable_auto_compaction" = "true"); """ sql """insert into var_agg values (1, '[1]', 1),(1, '{"a" : 1}', 1);""" sql """insert into var_agg values (2, '[2]', 2),(1, '{"a" : [[[1]]]}', 2);""" @@ -42,10 +42,10 @@ suite("regression_test_variant_agg"){ qt_sql1 "select k, cast(v['a'] as array) from var_agg where size(cast(v['a'] as array)) > 0 order by k, cast(v['a'] as string) asc" qt_sql2 "select k, cast(v as int), cast(v['b'] as string) from var_agg where length(cast(v['b'] as string)) > 4 order by k, cast(v as string), cast(v['b'] as string) " qt_sql3 "select k, v from var_agg order by k, cast(v as string) limit 5" - qt_sql4 "select v['b'], v['b']['c'], cast(v as int) from var_agg where cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k,cast(v as string) desc limit 10000;" + qt_sql4 "select v['b'], v['b']['c'], cast(v as int) from var_agg where cast(v['b'] as string) is not null and length(v['b']) >4 order by k,cast(v as string) desc limit 10000;" qt_sql5 "select v['b'] from var_agg where cast(v['b'] as int) > 0;" - qt_sql6 "select cast(v['b'] as string) from var_agg where cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k, cast(v['b'] as string) " - qt_sql7 "select * from var_agg where cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k, cast(v['b'] as string) " + qt_sql6 "select cast(v['b'] as string) from var_agg where cast(v['b'] as string) is not null and length(v['b']) >4 order by k, cast(v['b'] as string) " + qt_sql7 "select * from var_agg where cast(v['b'] as string) is not null and length(v['b']) >4 order by k, cast(v['b'] as string) " qt_sql8 "select * from var_agg order by 1, cast(2 as string), 3" sql "alter table var_agg drop column s" sql """insert into var_agg select 5, '{"a" : 1234, "xxxx" : "fffff", "point" : 42000}' as json_str diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy index 85a6f3c9a2..581d724fb0 100644 --- a/regression-test/suites/variant_p0/delete_update.groovy +++ b/regression-test/suites/variant_p0/delete_update.groovy @@ -53,7 +53,7 @@ suite("regression_test_variant_delete_and_update", "variant_type"){ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 - properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "true", "store_row_column" = "true"); + properties("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """ sql "insert into var_delete_update_mow select k, cast(v as string), cast(v as string) from var_delete_update" sql "delete from ${table_name} where k = 1" diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy index ee2607756b..7988f5bc22 100644 --- a/regression-test/suites/variant_p0/desc.groovy +++ b/regression-test/suites/variant_p0/desc.groovy @@ -257,6 +257,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){ sql "desc large_tablets" } finally { // reset flags - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") } } diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index d737a1e1c8..2c9a9ed662 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -97,9 +97,9 @@ suite("regression_test_variant", "p0"){ qt_sql1 "select k, cast(v['a'] as array) from ${table_name} where size(cast(v['a'] as array)) > 0 order by k, cast(v['a'] as string) asc" qt_sql2 "select k, cast(v as int), cast(v['b'] as string) from ${table_name} where length(cast(v['b'] as string)) > 4 order by k, cast(v as string), cast(v['b'] as string) " qt_sql3 "select k, v from ${table_name} order by k, cast(v as string) limit 5" - qt_sql4 "select v['b'], v['b']['c'], cast(v as int) from ${table_name} where cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k,cast(v as string) desc limit 10000;" + qt_sql4 "select v['b'], v['b']['c'], cast(v as int) from ${table_name} where cast(v['b'] as string) != 'null' and cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k,cast(v as string) desc limit 10000;" qt_sql5 "select v['b'] from ${table_name} where cast(v['b'] as int) > 0;" - qt_sql6 "select cast(v['b'] as string) from ${table_name} where cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k, cast(v['b'] as string) " + qt_sql6 "select cast(v['b'] as string) from ${table_name} where cast(v['b'] as string) != 'null' and cast(v['b'] as string) is not null and cast(v['b'] as string) != '{}' order by k, cast(v['b'] as string) " // verify table_name } sql "insert into simple_variant_DUPLICATE select k, cast(v as string) from simple_variant_UNIQUE;" diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 21fe5eb007..58e5ef4f84 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -166,6 +166,6 @@ suite("load_p2", "variant_type,p2"){ qt_sql("select count() from github_events") } finally { // reset flags - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + // set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") } }