From 7af79e1df58aa70437eb34e99b48582e98517c7d Mon Sep 17 00:00:00 2001 From: long2ice Date: Fri, 6 May 2022 15:27:34 +0800 Subject: [PATCH] [Feature][dbt] add partition_type support (#9389) --- extension/dbt-doris/dbt/adapters/doris/impl.py | 8 +++++++- .../dbt/include/doris/macros/adapters/relation.sql | 11 ++++------- .../macros/materializations/table/create_table_as.sql | 11 ++++++----- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/extension/dbt-doris/dbt/adapters/doris/impl.py b/extension/dbt-doris/dbt/adapters/doris/impl.py index 782e2ab36f..761aac6c99 100644 --- a/extension/dbt-doris/dbt/adapters/doris/impl.py +++ b/extension/dbt-doris/dbt/adapters/doris/impl.py @@ -46,10 +46,16 @@ class Engine(str, Enum): iceberg = "iceberg" +class PartitionType(str, Enum): + list = "LIST" + range = "RANGE" + + class DorisConfig(AdapterConfig): - engine: Engine = Engine.olap + engine: Engine duplicate_key: Tuple[str] partition_by: Tuple[str] + partition_type: PartitionType partition_by_init: List[str] distributed_by: Tuple[str] buckets: int diff --git a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql index ed53ea6fe1..290b9a72d3 100644 --- a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql +++ b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql @@ -17,23 +17,20 @@ {% macro doris__engine() -%} {% set label = 'ENGINE' %} - {% set engine = config.get('engine', validator=validation.any[basestring]) %} - {% if engine is not none %} + {% set engine = config.get('engine', 'OLAP', validator=validation.any[basestring]) %} {{ label }} = {{ engine }} - {% else %} - {{ label }} = OLAP - {% endif %} {%- endmacro %} {% macro doris__partition_by() -%} {% set cols = config.get('partition_by') %} + {% set partition_type = config.get('partition_type', 'RANGE') %} {% if cols is not none %} - PARTITION BY RANGE ( + PARTITION BY {{ partition_type }} ( {% for col in cols %} {{ col }}{% if not loop.last %},{% endif %} {% endfor %} )( - {% set init = config.get('partition_by_init',validator=validation.any[list]) %} + {% set init = config.get('partition_by_init', validator=validation.any[list]) %} {% if init is not none %} {% for row in init %} {{ row }}{% if not loop.last %},{% endif %} diff --git a/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql b/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql index 83d7648882..2442571427 100644 --- a/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql +++ b/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql @@ -16,11 +16,12 @@ -- under the License. {% macro doris__create_table_as(temporary, relation, sql) -%} - {% set sql_header = config.get('sql_header', none) %} - - {{ sql_header if sql_header is not none }} - create table {{ relation.include(database=False) }} + {% set sql_header = config.get('sql_header', none) %} + {% set table = relation.include(database=False) %} + {{ sql_header if sql_header is not none }} + create table {{ table }} {{ doris__partition_by() }} {{ doris__distributed_by() }} - {{ doris__properties() }} as {{ sql }} + {{ doris__properties() }} as {{ sql }}; + insert into {{ table }} {{ sql }}; {%- endmacro %}