Skip to content

Commit df19a7a

Browse files
prdpsvsprdpsvs
authored
Merge pull request #201 from microsoft/dbtsynapse1.5_dataroots
v1.5.0rc1
2 parents 29f910a + 46dd035 commit df19a7a

25 files changed

+1828
-25
lines changed

‎CHANGELOG.md

+13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,17 @@
11
# Changelog
2+
## v.1.5.0rc1
3+
4+
* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0)
5+
* Add support for model contracts by adapting `create_table_as` and `create_view_as` macros
6+
* Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class.
7+
* Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse
8+
* Add adapter tests zones
9+
- caching
10+
- column_types
11+
- constraints
12+
- hooks
13+
- simple_copy
14+
215
## v1.4.1rc1
316

417
#### Under the hood

‎dbt/adapters/synapse/__init__.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from dbt.adapters.base import AdapterPlugin
22

33
from dbt.adapters.synapse.synapse_adapter import SynapseAdapter
4+
from dbt.adapters.synapse.synapse_column import SynapseColumn
45
from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
56
from dbt.adapters.synapse.synapse_credentials import SynapseCredentials
67
from dbt.include import synapse
@@ -12,4 +13,10 @@
1213
dependencies=["fabric"],
1314
)
1415

15-
__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"]
16+
__all__ = [
17+
"Plugin",
18+
"SynapseConnectionManager",
19+
"SynapseColumn",
20+
"SynapseAdapter",
21+
"SynapseCredentials",
22+
]

‎dbt/adapters/synapse/__version__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version = "1.4.1rc1"
1+
version = "1.5.0rc1"

‎dbt/adapters/synapse/synapse_adapter.py

+56-2
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
1+
from enum import Enum
2+
from typing import Any, Dict, List, Optional
3+
14
from dbt.adapters.base.relation import BaseRelation
2-
from dbt.adapters.cache import _make_ref_key_msg
5+
from dbt.adapters.cache import _make_ref_key_dict
36
from dbt.adapters.fabric import FabricAdapter
47
from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME
8+
from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType
59
from dbt.events.functions import fire_event
610
from dbt.events.types import SchemaCreation
711

12+
from dbt.adapters.synapse.synapse_column import SynapseColumn
813
from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
914

1015

1116
class SynapseAdapter(FabricAdapter):
1217
ConnectionManager = SynapseConnectionManager
18+
Column = SynapseColumn
1319

1420
def create_schema(self, relation: BaseRelation) -> None:
1521
relation = relation.without_identifier()
16-
fire_event(SchemaCreation(relation=_make_ref_key_msg(relation)))
22+
fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
1723
macro_name = CREATE_SCHEMA_MACRO_NAME
1824
kwargs = {
1925
"relation": relation,
@@ -25,3 +31,51 @@ def create_schema(self, relation: BaseRelation) -> None:
2531

2632
self.execute_macro(macro_name, kwargs=kwargs)
2733
self.commit_if_has_connection()
34+
35+
class ConstraintSupport(str, Enum):
36+
ENFORCED = "enforced"
37+
NOT_ENFORCED = "not_enforced"
38+
NOT_SUPPORTED = "not_supported"
39+
40+
# https://learn.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-table-constraints#table-constraints
41+
CONSTRAINT_SUPPORT = {
42+
ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, # no CHECK support for Synapse
43+
ConstraintType.not_null: ConstraintSupport.ENFORCED,
44+
ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
45+
ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
46+
ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, # no FK support for Synapse
47+
}
48+
49+
@classmethod
50+
def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
51+
"""Render the given constraint as DDL text.
52+
Should be overriden by adapters which need custom constraint rendering."""
53+
if constraint.type == ConstraintType.check and constraint.expression:
54+
return f"check {constraint.expression}"
55+
elif constraint.type == ConstraintType.not_null:
56+
return "not null"
57+
elif constraint.type == ConstraintType.unique:
58+
return "unique NOT ENFORCED"
59+
elif constraint.type == ConstraintType.primary_key:
60+
return "primary key NONCLUSTERED NOT ENFORCED"
61+
elif constraint.type == ConstraintType.foreign_key:
62+
return "foreign key"
63+
elif constraint.type == ConstraintType.custom and constraint.expression:
64+
return constraint.expression
65+
else:
66+
return None
67+
68+
@classmethod
69+
def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
70+
rendered_column_constraints = []
71+
72+
for v in raw_columns.values():
73+
rendered_column_constraint = [f"{v['name']} {v['data_type']}"]
74+
for con in v.get("constraints", None):
75+
constraint = cls._parse_column_constraint(con)
76+
c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
77+
if c is not None:
78+
rendered_column_constraint.append(c)
79+
rendered_column_constraints.append(" ".join(rendered_column_constraint))
80+
81+
return rendered_column_constraints

‎dbt/adapters/synapse/synapse_column.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from dbt.adapters.fabric import FabricColumn
2+
3+
4+
class SynapseColumn(FabricColumn):
5+
# extending list of integer types for synapse
6+
def is_integer(self) -> bool:
7+
return self.dtype.lower() in [
8+
# real types
9+
"smallint",
10+
"bigint",
11+
"tinyint",
12+
"serial",
13+
"bigserial",
14+
"int",
15+
"bit",
16+
]

‎dbt/include/synapse/macros/adapters/persist_docs.sql

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer
2+
https://github.com/dbt-msft/dbt-sqlserver/issues/134
3+
https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16
4+
#}

‎dbt/include/synapse/macros/adapters/show.sql

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{% macro get_show_sql(compiled_code, sql_header, limit) -%}
2+
{%- if sql_header -%}
3+
{{ sql_header }}
4+
{%- endif -%}
5+
{%- if limit is not none -%}
6+
{{ get_limit_subquery_sql(compiled_code, limit) }}
7+
{%- else -%}
8+
{{ compiled_code }}
9+
{%- endif -%}
10+
{% endmacro %}
11+
12+
{% macro get_limit_subquery_sql(sql, limit) %}
13+
{{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}
14+
{% endmacro %}
15+
16+
{# Synapse doesnt support ANSI LIMIT clause #}
17+
{% macro synapse__get_limit_subquery_sql(sql, limit) %}
18+
select top {{ limit }} *
19+
from (
20+
{{ sql }}
21+
) as model_limit_subq
22+
{% endmacro %}

‎dbt/include/synapse/macros/materializations/models/table/create_table_as.sql

+29-10
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,35 @@
1010

1111
{{ synapse__drop_relation_script(relation) }}
1212

13-
EXEC('create view [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}] as
14-
{{ temp_view_sql }}
15-
');
16-
17-
CREATE TABLE {{ relation.include(database=False) }}
18-
WITH(
19-
DISTRIBUTION = {{dist}},
20-
{{index}}
21-
)
22-
AS (SELECT * FROM [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}])
13+
{{ synapse__create_view_as(tmp_relation, sql) }}
14+
15+
{% set contract_config = config.get('contract') %}
16+
17+
{% if contract_config.enforced %}
18+
19+
{{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
20+
21+
CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}]
22+
{{ synapse__build_columns_constraints(tmp_relation) }}
23+
WITH(
24+
DISTRIBUTION = {{dist}},
25+
{{index}}
26+
)
27+
{{ get_assert_columns_equivalent(sql) }}
28+
29+
{% set listColumns %}
30+
{% for column in model['columns'] %}
31+
{{ "["~column~"]" }}{{ ", " if not loop.last }}
32+
{% endfor %}
33+
{%endset%}
34+
{{ synapse__build_model_constraints(relation) }}
35+
36+
INSERT INTO [{{relation.schema}}].[{{relation.identifier}}]
37+
({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]
38+
39+
{%- else %}
40+
EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);');
41+
{% endif %}
2342

2443
{{ synapse__drop_relation_script(tmp_relation) }}
2544

‎dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{% macro synapse__build_columns_constraints(relation) %}
2+
{# loop through user_provided_columns to create DDL with data types and constraints #}
3+
{%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}
4+
(
5+
{% for c in raw_column_constraints -%}
6+
{{ c }}{{ "," if not loop.last }}
7+
{% endfor %}
8+
)
9+
{% endmacro %}
10+
11+
{% macro synapse__build_model_constraints(relation) %}
12+
{# loop through user_provided_columns to create DDL with data types and constraints #}
13+
{%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}
14+
{% for c in raw_model_constraints -%}
15+
alter table {{ relation.include(database=False) }} {{c}};
16+
{% endfor -%}
17+
{% endmacro %}

‎dbt/include/synapse/macros/materializations/models/view/create_view_as.sql

+13-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
11
{% macro synapse__create_view_as(relation, sql) -%}
2-
create view {{ relation.include(database=False) }} as
3-
{{ sql }}
2+
3+
{%- set temp_view_sql = sql.replace("'", "''") -%}
4+
5+
{% set contract_config = config.get('contract') %}
6+
7+
{{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
8+
9+
{% if contract_config.enforced %}
10+
{{ get_assert_columns_equivalent(sql) }}
11+
{%- endif %}
12+
13+
EXEC('create view {{ relation.include(database=False) }} as {{ temp_view_sql }};');
14+
415
{% endmacro %}

‎dev_requirements.txt

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
pytest==7.2.0
1+
pytest==7.4.0
22
twine==4.0.2
3-
wheel==0.40.0
3+
wheel==0.41.1
44
pre-commit==2.21.0;python_version<"3.8"
5-
pre-commit==3.3.1;python_version>="3.8"
5+
pre-commit==3.3.3;python_version>="3.8"
66
pytest-dotenv==0.5.2
7-
dbt-tests-adapter~=1.4.9
7+
dbt-tests-adapter~=1.5.9
88
aiohttp==3.8.3
99
azure-mgmt-synapse==2.0.0
1010
flaky==3.7.0
11-
pytest-xdist==3.3.0
11+
pytest-xdist==3.3.1
1212
-e .

‎setup.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
"Anders Swanson",
1616
"Sam Debruyn",
1717
]
18-
dbt_version = "1.4"
19-
dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3"
18+
dbt_version = "1.5"
19+
dbt_fabric_requirement = "dbt-fabric~=1.5.0"
2020
description = """An Azure Synapse adapter plugin for dbt"""
2121

2222
this_directory = os.path.abspath(os.path.dirname(__file__))

‎tests/functional/adapter/data/seed_model.sql

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
create table {schema}.on_model_hook
2+
(
3+
test_state VARCHAR(100), -- start|end
4+
target_dbname VARCHAR(100),
5+
target_host VARCHAR(100),
6+
target_name VARCHAR(100),
7+
target_schema VARCHAR(100),
8+
target_type VARCHAR(100),
9+
target_user VARCHAR(100),
10+
target_pass VARCHAR(100),
11+
target_threads INTEGER,
12+
run_started_at VARCHAR(100),
13+
invocation_id VARCHAR(100)
14+
)
15+
WITH(
16+
DISTRIBUTION = ROUND_ROBIN,
17+
HEAP
18+
)

‎tests/functional/adapter/data/seed_run.sql

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
if object_id ('{schema}.on_run_hook') is not null
2+
drop table {schema}.on_run_hook;
3+
4+
create table {schema}.on_run_hook
5+
(
6+
test_state VARCHAR(100), -- start|end
7+
target_dbname VARCHAR(100),
8+
target_host VARCHAR(100),
9+
target_name VARCHAR(100),
10+
target_schema VARCHAR(100),
11+
target_type VARCHAR(100),
12+
target_user VARCHAR(100),
13+
target_pass VARCHAR(100),
14+
target_threads INTEGER,
15+
run_started_at VARCHAR(100),
16+
invocation_id VARCHAR(100)
17+
)
18+
WITH(
19+
DISTRIBUTION = ROUND_ROBIN,
20+
HEAP
21+
);

0 commit comments

Comments
 (0)