dlt-dbt-databricks/databricks.yml at main · AndreaBozzo/dlt-dbt-databricks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
# Databricks Asset Bundle (DAB) — packages the dlt → dbt pipeline as a deployable Databricks Job.
# The job runs dlt ingestion, then (on success) dbt build, mirroring orchestration/run_e2e.py but
# as production-grade, version-controlled infrastructure.
#
#   databricks bundle validate --var warehouse_id=YOUR_WAREHOUSE_ID
#   databricks bundle deploy -t dev --var warehouse_id=YOUR_WAREHOUSE_ID
#   databricks bundle run dlt_dbt_pipeline -t dev
#
# See orchestration/README.md for the rationale.

bundle:
  name: dlt_dbt_databricks

variables:
  warehouse_id:
    description: "SQL warehouse id used by the dbt task."
    default: YOUR_WAREHOUSE_ID
  catalog:
    description: "Unity Catalog catalog for outputs."
    default: workspace
  schema:
    description: "Unity Catalog schema where dbt builds models."
    default: analytics

targets:
  dev:
    mode: development
    default: true
  prod:
    mode: production

resources:
  jobs:
    dlt_dbt_pipeline:
      name: "dlt-dbt-databricks [${bundle.target}]"
      tasks:
        # 1) dlt ingestion. Inside a Databricks job, dlt can resolve credentials from the runtime
        #    (zero-config) — no PAT in the bundle.
        - task_key: dlt_ingest
          environment_key: default
          spark_python_task:
            python_file: ingestion/pipelines/rest_api_to_databricks.py
            parameters:
              - "--catalog"
              - "${var.catalog}"
              - "--dataset-name"
              - "raw"
              - "--load-mode"
              - "spark"
        # 2) dbt build — only after ingestion succeeds. Runs against the SQL warehouse.
        - task_key: dbt_build
          depends_on:
            - task_key: dlt_ingest
          environment_key: default
          dbt_task:
            project_directory: transformation/dbt_databricks
            catalog: ${var.catalog}
            schema: ${var.schema}
            warehouse_id: ${var.warehouse_id}
            commands:
              - "dbt deps"
              - "dbt build"
        # 3) Agentic quality gate — dbt tests already gate correctness; this task adds the
        #    metric thresholds (volume, freshness, allowed-ratio) and fails the job on `block`.
        - task_key: quality_gate
          depends_on:
            - task_key: dbt_build
          environment_key: default
          spark_python_task:
            python_file: orchestration/gate_on_databricks.py
            parameters:
              - "--catalog"
              - "${var.catalog}"
              - "--schema"
              - "${var.schema}"
              - "--raw-schema"
              - "raw"
      environments:
        - environment_key: default
          spec:
            environment_version: "2"
            dependencies:
              - "dlt[databricks]>=1.32.0,<2"
              - "dbt-databricks>=1.13.0,<2"
              - "pyarrow>=18"
              - "python-dotenv>=1.0"