dlt-dbt-databricks/.env.example at main · AndreaBozzo/dlt-dbt-databricks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# Copy to `.env` and fill in real values. `.env` is gitignored.
# Load it into your shell before running examples:
#   - uv run automatically picks up .env via python-dotenv in the scripts, OR
#   - export manually:  set -a; source .env; set +a   (bash)
#
# ─────────────────────────────────────────────────────────────────────────────
# Databricks workspace (shared by dlt and dbt)
# ─────────────────────────────────────────────────────────────────────────────
# Workspace hostname WITHOUT https://   e.g. dbc-a1b2c3d4-e5f6.cloud.databricks.com
DATABRICKS_HOST=
# SQL warehouse HTTP path. Find under: SQL Warehouses > <your warehouse> > Connection details
#   e.g. /sql/1.0/warehouses/abcd1234efgh5678
DATABRICKS_HTTP_PATH=
# SQL warehouse id, used by Databricks Asset Bundle dbt tasks.
# If blank, tooling can derive it from DATABRICKS_HTTP_PATH when it follows /sql/1.0/warehouses/<id>.
DATABRICKS_WAREHOUSE_ID=
# Personal access token (User Settings > Developer > Access tokens).
# For production prefer OAuth M2M (see docs/setup-databricks.md).
DATABRICKS_TOKEN=
# Unity Catalog catalog + the schema dbt builds models into.
DATABRICKS_CATALOG=workspace
DATABRICKS_SCHEMA=analytics

# ─────────────────────────────────────────────────────────────────────────────
# dlt destination credentials
# dlt reads env vars in the form DESTINATION__DATABRICKS__CREDENTIALS__<FIELD>.
# These intentionally mirror the values above so you only set them once.
# (Alternatively put them in ingestion/.dlt/secrets.toml — see that file.)
# ─────────────────────────────────────────────────────────────────────────────
DESTINATION__DATABRICKS__CREDENTIALS__SERVER_HOSTNAME=${DATABRICKS_HOST}
DESTINATION__DATABRICKS__CREDENTIALS__HTTP_PATH=${DATABRICKS_HTTP_PATH}
DESTINATION__DATABRICKS__CREDENTIALS__ACCESS_TOKEN=${DATABRICKS_TOKEN}
DESTINATION__DATABRICKS__CREDENTIALS__CATALOG=${DATABRICKS_CATALOG}
# The Unity Catalog schema dlt LOADS INTO (the dbt sources read from here).
DLT_DATASET_NAME=raw

# Optional Zerobus append example (Linux/Windows only). The target catalog must use managed cloud
# object storage, not Databricks default storage. Prefer a dedicated OAuth service principal.
# ZEROBUS_CATALOG=<managed-storage-catalog>
# ZEROBUS_DATASET_NAME=<schema>
# DESTINATION__DATABRICKS__ZEROBUS__ENDPOINT_URL=https://<your-zerobus-endpoint>
# DESTINATION__DATABRICKS__ZEROBUS__CREDENTIALS__CLIENT_ID=
# DESTINATION__DATABRICKS__ZEROBUS__CREDENTIALS__CLIENT_SECRET=

# ─────────────────────────────────────────────────────────────────────────────
# SQL source for sql_database_to_databricks.py
# Default = EBI's PUBLIC read-only RNAcentral Postgres (real hosted DB, safe to share).
# Security scanners may flag this because it is password-shaped; see SECURITY.md for context.
# Swap for your own operational DB connection string.
# ─────────────────────────────────────────────────────────────────────────────
SOURCES__SQL_DATABASE__CREDENTIALS=postgresql://reader:NWDMCE5xdipIjRrp@hh-pgsql-public.ebi.ac.uk:5432/pfmegrnargs