From 4b5ae3cc3b0a42feee0bfc248ce2bb4ebd98437d Mon Sep 17 00:00:00 2001 From: Sreerama Yeshwanth Gowd Date: Mon, 27 Jul 2026 08:55:07 +0530 Subject: [PATCH 1/5] fix: harden credential redaction in logged SQL Signed-off-by: Sreerama Yeshwanth Gowd --- dbt/adapters/databricks/utils.py | 30 ++++++++++------ tests/unit/test_utils.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 10 deletions(-) diff --git a/dbt/adapters/databricks/utils.py b/dbt/adapters/databricks/utils.py index 1eb5b918d..2a406c2db 100644 --- a/dbt/adapters/databricks/utils.py +++ b/dbt/adapters/databricks/utils.py @@ -17,9 +17,18 @@ A = TypeVar("A", bound=BaseAdapter) -CREDENTIAL_IN_COPY_INTO_REGEX = re.compile( - r"(?<=credential)\s*?\((\s*?'\w*?'\s*?=\s*?'.*?'\s*?(?:,\s*?'\w*?'\s*?=\s*?'.*?'\s*?)*?)\)" +_SECRET_OPTION_KEY = r"'[^']+'" +# '.*?' here backtracks exponentially on an unterminated clause; this form stays linear. +_SECRET_OPTION_VALUE = r"'(?:[^']|'')*'" +_SECRET_OPTION = _SECRET_OPTION_KEY + r"\s*=\s*" + _SECRET_OPTION_VALUE + +SECRET_CLAUSE_IN_COPY_INTO_REGEX = re.compile( + r"(credential|encryption)\s*\(\s*" + r"(" + _SECRET_OPTION + r"(?:\s*,\s*" + _SECRET_OPTION + r")*)" + r"\s*\)", + re.IGNORECASE, ) +SECRET_OPTION_KEY_REGEX = re.compile("(" + _SECRET_OPTION_KEY + r")\s*=\s*" + _SECRET_OPTION_VALUE) def redact_credentials(sql: str) -> str: @@ -27,16 +36,17 @@ def redact_credentials(sql: str) -> str: return redacted +def _redact_secret_clause(match: "re.Match[str]") -> str: + keys = SECRET_OPTION_KEY_REGEX.findall(match.group(2)) + return f"{match.group(1)} (" + ", ".join(f"{key} = '[REDACTED]'" for key in keys) + ")" + + def _redact_credentials_in_copy_into(sql: str) -> str: - m = CREDENTIAL_IN_COPY_INTO_REGEX.search(sql, re.MULTILINE) - if m: - redacted = ", ".join( - f"{key.strip()} = '[REDACTED]'" - for key, _ in (pair.strip().split("=", 1) for pair in m.group(1).split(",")) - ) - return f"{sql[: m.start()]} ({redacted}){sql[m.end() :]}" - else: + # Cheap substring test first; the case-insensitive scan is much slower on large statements. + lowered = sql.lower() + if "credential" not in lowered and "encryption" not in lowered: return sql + return SECRET_CLAUSE_IN_COPY_INTO_REGEX.sub(_redact_secret_clause, sql) def remove_undefined(v: Any) -> Any: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cc9fb67ab..39b29e97d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -51,6 +51,65 @@ def test_redact_credentials__multiple_credentials(self): ) assert redact_credentials(sql) == expected + def test_redact_credentials__uppercase_credential(self): + sql = "copy into target_table\nfrom source_table\n WITH (CREDENTIAL ('KEY' = 'VALUE'))" + expected = ( + "copy into target_table\nfrom source_table\n WITH (CREDENTIAL ('KEY' = '[REDACTED]'))" + ) + assert redact_credentials(sql) == expected + + def test_redact_credentials__encryption(self): + sql = ( + "copy into target_table\n" + "from source_table\n" + " WITH (encryption ('TYPE' = 'AWS_SSE_C', 'MASTER_KEY' = 'VALUE'))" + ) + expected = ( + "copy into target_table\n" + "from source_table\n" + " WITH (encryption ('TYPE' = '[REDACTED]', 'MASTER_KEY' = '[REDACTED]'))" + ) + assert redact_credentials(sql) == expected + + def test_redact_credentials__credential_and_encryption(self): + sql = ( + "copy into target_table\n" + "from source_table\n" + " WITH (credential ('KEY' = 'VALUE') encryption ('MASTER_KEY' = 'VALUE'))" + ) + expected = ( + "copy into target_table\n" + "from source_table\n" + " WITH (credential ('KEY' = '[REDACTED]') encryption ('MASTER_KEY' = '[REDACTED]'))" + ) + assert redact_credentials(sql) == expected + + def test_redact_credentials__value_with_comma(self): + sql = "copy into target_table\n WITH (credential ('KEY' = 'VALUE,WITH,COMMAS'))" + expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" + assert redact_credentials(sql) == expected + + def test_redact_credentials__value_with_newline(self): + sql = "copy into target_table\n WITH (credential ('KEY' = 'VALUE\nCONTINUED'))" + expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" + assert redact_credentials(sql) == expected + + def test_redact_credentials__key_with_dots(self): + sql = "copy into target_table\n WITH (credential ('fs.azure.account.key' = 'VALUE'))" + expected = ( + "copy into target_table\n WITH (credential ('fs.azure.account.key' = '[REDACTED]'))" + ) + assert redact_credentials(sql) == expected + + def test_redact_credentials__prefixed_keyword(self): + sql = "copy into target_table\n WITH (storage_credential ('KEY' = 'VALUE'))" + expected = "copy into target_table\n WITH (storage_credential ('KEY' = '[REDACTED]'))" + assert redact_credentials(sql) == expected + + def test_redact_credentials__non_option_clause(self): + sql = "select * from target_table where credential_id = 1" + assert redact_credentials(sql) == sql + def test_remove_ansi(self): test_string = """Python model failed with traceback as: --------------------------------------------------------------------------- From 8e69ed5a53f37f4854a7c9dda031a4d82d6ce3a7 Mon Sep 17 00:00:00 2001 From: Sreerama Yeshwanth Gowd Date: Mon, 27 Jul 2026 08:56:29 +0530 Subject: [PATCH 2/5] chore: update CHANGELOG for #1610 Signed-off-by: Sreerama Yeshwanth Gowd --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc772852..d34fe1d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Fixes +- Redact all `credential` and `encryption` clauses in logged SQL, regardless of keyword case (thanks @SreeramaYeshwanthGowd!) ([#1610](https://github.com/databricks/dbt-databricks/pull/1610) resolves [#1609](https://github.com/databricks/dbt-databricks/issues/1609)) - Allow dropping a column that has governed tags ([#1597](https://github.com/databricks/dbt-databricks/pull/1597) resolves [#1323](https://github.com/databricks/dbt-databricks/issues/1323)) - Fix view materialization incorrectly producing a no-op instead of forcing recreation when `--full-refresh` is provided alongside `view_update_via_alter: true` and `use_materialization_v2: true` ([#1456](https://github.com/databricks/dbt-databricks/pull/1456) resolves [#1404](https://github.com/databricks/dbt-databricks/issues/1404)) - Support `dbt clone` and rebuilds over an existing shallow clone ([#1592](https://github.com/databricks/dbt-databricks/pull/1592) resolves [#1165](https://github.com/databricks/dbt-databricks/issues/1165)) From 0e646e141037f8235652d184dedff5c1e6f5bffd Mon Sep 17 00:00:00 2001 From: Sreerama Yeshwanth Gowd Date: Mon, 27 Jul 2026 11:42:07 +0530 Subject: [PATCH 3/5] fix: keep redacting values that contain a quote character Signed-off-by: Sreerama Yeshwanth Gowd --- dbt/adapters/databricks/utils.py | 5 +++-- tests/unit/test_utils.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/databricks/utils.py b/dbt/adapters/databricks/utils.py index 2a406c2db..c16b0c2f1 100644 --- a/dbt/adapters/databricks/utils.py +++ b/dbt/adapters/databricks/utils.py @@ -18,8 +18,9 @@ _SECRET_OPTION_KEY = r"'[^']+'" -# '.*?' here backtracks exponentially on an unterminated clause; this form stays linear. -_SECRET_OPTION_VALUE = r"'(?:[^']|'')*'" +# A closing quote is one followed by a delimiter, so any other quote is part of the value. Spelling +# that out keeps the match linear, where '.*?' backtracks exponentially on an unterminated clause. +_SECRET_OPTION_VALUE = r"'(?:[^']|'(?!\s*[,)]))*'" _SECRET_OPTION = _SECRET_OPTION_KEY + r"\s*=\s*" + _SECRET_OPTION_VALUE SECRET_CLAUSE_IN_COPY_INTO_REGEX = re.compile( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 39b29e97d..2303ea2f3 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -94,6 +94,16 @@ def test_redact_credentials__value_with_newline(self): expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" assert redact_credentials(sql) == expected + def test_redact_credentials__value_with_quote(self): + sql = "copy into target_table\n WITH (credential ('KEY' = 'VALUE'WITH'QUOTES'))" + expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" + assert redact_credentials(sql) == expected + + def test_redact_credentials__value_with_escaped_quote(self): + sql = "copy into target_table\n WITH (credential ('KEY' = 'VALUE\\'ESCAPED'))" + expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" + assert redact_credentials(sql) == expected + def test_redact_credentials__key_with_dots(self): sql = "copy into target_table\n WITH (credential ('fs.azure.account.key' = 'VALUE'))" expected = ( From 602987d035969920af080cdc196616bcfd8a4e8d Mon Sep 17 00:00:00 2001 From: Shubham Dhal Date: Wed, 5 Aug 2026 18:46:21 +0530 Subject: [PATCH 4/5] fix: redact escaped credentials in copy into logs --- dbt/adapters/databricks/utils.py | 5 ++-- tests/unit/test_utils.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/databricks/utils.py b/dbt/adapters/databricks/utils.py index c16b0c2f1..dd92a3b24 100644 --- a/dbt/adapters/databricks/utils.py +++ b/dbt/adapters/databricks/utils.py @@ -18,9 +18,8 @@ _SECRET_OPTION_KEY = r"'[^']+'" -# A closing quote is one followed by a delimiter, so any other quote is part of the value. Spelling -# that out keeps the match linear, where '.*?' backtracks exponentially on an unterminated clause. -_SECRET_OPTION_VALUE = r"'(?:[^']|'(?!\s*[,)]))*'" +# The alternatives are deliberately non-overlapping to keep matching linear on malformed input. +_SECRET_OPTION_VALUE = r"'(?:\\.|''|[^'\\]|'(?!'|\s*[,)]))*'" _SECRET_OPTION = _SECRET_OPTION_KEY + r"\s*=\s*" + _SECRET_OPTION_VALUE SECRET_CLAUSE_IN_COPY_INTO_REGEX = re.compile( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2303ea2f3..9363a4d42 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,3 +1,4 @@ +import dbt.adapters.databricks.utils as databricks_utils from dbt.adapters.databricks.utils import ( is_cluster_http_path, quote, @@ -104,6 +105,56 @@ def test_redact_credentials__value_with_escaped_quote(self): expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" assert redact_credentials(sql) == expected + def test_redact_credentials__escaped_quote_before_delimiter(self): + cases = [ + "copy into target_table\n WITH (credential ('KEY' = 'PREFIX'',SUFFIX'))", + "copy into target_table\n WITH (credential ('KEY' = 'PREFIX'')SUFFIX'))", + "copy into target_table\n WITH (credential ('KEY' = 'PREFIX\\',SUFFIX'))", + "copy into target_table\n WITH (credential ('KEY' = 'PREFIX\\')SUFFIX'))", + ] + expected = "copy into target_table\n WITH (credential ('KEY' = '[REDACTED]'))" + + for sql in cases: + redacted = redact_credentials(sql) + assert redacted == expected + assert "PREFIX" not in redacted + assert "SUFFIX" not in redacted + + def test_redact_credentials__malformed_secret_clause_is_unchanged(self): + sql = "copy into target_table\n WITH (credential ('KEY' = 'PREFIX',SUFFIX')) trailing SQL" + + assert redact_credentials(sql) == sql + + def test_redact_credentials__secretless_clause_is_unchanged(self): + cases = [ + "copy into target_table WITH (credential ())", + "select credential('public literal') as x, 42 as y", + "select my_encryption('public literal') as x", + ] + + for sql in cases: + assert redact_credentials(sql) == sql + + def test_redact_credentials__unquoted_key_is_unchanged(self): + sql = "copy into target_table WITH (credential (KEY = 'SECRET')) trailing SQL" + + assert redact_credentials(sql) == sql + + def test_redact_credentials__large_unterminated_clause(self): + sql = "credential (" + ", ".join("'KEY' = 'VALUE'" for _ in range(1_000)) + + assert redact_credentials(sql) == sql + + def test_redact_credentials__large_ordinary_statement_uses_fast_path(self, monkeypatch): + class UnexpectedRegex: + def sub(self, replacement, sql): + raise AssertionError("ordinary SQL should bypass the secret-clause regex") + + monkeypatch.setattr(databricks_utils, "SECRET_CLAUSE_IN_COPY_INTO_REGEX", UnexpectedRegex()) + sql = "select 1 -- " + "x" * 1_000_000 + + assert redact_credentials(sql) == sql + def test_redact_credentials__key_with_dots(self): sql = "copy into target_table\n WITH (credential ('fs.azure.account.key' = 'VALUE'))" expected = ( From 3b1e95ff6a74f7271c7670893634a212461e127b Mon Sep 17 00:00:00 2001 From: Shubham Dhal Date: Thu, 6 Aug 2026 20:25:52 +0530 Subject: [PATCH 5/5] fix: keep redaction errors from blocking execution --- dbt/adapters/databricks/utils.py | 6 ++++-- tests/unit/test_utils.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/databricks/utils.py b/dbt/adapters/databricks/utils.py index dd92a3b24..0fba562c5 100644 --- a/dbt/adapters/databricks/utils.py +++ b/dbt/adapters/databricks/utils.py @@ -32,8 +32,10 @@ def redact_credentials(sql: str) -> str: - redacted = _redact_credentials_in_copy_into(sql) - return redacted + try: + return _redact_credentials_in_copy_into(sql) + except Exception: + return sql def _redact_secret_clause(match: "re.Match[str]") -> str: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9363a4d42..2cb7c3042 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -155,6 +155,20 @@ def sub(self, replacement, sql): assert redact_credentials(sql) == sql + def test_redact_credentials__internal_error_fails_open(self, monkeypatch): + sql = "copy into target_table WITH (credential ('KEY' = 'SYNTHETIC_SECRET'))" + + def raise_internal_error(sql: str) -> str: + raise RuntimeError("synthetic redactor failure") + + monkeypatch.setattr( + databricks_utils, + "_redact_credentials_in_copy_into", + raise_internal_error, + ) + + assert redact_credentials(sql) == sql + def test_redact_credentials__key_with_dots(self): sql = "copy into target_table\n WITH (credential ('fs.azure.account.key' = 'VALUE'))" expected = (