diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 724f5a0b87ce3..50b2399564e0d 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -143,6 +143,7 @@ body: - cohere - common-ai - common-compat + - common-dataquality - common-io - common-messaging - common-sql diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 2848323162932..47f3c63d7dcce 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -141,6 +141,9 @@ labelPRBasedOnFilePath: provider:docker: - providers/docker/** + provider:common-dataquality: + - providers/common/dataquality/** + provider:edge: - providers/edge3/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index c2bb02ba55dbd..e5e67dfc6bab5 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -397,6 +397,8 @@ pre-installed when Airflow is installed. +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | common-compat | ``pip install 'apache-airflow[common-compat]'`` | Compatibility code for old Airflow | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ +| common-dataquality | ``pip install 'apache-airflow[common-dataquality]'``| Data Quality hooks and operators | | ++---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | common-io | ``pip install 'apache-airflow[common-io]'`` | Core IO Operators | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ | common-messaging | ``pip install 'apache-airflow[common-messaging]'`` | Core Messaging Operators | | diff --git a/airflow-core/tests/unit/always/test_project_structure.py b/airflow-core/tests/unit/always/test_project_structure.py index 750966a320221..a3626e17cf410 100644 --- a/airflow-core/tests/unit/always/test_project_structure.py +++ b/airflow-core/tests/unit/always/test_project_structure.py @@ -107,6 +107,7 @@ def test_providers_modules_should_have_tests(self): "providers/common/compat/tests/unit/common/compat/standard/test_utils.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_base_provider.py", "providers/common/messaging/tests/unit/common/messaging/providers/test_sqs.py", + "providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py", "providers/edge3/tests/unit/edge3/cli/test_example_extended_sysinfo.py", "providers/edge3/tests/unit/edge3/models/test_edge_job.py", "providers/edge3/tests/unit/edge3/models/test_edge_logs.py", diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index ca5cb1775bcc0..99bc5dbc0de1e 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -244,14 +244,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 7e86964622680..75dd9355bcff2 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -22cd66d36e80473f4fac0fa311c5e846 +e22969fb5e92a3efafb744881fea5ef8 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 09e78ba88111a..9df4462030484 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -153,14 +153,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 672e05eb6b26f..152f412a472d3 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -2700b8a52b9a04bb3cf5e365e70c2f40 +963d8b3e84cb64aa0d69026638b9fb1b diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index cef6e4d3b1d14..7e2dae4729d23 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -165,14 +165,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 6e02c700afdf5..c400d2c1da050 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -4cac13b21eee8b732a46c5a15aec7a4b +b4c6b4c23a30b7f750a9a8316abd2e91 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 2f9692be54fa0..78c943d5732d6 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -154,14 +154,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 6afb289b5854f..7c5cc1b0fb397 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -bd80bf2dd63a27111b8c5745df5dfb86 +ef90cd93a64627b874f63cca845bddbf diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 64efb0af6b65b..4199019d5c56f 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -173,15 +173,15 @@ │apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | │ │apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ │apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ -│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ -│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ -│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ -│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ -│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ -│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ -│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ -│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ -│yandex | ydb | zendesk)│ +│| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | │ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | │ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http │ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql │ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | │ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | │ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | │ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 7c14e04e63151..bf322a8137ed0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -5865810f5a4b12c9199267b7dd6459b3 +0e161f1be5f2a5b8c6094dcf7859f1c8 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 27a1d92de9a4b..2a119b59f74a7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -192,14 +192,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index f054c03806e05..9cc84f8ed57aa 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -a27c1726f5902e5fdb501ecdee226476 +40982121f81ca247791eee99e22d7214 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index b9c71f9c81aec..4785a550ef45e 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -213,14 +213,14 @@ [OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | -clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        -common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   -facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    -ydb | zendesk]...                                                                                                      +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io |      +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index a54080b6bfbb5..cab1aa0379851 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -c33a2f6d00a3a8dbec8b56c1c2d88d54 +0d1c20ffd24edc17351c5f132a1cc968 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index 9416d6183c582..59c03ad4d408a 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -192,14 +192,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index b6433c39bb9be..585d41d6275cb 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -cc4ad6886b5ff0da1d4f819d2c5dc80d +217c089e723c5584da63135bbf011504 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index a9d2e173ef2b8..8e1ee534c9b46 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -187,15 +187,15 @@ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | │ -│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ -│github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | │ -│jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | │ -│mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle │ -│| pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce │ -│| samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | │ -│standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | │ -│zendesk)│ +│common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks | │ +│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ +│facebook | ftp | git | github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | │ +│informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ +│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ +│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ +│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ +│yandex | ydb | zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     │ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index bb3262922cec4..ca3930b91a1a2 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -84f322f2ffa9ded046c2d3678135a8f7 +7cbc3f1d6c0c4f1745702d867f9e7fdc diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index dfb0027b3d7ef..13e7e9bf14620 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -207,14 +207,14 @@ apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk | jdbc | jenkins |   -keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       -openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       -postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     -snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate |  -yandex | ydb | zendesk]...                                                                                             +| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   +datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   +ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 4c6ec79dba124..a86438aee4e27 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -e784b5234a05d958f84df96cd5d8d562 +cb87f41a19fbc8895cd38aa6351139ae diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 0e8268c8de4cc..95bdf7999a8de 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -843,7 +843,7 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", - "remove-providers": "anthropic common.messaging edge3 fab git keycloak informatica common.ai opensearch", + "remove-providers": "anthropic common.messaging common.dataquality edge3 fab git keycloak informatica common.ai opensearch", "run-unit-tests": "true", }, { diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 1e0e2644ffd5d..d90dd5612de47 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2820,7 +2820,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.dataquality common.sql databricks elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 2b7c13ee66036..fb42c923efa4e 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -292,6 +292,7 @@ conda conf Config config +ConfigDict configfile configMap configmap @@ -400,6 +401,7 @@ dataprep Dataproc dataproc DataprocDiagnoseClusterOperator +dataquality DataScan dataScans Dataset diff --git a/providers/common/dataquality/.gitignore b/providers/common/dataquality/.gitignore new file mode 100644 index 0000000000000..f454b41e73c0a --- /dev/null +++ b/providers/common/dataquality/.gitignore @@ -0,0 +1,3 @@ +# Do not commit hash files, this is just to speed-up local builds +www-hash.txt +*.iml diff --git a/providers/common/dataquality/.pre-commit-config.yaml b/providers/common/dataquality/.pre-commit-config.yaml new file mode 100644 index 0000000000000..173d5a5527520 --- /dev/null +++ b/providers/common/dataquality/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: local + hooks: + - id: generate-common-dataquality-ruleset-schema + name: Generate Data Quality RuleSet schema + language: python + entry: ../../../scripts/ci/prek/generate_common_dataquality_ruleset_schema.py + pass_filenames: false + always_run: true + files: > + (?x) + ^src/airflow/providers/common/dataquality/rules/.*\.py$| + ^src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset\.schema\.json$ diff --git a/providers/common/dataquality/LICENSE b/providers/common/dataquality/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/common/dataquality/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/common/dataquality/NOTICE b/providers/common/dataquality/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/common/dataquality/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/common/dataquality/README.rst b/providers/common/dataquality/README.rst new file mode 100644 index 0000000000000..b9e552d12d263 --- /dev/null +++ b/providers/common/dataquality/README.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Package ``apache-airflow-providers-common-dataquality`` + +Release: ``0.1.0`` + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== diff --git a/providers/common/dataquality/docs/agents.rst b/providers/common/dataquality/docs/agents.rst new file mode 100644 index 0000000000000..8a0d160a4e2f2 --- /dev/null +++ b/providers/common/dataquality/docs/agents.rst @@ -0,0 +1,55 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:agents: + +Generating rules with an LLM +============================== + +Writing a :class:`~airflow.providers.common.dataquality.rules.RuleSet` by hand for every table doesn't scale. +An LLM can propose one from a table's column definitions instead, given the check catalog as +context. + +The ``dataquality-rule-authoring`` skill +---------------------------------------- + +This provider ships an `Agent Skill `__ at +``airflow/providers/common/dataquality/skills/dataquality-rule-authoring/``: a ``SKILL.md`` documenting the +``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema +(``references/ruleset.schema.json``) for validation. + +Point ``common.ai``'s :doc:`AgentSkillsToolset ` at +it, and give the model ``output_type=RuleSet`` so pydantic-ai validates -- and self-corrects -- +its output before the task completes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_task_dq_generate_ruleset_with_llm] + :end-before: [END howto_task_dq_generate_ruleset_with_llm] + +Requires ``apache-airflow-providers-common-ai[skills]`` and a configured ``llm_conn_id``. + +Wiring the result into a check +--------------------------------- + +``@task.dq_check`` can leave ``ruleset=`` unset and return the LLM task's result at execution +time instead: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_decorator_dq_check_llm_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_llm_runtime_ruleset] diff --git a/providers/common/dataquality/docs/assets.rst b/providers/common/dataquality/docs/assets.rst new file mode 100644 index 0000000000000..ce88628d334fb --- /dev/null +++ b/providers/common/dataquality/docs/assets.rst @@ -0,0 +1,66 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:assets: + +Assets and quality gating +============================ + +Quality rules can travel with the :class:`~airflow.sdk.Asset` they describe instead of being +scattered across every Dag that checks it, and a downstream consumer Dag can refuse to run when +the data it was triggered by did not meet a minimum quality bar. + +Attaching a ruleset to an asset +---------------------------------- + +:func:`~airflow.providers.common.dataquality.assets.asset_quality` stores ruleset, connection, and table +configuration inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is serialized with the +Dag and needs no Airflow core changes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_asset_quality] + :end-before: [END howto_asset_quality] + +Pass the resulting asset to :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` +via ``asset=`` (see :doc:`operators`) instead of ``table``/``ruleset``/``conn_id``: the operator +resolves all three from the asset's config, adds the asset to its own outlets, and attaches its +summary -- including the quality ``score`` used below -- to the asset event. + +Gating a consumer Dag on quality +------------------------------------ + +:func:`~airflow.providers.common.dataquality.assets.require_quality` builds a ``@task.short_circuit`` task that +reads the ``score`` off the asset event that triggered the current run, and skips every +downstream task when that event has no quality summary at all, or its score is below +``min_score``: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_require_quality] + :end-before: [END howto_require_quality] + +Put the gate first in a Dag scheduled by the asset, and chain everything else after it: + +.. code-block:: python + + with DAG("orders_consumer", schedule=orders_asset) as consumer: + gate = require_quality(orders_asset, min_score=0.95) + gate >> process_orders() + +``min_score`` must be between ``0`` and ``1``; the check considers only the *most recent* +triggering event for the asset when a run was triggered by several. diff --git a/providers/common/dataquality/docs/changelog.rst b/providers/common/dataquality/docs/changelog.rst new file mode 100644 index 0000000000000..9b489866efc4d --- /dev/null +++ b/providers/common/dataquality/docs/changelog.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there + are some breaking changes and you want to add an explanation to the users on how they + are supposed to deal with them. The changelog is updated and maintained semi-automatically + by release manager. + +``apache-airflow-providers-common-dataquality`` + + +Changelog +--------- + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/common/dataquality/docs/commits.rst b/providers/common/dataquality/docs/commits.rst new file mode 100644 index 0000000000000..f70c4b70a34e2 --- /dev/null +++ b/providers/common/dataquality/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-common-dataquality +------------------------------------------------------ + +``Data Quality Provider`` + + +This is detailed commit list of changes for versions provider package: ``dataquality``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/common/dataquality/docs/conf.py b/providers/common/dataquality/docs/conf.py new file mode 100644 index 0000000000000..222b45280f1cf --- /dev/null +++ b/providers/common/dataquality/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-common-dataquality" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/common/dataquality/docs/configurations-ref.rst b/providers/common/dataquality/docs/configurations-ref.rst new file mode 100644 index 0000000000000..ea8e668d75793 --- /dev/null +++ b/providers/common/dataquality/docs/configurations-ref.rst @@ -0,0 +1,19 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/providers-configurations-ref.rst +.. include:: /../../../../devel-common/src/sphinx_exts/includes/sections-and-options.rst diff --git a/providers/common/dataquality/docs/decorators.rst b/providers/common/dataquality/docs/decorators.rst new file mode 100644 index 0000000000000..1ec79dd5bd0b9 --- /dev/null +++ b/providers/common/dataquality/docs/decorators.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/decorator:dq_check: + +``@task.dq_check`` +===================== + +``@task.dq_check`` wraps :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` in +the TaskFlow API. ``ruleset`` may be declared as a decorator argument when it exists at +Dag-parse time, or returned by the decorated function as a runtime ruleset. ``table`` or +``asset`` are declared as decorator arguments exactly like the plain operator. The decorated +function is optional plumbing on top: return ``None`` to run the check exactly as declared. + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_decorator_dq_check] + :end-before: [END howto_decorator_dq_check] + +Runtime rule sets +-------------------- + +Return a :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a YAML path to use a +ruleset that is only known at task-execution time -- for example one produced by an upstream +task, loaded from a Variable, or generated by an LLM. Return ``None`` to use the ruleset declared +on the decorator. + +Swapping in a different ruleset at execution time: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py + :language: python + :start-after: [START howto_decorator_dq_check_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_runtime_ruleset] + +Everything documented for the plain operator in :doc:`operators` -- ``fail_on``, persistence, +``asset``, ``partition_clause``, ``custom_sql`` -- applies unchanged; the decorator only adds the +optional runtime ruleset step before the check runs. diff --git a/providers/common/dataquality/docs/index.rst b/providers/common/dataquality/docs/index.rst new file mode 100644 index 0000000000000..0e26c1be95b17 --- /dev/null +++ b/providers/common/dataquality/docs/index.rst @@ -0,0 +1,175 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-common-dataquality`` +=============================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Rules, rule sets, and checks + Operators + Decorators + Assets and quality gating + Generating rules with an LLM + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Configuration + Python API <_api/airflow/providers/common/dataquality/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/common/dataquality/index> + + +apache-airflow-providers-common-dataquality package +--------------------------------------------------- + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +See :doc:`rules` for the built-in check catalog (and its cross-database caveats), +:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to +an asset and gating a downstream Dag on its quality score. + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-common-dataquality package +------------------------------------------------------ + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. +Checks run through ``common.sql`` DB-API hooks; results are persisted to a +configurable results store (object storage or local files) so task, run, +and rule-level quality can be inspected over time. + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-common-dataquality 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-dataquality 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/common/dataquality/docs/installing-providers-from-sources.rst b/providers/common/dataquality/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/common/dataquality/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/common/dataquality/docs/operators.rst b/providers/common/dataquality/docs/operators.rst new file mode 100644 index 0000000000000..ab5eec1669b74 --- /dev/null +++ b/providers/common/dataquality/docs/operators.rst @@ -0,0 +1,119 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:DQCheckOperator: + +``DQCheckOperator`` +===================== + +Use :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` to run a +:doc:`ruleset ` against a table and persist per-rule results to the configured results +store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing +doesn't stop the others from running, and the full set of results is always written before the +task decides whether to fail. + +Basic usage +------------ + +Pass a connection, table, and ruleset directly: + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dq_check] + :end-before: [END howto_operator_dq_check] + +Parameters +----------- + +- ``ruleset`` -- a :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path to a + YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. +- ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the + asset's name). +- ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with + :func:`~airflow.providers.common.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, + ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's + outlets so its asset events carry the check summary -- see :doc:`assets`. +- ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. + ``"ds = '{{ ds }}'"`` (templated). +- ``fail_on`` -- severity that fails the task: + + - ``error`` (default) -- only ``error``-severity rule failures fail the task; ``warn`` + failures are recorded but don't fail it. + - ``warn`` -- any rule failure (``error`` or ``warn`` severity) fails the task. + - ``never`` -- rule failures never fail the task; only an execution error (the check query + itself failing) does. + +- ``conn_id`` -- connection to the database to check, any ``common.sql`` ``DbApiHook``- + compatible type. +- ``database`` -- optional database/schema, overriding the connection's default. + +Regardless of ``fail_on``, an execution error -- the check query itself failing, as opposed to +a rule failing its condition -- always fails the task; there's no way to check data whose query +can't even run. + +Persisting results +-------------------- + +Results are persisted to the backend configured under ``[common.dataquality] results_path`` (see +:doc:`configurations-ref`). When that's unset, checks still run and the task still passes or +fails normally -- only persisted data quality history is unavailable. There is no +per-operator override: every check in a deployment shares one results store, so history stays +available across tasks and Dags without stitching together several stores. + +Checking an asset +-------------------- + +Attach a ruleset to an :class:`~airflow.sdk.Asset` with +:func:`~airflow.providers.common.dataquality.assets.asset_quality`, then pass the asset instead of ``table``/ +``ruleset``/``conn_id``: + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :start-after: [START howto_operator_dq_check_asset] + :end-before: [END howto_operator_dq_check_asset] + +The operator adds the asset to its own outlets automatically, and attaches the check's summary +(including its quality ``score``) to the asset event -- which is what makes +:func:`~airflow.providers.common.dataquality.assets.require_quality` (see :doc:`assets`) able to gate a +downstream consumer Dag on it. + +``custom_sql`` checks +------------------------ + +Built-in checks are all single-column. For cross-column comparisons, joins, or anything the +catalog doesn't cover, use ``custom_sql``: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` grammar. + +Loading a ruleset from YAML +------------------------------ + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +TaskFlow decorator +-------------------- + +See :doc:`decorators` for the ``@task.dq_check`` equivalent, including runtime rule sets. diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst new file mode 100644 index 0000000000000..9ee6b8fd0a254 --- /dev/null +++ b/providers/common/dataquality/docs/rules.rst @@ -0,0 +1,221 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:rules: + +Rules, rule sets, and checks +============================ + +Rules are data, not code. A :class:`~airflow.providers.common.dataquality.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.common.dataquality.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` +(see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they +can also be proposed by an LLM -- see :doc:`agents`. + +.. code-block:: python + + from airflow.providers.common.dataquality.rules import DQRule, RuleSet + + orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + ), + ), + ) + +``DQRule`` fields +------------------ + +- ``name`` -- unique within its ruleset. Shows up in data quality results and logs. +- ``check`` -- one of the built-in checks below, or ``custom_sql``. +- ``condition`` -- the pass/fail condition for the observed value; see `Conditions`_. +- ``column`` -- target column. Required for column-level built-in checks, unused for + ``row_count`` and ``custom_sql``. +- ``sql`` -- a SQL statement returning a single scalar. Required for, and only valid with, + ``check="custom_sql"``. May reference the table being checked as ``{table}``. +- ``severity`` -- ``error`` (default) or ``warn``. Controls whether a failing rule fails the + task, subject to the operator's ``fail_on`` (see :doc:`operators`). +- ``partition_clause`` -- extra SQL predicate ANDed into this rule's ``WHERE`` clause, e.g. + ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. +- ``previous_name`` -- set when renaming a rule so its execution history stays continuous + (see `Identity and history`_). +- ``description`` -- optional human-readable text shown in data quality results. + When omitted, the provider generates a short default description from the rule and condition. +- ``dimension`` -- one of ``completeness``, ``uniqueness``, ``validity``, ``freshness``, + ``volume``, ``consistency``. Defaults to the check's catalog dimension (see the table below; + ``validity`` for ``custom_sql``) when left unset. Only set this explicitly for a ``custom_sql`` + rule that measures something the default dimension doesn't capture. + +Built-in checks +---------------- + +Each built-in check is backed by a plain SQL expression, rendered with ``{column}`` for +column-level checks: + +.. list-table:: + :header-rows: 1 + + * - Check + - SQL expression + - Column required + - Default dimension + * - ``null_count`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)`` + - yes + - ``completeness`` + * - ``null_ratio`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)`` + - yes + - ``completeness`` + * - ``distinct_count`` + - ``COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``unique_violations`` + - ``COUNT({column}) - COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``min`` + - ``MIN({column})`` + - yes + - ``validity`` + * - ``max`` + - ``MAX({column})`` + - yes + - ``validity`` + * - ``mean`` + - ``AVG({column})`` + - yes + - ``validity`` + * - ``row_count`` + - ``COUNT(*)`` + - no + - ``volume`` + +.. note:: + + ``null_ratio`` divides by ``COUNT(*)`` with no guard against an empty table (or an empty + partition, when combined with ``partition_clause``); running it against zero rows is a + division-by-zero at the database level, not a clean "not applicable" result. Guard with a + ``row_count`` rule upstream, or a ``partition_clause`` that guarantees a non-empty set. A + portable guard would wrap the denominator in ``NULLIF(COUNT(*), 0)``, but ``NULLIF`` isn't + supported by every ``DbApiHook`` (see `Supported checks and databases`_) -- this is one + instance of the general rule below: if a built-in check's SQL doesn't work against your + database, express it as ``custom_sql`` instead. + +Conditions +----------- + +A :class:`~airflow.providers.common.dataquality.rules.Condition` is the pass/fail rule applied to the observed +value, using the same grammar as the ``common.sql`` check operators: + +- ``equal_to`` -- exact match. Cannot be combined with any other comparison. +- ``greater_than`` / ``geq_to`` -- lower bound, exclusive/inclusive. +- ``less_than`` / ``leq_to`` -- upper bound, exclusive/inclusive. +- ``tolerance`` -- a percentage that widens ``equal_to`` into a range; only valid together + with ``equal_to``. + +``greater_than``/``less_than``/``geq_to``/``leq_to`` may be combined to express a range +(e.g. ``{"geq_to": 0, "leq_to": 10}``). + +``custom_sql``: the escape hatch +---------------------------------- + +Built-in checks are all single-column. The moment a rule needs to compare two columns, join +another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL +statement that resolves to a single scalar, evaluated exactly like a built-in check: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +Supported checks and databases +-------------------------------- + +Built-in checks are plain SQL expressions executed through whichever ``common.sql`` +:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook` your connection resolves to. +Airflow does not validate those expressions per database dialect. + +The catalog intentionally uses simple expressions that work on common relational databases +and many distributed SQL engines, but support is not guaranteed for every ``DbApiHook``. +Some hooks expose non-standard SQL layers and may reject or evaluate an expression differently. + +For example, ``null_ratio`` is currently rendered as: + +.. code-block:: text + + SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) + +This is simple and portable, but it is not guarded against an empty table or an empty +partition. A database-specific version could use a different expression, for example: + +.. code-block:: text + + CASE WHEN COUNT(*) = 0 THEN NULL + ELSE SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) END + +If a built-in check does not work for your database or you need different empty-table +semantics, use ``custom_sql`` and write the expression for your dialect. + +Loading rules from YAML +------------------------- + +Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.common.dataquality.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` +at Dag-parse time. This keeps rules editable by people who don't write Python. + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml + :language: yaml + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +Identity and history +---------------------- + +Each rule has a stable ``rule_uid``, a hash of the fields that define *what is being measured*: +``name`` (or ``previous_name``, if set), ``check``, ``column``, ``sql``, and ``condition``. +Everything else -- including ``severity``, ``partition_clause``, ``description``, and +``dimension`` -- can change between Dag runs without breaking the rule's execution history, +because it isn't part of the identity hash. + +Renaming a rule outright would normally start a new history under the new name; set +``previous_name`` to the old name for one deploy to carry the old identity forward instead: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", # renamed from order_id_unique + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ) diff --git a/providers/common/dataquality/docs/security.rst b/providers/common/dataquality/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/common/dataquality/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/common/dataquality/provider.yaml b/providers/common/dataquality/provider.yaml new file mode 100644 index 0000000000000..80969a57621e7 --- /dev/null +++ b/providers/common/dataquality/provider.yaml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-common-dataquality +name: Data Quality +description: | + ``Data Quality Provider`` + + Declarative data quality rules with durable, per-rule execution history. + Checks run through ``common.sql`` DB-API hooks; results are persisted to a + configurable results store (object storage or local files) so task, run, + and rule-level quality can be inspected over time. + +state: ready +lifecycle: incubation +source-date-epoch: 1751587200 +build-system: flit_core + +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 + +integrations: + - integration-name: Data Quality + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/ + how-to-guide: + - /docs/apache-airflow-providers-common-dataquality/operators.rst + tags: [software] + +operators: + - integration-name: Data Quality + python-modules: + - airflow.providers.common.dataquality.operators.dq_check + +task-decorators: + - class-name: airflow.providers.common.dataquality.decorators.dq_check.dq_check_task + name: dq_check + +config: + common.dataquality: + description: | + Configuration for the Data Quality provider results store. + options: + results_path: + description: | + Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality + results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. + When unset, checks still run but no history is persisted. + version_added: 0.1.0 + type: string + example: s3://data-platform/airflow-dq + default: ~ + results_conn_id: + description: | + Optional Airflow connection used to access ``results_path``. + version_added: 0.1.0 + type: string + example: aws_default + default: ~ diff --git a/providers/common/dataquality/pyproject.toml b/providers/common/dataquality/pyproject.toml new file mode 100644 index 0000000000000..bbf3f9764da68 --- /dev/null +++ b/providers/common/dataquality/pyproject.toml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-common-dataquality" +version = "0.1.0" +description = "Provider package apache-airflow-providers-common-dataquality for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "common.dataquality", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.15.0", + "apache-airflow-providers-common-sql>=2.0.0", + "pydantic>=2.11.0", + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.common.dataquality.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.common.dataquality" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/common/__init__.py", + "tests/", +] diff --git a/providers/common/dataquality/src/airflow/__init__.py b/providers/common/dataquality/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/__init__.py b/providers/common/dataquality/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/common/__init__.py b/providers/common/dataquality/src/airflow/providers/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py new file mode 100644 index 0000000000000..484a92c3fad31 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "3.0.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-common-dataquality:{__version__}` needs Apache Airflow 3.0.0+" + ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py new file mode 100644 index 0000000000000..4655e9478ab89 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Asset-level data quality declarations. + +Quality configuration lives inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is +serialized with the Dag and needs no Airflow core changes. The rules travel with the asset +definition instead of being scattered across the Dags that check it. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.sdk import task + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from airflow.sdk import Asset + from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult + +log = logging.getLogger(__name__) + +DQ_EXTRA_KEY = "airflow.dataquality" +DQ_RESULT_EXTRA_KEY = "airflow.dataquality.result" + + +def asset_quality( + asset: Asset, + *, + ruleset: RuleSet | dict[str, Any] | str, + conn_id: str | None = None, + table: str | None = None, +) -> Asset: + """ + Attach data quality configuration to an asset, returning the same asset. + + :param asset: The asset the rules describe. + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file (resolved eagerly, at Dag-parse time). + :param conn_id: Default connection a check operator should use for this asset. + :param table: Default table to check; falls back to the asset name when unset. + + Usage:: + + orders = asset_quality( + Asset("orders", uri="postgres://warehouse/analytics/orders"), + ruleset=rules, + conn_id="warehouse", + table="analytics.orders", + ) + check = DQCheckOperator(task_id="dq", asset=orders) + """ + if isinstance(ruleset, str): + ruleset = RuleSet.from_file(ruleset) + elif isinstance(ruleset, dict): + ruleset = RuleSet.from_dict(ruleset) + config: dict[str, Any] = {"ruleset": ruleset.to_dict()} + if conn_id: + config["conn_id"] = conn_id + if table: + config["table"] = table + asset.extra[DQ_EXTRA_KEY] = config + return asset + + +def get_asset_quality_config(asset: Asset) -> dict[str, Any] | None: + """Return the raw ``airflow.dataquality`` config attached to an asset, if any.""" + config = asset.extra.get(DQ_EXTRA_KEY) + if not isinstance(config, dict): + return None + return config + + +def get_asset_ruleset(asset: Asset) -> RuleSet: + """Return the ruleset attached to an asset, raising when none is attached.""" + config = get_asset_quality_config(asset) + if not config or "ruleset" not in config: + raise DQRuleValidationError( + f"Asset {asset.name!r} has no data quality config; attach one with asset_quality()" + ) + return RuleSet.from_dict(config["ruleset"]) + + +def _quality_score_passes( + asset: Asset, + min_score: float, + triggering_asset_events: Mapping[Asset, Sequence[AssetEventDagRunReferenceResult]], +) -> bool: + """Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag.""" + events = triggering_asset_events.get(asset, []) + if not events: + log.warning( + "require_quality(%s): no triggering event for this run; skipping downstream tasks", + asset.name, + ) + return False + + summary = events[-1].extra.get(DQ_RESULT_EXTRA_KEY) + score = summary.get("score") if isinstance(summary, dict) else None + if not isinstance(score, (int, float)) or isinstance(score, bool): + log.warning( + "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", + asset.name, + ) + return False + + if score < min_score: + log.warning( + "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", + asset.name, + score, + min_score, + ) + return False + + return True + + +def require_quality( + asset: Asset, + *, + min_score: float, + task_id: str | None = None, +) -> Any: + """ + Gate a Dag run on the data quality score attached to one of its triggering asset events. + + Reads the summary a :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` + attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dataquality.result"]`` + (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task + — when that summary is missing or its score is below ``min_score``. Call it inside a Dag + scheduled by ``asset``:: + + with DAG("orders_consumer", schedule=orders) as consumer: + start = require_quality(orders, min_score=0.95) + start >> process_orders() + + :param asset: The asset whose triggering event carries the quality summary. + :param min_score: Minimum required score in ``[0, 1]``; the run proceeds only when the + triggering event's score is at least this value. + :param task_id: Task id for the generated gate task. Defaults to + ``f"require_quality_{asset.name}"`` so gating on several assets in one Dag doesn't + collide on task id. + """ + if not 0 <= min_score <= 1: + raise ValueError(f"min_score must be between 0 and 1, got {min_score!r}") + gate_task_id = task_id or f"require_quality_{asset.name}" + + @task.short_circuit(task_id=gate_task_id) + def _require_quality(**context: Any) -> bool: + return _quality_score_passes(asset, min_score, context["triggering_asset_events"]) + + return _require_quality() diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py new file mode 100644 index 0000000000000..a41919c9f627a --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend + +__all__ = ["ObjectStorageResultsBackend", "get_backend_from_config"] + + +def get_backend_from_config() -> ObjectStorageResultsBackend | None: + """Build the results backend configured under ``[common.dataquality]``, or ``None`` when not configured.""" + from airflow.providers.common.compat.sdk import conf + + results_path = conf.get("common.dataquality", "results_path", fallback=None) + if not results_path: + return None + conn_id = conf.get("common.dataquality", "results_conn_id", fallback=None) + return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py new file mode 100644 index 0000000000000..16b963548b094 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -0,0 +1,250 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Object-storage results backend. + +Each DQ check writes a keyed JSON document plus read indexes optimized for the UI: + + runs/by_task/dag_id=/task_id=/date=<2026-07-04>/.json + Canonical run record: ``{"run": ..., "results": [...], "summary": ...}``. + + runs/by_task_instance/dag_id=/task_id=/__.json + Latest result for a task-instance page. Last write wins across retries. + + rules/by_task_rule/dag_id=/task_id=/rule_uid=/__.json + One rule result plus run context: ``{"run": ..., "result": ...}``. + +The duplicate files are intentional read indexes: DQ tasks write once, while the UI reads +many times. Keeping these indexes avoids scanning all task runs for common UI views. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary +from airflow.sdk import ObjectStoragePath + +log = logging.getLogger(__name__) + + +class ObjectStorageResultsBackend: + """Persist DQ results as JSON files via ``ObjectStoragePath``.""" + + def __init__(self, results_path: str, conn_id: str | None = None) -> None: + self.root = ObjectStoragePath(results_path, conn_id=conn_id) + + def write_run(self, run: DQRun, results: list[RuleResult]) -> None: + timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() + payload = self._build_run_payload(run, results) + + self._write_run_file(run, timestamp[:10], payload) + self._write_task_instance_index(run, payload) + self._write_rule_indexes(run, results, timestamp) + + def read_task_rule_history( + self, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None + ) -> dict[str, Any]: + """Return recent results for one rule produced by one task, newest first.""" + rule_dir = ( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"rule_uid={rule_uid}" + ) + return self._read_rule_history_dir(rule_dir, limit, before) + + def read_task_runs( + self, dag_id: str, task_id: str, limit: int = 50, before: str | None = None + ) -> dict[str, Any]: + """ + Return recent data quality runs for one task, newest first. + + Returns ``{"items": [...], "next_cursor": ...}``. ``before`` is the opaque + ``next_cursor`` from the previous page, so "load more" only reads runs it hasn't + shown yet. ``next_cursor`` is set by reading one extra entry past ``limit``: cheap + on every page except the last one, where there's no way to confirm history is + exhausted without walking to the end. + + ``date=`` partition names sort correctly as plain strings, so directories are walked + newest-first and scanning stops as soon as ``limit + 1`` matching runs have been + collected — a task with years of history doesn't pay for a full scan on every page. + """ + task_dir = self.root / "runs" / "by_task" / f"dag_id={dag_id}" / f"task_id={task_id}" + if not task_dir.exists(): + return {"items": [], "next_cursor": None} + + date_dirs = sorted((path for path in task_dir.iterdir() if path.is_dir()), reverse=True) + runs = [] + for date_dir in date_dirs: + for path in date_dir.iterdir(): + if not path.name.endswith(".json"): + continue + payload = self._read_json(path) + if payload is None: + continue + cursor = self._get_run_payload_cursor(payload) + if before is not None and cursor >= before: + continue + runs.append(payload) + if len(runs) > limit: + break + + ordered = sorted(runs, key=self._get_run_payload_cursor, reverse=True) + page = ordered[:limit] + next_cursor = self._get_run_payload_cursor(page[-1]) if len(ordered) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + def read_by_task_instance( + self, dag_id: str, task_id: str, run_id: str, map_index: int = -1 + ) -> dict[str, Any]: + """Read the latest run for one task instance as ``{"run": ..., "results": ..., "summary": ...}``.""" + path = ( + self.root + / "runs" + / "by_task_instance" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"{self._get_safe_key(run_id)}__{map_index}.json" + ) + return self._read_json_or_raise(path) + + def _write_run_file(self, run: DQRun, date_part: str, payload: dict[str, Any]) -> None: + run_dir = ( + self.root + / "runs" + / "by_task" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"date={date_part}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / f"{run.run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _write_task_instance_index(self, run: DQRun, payload: dict[str, Any]) -> None: + ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" + ti_dir.mkdir(parents=True, exist_ok=True) + (ti_dir / f"{self._get_safe_key(run.run_id)}__{run.map_index}.json").write_text( + json.dumps(payload, default=str) + ) + + def _write_rule_indexes(self, run: DQRun, results: list[RuleResult], timestamp: str) -> None: + run_context = self._build_run_context(run) + compact_ts = self._get_safe_key(timestamp) + for result in results: + payload = {"run": run_context, "result": result.to_dict()} + self._write_rule_index( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"rule_uid={result.rule_uid}", + compact_ts, + run.run_uid, + payload, + ) + + def _write_rule_index( + self, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] + ) -> None: + rule_dir.mkdir(parents=True, exist_ok=True) + (rule_dir / f"{compact_ts}__{run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _read_rule_history_dir( + self, rule_dir: ObjectStoragePath, limit: int, before: str | None = None + ) -> dict[str, Any]: + """ + Read rule-result records newest-first, as ``{"items": [...], "next_cursor": ...}``. + + Reads one entry past ``limit`` to determine ``next_cursor``: cheap on every page + except the last one, where confirming there's nothing older means reading to the end. + """ + if not rule_dir.exists(): + return {"items": [], "next_cursor": None} + + history: list[dict[str, Any]] = [] + for path in sorted(rule_dir.iterdir(), key=lambda p: p.name, reverse=True): + payload = self._read_json(path) + if payload is None: + continue + record = {**payload["result"], "run": payload["run"]} + cursor = self._get_rule_history_cursor(record) + if before is not None and cursor >= before: + continue + history.append(record) + if len(history) > limit: + break + + page = history[:limit] + next_cursor = self._get_rule_history_cursor(page[-1]) if len(history) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + @staticmethod + def _get_safe_key(value: str) -> str: + """Sanitize a value (for example an Airflow ``run_id``) for an object key segment.""" + return value.replace("/", "_").replace(":", "_").replace("+", "_") + + @staticmethod + def _build_run_payload(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + result_records = [result.to_dict() for result in results] + return { + "run": run.to_dict(), + "results": result_records, + "summary": build_summary(run, results), + } + + @staticmethod + def _build_run_context(run: DQRun) -> dict[str, Any]: + return { + "run_uid": run.run_uid, + "dag_id": run.dag_id, + "task_id": run.task_id, + "run_id": run.run_id, + "map_index": run.map_index, + "started_at": run.started_at, + "table_ref": run.table_ref, + } + + @staticmethod + def _get_run_payload_cursor(payload: dict[str, Any]) -> str: + run = payload["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _get_rule_history_cursor(record: dict[str, Any]) -> str: + run = record["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _build_cursor(started_at: str | None, run_uid: str | None) -> str: + return f"{started_at or ''}|{run_uid or ''}" + + def _read_json_or_raise(self, path: ObjectStoragePath) -> dict[str, Any]: + return json.loads(path.read_text()) + + def _read_json(self, path: ObjectStoragePath) -> dict[str, Any] | None: + try: + return self._read_json_or_raise(path) + except (OSError, json.JSONDecodeError): + log.warning("Skipping unreadable DQ result file %s", path) + return None diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py new file mode 100644 index 0000000000000..6bd05a2e08e8f --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""TaskFlow decorator for :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Any, ClassVar + +from airflow.providers.common.compat.sdk import ( + DecoratedOperator, + context_merge, + determine_kwargs, + task_decorator_factory, +) +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.sdk import Context + from airflow.sdk.bases.decorator import TaskDecorator + + +class _DQCheckDecoratedOperator(DecoratedOperator, DQCheckOperator): + """ + Wraps a callable that optionally returns a runtime ruleset for a data quality check. + + :param python_callable: A reference to a callable returning ``None`` or a ruleset for this run. + :param op_args: Positional arguments for the callable. + :param op_kwargs: Keyword arguments for the callable. + """ + + template_fields: Sequence[str] = ( + *DecoratedOperator.template_fields, + *DQCheckOperator.template_fields, + ) + template_fields_renderers: ClassVar[dict[str, str]] = { + **DecoratedOperator.template_fields_renderers, + } + + custom_operator_name = "@task.dq_check" + + def __init__( + self, + *, + python_callable: Callable, + op_args: Collection[Any] | None = None, + op_kwargs: Mapping[str, Any] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("ruleset", SET_DURING_EXECUTION) + super().__init__(python_callable=python_callable, op_args=op_args, op_kwargs=op_kwargs, **kwargs) + + def execute(self, context: Context) -> Any: + context_merge(context, self.op_kwargs) + kwargs = determine_kwargs(self.python_callable, self.op_args, context) + ruleset = self.python_callable(*self.op_args, **kwargs) + if ruleset is not None: + if not isinstance(ruleset, (RuleSet, dict, str)): + raise TypeError( + f"{self.custom_operator_name} function must return a RuleSet, dict, path, or None, " + f"got {type(ruleset).__name__}" + ) + self.ruleset = ruleset + return DQCheckOperator.execute(self, context) + + +def dq_check_task( + python_callable: Callable | None = None, + **kwargs, +) -> TaskDecorator: + """ + Turn a function into a data quality check task. + + ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned + by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` + exactly as on :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`. The + function itself is optional plumbing: return ``None`` to run the check as declared, or a + ruleset to use for this run. + + Usage:: + + @task.dq_check(conn_id="warehouse", ruleset=rules) + def orders_quality(ds=None): + return None + + :param python_callable: Function to decorate. + """ + return task_decorator_factory( + python_callable=python_callable, + decorated_operator_class=_DQCheckDecoratedOperator, + **kwargs, + ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py new file mode 100644 index 0000000000000..cf6a86506a954 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.engines.sql import Observation, SQLDQEngine + +__all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py new file mode 100644 index 0000000000000..d9e22ce776258 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""SQL execution engine: compiles a ruleset into check queries run through a DB-API hook.""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.dataquality.rules import CUSTOM_SQL_CHECK +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS + +if TYPE_CHECKING: + from airflow.providers.common.dataquality.rules import DQRule, RuleSet + from airflow.providers.common.sql.hooks.sql import DbApiHook + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Observation: + """Raw value a rule observed, before its condition is evaluated.""" + + rule: DQRule + observed_value: Any = None + duration_ms: float | None = None + error_message: str | None = None + sql: str | None = None + + +class SQLDQEngine: + """ + Runs built-in checks as a single UNION ALL query and custom SQL rules individually. + + Table, column, and partition-clause values come from the Dag author and are interpolated + into SQL the same way the ``common.sql`` check operators do — they are trusted input. + """ + + check_sql_template = "SELECT '{rule_uid}' AS rule_uid, {expression} AS observed FROM {table}{where}" + + def __init__(self, hook: DbApiHook) -> None: + self.hook = hook + + def measure(self, ruleset: RuleSet, table: str, partition_clause: str | None = None) -> list[Observation]: + builtin_rules = [rule for rule in ruleset.rules if rule.check != CUSTOM_SQL_CHECK] + custom_rules = [rule for rule in ruleset.rules if rule.check == CUSTOM_SQL_CHECK] + + observations = [] + if builtin_rules: + observations.extend(self._measure_builtin(builtin_rules, table, partition_clause)) + for rule in custom_rules: + observations.append(self._measure_custom(rule, table)) + return observations + + def build_batch_sql(self, rules: list[DQRule], table: str, partition_clause: str | None) -> str: + return " UNION ALL ".join(self.build_rule_sql(rule, table, partition_clause) for rule in rules) + + def build_rule_sql(self, rule: DQRule, table: str, partition_clause: str | None = None) -> str: + """Build the SQL used to measure one built-in rule.""" + expression = CHECK_SPECS[rule.check].expression.format(column=rule.column) + predicates = [p for p in (partition_clause, rule.partition_clause) if p] + where = f" WHERE {' AND '.join(predicates)}" if predicates else "" + return self.check_sql_template.format( + rule_uid=rule.rule_uid, expression=expression, table=table, where=where + ) + + def _measure_builtin( + self, rules: list[DQRule], table: str, partition_clause: str | None + ) -> list[Observation]: + sql = self.build_batch_sql(rules, table, partition_clause) + log.info("Running %d built-in checks against %s", len(rules), table) + started = time.monotonic() + try: + records = self.hook.get_records(sql) + except Exception as e: + elapsed_ms = (time.monotonic() - started) * 1000 + log.exception("Check query failed for table %s", table) + return [ + Observation( + rule=rule, + duration_ms=elapsed_ms, + error_message=str(e), + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + elapsed_ms = (time.monotonic() - started) * 1000 + observed_by_uid = {str(row[0]): row[1] for row in records or []} + return [ + Observation( + rule=rule, + observed_value=observed_by_uid.get(rule.rule_uid), + duration_ms=elapsed_ms, + error_message=None if rule.rule_uid in observed_by_uid else "No result returned for rule", + sql=self.build_rule_sql(rule, table, partition_clause), + ) + for rule in rules + ] + + def _measure_custom(self, rule: DQRule, table: str) -> Observation: + if rule.sql is None: + raise ValueError(f"Rule {rule.name!r} has no SQL to execute") + sql = rule.sql.replace("{table}", table) + log.info("Running custom SQL check %s", rule.name) + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + log.exception("Custom SQL check %s failed", rule.name) + return Observation( + rule=rule, + duration_ms=(time.monotonic() - started) * 1000, + error_message=str(e), + sql=sql, + ) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="Query returned no rows", sql=sql + ) + return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py new file mode 100644 index 0000000000000..8b2dcb32f9d13 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``custom_sql`` for checks the built-in catalog can't express. + +Built-in checks (``null_count``, ``min``, ``max``, ...) are all single-column. The moment a +rule needs to compare two columns, join against another table, or use a SQL function the +built-in catalog doesn't cover (or that your database's ``DbApiHook`` doesn't support -- see +"Supported checks and databases" in the provider docs), ``custom_sql`` is the escape hatch: +any statement that resolves to a single scalar, evaluated the same way as a built-in check. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_check_custom_sql" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_custom_sql_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_custom_sql] +custom_sql_ruleset = RuleSet( + name="orders_custom_sql", + rules=( + DQRule( + name="shipped_after_ordered", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string + # placeholder -- a cross-column comparison no single-column built-in can express. + sql="SELECT COUNT(*) FROM {table} WHERE shipped_at < ordered_at", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_ratio", + check="custom_sql", + sql=( + "SELECT CAST(SUM(CASE WHEN amount > 100 THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM {table}" + ), + condition={"leq_to": 0.5}, + severity=Severity.WARN, + ), + ), +) +# [END howto_operator_dq_check_custom_sql] + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + amount REAL, + ordered_at TEXT, + shipped_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ordered_at, shipped_at) VALUES + (1, 42.0, '2026-07-01', '2026-07-02'), + (2, 150.0, '2026-07-01', '2026-07-03'), + (3, 15.5, '2026-07-02', '2026-07-02'); + """, + ) + + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=custom_sql_ruleset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py new file mode 100644 index 0000000000000..41e194218775f --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``@task.dq_check`` with a runtime ruleset. + +``table`` (or ``asset``) is declared as a decorator argument, exactly like the plain operator. +``ruleset`` may be declared at Dag-parse time, or returned by the decorated function at +execution time. This is useful when an upstream task, variable, or generated value decides which +ruleset to run. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, task + +DAG_ID = "example_dq_check_decorator_dynamic" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_dynamic_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +orders_ruleset = RuleSet( + name="orders_dynamic", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +strict_ruleset = RuleSet( + name="orders_dynamic_strict", + rules=( + *orders_ruleset.rules, + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + ), +) + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL, ds TEXT);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ds) VALUES + (1, 10.0, '{{{{ ds }}}}'), + (2, 25.5, '{{{{ ds }}}}'), + (3, 7.25, '2020-01-01'); + """, + ) + + # [START howto_decorator_dq_check_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, ruleset=orders_ruleset, fail_on="never") + def check_with_ruleset_from_upstream(strict: bool = True): + """Swap in a stricter ruleset based on a signal only known at task-execution time.""" + return strict_ruleset if strict else None + + # [END howto_decorator_dq_check_runtime_ruleset] + + create_table >> insert_orders >> check_with_ruleset_from_upstream() diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py new file mode 100644 index 0000000000000..15676d32a76fc --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dataquality-rule-authoring`` skill. + +Requires the optional ``apache-airflow-providers-common-ai[skills]`` package and a configured +LLM connection (``llm_conn_id``); this Dag is not registered when common.ai isn't installed. + +An LLM is asked to propose data quality rules for a table's columns. Its ``system_prompt`` +points it at the ``dataquality-rule-authoring`` skill shipped with this provider (via +``AgentSkillsToolset``), so it knows the exact ``RuleSet``/``DQRule`` schema, the built-in check +catalog, and the ``Condition`` grammar, instead of guessing at field or check names. +``output_type=RuleSet`` makes pydantic-ai validate -- and self-correct -- the model's output +against the real ``RuleSet`` model before the task completes, so a malformed rule never reaches +``DQCheckOperator`` in the first place. + +The generated ``RuleSet`` is then returned by ``@task.dq_check`` at runtime, since the real +ruleset doesn't exist until the LLM task runs. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, task + +try: + from airflow.providers.common.ai.toolsets.skills import AgentSkillsToolset +except ImportError: + AgentSkillsToolset = None # type: ignore[assignment,misc] + +DAG_ID = "example_dq_llm_generated_ruleset" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_llm_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +# The skill ships next to this Dag's package, not next to this file -- resolve relative to +# __file__ so the path holds regardless of the Dag processor's working directory. +SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dataquality-rule-authoring" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +if AgentSkillsToolset is not None: + with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq", "ai"], + ) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + region TEXT, + created_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, region, created_at) VALUES + (1, 101, 10.0, 'US', '2026-07-01'), + (2, 102, 25.5, 'EU', '2026-07-02'), + (3, NULL, 7.25, 'US', '2026-07-03'); + """, + ) + + # [START howto_task_dq_generate_ruleset_with_llm] + @task.llm( + llm_conn_id="pydanticai_default", + system_prompt=( + "You are a data quality engineer. Before answering, consult the " + "dataquality-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " + "check catalog, and the Condition grammar -- do not invent field or check names." + ), + output_type=RuleSet, + agent_params={"toolsets": [AgentSkillsToolset(sources=[str(SKILLS_DIR)])]}, + ) + def generate_ruleset(): + return ( + "Generate a RuleSet named 'orders_llm_quality' for a table with these columns: " + "order_id (integer, primary key), customer_id (integer, nullable foreign key), " + "amount (float, must be non-negative), region (short string, low-cardinality), " + "created_at (date). Include at least a not-null check on order_id, a uniqueness " + "check on order_id, and a row-count check." + ) + + # [END howto_task_dq_generate_ruleset_with_llm] + + # [START howto_decorator_dq_check_llm_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, fail_on="never") + def check_with_llm_ruleset(ruleset: RuleSet): + return ruleset + + # [END howto_decorator_dq_check_llm_runtime_ruleset] + + generated_ruleset = generate_ruleset() + checked = check_with_llm_ruleset(generated_ruleset) + + create_table >> insert_orders >> generated_ruleset >> checked diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py new file mode 100644 index 0000000000000..af721b10b89da --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAGs: gate a consumer Dag on the data quality score of the asset that triggers it. + +Two Dags, wired together only through the ``dq_gated_orders`` asset: + +- ``example_dq_require_quality_producer`` checks a table and produces the asset. + :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` attaches its summary + (including the quality ``score``) to the asset event automatically because the asset carries + quality config attached with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. +- ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by + :func:`~airflow.providers.common.dataquality.assets.require_quality`, reads the score off the triggering + asset event and short-circuits -- skipping every downstream task -- when it's missing or + below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.assets import asset_quality, require_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, Asset, task + +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_gated_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_asset_quality] +gated_orders_ruleset = RuleSet( + name="gated_orders_quality", + rules=( + DQRule(name="order_id_not_null", check="null_count", column="order_id", condition={"equal_to": 0}), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition={"geq_to": 0}), + DQRule(name="row_count_present", check="row_count", condition={"greater_than": 0}), + ), +) + +gated_orders_asset = asset_quality( + Asset("dq_gated_orders", uri="file:///tmp/airflow_dq_example/gated_orders"), + ruleset=gated_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_asset_quality] + +with DAG( + dag_id="example_dq_require_quality_producer", + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as producer_dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # Producing check: no explicit outlets needed, DQCheckOperator adds the asset itself + # because it was passed via asset=. + check_orders = DQCheckOperator( + task_id="check_orders", + asset=gated_orders_asset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders + +# [START howto_require_quality] +with DAG( + dag_id="example_dq_require_quality_consumer", + schedule=gated_orders_asset, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as consumer_dag: + gate = require_quality(gated_orders_asset, min_score=0.95) + + @task + def process_orders(): + print("Processing orders -- quality gate passed.") + + gate >> process_orders() +# [END howto_require_quality] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py new file mode 100644 index 0000000000000..79ca15ae3826c --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: load a ruleset from a YAML file instead of declaring it in Python. + +A path string is accepted anywhere a :class:`~airflow.providers.common.dataquality.rules.RuleSet` is -- +``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and +:func:`~airflow.providers.common.dataquality.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` +without touching the Dag file. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_ruleset_from_yaml" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_yaml_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +RULESET_FILE = Path(__file__).parent / "orders_ruleset.yaml" + +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # [START howto_operator_dq_check_ruleset_from_yaml] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=str(RULESET_FILE), + fail_on="never", + ) + # [END howto_operator_dq_check_ruleset_from_yaml] + + create_table >> insert_orders >> check_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml new file mode 100644 index 0000000000000..049e69543bc32 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- + +name: orders_from_yaml +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 + dimension: completeness + - name: amount_min_ge_zero + check: min + column: amount + condition: + geq_to: 0 + - name: row_count_present + check: row_count + condition: + greater_than: 0 + severity: warn diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py new file mode 100644 index 0000000000000..9350308d46501 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class DQRuleValidationError(ValueError): + """Raised when a rule or ruleset definition is invalid.""" + + +class DQCheckFailedError(RuntimeError): + """Raised when a data quality check fails at or above the operator's ``fail_on`` severity.""" diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py new file mode 100644 index 0000000000000..d6be8d160587e --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-common-dataquality", + "name": "Data Quality", + "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", + "integrations": [ + { + "integration-name": "Data Quality", + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/", + "how-to-guide": ["/docs/apache-airflow-providers-common-dataquality/operators.rst"], + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Data Quality", + "python-modules": ["airflow.providers.common.dataquality.operators.dq_check"], + } + ], + "task-decorators": [ + { + "class-name": "airflow.providers.common.dataquality.decorators.dq_check.dq_check_task", + "name": "dq_check", + } + ], + "config": { + "common.dataquality": { + "description": "Configuration for the Data Quality provider results store.\n", + "options": { + "results_path": { + "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", + "version_added": "0.1.0", + "type": "string", + "example": "s3://data-platform/airflow-dq", + "default": None, + }, + "results_conn_id": { + "description": "Optional Airflow connection used to access ``results_path``.\n", + "version_added": "0.1.0", + "type": "string", + "example": "aws_default", + "default": None, + }, + }, + } + }, + } diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py new file mode 100644 index 0000000000000..048c2ff752049 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator + +__all__ = ["DQCheckOperator"] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py new file mode 100644 index 0000000000000..ebc5369e4e323 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py @@ -0,0 +1,255 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from collections.abc import Sequence +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, cast + +from airflow.providers.common.dataquality.assets import DQ_RESULT_EXTRA_KEY, get_asset_quality_config +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.results import ( + ERROR, + FAIL, + PASS, + WARN, + DQRun, + RuleResult, + build_summary, +) +from airflow.providers.common.dataquality.rules import RuleSet, describe_rule +from airflow.providers.common.sql.operators.sql import BaseSQLOperator +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.providers.common.dataquality.engines.sql import Observation + from airflow.providers.common.dataquality.rules.rule import Condition, Dimension + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +FAIL_ON_CHOICES = ("error", "warn", "never") + + +class DQCheckOperator(BaseSQLOperator): + """ + Run a ruleset against a table and persist per-rule results to the configured results store. + + Every rule is evaluated and recorded regardless of the task outcome. Execution errors + (a check query failing) always fail the task; rule failures fail the task according to + ``fail_on``. + + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file. Optional when ``asset`` carries one. + :param table: The table to run checks against. Optional when ``asset`` carries one + (falling back to the asset name). + :param asset: An asset decorated with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. + Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments + win) and is automatically added to the task's outlets so its asset events carry + the check summary. + :param partition_clause: Predicate ANDed into every built-in check's WHERE clause, + e.g. ``"ds = '{{ ds }}'"``. + :param fail_on: Severity that fails the task: ``error`` (default — warn-severity rule + failures are recorded but don't fail), ``warn`` (any rule failure fails), or + ``never`` (only execution errors fail). + :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). + :param database: Optional database/schema overriding the connection's default. + + Results are persisted to the backend configured under ``[common.dataquality] results_path``; when that's + unset, checks still run but no history is persisted. There is no per-operator override -- + all checks in a deployment share one results store. + """ + + template_fields: Sequence[str] = ("table", "partition_clause", *BaseSQLOperator.template_fields) + ui_color: str = "#87ceeb" + + def __init__( + self, + *, + ruleset: RuleSet | dict[str, Any] | str | None = None, + table: str | None = None, + asset: Asset | None = None, + partition_clause: str | None = None, + fail_on: str = "error", + **kwargs, + ) -> None: + if asset is not None: + config = get_asset_quality_config(asset) or {} + ruleset = ruleset if ruleset is not None else config.get("ruleset") + table = table or config.get("table") or asset.name + kwargs.setdefault("conn_id", config.get("conn_id")) + outlets = list(kwargs.get("outlets") or []) + if asset not in outlets: + outlets.append(asset) + kwargs["outlets"] = outlets + super().__init__(**kwargs) + if fail_on not in FAIL_ON_CHOICES: + raise ValueError(f"fail_on must be one of {FAIL_ON_CHOICES}, got {fail_on!r}") + if ruleset is None: + raise ValueError("ruleset is required, either directly or via an asset_quality() asset") + if not table: + raise ValueError("table is required, either directly or via an asset_quality() asset") + self.ruleset = ruleset + self.table = table + self.asset = asset + self.partition_clause = partition_clause + self.fail_on = fail_on + + def execute(self, context: Context) -> dict[str, Any]: + ruleset = self._resolve_ruleset() + started_at = datetime.now(tz=timezone.utc).isoformat() + engine = self._get_engine(self.get_db_hook()) + observations = engine.measure(ruleset, self.table, self.partition_clause) + results = [self._evaluate(observation) for observation in observations] + finished_at = datetime.now(tz=timezone.utc).isoformat() + + run = self._build_run(context, ruleset, started_at, finished_at) + summary = build_summary(run, results) + self._log_results(results, summary) + self._persist(run, results) + self._attach_to_outlet_events(context, summary) + self._raise_for_failures(results, summary) + return summary + + def _get_engine(self, hook: DbApiHook) -> SQLDQEngine: + return SQLDQEngine(hook) + + def _resolve_ruleset(self) -> RuleSet: + if self.ruleset is SET_DURING_EXECUTION: + raise ValueError("ruleset is required, either directly or returned by @task.dq_check") + if isinstance(self.ruleset, RuleSet): + return self.ruleset + if isinstance(self.ruleset, dict): + return RuleSet.from_dict(self.ruleset) + return RuleSet.from_file(self.ruleset) + + @staticmethod + def _evaluate(observation: Observation) -> RuleResult: + rule = observation.rule + condition = cast("Condition", rule.condition) + dimension = cast("Dimension", rule.dimension) + error_message = observation.error_message + if error_message is None: + try: + passed = condition.evaluate(observation.observed_value) + except (TypeError, ValueError) as e: + passed = False + error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" + else: + passed = False + + if error_message is not None: + status = ERROR + elif passed: + status = PASS + else: + status = WARN if rule.severity == "warn" else FAIL + observed = observation.observed_value + if observed is not None and not isinstance(observed, int | float | str): + observed = str(observed) + return RuleResult( + rule_uid=rule.rule_uid, + rule_name=rule.name, + status=status, + observed_value=observed, + condition=condition.to_dict(), + dimension=dimension.value, + severity=rule.severity.value, + duration_ms=observation.duration_ms, + error_message=error_message, + description=rule.description or describe_rule(rule), + sql=observation.sql, + ) + + def _build_run(self, context: Context, ruleset: RuleSet, started_at: str, finished_at: str) -> DQRun: + ti = context["ti"] + return DQRun( + dag_id=ti.dag_id, + task_id=ti.task_id, + run_id=ti.run_id, + try_number=ti.try_number, + map_index=ti.map_index if ti.map_index is not None else -1, + ruleset_name=ruleset.name, + table_ref=self.table, + asset_names=tuple(self._outlet_asset_names()), + started_at=started_at, + finished_at=finished_at, + ) + + def _outlet_asset_names(self) -> list[str]: + names = [] + for outlet in self.outlets or []: + name = getattr(outlet, "name", None) + if name: + names.append(name) + return names + + def _persist(self, run: DQRun, results: list[RuleResult]) -> None: + backend = get_backend_from_config() + if backend is None: + self.log.info("No [common.dataquality] results_path configured; skipping results persistence") + return + # Persistence is best-effort: an unreachable results store leaves a gap in + # history but must not change the outcome of the check itself. + try: + backend.write_run(run, results) + except Exception: + self.log.exception("Failed to persist data quality results; continuing") + + def _attach_to_outlet_events(self, context: Context, summary: dict[str, Any]) -> None: + try: + outlet_events = context["outlet_events"] + except KeyError: + return + for outlet in self.outlets or []: + if getattr(outlet, "name", None): + try: + outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary + except Exception: + self.log.warning("Could not attach dq summary to outlet event for %s", outlet) + + def _log_results(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + for result in results: + self.log.info( + "Rule %s [%s/%s]: %s (observed=%s, condition=%s)", + result.rule_name, + result.dimension, + result.severity, + result.status.upper(), + result.observed_value, + result.condition, + ) + self.log.info("Data quality score: %s", summary["score"]) + + def _raise_for_failures(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + errored = [r.rule_name for r in results if r.status == ERROR] + if errored: + raise DQCheckFailedError(f"Check execution errored for rules: {errored}") + failed = [r.rule_name for r in results if r.status == FAIL] + warned = [r.rule_name for r in results if r.status == WARN] + if self.fail_on == "error" and failed: + raise DQCheckFailedError(f"Data quality rules failed: {failed} (score={summary['score']})") + if self.fail_on == "warn" and (failed or warned): + raise DQCheckFailedError( + f"Data quality rules failed: {failed + warned} (score={summary['score']})" + ) + if failed or warned: + self.log.warning("Rule failures below fail_on=%s threshold: %s", self.fail_on, failed + warned) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py new file mode 100644 index 0000000000000..5fbcf04573cc7 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Result records produced by a data quality check run — immutable facts, JSON-serializable.""" + +from __future__ import annotations + +import uuid +from dataclasses import asdict, dataclass, field +from typing import Any + +PASS = "pass" +WARN = "warn" +FAIL = "fail" +ERROR = "error" + +# Weight of a warn-severity failure in the run score, relative to an error-severity failure. +WARN_SCORE_WEIGHT = 0.25 + + +@dataclass(frozen=True) +class RuleResult: + """Outcome of evaluating one rule in one run.""" + + rule_uid: str + rule_name: str + status: str # pass | warn | fail | error + observed_value: float | str | None = None + condition: dict[str, Any] = field(default_factory=dict) + dimension: str = "validity" + severity: str = "error" + duration_ms: float | None = None + error_message: str | None = None + description: str | None = None + sql: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleResult: + return cls(**data) + + +@dataclass(frozen=True) +class DQRun: + """One execution of a ruleset by one task instance.""" + + dag_id: str + task_id: str + run_id: str + try_number: int = 1 + map_index: int = -1 + run_uid: str = field(default_factory=lambda: uuid.uuid4().hex) + ruleset_name: str | None = None + table_ref: str | None = None + asset_names: tuple[str, ...] = () + started_at: str | None = None + finished_at: str | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["asset_names"] = list(self.asset_names) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRun: + data = {**data, "asset_names": tuple(data.get("asset_names", ()))} + return cls(**data) + + +def compute_score(results: list[RuleResult]) -> float | None: + """ + Weighted pass rate for a run in [0, 1]. + + Error-severity failures (and execution errors) count fully against the score; + warn-severity failures count at ``WARN_SCORE_WEIGHT``. + """ + if not results: + return None + penalty = 0.0 + for result in results: + if result.status in (FAIL, ERROR): + penalty += 1.0 + elif result.status == WARN: + penalty += WARN_SCORE_WEIGHT + return round(1.0 - penalty / len(results), 4) + + +def build_summary(run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + """Compact run summary attached to XCom and outlet asset events.""" + return { + "run_uid": run.run_uid, + "ruleset": run.ruleset_name, + "table": run.table_ref, + "score": compute_score(results), + "passed": sum(1 for r in results if r.status == PASS), + "warned": sum(1 for r in results if r.status == WARN), + "failed": sum(1 for r in results if r.status == FAIL), + "errored": sum(1 for r in results if r.status == ERROR), + "failed_rules": sorted(r.rule_name for r in results if r.status in (FAIL, ERROR)), + "warned_rules": sorted(r.rule_name for r in results if r.status == WARN), + } diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py new file mode 100644 index 0000000000000..e50c9a99f43b9 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.common.dataquality.rules.rule import ( + CUSTOM_SQL_CHECK, + Condition, + DQRule, + RuleSet, + Severity, + describe_rule, +) + +__all__ = [ + "CHECK_SPECS", + "CUSTOM_SQL_CHECK", + "CheckSpec", + "Condition", + "DQRule", + "Dimension", + "RuleSet", + "Severity", + "describe_rule", +] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py new file mode 100644 index 0000000000000..11e73a5208f30 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Built-in check catalog: one place for each check's SQL expression and metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class Dimension(str, Enum): + """Data quality dimension assigned to a rule result.""" + + COMPLETENESS = "completeness" + UNIQUENESS = "uniqueness" + VALIDITY = "validity" + FRESHNESS = "freshness" + VOLUME = "volume" + CONSISTENCY = "consistency" + + +@dataclass(frozen=True) +class CheckSpec: + """ + Metadata for one built-in check. + + :param expression: SQL expression template. Column-level checks are rendered with + ``{column}``; table-level checks (``requires_column=False``) ignore it. + :param dimension: Default value for ``DQRule.dimension`` when a rule doesn't set it + explicitly. ``DQRule.dimension`` is a settable field; this is only the fallback. + :param requires_column: Whether a rule using this check must set ``column``. + :param default_condition: Default ``DQRule.condition`` (as a dict) for a rule that doesn't + set one explicitly; ``None`` means the rule must always specify a condition. + """ + + expression: str + dimension: Dimension = Dimension.VALIDITY + requires_column: bool = True + default_condition: dict[str, Any] | None = None + + +CHECK_SPECS: dict[str, CheckSpec] = { + "null_count": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)", + dimension=Dimension.COMPLETENESS, + ), + "null_ratio": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)", + dimension=Dimension.COMPLETENESS, + ), + "distinct_count": CheckSpec( + expression="COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "unique_violations": CheckSpec( + expression="COUNT({column}) - COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "min": CheckSpec(expression="MIN({column})"), + "max": CheckSpec(expression="MAX({column})"), + "mean": CheckSpec(expression="AVG({column})"), + "row_count": CheckSpec( + expression="COUNT(*)", + dimension=Dimension.VOLUME, + requires_column=False, + ), +} diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py new file mode 100644 index 0000000000000..f65a4d67be33e --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -0,0 +1,279 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Declarative data quality rule model: rules are data, not code.""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from typing import Any, cast + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, Dimension + +CUSTOM_SQL_CHECK = "custom_sql" + + +class Severity(str, Enum): + """Severity used to decide whether a failing rule fails the task.""" + + WARN = "warn" + ERROR = "error" + + +class Condition(BaseModel): + """ + Pass/fail condition evaluated against a rule's observed value. + + Uses the same grammar as the ``common.sql`` check operators: ``equal_to``, + ``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage + ``tolerance`` that widens ``equal_to`` into a range. + + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + equal_to: float | None = None + greater_than: float | None = None + less_than: float | None = None + geq_to: float | None = None + leq_to: float | None = None + tolerance: float | None = None + + @model_validator(mode="after") + def _validate_comparisons(self) -> Condition: + comparisons = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + } + set_comparisons = {name for name, value in comparisons.items() if value is not None} + if not set_comparisons: + raise ValueError(f"Condition needs at least one comparison out of: {', '.join(comparisons)}") + if self.equal_to is not None and len(set_comparisons) > 1: + raise ValueError("equal_to cannot be combined with other comparisons") + if self.tolerance is not None and self.equal_to is None: + raise ValueError("tolerance is only supported together with equal_to") + return self + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Condition: + return cls(**data) + + def to_dict(self) -> dict[str, float]: + data = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + "tolerance": self.tolerance, + } + return {k: v for k, v in data.items() if v is not None} + + def evaluate(self, observed: Any) -> bool: + if observed is None: + return False + value = float(observed) + if self.equal_to is not None: + if self.tolerance is not None: + low = self.equal_to * (1 - self.tolerance) + high = self.equal_to * (1 + self.tolerance) + return min(low, high) <= value <= max(low, high) + return value == self.equal_to + return ( + (self.greater_than is None or value > self.greater_than) + and (self.less_than is None or value < self.less_than) + and (self.geq_to is None or value >= self.geq_to) + and (self.leq_to is None or value <= self.leq_to) + ) + + +class DQRule(BaseModel): + """ + A single, named data quality rule. + + :param name: Rule name, unique within its ruleset. + :param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in + checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``; + see "Supported checks and databases" in the provider docs. Use ``custom_sql`` if a + built-in check doesn't fit your database's dialect. + :param condition: Pass condition for the observed value (``Condition`` or its dict form). + Optional only for checks whose ``CheckSpec.default_condition`` is set; every current + built-in check requires one explicitly. + :param column: Target column; required for column-level built-in checks. + :param sql: SQL statement returning a single scalar; required for ``custom_sql``. + May reference the target table as ``{table}``. + :param severity: ``error`` (fails the task by default) or ``warn`` (recorded only). + :param partition_clause: Extra predicate ANDed into the check's WHERE clause. + :param previous_name: Set when renaming a rule, to keep its history continuous. + :param description: Human-readable description shown in results and the UI. When omitted, + the provider generates a short default description from the rule and condition. + + Invalid input raises pydantic's own :class:`~pydantic.ValidationError`. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + check: str = Field(description="One of the built-in checks, or custom_sql.") + condition: Condition | dict[str, Any] | None = None + column: str | None = None + sql: str | None = None + severity: Severity = Severity.ERROR + partition_clause: str | None = None + previous_name: str | None = None + description: str | None = Field( + default=None, + description=( + "Human-readable description shown in results and the UI. When omitted, the provider " + "generates a short default description from the rule and condition." + ), + ) + dimension: Dimension | None = Field( + default=None, + description=( + "Data quality dimension recorded for this rule's results. Defaults to the check's " + "catalog dimension (validity for custom_sql) when not set explicitly." + ), + ) + + @model_validator(mode="after") + def _validate(self) -> DQRule: + if not self.name: + raise ValueError("Rule name cannot be empty") + catalog_spec = None + if self.check == CUSTOM_SQL_CHECK: + if not self.sql: + raise ValueError(f"Rule {self.name!r}: custom_sql check requires 'sql'") + else: + catalog_spec = CHECK_SPECS.get(self.check) + if catalog_spec is None: + supported = sorted([*CHECK_SPECS, CUSTOM_SQL_CHECK]) + raise ValueError(f"Rule {self.name!r}: unknown check {self.check!r}; supported: {supported}") + if self.sql: + raise ValueError(f"Rule {self.name!r}: 'sql' is only valid with custom_sql") + if catalog_spec.requires_column and not self.column: + raise ValueError(f"Rule {self.name!r}: check {self.check!r} requires 'column'") + if self.condition is None and catalog_spec.default_condition is not None: + object.__setattr__(self, "condition", Condition.from_dict(catalog_spec.default_condition)) + if self.condition is None: + raise ValueError(f"Rule {self.name!r}: condition is required for check {self.check!r}") + if self.dimension is None: + object.__setattr__( + self, "dimension", catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + ) + return self + + @property + def rule_uid(self) -> str: + """Stable identity across runs: survives severity/dimension tweaks and Dag refactors.""" + condition = cast("Condition", self.condition) + identity = { + "name": self.previous_name or self.name, + "check": self.check, + "column": self.column, + "sql": self.sql, + "condition": condition.to_dict(), + } + digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + return digest[:16] + + def to_dict(self) -> dict[str, Any]: + condition = cast("Condition", self.condition) + data: dict[str, Any] = { + "name": self.name, + "check": self.check, + "condition": condition.to_dict(), + "severity": self.severity.value, + } + for optional in ("column", "sql", "partition_clause", "previous_name", "description"): + value = getattr(self, optional) + if value is not None: + data[optional] = value + # Only emit dimension when it overrides the check's catalog default, so a rule that + # never set one explicitly round-trips through to_dict()/from_dict() unchanged. + catalog_spec = CHECK_SPECS.get(self.check) + default_dimension = catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + if self.dimension != default_dimension: + data["dimension"] = cast("Dimension", self.dimension).value + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRule: + return cls(**data) + + +def describe_rule(rule: DQRule) -> str: + """Return a default human-readable description for a rule.""" + condition = cast("Condition", rule.condition).to_dict() + subject = rule.column or rule.name.replace("_", " ") + if "equal_to" in condition: + return f"{subject} should equal {condition['equal_to']}" + if "geq_to" in condition: + return f"{subject} should be greater than or equal to {condition['geq_to']}" + if "greater_than" in condition: + return f"{subject} should be greater than {condition['greater_than']}" + if "leq_to" in condition: + return f"{subject} should be less than or equal to {condition['leq_to']}" + if "less_than" in condition: + return f"{subject} should be less than {condition['less_than']}" + return subject + + +class RuleSet(BaseModel): + """A named collection of rules, typically attached to one table or asset.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + rules: tuple[DQRule, ...] = () + + @model_validator(mode="after") + def _validate(self) -> RuleSet: + if not self.name: + raise ValueError("RuleSet name cannot be empty") + names = [rule.name for rule in self.rules] + duplicates = {name for name in names if names.count(name) > 1} + if duplicates: + raise ValueError(f"Duplicate rule names in ruleset {self.name!r}: {sorted(duplicates)}") + return self + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "rules": [rule.to_dict() for rule in self.rules]} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleSet: + rules = [ + rule if isinstance(rule, DQRule) else DQRule.from_dict(rule) for rule in data.get("rules", []) + ] + return cls(name=data.get("name", ""), rules=tuple(rules)) + + @classmethod + def from_file(cls, path: str) -> RuleSet: + """Load a ruleset from a YAML (or JSON, being a YAML subset) file.""" + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise DQRuleValidationError(f"Ruleset file {path!r} must contain a mapping at top level") + return cls.from_dict(data) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md new file mode 100644 index 0000000000000..97dc9762b7fa2 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -0,0 +1,174 @@ +--- +name: dataquality-rule-authoring +description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dataquality provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." +--- + + + +# dataquality rule authoring + +You are producing a **RuleSet**: a JSON object naming a table's data quality checks. This +document is the schema and the full list of valid values -- do not guess field names or check +names, and do not invent checks that aren't in the catalog below. + +## Output shape + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + } + ] +} +``` + +`name` (ruleset name) and `rules` (array) are the only top-level keys. Every rule name must be +unique within the ruleset. + +## `DQRule` fields + +| Field | Required | Notes | +|---|---|---| +| `name` | yes | Unique within the ruleset. Shows up in data quality results and logs. | +| `check` | yes | One of the catalog names below, or `"custom_sql"`. Exact string match -- there is no fuzzy matching. | +| `condition` | yes* | See `Condition` grammar below. *Every catalog check currently requires one explicitly (no defaults) -- always include it. | +| `column` | check-dependent | Required for every catalog check except `row_count`. Not used with `custom_sql`. | +| `sql` | `custom_sql` only | A SQL statement returning a single scalar. May reference the checked table as `{table}`. Invalid together with any catalog check. | +| `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | +| `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | +| `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | +| `description` | no | Human-readable text shown in data quality results. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | +| `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | + +**Do not add any key not listed above.** The schema rejects unrecognized fields. + +## Built-in check catalog + +| `check` | SQL expression | needs `column` | typical use | +|---|---|---|---| +| `null_count` | `SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)` | yes | count of nulls in a column | +| `null_ratio` | `SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)` | yes | fraction of nulls, 0.0-1.0 | +| `distinct_count` | `COUNT(DISTINCT {column})` | yes | number of distinct values | +| `unique_violations` | `COUNT({column}) - COUNT(DISTINCT {column})` | yes | 0 means the column is fully unique | +| `min` | `MIN({column})` | yes | minimum value | +| `max` | `MAX({column})` | yes | maximum value | +| `mean` | `AVG({column})` | yes | average value | +| `row_count` | `COUNT(*)` | no | total row count -- omit `column` entirely | + +These are the *only* built-in check names. Anything else -- comparing two columns, joining +another table, checking a format/regex, freshness against `now()`, referential integrity -- must +be written as `custom_sql`. + +## `Condition` grammar + +`condition` is an object with these optional numeric keys; at least one is required: + +- `equal_to` -- exact match. Cannot be combined with any other key below. +- `greater_than` / `geq_to` -- lower bound, exclusive / inclusive. +- `less_than` / `leq_to` -- upper bound, exclusive / inclusive. +- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens `equal_to` into a range. Only valid + together with `equal_to`. + +Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, e.g. +`{"geq_to": 0, "leq_to": 100}`. Never combine `equal_to` with another comparison key (other than +`tolerance`). + +## `custom_sql`: when a catalog check doesn't fit + +Use `check: "custom_sql"` with a `sql` statement resolving to a single scalar whenever: + +- The check needs more than one column (e.g. `end_date >= start_date`), a join, or a subquery. +- A catalog expression doesn't run correctly against the target database's SQL dialect (for + example, some engines don't support `NULLIF`, or evaluate `CASE`/`COUNT DISTINCT` + differently) -- write the equivalent expression for that dialect instead. +- The check is conceptually one of the checks above but needs different null/empty-table + semantics than the catalog expression provides. + +Reference the table being checked as `{table}` in the SQL: + +```json +{ + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE order_date > CURRENT_DATE", + "condition": {"equal_to": 0} +} +``` + +## Worked example + +Given these column definitions for an `orders` table -- +`order_id` (integer, primary key), `customer_id` (integer, nullable foreign key), +`amount` (decimal, must be non-negative), `region` (string, low-cardinality), +`created_at` (timestamp) -- a reasonable ruleset: + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "order_id_unique", + "check": "unique_violations", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "customer_id_null_ratio_low", + "check": "null_ratio", + "column": "customer_id", + "condition": {"leq_to": 0.05}, + "severity": "warn" + }, + { + "name": "amount_non_negative", + "check": "min", + "column": "amount", + "condition": {"geq_to": 0} + }, + { + "name": "region_cardinality_reasonable", + "check": "distinct_count", + "column": "region", + "condition": {"leq_to": 20} + }, + { + "name": "table_not_empty", + "check": "row_count", + "condition": {"greater_than": 0} + }, + { + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE created_at > CURRENT_TIMESTAMP", + "condition": {"equal_to": 0} + } + ] +} +``` + +## Reference schema + +`references/ruleset.schema.json` in this skill's directory is the pydantic-generated JSON +Schema for this exact shape (`RuleSet.model_json_schema()`), useful for validating output +structurally. It does not enumerate valid `check` values (that field is a plain string) -- +this document is the source of truth for which check names exist. + +## How this is consumed + +The generated JSON is typically produced by an LLM task (e.g. `@task.llm(output_type=RuleSet, ...)` +from `common.ai`) and passed straight to `DQCheckOperator(ruleset=...)`, `@task.dq_check(ruleset=...)`, +or `asset_quality(ruleset=...)` -- all three accept a `RuleSet`, its dict form, or a path to a +YAML file written in this same shape. Invalid output raises a `pydantic.ValidationError` +describing exactly which field or value was wrong. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json new file mode 100644 index 0000000000000..a8f4387981d6d --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json @@ -0,0 +1,241 @@ +{ + "$defs": { + "Condition": { + "additionalProperties": false, + "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens ``equal_to`` into a range.", + "properties": { + "equal_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Equal To" + }, + "greater_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Greater Than" + }, + "less_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Less Than" + }, + "geq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Geq To" + }, + "leq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Leq To" + }, + "tolerance": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tolerance" + } + }, + "title": "Condition", + "type": "object" + }, + "DQRule": { + "additionalProperties": false, + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "check": { + "description": "One of the built-in checks, or custom_sql.", + "title": "Check", + "type": "string" + }, + "condition": { + "anyOf": [ + { + "$ref": "#/$defs/Condition" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Condition" + }, + "column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Column" + }, + "sql": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sql" + }, + "severity": { + "$ref": "#/$defs/Severity", + "default": "error" + }, + "partition_clause": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Partition Clause" + }, + "previous_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Previous Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition.", + "title": "Description" + }, + "dimension": { + "anyOf": [ + { + "$ref": "#/$defs/Dimension" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data quality dimension recorded for this rule's results. Defaults to the check's catalog dimension (validity for custom_sql) when not set explicitly." + } + }, + "required": [ + "name", + "check" + ], + "title": "DQRule", + "type": "object" + }, + "Dimension": { + "description": "Data quality dimension assigned to a rule result.", + "enum": [ + "completeness", + "uniqueness", + "validity", + "freshness", + "volume", + "consistency" + ], + "title": "Dimension", + "type": "string" + }, + "Severity": { + "description": "Severity used to decide whether a failing rule fails the task.", + "enum": [ + "warn", + "error" + ], + "title": "Severity", + "type": "string" + } + }, + "additionalProperties": false, + "description": "A named collection of rules, typically attached to one table or asset.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "rules": { + "default": [], + "items": { + "$ref": "#/$defs/DQRule" + }, + "title": "Rules", + "type": "array" + } + }, + "required": [ + "name" + ], + "title": "RuleSet", + "type": "object" +} diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py new file mode 100644 index 0000000000000..b47842705282b --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/version_compat.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY +# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS +# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT +# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE +# +from __future__ import annotations + + +def get_base_airflow_version_tuple() -> tuple[int, int, int]: + from packaging.version import Version + + from airflow import __version__ + + airflow_version = Version(__version__) + return airflow_version.major, airflow_version.minor, airflow_version.micro + + +AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0) + +__all__ = ["AIRFLOW_V_3_1_PLUS"] diff --git a/providers/common/dataquality/tests/conftest.py b/providers/common/dataquality/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/common/dataquality/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/common/dataquality/tests/system/__init__.py b/providers/common/dataquality/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/system/common/__init__.py b/providers/common/dataquality/tests/system/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/system/common/dataquality/__init__.py b/providers/common/dataquality/tests/system/common/dataquality/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/dataquality/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py new file mode 100644 index 0000000000000..cfd434c8ab431 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py @@ -0,0 +1,354 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG for the Data Quality provider's ``DQCheckOperator``. + +Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra +infrastructure required). Results are persisted through the ``[common.dataquality] results_path`` config +option, seeded here via ``AIRFLOW__DQ__RESULTS_PATH`` so the history can be inspected +afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set +``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. + +Trigger the Dag repeatedly with different ``dq_scenario`` values to create useful +persisted data quality history: + +- ``pass`` inserts clean data. +- ``negative`` inserts a negative amount, failing ``amount_non_negative``. +- ``duplicate_order`` inserts a duplicate ``order_id``, failing uniqueness. +- ``nulls`` inserts null ``order_id``/``customer_id`` values and a high discount null ratio. +- ``outlier`` inserts amounts above the allowed maximum. +- ``small_table`` inserts too few rows for volume checks. +- ``zero_quantity`` inserts an invalid quantity minimum. +- ``empty`` leaves the table empty. +- ``mixed`` combines several bad values to show multiple failures in one run. + +Demonstrates +three equivalent ways to run the same ruleset: + +- Plain ``DQCheckOperator`` with an explicit ``table``/``conn_id``. +- ``asset_quality()`` attaching the ruleset to an ``Asset``, then ``DQCheckOperator(asset=...)`` + picking up ``table``/``conn_id``/``ruleset`` from it and adding the asset to the task's + outlets automatically. +- A second asset-producing ``DQCheckOperator`` with stricter rules, so the Browse > Data Quality + page shows multiple producers for the same asset with both pass and fail outcomes. +- The ``@task.dq_check`` TaskFlow decorator. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, Asset, task + +DAG_ID = "example_dq_check" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_example_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +# DQCheckOperator has no per-operator results-store override; every check in a deployment +# shares the one store configured under [common.dataquality] results_path. setdefault() so a real deployment's +# own config is never overridden. +os.environ.setdefault("AIRFLOW__DQ__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_ruleset] +orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="customer_id_not_null", + check="null_count", + column="customer_id", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="discount_null_ratio", + check="null_ratio", + column="discount", + condition={"leq_to": 0.25}, + severity=Severity.WARN, + ), + DQRule( + name="region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 2}, + severity=Severity.WARN, + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition={"geq_to": 0}, + ), + DQRule( + name="amount_max_le_100", + check="max", + column="amount", + condition={"leq_to": 100}, + ), + DQRule( + name="amount_mean_between_5_and_60", + check="mean", + column="amount", + condition={"geq_to": 5, "leq_to": 60}, + ), + DQRule( + name="quantity_min_ge_one", + check="min", + column="quantity", + condition={"geq_to": 1}, + ), + DQRule( + name="quantity_max_le_ten", + check="max", + column="quantity", + condition={"leq_to": 10}, + ), + DQRule( + name="amount_non_negative", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string placeholder. + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, + ), + DQRule( + name="high_value_order_count", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount > 100", + condition={"equal_to": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_present", + check="row_count", + condition={"greater_than": 0}, + severity=Severity.WARN, + ), + DQRule( + name="row_count_at_least_three", + check="row_count", + condition={"geq_to": 3}, + ), + ), +) +# [END howto_operator_dq_check_ruleset] + +# [START howto_operator_dq_check_asset] +orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_operator_dq_check_asset] + +strict_orders_ruleset = RuleSet( + name="orders_strict_quality", + rules=( + DQRule( + name="strict_order_id_not_null", + check="null_count", + column="order_id", + condition={"equal_to": 0}, + ), + DQRule( + name="strict_amount_max_le_20", + check="max", + column="amount", + condition={"leq_to": 20}, + ), + DQRule( + name="strict_row_count_equal_two", + check="row_count", + condition={"equal_to": 2}, + ), + DQRule( + name="strict_region_distinct_count", + check="distinct_count", + column="region", + condition={"geq_to": 4}, + severity=Severity.WARN, + ), + ), +) + +strict_orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=strict_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) + + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + discount REAL, + quantity INTEGER, + region TEXT + ); + """, + ], + ) + + clear_table = SQLExecuteQueryOperator( + task_id="clear_table", + conn_id=CONN_ID, + sql=f"DELETE FROM {TABLE_NAME};", + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + {{% set scenario = dag_run.conf.get("dq_scenario", "pass") if dag_run else "pass" %}} + {{% if scenario == "negative" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, -25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "duplicate_order" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (1, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "nulls" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 1, 'US'), + (NULL, NULL, 25.5, NULL, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "outlier" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 120.0, 0.10, 2, 'EU'), + (3, 103, 150.0, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "small_table" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'); + {{% elif scenario == "zero_quantity" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 0, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "mixed" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 0, 'US'), + (1, NULL, -25.5, NULL, 2, 'US'), + (NULL, 103, 150.0, NULL, 11, 'US'), + (4, 104, 55.0, 0.20, 4, 'US'); + {{% elif scenario == "empty" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) + SELECT NULL, NULL, NULL, NULL, NULL, NULL WHERE 0; + {{% else %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% endif %}} + """, + ) + + # [START howto_operator_dq_check] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + # [END howto_operator_dq_check] + + check_orders_via_asset = DQCheckOperator( + task_id="check_orders_via_asset", + asset=orders_asset, + fail_on="never", + ) + + check_orders_strict_via_asset = DQCheckOperator( + task_id="check_orders_strict_via_asset", + asset=strict_orders_asset, + fail_on="never", + ) + + # [START howto_decorator_dq_check] + @task.dq_check( + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + def check_orders_decorated(): + return None + + # [END howto_decorator_dq_check] + + ( + create_table + >> clear_table + >> insert_orders + >> [check_orders, check_orders_via_asset, check_orders_strict_via_asset, check_orders_decorated()] + ) + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/common/dataquality/tests/unit/__init__.py b/providers/common/dataquality/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/unit/common/__init__.py b/providers/common/dataquality/tests/unit/common/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py new file mode 100644 index 0000000000000..e3743e6859b56 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -0,0 +1,364 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.results import DQRun, RuleResult +from airflow.providers.common.dataquality.rules import Severity + +from tests_common.test_utils.config import conf_vars + + +def make_run(run_uid: str = "abc123", started_at: str = "2026-07-04T06:00:00+00:00") -> DQRun: + return DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid=run_uid, + ruleset_name="orders", + table_ref="analytics.orders", + asset_names=("dq_example_orders",), + started_at=started_at, + finished_at="2026-07-04T06:00:03+00:00", + ) + + +def make_result(rule_uid: str = "rule-1", status: str = "pass") -> RuleResult: + return RuleResult( + rule_uid=rule_uid, + rule_name="ids_not_null", + status=status, + observed_value=0, + condition={"equal_to": 0}, + dimension="completeness", + severity=Severity.ERROR, + duration_ms=12.5, + ) + + +class TestObjectStorageResultsBackend: + @pytest.fixture + def backend(self, tmp_path): + return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + def test_write_run_creates_run_file_with_run_and_results(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + assert record["run"]["run_uid"] == "abc123" + assert record["run"]["dag_id"] == "orders_pipeline" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["results"][0]["status"] == "pass" + + def test_write_run_stores_keyed_json_payload(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "runs" + / "by_task" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "date=2026-07-04" + / "abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["run_uid"] == "abc123" + assert payload["results"][0]["rule_uid"] == "rule-1" + assert payload["summary"]["passed"] == 1 + + def test_write_run_stores_task_rule_index(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = ( + backend.root + / "rules" + / "by_task_rule" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "rule_uid=rule-1" + / "2026-07-04T06_00_00_00_00__abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["dag_id"] == "orders_pipeline" + assert payload["run"]["task_id"] == "dq" + assert payload["result"]["rule_uid"] == "rule-1" + + def test_write_run_does_not_store_global_rule_index(self, backend): + backend.write_run(make_run(), [make_result()]) + + path = backend.root / "rules" / "by_rule" / "rule_uid=rule-1" + + assert not path.exists() + + def test_rule_history_is_newest_first(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1") + + assert [record["status"] for record in result["items"]] == ["fail", "pass"] + assert result["items"][0]["run"]["run_uid"] == "run2" + assert result["next_cursor"] is None + + def test_rule_history_includes_task_instance_route_context(self, backend): + backend.write_run(make_run(), [make_result()]) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert history[0]["run"]["dag_id"] == "orders_pipeline" + assert history[0]["run"]["task_id"] == "dq" + assert history[0]["run"]["run_id"] == "scheduled__2026-07-04" + assert history[0]["run"]["map_index"] == -1 + + def test_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_concurrent_style_writes_do_not_collide(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run(make_run(run_uid="run2"), [make_result()]) + + runs = backend.read_task_runs("orders_pipeline", "dq")["items"] + assert {run["run"]["run_uid"] for run in runs} == {"run1", "run2"} + + def test_read_by_task_instance_matches_run(self, backend): + backend.write_run(make_run(), [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "abc123" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_unknown_raises(self, backend): + with pytest.raises(FileNotFoundError): + backend.read_by_task_instance("orders_pipeline", "dq", "no_such_run") + + def test_read_by_task_instance_last_write_wins_across_retries(self, backend): + first = make_run(run_uid="try1") + backend.write_run(first, [make_result(status="fail")]) + second = make_run(run_uid="try2") + backend.write_run(second, [make_result(status="pass")]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", "scheduled__2026-07-04") + + assert record["run"]["run_uid"] == "try2" + assert record["results"][0]["status"] == "pass" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_sanitizes_run_id(self, backend): + run_id = "manual__2026-07-04T06:00:00+00:00" + run = DQRun(dag_id="orders_pipeline", task_id="dq", run_id=run_id, run_uid="abc123") + backend.write_run(run, [make_result()]) + + record = backend.read_by_task_instance("orders_pipeline", "dq", run_id) + + assert record["run"]["run_id"] == run_id + + def test_read_task_runs_returns_newest_first_with_summaries(self, backend): + backend.write_run( + make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + [make_result(status="pass")], + ) + backend.write_run( + make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + [make_result(status="fail")], + ) + + result = backend.read_task_runs("orders_pipeline", "dq") + runs = result["items"] + + assert [record["run"]["run_uid"] for record in runs] == ["run2", "run1"] + assert runs[0]["summary"]["failed"] == 1 + assert runs[1]["summary"]["passed"] == 1 + assert result["next_cursor"] is None + + def test_read_task_runs_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_runs_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_runs("orders_pipeline", "dq", limit=2) + second_page = backend.read_task_runs( + "orders_pipeline", "dq", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monkeypatch): + for day in range(1, 5): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs("orders_pipeline", "dq", limit=1) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:00+00:00|run4" + # Determining next_cursor reads one entry past the limit (date=2026-07-03's run3), but + # no further — date=2026-07-02/01 must not be read. + assert len(read_paths) == 2 + + def test_read_task_rule_history_filters_to_task(self, backend): + backend.write_run(make_run(run_uid="run1"), [make_result()]) + backend.write_run( + DQRun( + dag_id="other_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid="run2", + started_at="2026-07-04T07:00:00+00:00", + ), + [make_result()], + ) + + history = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1")["items"] + + assert len(history) == 1 + assert history[0]["run"]["dag_id"] == "orders_pipeline" + + def test_read_task_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + result = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run3", "run2"] + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + [make_result()], + ) + + first_page = backend.read_task_rule_history("orders_pipeline", "dq", "rule-1", limit=2) + second_page = backend.read_task_rule_history( + "orders_pipeline", "dq", "rule-1", limit=2, before=first_page["next_cursor"] + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + +class TestGetBackendFromConfig: + @conf_vars({("common.dataquality", "results_path"): None}) + def test_no_results_path_returns_none(self): + assert get_backend_from_config() is None + + def test_results_path_builds_object_storage_backend(self, tmp_path): + with conf_vars({("common.dataquality", "results_path"): f"file://{tmp_path}"}): + backend = get_backend_from_config() + assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py new file mode 100644 index 0000000000000..ecd65c42d01c2 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +RULESET = RuleSet(name="orders", rules=(NULLS,)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch( + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, + ): + yield backend + + +def make_decorated_operator(records, python_callable, **kwargs): + kwargs.setdefault("task_id", "dq") + if not kwargs.pop("omit_ruleset", False): + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = _DQCheckDecoratedOperator(conn_id="warehouse", python_callable=python_callable, **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +class TestDQCheckDecoratedOperator: + def test_custom_operator_name(self): + assert _DQCheckDecoratedOperator.custom_operator_name == "@task.dq_check" + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_none_return_runs_check_as_declared(self, mock_get_db_hook): + operator, hook = make_decorated_operator(records=[(NULLS.rule_uid, 0)], python_callable=lambda: None) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert operator.table == "orders" + assert summary["passed"] == 1 + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_ruleset_can_be_provided_only_at_runtime(self, mock_get_db_hook): + other_rule = DQRule(name="row_count_ok", check="row_count", condition={"greater_than": 0}) + other_ruleset = RuleSet(name="dynamic", rules=(other_rule,)) + + operator, hook = make_decorated_operator( + records=[(other_rule.rule_uid, 10)], + python_callable=lambda: other_ruleset, + omit_ruleset=True, + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 1 + + def test_missing_runtime_ruleset_raises_value_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: None, omit_ruleset=True) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(ValueError, match="ruleset is required"): + operator.execute(make_context()) + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_op_kwargs_are_passed_to_callable(self, mock_get_db_hook): + seen = {} + + def resolve_target(suffix): + seen["suffix"] = suffix + return None + + operator, hook = make_decorated_operator( + records=[(NULLS.rule_uid, 0)], + python_callable=resolve_target, + op_kwargs={"suffix": "eu"}, + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + assert seen["suffix"] == "eu" + + def test_invalid_return_raises_type_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: 42) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(TypeError, match="must return a RuleSet"): + operator.execute(make_context()) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py new file mode 100644 index 0000000000000..699fc46ecaef9 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook + + +@pytest.fixture +def hook(): + return mock.create_autospec(DbApiHook, instance=True) + + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) +CUSTOM = DQRule( + name="negatives", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition={"equal_to": 0}, +) + + +class TestBuildBatchSql: + def test_batch_sql_unions_one_select_per_rule(self, hook): + engine = SQLDQEngine(hook) + sql = engine.build_batch_sql([NULLS, VOLUME], "orders", None) + assert sql.count("UNION ALL") == 1 + assert f"'{NULLS.rule_uid}' AS rule_uid" in sql + assert "SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END)" in sql + assert "COUNT(*)" in sql + assert "WHERE" not in sql + + def test_partition_clauses_are_anded(self, hook): + rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + partition_clause="region = 'EU'", + ) + sql = SQLDQEngine(hook).build_batch_sql([rule], "orders", "ds = '2026-07-04'") + assert "WHERE ds = '2026-07-04' AND region = 'EU'" in sql + + +class TestMeasure: + def test_builtin_rules_measured_from_single_query(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_records.assert_called_once() + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["volume"].observed_value == 42 + assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(NULLS, "orders") + assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(VOLUME, "orders") + assert all(obs.error_message is None for obs in observations) + + def test_query_failure_yields_error_observations(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert len(observations) == 2 + assert all(obs.error_message == "connection refused" for obs in observations) + assert all(obs.observed_value is None for obs in observations) + assert all(obs.sql for obs in observations) + + def test_missing_rule_in_result_is_an_error(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].error_message is None + assert by_name["volume"].error_message == "No result returned for rule" + + def test_custom_sql_rule_runs_individually_with_table_substituted(self, hook): + hook.get_first.return_value = (3,) + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + hook.get_first.assert_called_once_with("SELECT COUNT(*) FROM orders WHERE amount < 0") + hook.get_records.assert_not_called() + assert observations[0].observed_value == 3 + assert observations[0].sql == "SELECT COUNT(*) FROM orders WHERE amount < 0" + + def test_custom_sql_no_rows_is_an_error(self, hook): + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset, "orders") + + assert observations[0].error_message == "Query returned no rows" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py new file mode 100644 index 0000000000000..dee1709da10fe --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -0,0 +1,251 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import DQRule, RuleSet, Severity +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition={"equal_to": 0}) +VOLUME_WARN = DQRule( + name="volume", check="row_count", condition={"greater_than": 100}, severity=Severity.WARN +) +RULESET = RuleSet(name="orders", rules=(NULLS, VOLUME_WARN)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +def make_operator(records, **kwargs): + kwargs.setdefault("task_id", "dq") + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = DQCheckOperator(conn_id="warehouse", **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch( + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, + ): + yield backend + + +class TestDQCheckOperatorConstruction: + def test_requires_ruleset(self): + with pytest.raises(ValueError, match="ruleset is required"): + DQCheckOperator(task_id="dq", table="orders", conn_id="c") + + def test_requires_table(self): + with pytest.raises(ValueError, match="table is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, conn_id="c") + + def test_rejects_bad_fail_on(self): + with pytest.raises(ValueError, match="fail_on"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders", conn_id="c", fail_on="maybe") + + def test_asset_supplies_defaults_and_outlet(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset) + assert operator.table == "analytics.orders" + assert operator.conn_id == "warehouse" + assert asset in operator.outlets + + def test_explicit_arguments_beat_asset_config(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset, table="other_table", conn_id="other_conn") + assert operator.table == "other_table" + assert operator.conn_id == "other_conn" + + +class TestDQCheckOperatorExecute: + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_all_pass_returns_summary_and_persists(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + assert summary["failed"] == 0 + assert summary["score"] == 1.0 + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_error_severity_failure_fails_task(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="nulls"): + operator.execute(make_context()) + + # The failing result is persisted even though the task fails. + assert ( + results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"][0][ + "status" + ] + == "fail" + ) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_warn_severity_failure_does_not_fail_task_by_default(self, mock_get_db_hook): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["warned"] == 1 + assert summary["warned_rules"] == ["volume"] + assert summary["score"] == pytest.approx(0.875) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): + described_rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition={"equal_to": 0}, + description="Order IDs must always be present.", + ) + operator, hook = make_operator( + records=[(described_rule.rule_uid, 0)], + ruleset=RuleSet(name="orders", rules=(described_rule,)), + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", described_rule.rule_uid)[ + "items" + ] + assert history[0]["description"] == "Order IDs must always be present." + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_warn_fails_task_on_warn_failure(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)], fail_on="warn" + ) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="volume"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_never_records_failures_without_raising(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 5)], fail_on="never" + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["failed"] == 1 + assert summary["warned"] == 1 + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_execution_error_always_fails_task(self, mock_get_db_hook): + operator, hook = make_operator(records=None, fail_on="never") + hook.get_records.side_effect = RuntimeError("boom") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_non_numeric_observed_value_is_persisted_as_error(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, "not-a-number")], fail_on="never") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + history = results_backend.read_task_rule_history("orders_pipeline", "dq", NULLS.rule_uid)["items"] + assert history[0]["status"] == "error" + assert "Could not evaluate observed value" in history[0]["error_message"] + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with mock.patch( + "airflow.providers.common.dataquality.operators.dq_check.get_backend_from_config", + return_value=backend, + ): + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + backend.write_run.assert_called_once() + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_summary_attached_to_outlet_events(self, mock_get_db_hook): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="orders", + ) + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)], asset=asset + ) + mock_get_db_hook.return_value = hook + context = make_context() + + summary = operator.execute(context) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + extra = outlet_events.__getitem__.return_value.extra + extra.__setitem__.assert_called_once_with("airflow.dataquality.result", summary) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py new file mode 100644 index 0000000000000..a06ae0ef38e02 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.rules import CHECK_SPECS + + +class TestCheckSpecs: + def test_every_column_check_expression_renders(self): + for name, spec in CHECK_SPECS.items(): + if spec.requires_column: + assert "{column}" in spec.expression, name + + def test_row_count_is_the_only_table_level_check(self): + table_level = {name for name, spec in CHECK_SPECS.items() if not spec.requires_column} + assert table_level == {"row_count"} diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py new file mode 100644 index 0000000000000..8165e19ccaec7 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import airflow.providers.common.dataquality +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity, describe_rule + + +def test_skill_reference_schema_matches_live_model(): + """ + The dataquality-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for + agents to consult. Guard against it silently drifting out of sync with the real model -- + regenerate with ``json.dumps(RuleSet.model_json_schema(), indent=2)`` (plus a trailing + newline) if this fails after an intentional schema change. + """ + schema_path = ( + Path(airflow.providers.common.dataquality.__file__).parent + / "skills" + / "dataquality-rule-authoring" + / "references" + / "ruleset.schema.json" + ) + checked_in = json.loads(schema_path.read_text()) + assert checked_in == RuleSet.model_json_schema() + + +class TestCondition: + @pytest.mark.parametrize( + ("condition", "observed", "expected"), + [ + ({"equal_to": 0}, 0, True), + ({"equal_to": 0}, 1, False), + ({"equal_to": 100, "tolerance": 0.1}, 105, True), + ({"equal_to": 100, "tolerance": 0.1}, 111, False), + ({"greater_than": 5}, 6, True), + ({"greater_than": 5}, 5, False), + ({"less_than": 5}, 4, True), + ({"geq_to": 5}, 5, True), + ({"leq_to": 5}, 6, False), + ({"geq_to": 0, "leq_to": 10}, 5, True), + ({"geq_to": 0, "leq_to": 10}, 11, False), + ({"equal_to": 0}, None, False), + ], + ) + def test_evaluate(self, condition, observed, expected): + assert Condition.from_dict(condition).evaluate(observed) is expected + + @pytest.mark.parametrize( + "condition", + [ + {}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + {"greater_than": 0, "tolerance": 0.1}, + {"nonsense": 1}, + ], + ) + def test_invalid_conditions_rejected(self, condition): + with pytest.raises(ValidationError): + Condition.from_dict(condition) + + def test_to_dict_round_trip(self): + condition = Condition.from_dict({"geq_to": 1, "leq_to": 2}) + assert Condition.from_dict(condition.to_dict()) == condition + + +class TestDQRule: + def test_condition_dict_is_coerced(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + assert isinstance(rule.condition, Condition) + + def test_rule_uid_is_stable_across_severity(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + tweaked = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + severity=Severity.WARN, + ) + assert base.rule_uid == tweaked.rule_uid + + def test_rule_uid_is_stable_across_description(self): + base = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + described = DQRule( + name="r", + check="null_count", + column="c", + condition={"equal_to": 0}, + description="Column c must be populated.", + ) + assert base.rule_uid == described.rule_uid + + def test_rule_uid_changes_with_condition(self): + one = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) + two = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 1}) + assert one.rule_uid != two.rule_uid + + def test_previous_name_keeps_uid(self): + old = DQRule(name="old_name", check="null_count", column="c", condition={"equal_to": 0}) + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition={"equal_to": 0}, + previous_name="old_name", + ) + assert renamed.rule_uid == old.rule_uid + + @pytest.mark.parametrize( + "kwargs", + [ + {"name": "", "check": "row_count", "condition": {"equal_to": 0}}, + {"name": "r", "check": "no_such_check", "condition": {"equal_to": 0}}, + {"name": "r", "check": "null_count", "condition": {"equal_to": 0}}, # missing column + {"name": "r", "check": "custom_sql", "condition": {"equal_to": 0}}, # missing sql + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "sql": "SELECT 1"}, + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "severity": "fatal"}, + ], + ) + def test_invalid_rules_rejected(self, kwargs): + with pytest.raises(ValidationError): + DQRule(**kwargs) + + def test_row_count_needs_no_column(self): + rule = DQRule(name="volume", check="row_count", condition={"greater_than": 0}) + assert rule.column is None + + @pytest.mark.parametrize( + ("check", "column", "expected_dimension"), + [ + ("null_count", "c", "completeness"), + ("null_ratio", "c", "completeness"), + ("distinct_count", "c", "uniqueness"), + ("unique_violations", "c", "uniqueness"), + ("min", "c", "validity"), + ("max", "c", "validity"), + ("mean", "c", "validity"), + ("row_count", None, "volume"), + ], + ) + def test_default_dimension_comes_from_check_catalog(self, check, column, expected_dimension): + rule = DQRule(name="r", check=check, column=column, condition={"equal_to": 0}) + assert rule.dimension.value == expected_dimension + + def test_custom_sql_dimension_can_be_overridden(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.dimension.value == "freshness" + + def test_invalid_dimension_rejected(self): + with pytest.raises(ValidationError): + DQRule(name="r", check="row_count", condition={"greater_than": 0}, dimension="not_a_dimension") + + def test_explicit_dimension_matching_default_is_omitted_from_to_dict(self): + rule = DQRule( + name="r", check="null_count", column="c", condition={"equal_to": 0}, dimension="completeness" + ) + assert "dimension" not in rule.to_dict() + + def test_dimension_override_round_trips(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition={"equal_to": 0}, + dimension="freshness", + ) + assert rule.to_dict()["dimension"] == "freshness" + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_missing_condition_without_catalog_default_raises(self): + with pytest.raises(ValidationError, match="condition is required"): + DQRule(name="r", check="null_count", column="c") + + def test_to_dict_round_trip(self): + rule = DQRule( + name="r", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE x < 0", + condition={"equal_to": 0}, + severity=Severity.WARN, + partition_clause="ds = '2026-07-04'", + description="No rows should have negative x.", + ) + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_description_is_serialized(self): + rule = DQRule( + name="r", + check="row_count", + condition={"greater_than": 0}, + description="Orders table should not be empty.", + ) + assert rule.to_dict()["description"] == "Orders table should not be empty." + + def test_default_description_uses_column_when_available(self): + rule = DQRule(name="amount_min", check="min", column="amount", condition={"geq_to": 0}) + assert describe_rule(rule) == "amount should be greater than or equal to 0.0" + + +class TestRuleSet: + def test_duplicate_rule_names_rejected(self): + rule = DQRule(name="r", check="row_count", condition={"greater_than": 0}) + with pytest.raises(ValidationError, match="Duplicate rule names"): + RuleSet(name="s", rules=(rule, rule)) + + def test_from_dict_round_trip(self): + ruleset = RuleSet( + name="orders", + rules=( + DQRule(name="volume", check="row_count", condition={"greater_than": 0}), + DQRule(name="ids", check="null_count", column="id", condition={"equal_to": 0}), + ), + ) + assert RuleSet.from_dict(ruleset.to_dict()) == ruleset + + def test_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: ids_not_null + check: null_count + column: id + condition: + equal_to: 0 + - name: volume + check: row_count + condition: + greater_than: 100 + severity: warn + """ + ) + ruleset = RuleSet.from_file(str(ruleset_file)) + assert ruleset.name == "orders" + assert [rule.name for rule in ruleset.rules] == ["ids_not_null", "volume"] + assert ruleset.rules[1].severity.value == "warn" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py new file mode 100644 index 0000000000000..3c9f0e8935897 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.common.dataquality.assets import ( + DQ_EXTRA_KEY, + DQ_RESULT_EXTRA_KEY, + _quality_score_passes, + asset_quality, + get_asset_quality_config, + get_asset_ruleset, + require_quality, +) +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import DQRule, RuleSet +from airflow.sdk import Asset +from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult +from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor + +RULESET = RuleSet( + name="orders", + rules=(DQRule(name="volume", check="row_count", condition={"greater_than": 0}),), +) + +ORDERS = Asset("orders") + + +def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: + event = { + "asset": {"name": asset.name, "uri": asset.uri, "extra": {}}, + "extra": extra, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "manual__2026-07-04", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + return TriggeringAssetEventsAccessor.build([AssetEventDagRunReferenceResult.model_validate(event)]) + + +class TestAssetQuality: + def test_config_stored_under_dq_extra_key(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + config = asset.extra[DQ_EXTRA_KEY] + assert config["conn_id"] == "warehouse" + assert config["table"] == "analytics.orders" + assert config["ruleset"] == RULESET.to_dict() + + def test_config_is_json_serializable(self): + import json + + asset = asset_quality(Asset("orders"), ruleset=RULESET) + json.dumps(asset.extra) + + def test_ruleset_round_trips_through_asset(self): + asset = asset_quality(Asset("orders"), ruleset=RULESET) + assert get_asset_ruleset(asset) == RULESET + + def test_ruleset_from_yaml_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: volume + check: row_count + condition: + greater_than: 0 + """ + ) + asset = asset_quality(Asset("orders"), ruleset=str(ruleset_file)) + assert get_asset_ruleset(asset).rules[0].name == "volume" + + def test_get_config_returns_none_without_quality(self): + assert get_asset_quality_config(Asset("plain")) is None + + def test_get_ruleset_raises_without_quality(self): + with pytest.raises(DQRuleValidationError, match="no data quality config"): + get_asset_ruleset(Asset("plain")) + + +class TestQualityScorePasses: + """ + Covers the pure decision logic behind ``require_quality()``. + + Kept independent of ``@task.short_circuit`` / Dag wiring, which is exercised by the + standard provider's own tests — this only needs to prove "given this triggering event, + does the gate pass or fail". + """ + + def test_passes_when_score_at_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}) + assert _quality_score_passes(ORDERS, 0.95, events) is True + + def test_fails_when_score_below_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.8}}) + assert _quality_score_passes(ORDERS, 0.95, events) is False + + def test_fails_when_no_triggering_event_for_asset(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 1.0}}, asset=Asset("other")) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_event_has_no_dq_summary(self): + events = make_triggering_events(extra={}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_summary_has_no_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"passed": 3}}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_uses_most_recent_event_when_several_present(self): + first = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run1", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-03T06:00:00Z", + } + ) + second = AssetEventDagRunReferenceResult.model_validate( + { + "asset": {"name": ORDERS.name, "uri": ORDERS.uri, "extra": {}}, + "extra": {DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": "run2", + "source_map_index": -1, + "source_aliases": [], + "timestamp": "2026-07-04T06:00:00Z", + } + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events) is True + + +class TestRequireQuality: + def test_rejects_min_score_below_zero(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=-0.1) + + def test_rejects_min_score_above_one(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=1.1) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_results.py b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py new file mode 100644 index 0000000000000..37147de4416cb --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary, compute_score + + +def make_result(status: str, name: str = "r") -> RuleResult: + return RuleResult(rule_uid=f"uid-{name}", rule_name=name, status=status) + + +class TestComputeScore: + @pytest.mark.parametrize( + ("statuses", "expected"), + [ + ([], None), + (["pass", "pass"], 1.0), + (["pass", "fail"], 0.5), + (["pass", "error"], 0.5), + (["pass", "pass", "pass", "warn"], 0.9375), + (["fail", "fail"], 0.0), + ], + ) + def test_score(self, statuses, expected): + results = [make_result(status, name=f"r{i}") for i, status in enumerate(statuses)] + assert compute_score(results) == expected + + +class TestBuildSummary: + def test_summary_counts_and_rule_names(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", ruleset_name="orders", table_ref="orders") + results = [ + make_result("pass", "a"), + make_result("warn", "b"), + make_result("fail", "c"), + make_result("error", "d"), + ] + + summary = build_summary(run, results) + + assert summary["passed"] == 1 + assert summary["warned"] == 1 + assert summary["failed"] == 1 + assert summary["errored"] == 1 + assert summary["failed_rules"] == ["c", "d"] + assert summary["warned_rules"] == ["b"] + assert summary["run_uid"] == run.run_uid + + +class TestSerialization: + def test_rule_result_round_trip(self): + result = RuleResult( + rule_uid="u", + rule_name="r", + status="fail", + observed_value=3, + condition={"equal_to": 0}, + description="r should equal 0", + duration_ms=1.5, + sql="SELECT 3", + ) + assert RuleResult.from_dict(result.to_dict()) == result + + def test_run_round_trip(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", asset_names=("orders",)) + assert DQRun.from_dict(run.to_dict()) == run diff --git a/pyproject.toml b/pyproject.toml index 42a9e93a80130..cbfec3bf747f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,6 +191,9 @@ apache-airflow = "airflow.__main__:main" "common.compat" = [ "apache-airflow-providers-common-compat>=1.2.1" ] +"common.dataquality" = [ + "apache-airflow-providers-common-dataquality>=0.1.0" +] "common.io" = [ "apache-airflow-providers-common-io>=1.4.2" ] @@ -441,6 +444,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-cohere>=1.4.0", "apache-airflow-providers-common-ai>=0.1.0", "apache-airflow-providers-common-compat>=1.2.1", + "apache-airflow-providers-common-dataquality>=0.1.0", "apache-airflow-providers-common-io>=1.4.2", "apache-airflow-providers-common-messaging>=2.0.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-common-sql>=1.18.0", @@ -1150,6 +1154,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/common/ai/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/src", "$MYPY_CONFIG_FILE_DIR/providers/common/compat/tests", + "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/src", + "$MYPY_CONFIG_FILE_DIR/providers/common/dataquality/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/io/src", "$MYPY_CONFIG_FILE_DIR/providers/common/io/tests", "$MYPY_CONFIG_FILE_DIR/providers/common/messaging/src", @@ -1464,6 +1470,7 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1610,6 +1617,7 @@ apache-airflow-providers-cncf-kubernetes = false apache-airflow-providers-cohere = false apache-airflow-providers-common-ai = false apache-airflow-providers-common-compat = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-io = false apache-airflow-providers-common-messaging = false apache-airflow-providers-common-sql = false @@ -1771,6 +1779,7 @@ apache-airflow-providers-cncf-kubernetes = { workspace = true } apache-airflow-providers-cohere = { workspace = true } apache-airflow-providers-common-ai = { workspace = true } apache-airflow-providers-common-compat = { workspace = true } +apache-airflow-providers-common-dataquality = { workspace = true } apache-airflow-providers-common-io = { workspace = true } apache-airflow-providers-common-messaging = { workspace = true } apache-airflow-providers-common-sql = { workspace = true } @@ -1912,6 +1921,7 @@ members = [ "providers/cohere", "providers/common/ai", "providers/common/compat", + "providers/common/dataquality", "providers/common/io", "providers/common/messaging", "providers/common/sql", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index ed2e4b1bed789..edb644a179a19 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -58,6 +58,7 @@ services: - ../../../empty:/opt/airflow/providers/cohere/src - ../../../empty:/opt/airflow/providers/common/ai/src - ../../../empty:/opt/airflow/providers/common/compat/src + - ../../../empty:/opt/airflow/providers/common/dataquality/src - ../../../empty:/opt/airflow/providers/common/io/src - ../../../empty:/opt/airflow/providers/common/messaging/src - ../../../empty:/opt/airflow/providers/common/sql/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index 1f6e47ba961c2..eb700548346da 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -71,6 +71,7 @@ services: - ../../../providers/cohere/tests:/opt/airflow/providers/cohere/tests - ../../../providers/common/ai/tests:/opt/airflow/providers/common/ai/tests - ../../../providers/common/compat/tests:/opt/airflow/providers/common/compat/tests + - ../../../providers/common/dataquality/tests:/opt/airflow/providers/common/dataquality/tests - ../../../providers/common/io/tests:/opt/airflow/providers/common/io/tests - ../../../providers/common/messaging/tests:/opt/airflow/providers/common/messaging/tests - ../../../providers/common/sql/tests:/opt/airflow/providers/common/sql/tests diff --git a/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py new file mode 100755 index 0000000000000..dab079ae734d9 --- /dev/null +++ b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Regenerate the Data Quality RuleSet JSON schema snapshot at +``providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json``. +""" + +from __future__ import annotations + +import subprocess +import sys + +from common_prek_utils import AIRFLOW_ROOT_PATH, console + +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "common" / "dataquality" +SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( + "src", + "airflow", + "providers", + "common", + "dataquality", + "skills", + "dataquality-rule-authoring", + "references", + "ruleset.schema.json", +) + +DUMP_SCHEMA = r""" +import json +import sys + +from airflow.providers.common.dataquality.rules import RuleSet + +sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) +sys.stdout.write("\n") +""" + + +def dump_schema() -> str: + """Run the schema-dump snippet in the dataquality provider project and return its stdout.""" + result = subprocess.run( + [ + "uv", + "run", + "--frozen", + "--no-progress", + "--project", + str(DQ_PROVIDER_PATH), + "python", + "-c", + DUMP_SCHEMA, + ], + cwd=DQ_PROVIDER_PATH, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema generation failed: {result.stderr}") + return result.stdout + + +def main() -> int: + try: + new_content = dump_schema() + except Exception as e: + console.print(f"[bold red]ERROR:[/] {e}") + return 1 + + if SCHEMA_PATH.exists(): + old_content = SCHEMA_PATH.read_text() + if old_content == new_content: + return 0 + else: + SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + + SCHEMA_PATH.write_text(new_content) + rel = SCHEMA_PATH.relative_to(AIRFLOW_ROOT_PATH) + console.print(f"[yellow]Regenerated[/] [cyan]{rel}[/]. Please review the diff and re-stage the file.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uv.lock b/uv.lock index f08cb4583d8b2..9bca05707d6b6 100644 --- a/uv.lock +++ b/uv.lock @@ -53,6 +53,7 @@ apache-airflow-providers-telegram = false apache-airflow-providers-celery = false apache-airflow-providers-docker = false apache-airflow-providers-sendgrid = false +apache-airflow-providers-common-dataquality = false apache-airflow-providers-common-ai = false apache-airflow = false apache-airflow-shared-observability = false @@ -210,6 +211,7 @@ members = [ "apache-airflow-providers-common-sql", "apache-airflow-providers-databricks", "apache-airflow-providers-datadog", + "apache-airflow-providers-common-dataquality", "apache-airflow-providers-dbt-cloud", "apache-airflow-providers-dingding", "apache-airflow-providers-discord", @@ -1039,6 +1041,7 @@ all = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks" }, { name = "apache-airflow-providers-datadog" }, + { name = "apache-airflow-providers-common-dataquality" }, { name = "apache-airflow-providers-dbt-cloud" }, { name = "apache-airflow-providers-dingding" }, { name = "apache-airflow-providers-discord" }, @@ -1235,6 +1238,9 @@ databricks = [ datadog = [ { name = "apache-airflow-providers-datadog" }, ] +common-dataquality = [ + { name = "apache-airflow-providers-common-dataquality" }, +] dbt-cloud = [ { name = "apache-airflow-providers-dbt-cloud" }, ] @@ -1646,6 +1652,8 @@ requires-dist = [ { name = "apache-airflow-providers-databricks", marker = "extra == 'databricks'", editable = "providers/databricks" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'all'", editable = "providers/datadog" }, { name = "apache-airflow-providers-datadog", marker = "extra == 'datadog'", editable = "providers/datadog" }, + { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'all'", editable = "providers/common/dataquality" }, + { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'common-dataquality'", editable = "providers/common/dataquality" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'all'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dbt-cloud", marker = "extra == 'dbt-cloud'", editable = "providers/dbt/cloud" }, { name = "apache-airflow-providers-dingding", marker = "extra == 'all'", editable = "providers/dingding" }, @@ -1792,7 +1800,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.26" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "common-dataquality", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -4926,6 +4934,49 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-common-dataquality" +version = "0.1.0" +source = { editable = "providers/common/dataquality" } +dependencies = [ + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-dbt-cloud" version = "4.9.2"