diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py index 12f71b990fc1..08a1004d111e 100644 --- a/ingestion/src/metadata/workflow/base.py +++ b/ingestion/src/metadata/workflow/base.py @@ -32,6 +32,9 @@ IngestionPipeline, PipelineState, ) +from metadata.generated.schema.entity.services.ingestionPipelines.progressUpdate import ( + ProgressUpdateType, +) from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -289,6 +292,13 @@ def execute(self) -> None: pipeline_state = PipelineState.success self.timer.trigger() diagnostics.install(self) + # Emit a "run started" update immediately. The reporting timer's first + # tick is a full REPORTS_INTERVAL_SECONDS away, so without this a run + # that finishes inside that window would only ever emit its terminal + # event — and any live viewer would see nothing while it ran. This + # registers the run with the server up front so it is visible the moment + # it starts, regardless of duration. + self.send_progress_update(ProgressUpdateType.DISCOVERY) # `self.config` is typed Union[Any, Dict]; getattr keeps the static # checker happy without changing behavior (the Dict branch never # carries this attribute at runtime). diff --git a/ingestion/src/metadata/workflow/progress_render.py b/ingestion/src/metadata/workflow/progress_render.py index b3304e534d96..dcbb4f4ce56d 100644 --- a/ingestion/src/metadata/workflow/progress_render.py +++ b/ingestion/src/metadata/workflow/progress_render.py @@ -100,6 +100,12 @@ def global_counters(self) -> "List[Tuple[str, int, Optional[int]]]": # noqa: UP reported even when the active tree is momentarily empty.""" return self._registry.global_counters() + def assets_ingested(self) -> int: + """Monotonic run-total of leaf assets ingested (tables + stored + procedures + other leaf entities). Independent of the tree, so it stays + correct after scope pruning and is authoritative on the terminal event.""" + return self._registry.assets_ingested() + def _driver_counter( self, counters: "Optional[List[Tuple[str, int, Optional[int]]]]" = None, # noqa: UP006,UP045 diff --git a/ingestion/src/metadata/workflow/workflow_status_mixin.py b/ingestion/src/metadata/workflow/workflow_status_mixin.py index 0f60aa7348d5..96a43e81ffbb 100644 --- a/ingestion/src/metadata/workflow/workflow_status_mixin.py +++ b/ingestion/src/metadata/workflow/workflow_status_mixin.py @@ -203,6 +203,7 @@ def send_progress_update(self, update_type: ProgressUpdateType = ProgressUpdateT progress_data = reporter.payload() if reporter is not None else None counters = reporter.global_counters() if reporter is not None else [] eta_seconds = reporter.eta_seconds() if reporter is not None else None + total_assets = reporter.assets_ingested() if reporter is not None else None progress_update = ProgressUpdate( runId=self.run_id, @@ -213,6 +214,7 @@ def send_progress_update(self, update_type: ProgressUpdateType = ProgressUpdateT {"entityType": type_, "done": done, "total": total} for type_, done, total in counters ], estimatedSecondsRemaining=eta_seconds, + totalAssetsIngested=total_assets, stepName=None, currentEntity=None, message=None, diff --git a/ingestion/tests/unit/workflow/test_base_workflow.py b/ingestion/tests/unit/workflow/test_base_workflow.py index f6eeb2545f75..804c844a7503 100644 --- a/ingestion/tests/unit/workflow/test_base_workflow.py +++ b/ingestion/tests/unit/workflow/test_base_workflow.py @@ -249,10 +249,13 @@ def test_close_steps_runs_before_status_publishing_and_stop(self): workflow.execute() ordered_names = [mock_call[0] for mock_call in manager.mock_calls] - # `close_steps` is recorded twice: once from execute() before - # print_status, and once from inside stop() to keep the public - # cleanup contract. The second call is a no-op via _steps_closed. + # An initial `send_progress_update` fires at the very start so short + # runs (shorter than the reporting interval) still emit a "run started" + # event. `close_steps` is recorded twice: once from execute() before + # print_status, and once from inside stop() to keep the public cleanup + # contract. The second call is a no-op via _steps_closed. assert ordered_names == [ + "send_progress_update", "close_steps", "build_ingestion_status", "set_ingestion_pipeline_status", @@ -261,7 +264,8 @@ def test_close_steps_runs_before_status_publishing_and_stop(self): "stop", "close_steps", ] - mock_send_progress_update.assert_called_once_with(ProgressUpdateType.ERROR) + progress_types = [call.args[0] for call in mock_send_progress_update.call_args_list] + assert progress_types == [ProgressUpdateType.DISCOVERY, ProgressUpdateType.ERROR] def test_success_states_map_to_pipeline_complete_progress(self): workflow = SimpleWorkflow(config=config) @@ -281,7 +285,40 @@ def test_failed_workflow_sends_terminal_error_progress(self): ) as mock_send_progress_update: workflow.execute() - mock_send_progress_update.assert_called_once_with(ProgressUpdateType.ERROR) + progress_types = [call.args[0] for call in mock_send_progress_update.call_args_list] + assert progress_types == [ProgressUpdateType.DISCOVERY, ProgressUpdateType.ERROR] + + def test_execute_emits_initial_progress_before_work(self): + """A run shorter than the reporting interval still emits events: an + initial DISCOVERY at start (so the run is registered/visible live) and + a terminal update at the end — never zero events.""" + workflow = SimpleWorkflow(config=config) + + emitted = [] + original = workflow.send_progress_update + + def record(update_type=ProgressUpdateType.PROCESSING): + emitted.append((update_type, workflow.execute_internal_started)) + return original(update_type) + + workflow.execute_internal_started = False + real_execute_internal = workflow.execute_internal + + def flag_execute_internal(): + workflow.execute_internal_started = True + return real_execute_internal() + + with ( + patch.object(workflow, "send_progress_update", side_effect=record), + patch.object(workflow, "execute_internal", side_effect=flag_execute_internal), + ): + workflow.execute() + + # First emit is DISCOVERY and happens before execute_internal runs. + assert emitted[0][0] is ProgressUpdateType.DISCOVERY + assert emitted[0][1] is False + # At least the initial + a terminal event, so never zero. + assert len(emitted) >= 2 def test_stop_still_runs_when_print_status_raises(self): workflow = SimpleWorkflow(config=config) diff --git a/ingestion/tests/unit/workflow/test_progress_rendering.py b/ingestion/tests/unit/workflow/test_progress_rendering.py index bb026dbf8f29..0333f831387f 100644 --- a/ingestion/tests/unit/workflow/test_progress_rendering.py +++ b/ingestion/tests/unit/workflow/test_progress_rendering.py @@ -35,6 +35,30 @@ def _tree_snapshot(): return reg.snapshot() +class TestReporterAssetsIngested: + def test_counts_all_leaf_types(self): + reg = ProgressRegistry() + reg.open([], "Database", None) + reg.open(["db"], "DatabaseSchema", 1) + reg.open(["db", "sch"], "Table", 3) + reg.open(["db", "sch"], "StoredProcedure", 2) + for _ in range(3): + reg.advance(["db", "sch"], "Table") + for _ in range(2): + reg.advance(["db", "sch"], "StoredProcedure") + assert ProgressReporter(reg).assets_ingested() == 5 + + def test_survives_scope_pruning(self): + reg = ProgressRegistry() + reg.open([], "Database", None) + reg.open(["db", "sch"], "Table", 2) + reg.advance(["db", "sch"], "Table") + reg.advance(["db", "sch"], "Table") + reg.close(["db", "sch"]) + reg.close(["db"]) + assert ProgressReporter(reg).assets_ingested() == 2 + + class TestProgressRendering: def test_cli_renders_database_line(self): out = render_progress_tree(_tree_snapshot()) diff --git a/ingestion/tests/unit/workflow/test_status_mixin_progress.py b/ingestion/tests/unit/workflow/test_status_mixin_progress.py index daebf0434b7c..146a62d7ee75 100644 --- a/ingestion/tests/unit/workflow/test_status_mixin_progress.py +++ b/ingestion/tests/unit/workflow/test_status_mixin_progress.py @@ -13,6 +13,9 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +from metadata.generated.schema.entity.services.ingestionPipelines.progressUpdate import ( + ProgressUpdateType, +) from metadata.utils.progress_registry import ProgressRegistry from metadata.workflow.workflow_status_mixin import WorkflowStatusMixin @@ -60,3 +63,39 @@ def test_null_when_not_computable(self): reg.set_total("DatabaseSchema", 10) _Harness(reg, metadata).send_progress_update() assert _sent_update(metadata).estimatedSecondsRemaining is None + + +class TestTotalAssetsIngested: + def test_processing_update_carries_leaf_total(self): + reg = ProgressRegistry() + metadata = MagicMock() + reg.open([], "Database", None) + reg.open(["db", "sch"], "Table", 3) + reg.open(["db", "sch"], "StoredProcedure", 2) + for _ in range(3): + reg.advance(["db", "sch"], "Table") + for _ in range(2): + reg.advance(["db", "sch"], "StoredProcedure") + _Harness(reg, metadata).send_progress_update() + assert _sent_update(metadata).totalAssetsIngested == 5 + + def test_terminal_update_carries_final_total_after_pruning(self): + reg = ProgressRegistry() + metadata = MagicMock() + reg.open([], "Database", None) + reg.open(["db", "sch"], "Table", 2) + reg.advance(["db", "sch"], "Table") + reg.advance(["db", "sch"], "Table") + reg.close(["db", "sch"]) + reg.close(["db"]) + _Harness(reg, metadata).send_progress_update(ProgressUpdateType.PIPELINE_COMPLETE) + update = _sent_update(metadata) + assert update.updateType is ProgressUpdateType.PIPELINE_COMPLETE + assert update.totalAssetsIngested == 2 + + def test_null_when_no_registry(self): + metadata = MagicMock() + harness = _Harness(ProgressRegistry(), metadata) + harness._steps = [] + harness.send_progress_update() + assert _sent_update(metadata).totalAssetsIngested is None diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java index a1b15203de19..0b71b58a56ea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java @@ -1313,10 +1313,29 @@ public RestUtil.PutResponse updateProgress( return new RestUtil.PutResponse<>(Response.Status.OK, progressUpdate, ENTITY_FIELDS_CHANGED); } - progressTracker.updateProgress(fqn, runId, progressUpdate); + progressTracker.updateProgress( + fqn, runId, progressUpdate, discoveredPipeline(fqn, progressUpdate)); return new RestUtil.PutResponse<>(Response.Status.OK, progressUpdate, ENTITY_FIELDS_CHANGED); } + /** + * The pipeline entity to attach to the run's opening event, so the UI can discover a newly + * created agent from the stream itself. Loaded only for the DISCOVERY update (once per run) and + * never fails the progress update — a lookup error just omits the entity. + */ + private IngestionPipeline discoveredPipeline(String fqn, ProgressUpdate update) { + IngestionPipeline result = null; + if (update.getUpdateType() == ProgressUpdateType.DISCOVERY) { + try { + result = getByName(null, fqn, getFields("")); + } catch (Exception e) { + LOG.debug( + "Could not load ingestion pipeline {} for progress discovery: {}", fqn, e.getMessage()); + } + } + return result; + } + public RestUtil.PutResponse addOperationMetrics( String fqn, UUID runId, OperationMetricsBatch batch) { if (progressTracker == null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/IngestionProgressTracker.java b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/IngestionProgressTracker.java index a507951dd959..bb3f7997354e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/IngestionProgressTracker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/monitoring/IngestionProgressTracker.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.OperationMetricsBatch; import org.openmetadata.schema.entity.services.ingestionPipelines.ProgressUpdate; import org.openmetadata.schema.entity.services.ingestionPipelines.ProgressUpdateType; @@ -87,6 +88,11 @@ public IngestionProgressTracker(MeterRegistry meterRegistry) { } public void updateProgress(String pipelineFqn, UUID runId, ProgressUpdate update) { + updateProgress(pipelineFqn, runId, update, null); + } + + public void updateProgress( + String pipelineFqn, UUID runId, ProgressUpdate update, IngestionPipeline ingestionPipeline) { String key = buildKey(pipelineFqn, runId); progressUpdatesReceived.increment(); @@ -98,7 +104,7 @@ public void updateProgress(String pipelineFqn, UUID runId, ProgressUpdate update } notifyListeners(key, update); - routeToService(pipelineFqn, runId, update); + routeToService(pipelineFqn, runId, update, ingestionPipeline); } public void addMetricsBatch(String pipelineFqn, UUID runId, OperationMetricsBatch batch) { @@ -169,7 +175,8 @@ private String buildKey(String pipelineFqn, UUID runId) { return pipelineFqn + "/" + runId.toString(); } - private void routeToService(String pipelineFqn, UUID runId, ProgressUpdate update) { + private void routeToService( + String pipelineFqn, UUID runId, ProgressUpdate update, IngestionPipeline ingestionPipeline) { String serviceFqn = FullyQualifiedName.getParentFQN(pipelineFqn); if (!nullOrEmpty(serviceFqn)) { String runKey = buildKey(pipelineFqn, runId); @@ -177,7 +184,8 @@ private void routeToService(String pipelineFqn, UUID runId, ProgressUpdate updat new ServiceProgressEvent() .withPipelineFqn(pipelineFqn) .withRunId(runId.toString()) - .withEvent(update); + .withEvent(update) + .withIngestionPipeline(ingestionPipeline); recordActiveRun(serviceFqn, runKey, event); notifyServiceListeners(serviceFqn, event); dropRunIfTerminal(serviceFqn, runKey, update); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/IngestionProgressTrackerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/IngestionProgressTrackerTest.java index 0402f0792566..2fa1db82ce15 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/IngestionProgressTrackerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/monitoring/IngestionProgressTrackerTest.java @@ -23,6 +23,7 @@ import java.util.function.Consumer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.schema.entity.services.ingestionPipelines.OperationMetric; import org.openmetadata.schema.entity.services.ingestionPipelines.OperationMetricsBatch; import org.openmetadata.schema.entity.services.ingestionPipelines.ProgressNode; @@ -333,6 +334,40 @@ void testUnregisterServiceListenerStopsDelivery() { assertEquals(1, received.size()); } + @Test + void serviceEventCarriesIngestionPipelineWhenProvided() { + UUID run = UUID.randomUUID(); + List received = new ArrayList<>(); + tracker.registerServiceListener("svc", received::add); + + IngestionPipeline pipeline = + new IngestionPipeline().withName("metadata").withFullyQualifiedName("svc.metadata"); + tracker.updateProgress("svc.metadata", run, discovery(run), pipeline); + + assertEquals(1, received.size()); + assertNotNull(received.get(0).getIngestionPipeline()); + assertEquals("svc.metadata", received.get(0).getIngestionPipeline().getFullyQualifiedName()); + } + + @Test + void serviceEventHasNoIngestionPipelineByDefault() { + UUID run = UUID.randomUUID(); + List received = new ArrayList<>(); + tracker.registerServiceListener("svc", received::add); + + tracker.updateProgress("svc.metadata", run, processing(run, "p")); + + assertEquals(1, received.size()); + assertNull(received.get(0).getIngestionPipeline()); + } + + private static ProgressUpdate discovery(UUID runId) { + return new ProgressUpdate() + .withRunId(runId.toString()) + .withTimestamp(System.currentTimeMillis()) + .withUpdateType(ProgressUpdateType.DISCOVERY); + } + private static ProgressUpdate processing(UUID runId, String msg) { return new ProgressUpdate() .withRunId(runId.toString()) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/progressUpdate.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/progressUpdate.json index f55637339f43..0a5fde87f21a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/progressUpdate.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/progressUpdate.json @@ -91,6 +91,10 @@ "description": "Estimated seconds until the run completes; null when not yet computable", "type": ["integer", "null"] }, + "totalAssetsIngested": { + "description": "Monotonic count of leaf assets ingested so far (tables + stored procedures + other leaf entities). Survives scope pruning and is present on the terminal PIPELINE_COMPLETE event, so it carries the run's final asset total.", + "type": ["integer", "null"] + }, "currentEntity": { "description": "FQN of the entity currently being processed", "type": "string" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/serviceProgressEvent.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/serviceProgressEvent.json index 07054a986ac1..1bd4637ab3a5 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/serviceProgressEvent.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/ingestionPipelines/serviceProgressEvent.json @@ -17,6 +17,10 @@ "event": { "description": "The per-run progress update", "$ref": "progressUpdate.json" + }, + "ingestionPipeline": { + "description": "The ingestion pipeline (agent) this run belongs to. Populated on the DISCOVERY event that opens a run, so the UI can discover a newly created agent without a separate lookup.", + "$ref": "ingestionPipeline.json" } }, "required": ["pipelineFqn", "runId", "event"], diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/progressUpdate.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/progressUpdate.ts index ec2a9477d037..e147407f453f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/progressUpdate.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/progressUpdate.ts @@ -47,6 +47,12 @@ export interface ProgressUpdate { * When this update was created */ timestamp: number; + /** + * Monotonic count of leaf assets ingested so far (tables + stored procedures + other leaf + * entities). Survives scope pruning and is present on the terminal PIPELINE_COMPLETE event, + * so it carries the run's final asset total. + */ + totalAssetsIngested?: number | null; /** * Type of progress update */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/serviceProgressEvent.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/serviceProgressEvent.ts index fe4e2c98ce40..ee820c0fb4b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/serviceProgressEvent.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/serviceProgressEvent.ts @@ -18,6 +18,11 @@ export interface ServiceProgressEvent { * The per-run progress update */ event: ProgressUpdate; + /** + * The ingestion pipeline (agent) this run belongs to. Populated on the DISCOVERY event that + * opens a run, so the UI can discover a newly created agent without a separate lookup. + */ + ingestionPipeline?: IngestionPipeline; /** * Fully qualified name of the ingestion pipeline this event belongs to */ @@ -67,6 +72,12 @@ export interface ProgressUpdate { * When this update was created */ timestamp: number; + /** + * Monotonic count of leaf assets ingested so far (tables + stored procedures + other leaf + * entities). Survives scope pruning and is present on the terminal PIPELINE_COMPLETE event, + * so it carries the run's final asset total. + */ + totalAssetsIngested?: number | null; /** * Type of progress update */ @@ -126,3 +137,8524 @@ export enum ProgressUpdateType { Processing = "PROCESSING", StepComplete = "STEP_COMPLETE", } + +/** + * The ingestion pipeline (agent) this run belongs to. Populated on the DISCOVERY event that + * opens a run, so the UI can discover a newly created agent without a separate lookup. + * + * Ingestion Pipeline Config is used to set up a DAG and deploy. This entity is used to + * setup metadata/quality pipelines on Apache Airflow. + */ +export interface IngestionPipeline { + airflowConfig: AirflowConfig; + /** + * Type of the application when pipelineType is 'application'. + */ + applicationType?: string; + /** + * Change that led to this version of the entity. + */ + changeDescription?: ChangeDescription; + /** + * When `true` indicates the entity has been soft deleted. + */ + deleted?: boolean; + /** + * Indicates if the workflow has been successfully deployed to Airflow. + */ + deployed?: boolean; + /** + * Description of the Pipeline. + */ + description?: string; + /** + * Display Name that identifies this Pipeline. + */ + displayName?: string; + /** + * Domains the asset belongs to. When not set, the asset inherits the domain from the parent + * it belongs to. + */ + domains?: EntityReference[]; + /** + * True if the pipeline is ready to be run in the next schedule. False if it is paused. + */ + enabled?: boolean; + /** + * Enable real-time log streaming to the OpenMetadata server. When enabled, ingestion logs + * will be automatically shipped to the server's configured log storage backend (S3 or + * compatible). + */ + enableStreamableLogs?: boolean; + /** + * Followers of this entity. + */ + followers?: EntityReference[]; + /** + * Name that uniquely identifies a Pipeline. + */ + fullyQualifiedName?: string; + /** + * Link to this ingestion pipeline resource. + */ + href?: string; + /** + * Unique identifier that identifies this pipeline. + */ + id?: string; + /** + * Bot user that performed the action on behalf of the actual user. + */ + impersonatedBy?: string; + /** + * Change that lead to this version of the entity. + */ + incrementalChangeDescription?: ChangeDescription; + /** + * The ingestion agent responsible for executing the ingestion pipeline. + */ + ingestionRunner?: EntityReference; + /** + * Set the logging level for the workflow. + */ + loggerLevel?: LogLevels; + /** + * Name that identifies this pipeline instance uniquely. + */ + name: string; + openMetadataServerConnection?: OpenMetadataConnection; + /** + * Owners of this Pipeline. + */ + owners?: EntityReference[]; + /** + * List of the most recent executions and status for the Pipeline. + */ + pipelineStatuses?: PipelineStatus[]; + pipelineType: PipelineType; + /** + * The processing engine responsible for executing the ingestion pipeline logic. + */ + processingEngine?: EntityReference; + provider?: ProviderType; + /** + * Control if we want to flag the workflow as failed if we encounter any processing errors. + */ + raiseOnError?: boolean; + /** + * Link to the service (such as database, messaging, storage services, etc. for which this + * ingestion pipeline ingests the metadata from. + */ + service?: EntityReference; + sourceConfig: SourceConfig; + /** + * Last update time corresponding to the new version of the entity in Unix epoch time + * milliseconds. + */ + updatedAt?: number; + /** + * User who made the update. + */ + updatedBy?: string; + /** + * Metadata version of the entity. + */ + version?: number; +} + +/** + * Properties to configure the Airflow pipeline that will run the workflow. + */ +export interface AirflowConfig { + /** + * Concurrency of the Pipeline. + */ + concurrency?: number; + /** + * Email to notify workflow status. + */ + email?: string; + /** + * End Date of the pipeline. + */ + endDate?: Date; + /** + * Maximum Number of active runs. + */ + maxActiveRuns?: number; + /** + * pause the pipeline from running once the deploy is finished successfully. + */ + pausePipeline?: boolean; + /** + * Run past executions if the start date is in the past. + */ + pipelineCatchup?: boolean; + /** + * Timezone in which pipeline going to be scheduled. + */ + pipelineTimezone?: string; + /** + * Retry pipeline in case of failure. + */ + retries?: number; + /** + * Delay between retries in seconds. + */ + retryDelay?: number; + /** + * Scheduler Interval for the pipeline in cron format. + */ + scheduleInterval?: string; + /** + * Start date of the pipeline. + */ + startDate?: Date; + /** + * Default view in Airflow. + */ + workflowDefaultView?: string; + /** + * Default view Orientation in Airflow. + */ + workflowDefaultViewOrientation?: string; + /** + * Timeout for the workflow in seconds. + */ + workflowTimeout?: number; +} + +/** + * Change that led to this version of the entity. + * + * Description of the change. + * + * Change that lead to this version of the entity. + */ +export interface ChangeDescription { + changeSummary?: { [key: string]: ChangeSummary }; + /** + * Names of fields added during the version changes. + */ + fieldsAdded?: FieldChange[]; + /** + * Fields deleted during the version changes with old value before deleted. + */ + fieldsDeleted?: FieldChange[]; + /** + * Fields modified during the version changes with old and new values. + */ + fieldsUpdated?: FieldChange[]; + /** + * When a change did not result in change, this could be same as the current version. + */ + previousVersion?: number; +} + +export interface ChangeSummary { + changedAt?: number; + /** + * Name of the user or bot who made this change + */ + changedBy?: string; + changeSource?: ChangeSource; + [property: string]: any; +} + +/** + * The source of the change. This will change based on the context of the change (example: + * manual vs programmatic) + */ +export enum ChangeSource { + Automated = "Automated", + Derived = "Derived", + Ingested = "Ingested", + Manual = "Manual", + Propagated = "Propagated", + Suggested = "Suggested", +} + +export interface FieldChange { + /** + * Name of the entity field that changed. + */ + name?: string; + /** + * New value of the field. Note that this is a JSON string and use the corresponding field + * type to deserialize it. + */ + newValue?: any; + /** + * Previous value of the field. Note that this is a JSON string and use the corresponding + * field type to deserialize it. + */ + oldValue?: any; +} + +/** + * Domains the asset belongs to. When not set, the asset inherits the domain from the parent + * it belongs to. + * + * This schema defines the EntityReferenceList type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * The ingestion agent responsible for executing the ingestion pipeline. + * + * The processing engine responsible for executing the ingestion pipeline logic. + * + * Link to the service (such as database, messaging, storage services, etc. for which this + * ingestion pipeline ingests the metadata from. + * + * Service to be modified + */ +export interface EntityReference { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + +/** + * Set the logging level for the workflow. + * + * Supported logging levels + */ +export enum LogLevels { + Debug = "DEBUG", + Error = "ERROR", + Info = "INFO", + Warn = "WARN", +} + +/** + * OpenMetadata Connection Config + */ +export interface OpenMetadataConnection { + /** + * OpenMetadata server API version to use. + */ + apiVersion?: string; + /** + * OpenMetadata Server Authentication Provider. + */ + authProvider?: AuthProvider; + /** + * Cluster name to differentiate OpenMetadata Server instance + */ + clusterName?: string; + /** + * Regex to only include/exclude databases that matches the pattern. + */ + databaseFilterPattern?: FilterPattern; + /** + * Configuration for Sink Component in the OpenMetadata Ingestion Framework. + */ + elasticsSearch?: OpenMetadataServerConnectionElasticsSearch; + /** + * Validate Openmetadata Server & Client Version. + */ + enableVersionValidation?: boolean; + extraHeaders?: { [key: string]: string }; + /** + * Force the overwriting of any entity during the ingestion. + */ + forceEntityOverwriting?: boolean; + /** + * OpenMetadata Server Config. Must include API end point ex: http://localhost:8585/api + */ + hostPort: string; + /** + * Include Dashboards for Indexing + */ + includeDashboards?: boolean; + /** + * Include Database Services for Indexing + */ + includeDatabaseServices?: boolean; + /** + * Include Glossary Terms for Indexing + */ + includeGlossaryTerms?: boolean; + /** + * Include Messaging Services for Indexing + */ + includeMessagingServices?: boolean; + /** + * Include MlModels for Indexing + */ + includeMlModels?: boolean; + /** + * Include Pipelines for Indexing + */ + includePipelines?: boolean; + /** + * Include Pipeline Services for Indexing + */ + includePipelineServices?: boolean; + /** + * Include Tags for Policy + */ + includePolicy?: boolean; + /** + * Include Tables for Indexing + */ + includeTables?: boolean; + /** + * Include Tags for Indexing + */ + includeTags?: boolean; + /** + * Include Teams for Indexing + */ + includeTeams?: boolean; + /** + * Include Topics for Indexing + */ + includeTopics?: boolean; + /** + * Include Users for Indexing + */ + includeUsers?: boolean; + /** + * Limit the number of records for Indexing. + */ + limitRecords?: number; + /** + * Regex to only include/exclude schemas that matches the pattern. + */ + schemaFilterPattern?: FilterPattern; + /** + * Secrets Manager Loader for the Pipeline Service Client. + */ + secretsManagerLoader?: SecretsManagerClientLoader; + /** + * Secrets Manager Provider for OpenMetadata Server. + */ + secretsManagerProvider?: SecretsManagerProvider; + /** + * OpenMetadata Client security configuration. + */ + securityConfig?: OpenMetadataJWTClientConfig; + /** + * SSL Configuration for OpenMetadata Server + */ + sslConfig?: DbtSSLConfigClass; + /** + * If set to true, when creating a service during the ingestion we will store its Service + * Connection. Otherwise, the ingestion will create a bare service without connection + * details. + */ + storeServiceConnection?: boolean; + /** + * Flag to enable Data Insight Extraction + */ + supportsDataInsightExtraction?: boolean; + /** + * Flag to enable ElasticSearch Reindexing Extraction + */ + supportsElasticSearchReindexingExtraction?: boolean; + /** + * Regex to only include/exclude tables that matches the pattern. + */ + tableFilterPattern?: FilterPattern; + /** + * Service Type + */ + type?: OpenmetadataType; + /** + * Flag to verify SSL Certificate for OpenMetadata Server. + */ + verifySSL?: VerifySSL; +} + +/** + * OpenMetadata Server Authentication Provider. + * + * OpenMetadata Server Authentication Provider. Make sure configure same auth providers as + * the one configured on OpenMetadata server. + */ +export enum AuthProvider { + Auth0 = "auth0", + AwsCognito = "aws-cognito", + Azure = "azure", + Basic = "basic", + CustomOidc = "custom-oidc", + Google = "google", + LDAP = "ldap", + Okta = "okta", + Openmetadata = "openmetadata", + Saml = "saml", +} + +/** + * Regex to only include/exclude databases that matches the pattern. + * + * Regex to only fetch entities that matches the pattern. + * + * Regex to only include/exclude schemas that matches the pattern. + * + * Regex to only include/exclude tables that matches the pattern. + * + * Regex to only include/exclude stored procedures that matches the pattern. + * + * Regex to only fetch databases that matches the pattern. + * + * Regex to only fetch tables or databases that matches the pattern. + * + * Regex to only fetch stored procedures that matches the pattern. + * + * Regex exclude tables or databases that matches the pattern. + * + * Regex exclude or include charts that matches the pattern. + * + * Regex to exclude or include dashboards that matches the pattern. + * + * Regex exclude or include data models that matches the pattern. + * + * Regex to exclude or include projects that matches the pattern. + * + * Regex to only fetch topics that matches the pattern. + * + * Regex to only compute metrics for table that matches the given tag, tiers, gloassary + * pattern. + * + * Regex exclude pipelines. + * + * Regex to only fetch MlModels with names matching the pattern. + * + * Regex to only fetch containers that matches the pattern. + * + * Regex to only fetch buckets (top-level containers) that match the pattern. + * + * Regex to only compute metrics for containers that matches the given tag, tiers, glossary + * pattern. + * + * Regex to only include/exclude directories that matches the pattern. + * + * Regex to only include/exclude files that matches the pattern. + * + * Regex to only include/exclude spreadsheets that matches the pattern. + * + * Regex to only include/exclude worksheets that matches the pattern. + * + * Regex to only fetch search indexes that matches the pattern. + * + * Regex to only fetch api collections with names matching the pattern. + * + * Regex to only fetch api endpoints with names matching the pattern. + * + * Regex to exclude or include charts that matches the pattern. + * + * Regex to only include/exclude schemas that matches the pattern. System schemas + * (information_schema, _statistics_, sys) are excluded by default. + * + * Regex to include/exclude FHIR resource categories + * + * Regex to include/exclude FHIR resource types + * + * Regex to only include/exclude namespaces (sources/spaces) that match the pattern. In + * Dremio Cloud, namespaces are mapped as databases. + * + * Regex to only include/exclude folders that match the pattern. In Dremio Cloud, folders + * are mapped as schemas. + * + * Regex to only include/exclude tables that match the pattern. + * + * Regex to only include/exclude dictionaries (tables) that matches the pattern. + * + * Regex to only include/exclude IOMETE databases (e.g. 'default', 'finance_db') that match + * the pattern. In IOMETE, a database corresponds to an OpenMetadata schema. + * + * Regex to only include/exclude InfoAreas that match the pattern. + * + * Regex to only include/exclude InfoProviders (ADSOs, CompositeProviders) that match the + * pattern. + * + * Regex to only include/exclude domains that match the pattern. + * + * Regex to only include/exclude glossaries that match the pattern. + * + * Regex to filter MuleSoft applications by name. + * + * Regex to only include/exclude pipelines that matches the pattern. + * + * Regex to only include/exclude Process Chains that match the pattern. + * + * Regex to only include/exclude directories that match the pattern. + * + * Regex to only include/exclude files that match the pattern. + * + * Regex to only fetch servers with names matching the pattern + * + * Regex to only fetch tags that matches the pattern. + * + * Regex to only fetch MCP servers with names matching the pattern. + */ +export interface FilterPattern { + /** + * List of strings/regex patterns to match and exclude only database entities that match. + */ + excludes?: string[]; + /** + * List of strings/regex patterns to match and include only database entities that match. + */ + includes?: string[]; +} + +/** + * Configuration for Sink Component in the OpenMetadata Ingestion Framework. + */ +export interface OpenMetadataServerConnectionElasticsSearch { + config?: { [key: string]: any }; + /** + * Type of sink component ex: metadata + */ + type: string; +} + +/** + * Secrets Manager Loader for the Pipeline Service Client. + * + * OpenMetadata Secrets Manager Client Loader. Lets the client know how the Secrets Manager + * Credentials should be loaded from the environment. + */ +export enum SecretsManagerClientLoader { + Airflow = "airflow", + Env = "env", + Noop = "noop", +} + +/** + * Secrets Manager Provider for OpenMetadata Server. + * + * OpenMetadata Secrets Manager Provider. Make sure to configure the same secrets manager + * providers as the ones configured on the OpenMetadata server. + */ +export enum SecretsManagerProvider { + Aws = "aws", + AwsSsm = "aws-ssm", + AzureKv = "azure-kv", + DB = "db", + Gcp = "gcp", + InMemory = "in-memory", + Kubernetes = "kubernetes", + ManagedAws = "managed-aws", + ManagedAwsSsm = "managed-aws-ssm", + ManagedAzureKv = "managed-azure-kv", +} + +/** + * OpenMetadata Client security configuration. + * + * openMetadataJWTClientConfig security configs. + */ +export interface OpenMetadataJWTClientConfig { + /** + * OpenMetadata generated JWT token. + */ + jwtToken: string; +} + +/** + * SSL Configuration for OpenMetadata Server + * + * Client SSL configuration + * + * SSL Configuration details. + * + * CA certificate, client certificate, and private key for SSL validation. Required when + * verifySSL is 'validate'. + * + * SSL Configuration details for DB2 connection. Provide CA certificate for server + * validation, and optionally client certificate and key for mutual TLS authentication. + * + * SSL/TLS certificate configuration for client authentication. Provide CA certificate, + * client certificate, and private key for mutual TLS authentication. + * + * SSL Configuration details. Provide the CA certificate to validate the Informix server + * certificate. Paste the PEM content directly or upload the certificate file. + * + * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config + * connection. + * + * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry + * connection. + * + * SSL certificate configuration for validating the server certificate when fetching dbt + * artifacts. + * + * OpenMetadata Client configured to validate SSL certificates. + */ +export interface DbtSSLConfigClass { + /** + * The CA certificate used for SSL validation. + */ + caCertificate?: string; + /** + * The SSL certificate used for client authentication. + */ + sslCertificate?: string; + /** + * The private key associated with the SSL certificate. + */ + sslKey?: string; +} + +/** + * Service Type + * + * OpenMetadata service type + */ +export enum OpenmetadataType { + OpenMetadata = "OpenMetadata", +} + +/** + * Flag to verify SSL Certificate for OpenMetadata Server. + * + * Client SSL verification. Make sure to configure the SSLConfig if enabled. + * + * Client SSL verification. + * + * Client SSL verification. Use 'no-ssl' for plain HTTP, 'ignore' to skip certificate + * validation, 'validate' to verify against a CA certificate. + * + * SSL/TLS verification mode when fetching dbt artifacts over HTTPS. + */ +export enum VerifySSL { + Ignore = "ignore", + NoSSL = "no-ssl", + Validate = "validate", +} + +/** + * This defines runtime status of Pipeline. + */ +export interface PipelineStatus { + /** + * Pipeline configuration for this particular execution. + */ + config?: { [key: string]: any }; + /** + * endDate of the pipeline run for this particular execution. + */ + endDate?: number; + /** + * Metadata for the pipeline status. + */ + metadata?: { [key: string]: any }; + /** + * Pipeline status denotes if its failed or succeeded. + */ + pipelineState?: PipelineState; + /** + * Pipeline unique run ID. + */ + runId?: string; + /** + * startDate of the pipeline run for this particular execution. + */ + startDate?: number; + /** + * Ingestion Pipeline summary status. Informed at the end of the execution. + */ + status?: StepSummary[]; + /** + * executionDate of the pipeline run for this particular execution. + */ + timestamp?: number; +} + +/** + * Pipeline status denotes if its failed or succeeded. + */ +export enum PipelineState { + Failed = "failed", + PartialSuccess = "partialSuccess", + Queued = "queued", + Running = "running", + Stopped = "stopped", + Success = "success", +} + +/** + * Ingestion Pipeline summary status. Informed at the end of the execution. + * + * Summary for each step of the ingestion pipeline + * + * Defines the summary status of each step executed in an Ingestion Pipeline. + */ +export interface StepSummary { + /** + * Number of records with errors. + */ + errors?: number; + /** + * Sample of errors encountered in the step + */ + failures?: StackTraceError[]; + /** + * Number of filtered records. + */ + filtered?: number; + /** + * Step name + */ + name: string; + /** + * Operation metrics by category (db_queries, api_calls) -> operation -> entityType -> + * summary + */ + operationMetrics?: { [key: string]: { [key: string]: { [key: string]: OperationMetric } } }; + /** + * Detailed progress tracking by entity type (databases, schemas, tables, stored procedures) + */ + progress?: { [key: string]: Progress }; + /** + * Number of successfully processed records. + */ + records?: number; + /** + * Total time spent processing and sinking data to OpenMetadata (milliseconds) + */ + sinkTimeMs?: number; + /** + * Total time spent fetching data from source systems (milliseconds) + */ + sourceTimeMs?: number; + /** + * Number of successfully updated records. + */ + updated_records?: number; + /** + * Number of records raising warnings. + */ + warnings?: number; +} + +/** + * Represents a failure status + */ +export interface StackTraceError { + /** + * Error being handled + */ + error: string; + /** + * Name of the asset with the error + */ + name: string; + /** + * Exception stack trace + */ + stackTrace?: string; +} + +export interface OperationMetric { + /** + * Average time per operation in milliseconds + */ + avgTimeMs?: number; + /** + * Total number of operations + */ + count?: number; + /** + * Maximum operation time in milliseconds + */ + maxTimeMs?: number; + /** + * Minimum operation time in milliseconds + */ + minTimeMs?: number; + /** + * Total time spent in milliseconds + */ + totalTimeMs?: number; + [property: string]: any; +} + +export interface Progress { + /** + * Estimated remaining time in seconds for this entity type + */ + estimatedRemainingSeconds?: number; + /** + * Number of entities processed + */ + processed?: number; + /** + * Total number of entities discovered + */ + total?: number; + [property: string]: any; +} + +/** + * Type of Pipeline - metadata, usage + */ +export enum PipelineType { + Application = "application", + AutoClassification = "autoClassification", + DataInsight = "dataInsight", + Dbt = "dbt", + ElasticSearchReindex = "elasticSearchReindex", + Lineage = "lineage", + Metadata = "metadata", + PolicyAgent = "policyAgent", + Profiler = "profiler", + TestSuite = "TestSuite", + Usage = "usage", +} + +/** + * Type of provider of an entity. Some entities are provided by the `system`. Some are + * entities created and provided by the `user`. Typically `system` provide entities can't be + * deleted and can only be disabled. Some apps such as AutoPilot create entities with + * `automation` provider type. These entities can be deleted by the user. + */ +export enum ProviderType { + Automation = "automation", + System = "system", + User = "user", +} + +/** + * Additional connection configuration. + */ +export interface SourceConfig { + config?: Pipeline; +} + +/** + * DatabaseService Metadata Pipeline Configuration. + * + * DatabaseService Query Usage Pipeline Configuration. + * + * DatabaseService Query Lineage Pipeline Configuration. + * + * DashboardService Metadata Pipeline Configuration. + * + * MessagingService Metadata Pipeline Configuration. + * + * DatabaseService Profiler Pipeline Configuration. + * + * DatabaseService AutoClassification & Auto Classification Pipeline Configuration. + * + * PipelineService Metadata Pipeline Configuration. + * + * MlModelService Metadata Pipeline Configuration. + * + * StorageService Metadata Pipeline Configuration. + * + * StorageService AutoClassification Pipeline Configuration. + * + * DriveService Metadata Pipeline Configuration. + * + * SearchService Metadata Pipeline Configuration. + * + * TestSuite Pipeline Configuration. + * + * Data Insight Pipeline Configuration. + * + * DBT Pipeline Configuration. + * + * Application Pipeline Configuration. + * + * ApiService Metadata Pipeline Configuration. + * + * Apply a set of operations on a service + * + * McpService Metadata Pipeline Configuration. + * + * Policy Agent Pipeline Configuration. Applies access grants against the source system. + */ +export interface Pipeline { + /** + * Regex to only include/exclude databases that matches the pattern. + * + * Regex to only fetch databases that matches the pattern. + */ + databaseFilterPattern?: FilterPattern; + /** + * Extract JSON schema from JSON columns by sampling data. This requires SELECT permission + * on the tables. If disabled or if SELECT fails, JSON columns will be ingested without + * schema information. + */ + extractJsonSchema?: boolean; + /** + * Optional configuration to toggle the ingestion of source-specific custom properties (e.g. + * Iceberg table properties) onto the entity extension. When disabled, no custom property + * definitions are registered and no extension values are set. + */ + includeCustomProperties?: boolean; + /** + * Optional configuration to toggle the DDL Statements ingestion. + */ + includeDDL?: boolean; + /** + * Set the 'Include Owners' toggle to control whether to include owners to the ingested + * entity if the owner email matches with a user stored in the OM server as part of metadata + * ingestion. If the ingested entity already exists and has an owner, the owner will not be + * overwritten. + * + * Enabling a flag will replace the current owner with a new owner from the source during + * metadata ingestion, if the current owner is null. It is recommended to keep the flag + * enabled to obtain the owner information during the first metadata ingestion. + */ + includeOwners?: boolean; + /** + * Optional configuration to toggle the Stored Procedures ingestion. + */ + includeStoredProcedures?: boolean; + /** + * Optional configuration to turn off fetching metadata for tables. + */ + includeTables?: boolean; + /** + * Optional configuration to toggle the tags ingestion. + */ + includeTags?: boolean; + /** + * Optional configuration to turn off fetching metadata for views. + */ + includeViews?: boolean; + /** + * Use incremental Metadata extraction after the first execution. This is commonly done by + * getting the changes from Audit tables on the supporting databases. + */ + incremental?: IncrementalMetadataExtractionConfiguration; + /** + * Number of rows to sample for inferring JSON schema. A larger sample size provides more + * accurate schema inference but increases query time. + */ + jsonSchemaSampleSize?: number; + /** + * Optional configuration to soft delete databases in OpenMetadata if the source databases + * are deleted. Also, if the database is deleted, all the associated entities like schemas, + * tables, views, stored procedures, lineage, etc., with that database will be deleted + */ + markDeletedDatabases?: boolean; + /** + * Optional configuration to soft delete schemas in OpenMetadata if the source schemas are + * deleted. Also, if the schema is deleted, all the associated entities like tables, views, + * stored procedures, lineage, etc., with that schema will be deleted + */ + markDeletedSchemas?: boolean; + /** + * Optional configuration to soft delete stored procedures in OpenMetadata if the source + * stored procedures are deleted. Also, if the stored procedures is deleted, all the + * associated entities like lineage, etc., with that stored procedures will be deleted + */ + markDeletedStoredProcedures?: boolean; + /** + * This is an optional configuration for enabling soft deletion of tables. When this option + * is enabled, only tables that have been deleted from the source will be soft deleted, and + * this will apply solely to the schema that is currently being ingested via the pipeline. + * Any related entities such as test suites or lineage information that were associated with + * those tables will also be deleted. + */ + markDeletedTables?: boolean; + /** + * Set the 'Override Metadata' toggle to control whether to override the existing metadata + * in the OpenMetadata server with the metadata fetched from the source. If the toggle is + * set to true, the metadata fetched from the source will override the existing metadata in + * the OpenMetadata server. If the toggle is set to false, the metadata fetched from the + * source will not override the existing metadata in the OpenMetadata server. This is + * applicable for fields like description, tags, owner and displayName + * + * Set the 'Override Metadata' toggle to control whether to override the existing metadata + * in the OpenMetadata server with the metadata fetched from the source. + */ + overrideMetadata?: boolean; + /** + * Advanced configuration for managing owners at multiple hierarchy levels (Service, + * Database, Schema, Table) with custom mappings and inheritance rules. + */ + ownerConfig?: OwnerConfiguration; + /** + * Configuration to tune how far we want to look back in query logs to process Stored + * Procedures results. + * + * Configuration to tune how far we want to look back in query logs to process usage data. + * + * Configuration to tune how far we want to look back in query logs to process lineage data. + */ + queryLogDuration?: number; + /** + * Configuration to set the timeout for parsing the query in seconds. + */ + queryParsingTimeoutLimit?: number; + /** + * Regex to only include/exclude schemas that matches the pattern. + * + * Regex to only fetch tables or databases that matches the pattern. + */ + schemaFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude stored procedures that matches the pattern. + * + * Regex to only fetch stored procedures that matches the pattern. + */ + storedProcedureFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude tables that matches the pattern. + * + * Regex exclude tables or databases that matches the pattern. + */ + tableFilterPattern?: FilterPattern; + /** + * Number of Threads to use in order to parallelize Table ingestion. + * + * Number of Threads to use in order to parallelize lineage ingestion. + * + * Number of Threads to use in order to parallelize Drive ingestion. + */ + threads?: number; + /** + * Pipeline type + */ + type?: FluffyType; + /** + * Regex will be applied on fully qualified name (e.g + * service_name.db_name.schema_name.table_name) instead of raw name (e.g. table_name) + * + * Regex will be applied on fully qualified name (e.g service_name.container_name) instead + * of raw name (e.g. container_name) + * + * Regex will be applied on fully qualified name (e.g service_name.directory_name.file_name) + * instead of raw name (e.g. file_name) + */ + useFqnForFiltering?: boolean; + /** + * Configuration the condition to filter the query history. + */ + filterCondition?: string; + /** + * Configuration to process query cost + */ + processQueryCostAnalysis?: boolean; + /** + * Configuration to set the file path for query logs + */ + queryLogFilePath?: string; + /** + * Configuration to set the limit for query logs + */ + resultLimit?: number; + /** + * Temporary file name to store the query logs before processing. Absolute file path + * required. + */ + stageFileLocation?: string; + /** + * Set 'Cross Database Service Names' to process lineage with the database. + */ + crossDatabaseServiceNames?: string[]; + /** + * Handle Lineage for Snowflake Temporary and Transient Tables. + */ + enableTempTableLineage?: boolean; + /** + * Set the 'Incremental Lineage Processing' toggle to control whether to process lineage + * incrementally. + */ + incrementalLineageProcessing?: boolean; + /** + * Set the 'Override View Lineage' toggle to control whether to override the existing view + * lineage. + */ + overrideViewLineage?: boolean; + /** + * Configuration to set the timeout for parsing the query in seconds. + */ + parsingTimeoutLimit?: number; + /** + * Set the 'Process Cross Database Lineage' toggle to control whether to process table + * lineage across different databases. + */ + processCrossDatabaseLineage?: boolean; + /** + * Set the 'Process Query Lineage' toggle to control whether to process query lineage. + */ + processQueryLineage?: boolean; + /** + * Set the 'Process Stored ProcedureLog Lineage' toggle to control whether to process stored + * procedure lineage. + */ + processStoredProcedureLineage?: boolean; + /** + * Set the 'Process View Lineage' toggle to control whether to process view lineage. + */ + processViewLineage?: boolean; + /** + * Configuration for SQL query parser selection for lineage extraction. + */ + queryParserConfig?: QueryParserConfig; + /** + * Regex exclude or include charts that matches the pattern. + */ + chartFilterPattern?: FilterPattern; + /** + * Regex to exclude or include dashboards that matches the pattern. + */ + dashboardFilterPattern?: FilterPattern; + /** + * Regex exclude or include data models that matches the pattern. + */ + dataModelFilterPattern?: FilterPattern; + /** + * Optional configuration to toggle the ingestion of data models. + */ + includeDataModels?: boolean; + /** + * Optional Configuration to include/exclude draft dashboards. By default it will include + * draft dashboards + */ + includeDraftDashboard?: boolean; + /** + * Optional configuration to toggle the ingestion of usage metadata for dashboards. When + * enabled, usage statistics will be collected and ingested. + */ + includeUsage?: boolean; + /** + * Details required to generate Lineage + */ + lineageInformation?: LineageInformation; + /** + * Optional configuration to soft delete charts in OpenMetadata if the source charts are + * deleted. + */ + markDeletedCharts?: boolean; + /** + * Optional configuration to soft delete dashboards in OpenMetadata if the source dashboards + * are deleted. Also, if the dashboard is deleted, all the associated entities like lineage, + * etc., with that dashboard will be deleted + */ + markDeletedDashboards?: boolean; + /** + * Optional configuration to soft delete data models in OpenMetadata if the source data + * models are deleted. Also, if the data models is deleted, all the associated entities like + * lineage, etc., with that data models will be deleted + */ + markDeletedDataModels?: boolean; + /** + * Set the 'Override Lineage' toggle to control whether to override the existing lineage. + */ + overrideLineage?: boolean; + /** + * Regex to exclude or include projects that matches the pattern. + */ + projectFilterPattern?: FilterPattern; + /** + * Option to turn on/off generating sample data during metadata extraction. + */ + generateSampleData?: boolean; + /** + * Optional configuration to soft delete topics in OpenMetadata if the source topics are + * deleted. Also, if the topic is deleted, all the associated entities like sample data, + * lineage, etc., with that topic will be deleted + */ + markDeletedTopics?: boolean; + /** + * Regex to only fetch topics that matches the pattern. + */ + topicFilterPattern?: FilterPattern; + /** + * Regex to only compute metrics for table that matches the given tag, tiers, gloassary + * pattern. + * + * Regex to only compute metrics for containers that matches the given tag, tiers, glossary + * pattern. + */ + classificationFilterPattern?: FilterPattern; + /** + * Option to turn on/off column metric computation. If enabled, profiler will compute column + * level metrics. + */ + computeColumnMetrics?: boolean; + /** + * Option to turn on/off table metric computation. If enabled, profiler will compute table + * level metrics. + */ + computeTableMetrics?: boolean; + /** + * List of metrics to compute. If empty, then all metrics will be computed + */ + metrics?: MetricType[]; + processingEngine?: ProcessingEngine; + profileSampleConfig?: ProfileSampleConfig; + /** + * Whether to randomize the sample data or not. + */ + randomizedSample?: boolean; + /** + * Number of threads to use during metric computations + */ + threadCount?: number | null; + /** + * Profiler Timeout in Seconds + */ + timeoutSeconds?: number; + /** + * Use system tables to extract table metrics. Metrics that cannot be gathered from system + * tables will use the default methods. Using system tables can be faster but requires + * gathering statistics before running (for example using the ANALYZE procedure). More + * information can be found in the documentation: + * https://docs.openmetadata.org/latest/profler + */ + useStatistics?: boolean; + /** + * Language to use for auto classification recognizers. Use 'any' to run all recognizers + * regardless of their configured language. For specific languages, only recognizers that + * support that language will be used. + */ + classificationLanguage?: ClassificationLanguage; + /** + * Set the Confidence value for which you want the column to be tagged as PII. Confidence + * value ranges from 0 to 100. A higher number will yield less false positives but more + * false negatives. A lower number will yield more false positives but less false negatives. + */ + confidence?: number; + /** + * Optional configuration to automatically tag columns that might contain sensitive + * information + */ + enableAutoClassification?: boolean; + /** + * Number of sample rows to ingest when 'Generate Sample Data' is enabled + */ + sampleDataCount?: number; + /** + * Option to turn on/off storing sample data. If enabled, we will ingest sample data for + * each table. + * + * Option to turn on/off storing sample data. If enabled, we will ingest sample data for + * each structured container. + */ + storeSampleData?: boolean; + /** + * Optional configuration to turn off fetching lineage from pipelines. + */ + includeLineage?: boolean; + /** + * Optional configuration to toggle whether the un-deployed pipelines should be ingested or + * not. If set to false, only deployed pipelines will be ingested. + */ + includeUnDeployedPipelines?: boolean; + /** + * Optional configuration to soft delete Pipelines in OpenMetadata if the source Pipelines + * are deleted. Also, if the Pipeline is deleted, all the associated entities like lineage, + * etc., with that Pipeline will be deleted + */ + markDeletedPipelines?: boolean; + /** + * Set how owners from source metadata update Pipeline owners. In replace mode, resolved + * owners from the current source replace existing owners. In append mode, resolved owners + * are appended to active existing Pipeline owners. + */ + ownershipUpdateMode?: OwnershipUpdateMode; + /** + * Regex exclude pipelines. + */ + pipelineFilterPattern?: FilterPattern; + /** + * Number of days of pipeline run status history to ingest. Only runs within the last N days + * will be fetched. + */ + statusLookbackDays?: number; + /** + * Optional configuration to soft delete MlModels in OpenMetadata if the source MlModels are + * deleted. Also, if the MlModel is deleted, all the associated entities like lineage, etc., + * with that MlModels will be deleted + */ + markDeletedMlModels?: boolean; + /** + * Regex to only fetch MlModels with names matching the pattern. + */ + mlModelFilterPattern?: FilterPattern; + /** + * Regex to only fetch containers that matches the pattern. + */ + containerFilterPattern?: FilterPattern; + /** + * Fallback manifest applied to any bucket that does not have its own openmetadata.json + * file. If a bucket has a manifest file, that file takes precedence and this value is + * ignored for that bucket. Paste the same JSON you would place in a bucket's + * openmetadata.json file — entries accept literal paths or glob-style dataPath patterns. + */ + defaultManifest?: string; + /** + * Optional configuration to soft delete containers in OpenMetadata if the source containers + * are deleted. Also, if the topic is deleted, all the associated entities with that + * containers will be deleted + */ + markDeletedContainers?: boolean; + /** + * Global manifest source. When configured, entries here take precedence over any + * bucket-level openmetadata.json and over defaultManifest for buckets whose containerName + * matches. + */ + storageMetadataConfigSource?: StorageMetadataConfigurationSource; + /** + * Regex to only fetch buckets (top-level containers) that match the pattern. + */ + bucketFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude directories that matches the pattern. + */ + directoryFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude files that matches the pattern. + */ + fileFilterPattern?: FilterPattern; + /** + * Optional configuration to turn off fetching metadata for directories. + */ + includeDirectories?: boolean; + /** + * Optional configuration to turn off fetching metadata for files. + */ + includeFiles?: boolean; + /** + * Optional configuration to turn off fetching metadata for spreadsheets. + */ + includeSpreadsheets?: boolean; + /** + * Optional configuration to turn off fetching metadata for worksheets. + */ + includeWorksheets?: boolean; + /** + * Optional configuration to soft delete directories in OpenMetadata if the source + * directories are deleted. Also, if the directory is deleted, all the associated entities + * like files, spreadsheets, worksheets, lineage, etc., with that directory will be deleted + */ + markDeletedDirectories?: boolean; + /** + * Optional configuration to soft delete files in OpenMetadata if the source files are + * deleted. Also, if the file is deleted, all the associated entities like lineage, etc., + * with that file will be deleted + */ + markDeletedFiles?: boolean; + /** + * Optional configuration to soft delete spreadsheets in OpenMetadata if the source + * spreadsheets are deleted. Also, if the spreadsheet is deleted, all the associated + * entities like worksheets, lineage, etc., with that spreadsheet will be deleted + */ + markDeletedSpreadsheets?: boolean; + /** + * Optional configuration to soft delete worksheets in OpenMetadata if the source worksheets + * are deleted. Also, if the worksheet is deleted, all the associated entities like lineage, + * etc., with that worksheet will be deleted + */ + markDeletedWorksheets?: boolean; + /** + * Regex to only include/exclude spreadsheets that matches the pattern. + */ + spreadsheetFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude worksheets that matches the pattern. + */ + worksheetFilterPattern?: FilterPattern; + /** + * Enable the 'Include Index Template' toggle to manage the ingestion of index template data. + */ + includeIndexTemplate?: boolean; + /** + * Optional configuration to turn off fetching sample data for search index. + */ + includeSampleData?: boolean; + /** + * Optional configuration to soft delete search indexes in OpenMetadata if the source search + * indexes are deleted. Also, if the search index is deleted, all the associated entities + * like lineage, etc., with that search index will be deleted + */ + markDeletedSearchIndexes?: boolean; + /** + * No. of records of sample data we want to ingest. + */ + sampleSize?: number; + /** + * Regex to only fetch search indexes that matches the pattern. + */ + searchIndexFilterPattern?: FilterPattern; + /** + * Fully qualified name of the entity to be tested, if we're working with a basic suite. + */ + entityFullyQualifiedName?: string; + /** + * Service connections to be used for the logical test suite. + */ + serviceConnections?: ServiceConnections[]; + /** + * List of test cases to be executed on the entity. If null, all test cases will be executed. + */ + testCases?: string[]; + /** + * Maximum number of events entities in a batch (Default 1000). + */ + batchSize?: number; + /** + * Certificate path to be added in configuration. The path should be local in the Ingestion + * Container. + */ + caCerts?: string; + recreateIndex?: boolean; + /** + * Region name. Required when using AWS Credentials. + */ + regionName?: string; + /** + * Recreate Indexes with updated Language + */ + searchIndexMappingLanguage?: SearchIndexMappingLanguage; + /** + * Connection Timeout + */ + timeout?: number; + /** + * Indicates whether to use aws credentials when connecting to OpenSearch in AWS. + */ + useAwsCredentials?: boolean; + /** + * Indicates whether to use SSL when connecting to ElasticSearch. By default, we will ignore + * SSL settings. + */ + useSSL?: boolean; + /** + * Indicates whether to verify certificates when using SSL connection to ElasticSearch. + * Ignored by default. Is set to true, make sure to send the certificates in the property + * `CA Certificates`. + */ + verifyCerts?: boolean; + /** + * Custom OpenMetadata Classification name for dbt tags. + */ + dbtClassificationName?: string; + /** + * Available sources to fetch DBT catalog and manifest files. + */ + dbtConfigSource?: DBTConfigurationSource; + /** + * Optional configuration to update the description from DBT or not + */ + dbtUpdateDescriptions?: boolean; + /** + * Optional configuration to update the owners from DBT or not + */ + dbtUpdateOwners?: boolean; + /** + * Optional configuration to search across databases for tables or not + */ + searchAcrossDatabases?: boolean; + /** + * Regex to only fetch tags that matches the pattern. + */ + tagFilterPattern?: FilterPattern; + /** + * Application configuration + */ + appConfig?: any[] | boolean | number | null | CollateAIAppConfig | string; + /** + * Application private configuration + */ + appPrivateConfig?: PrivateConfig; + /** + * Source Python Class Name to run the application + */ + sourcePythonClass?: string; + /** + * Regex to only fetch api collections with names matching the pattern. + */ + apiCollectionFilterPattern?: FilterPattern; + /** + * Regex to only fetch api endpoints with names matching the pattern. + */ + apiEndpointFilterPattern?: FilterPattern; + /** + * Optional configuration to soft delete api collections in OpenMetadata if the source + * collections are deleted. Also, if the collection is deleted, all the associated entities + * like endpoints, etc., with that collection will be deleted + */ + markDeletedApiCollections?: boolean; + /** + * Optional value of the ingestion runner name responsible for running the workflow + */ + ingestionRunner?: string; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: EntityReference; + /** + * Regex to only fetch MCP servers with names matching the pattern. + */ + serverFilterPattern?: FilterPattern; + /** + * List of access grants to apply on the source. + */ + policies?: Policy[]; +} + +/** + * Configuration for the CollateAI External Application. + * + * Configuration for the Automator External Application. + * + * This schema defines the Slack App Information + * + * Configuration for the Collate AI Quality Agent. + * + * No configuration needed to instantiate the Data Insights Pipeline. The logic is handled + * in the backend. + * + * Search Indexing App. + * + * Cache Warmup Application Configuration. + * + * Configuration for the AutoPilot Application. + */ +export interface CollateAIAppConfig { + /** + * Query filter to be passed to ES. E.g., + * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domains.displayName.keyword":"DG + * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. + */ + filter?: string; + /** + * Patch the description if it is empty, instead of raising a suggestion + * + * Patch the tier if it is empty, instead of raising a suggestion + */ + patchIfEmpty?: boolean; + /** + * Application Type + */ + type?: CollateAIAppConfigType; + /** + * Action to take on those entities. E.g., propagate description through lineage, auto + * tagging, etc. + */ + actions?: Action[]; + /** + * Entities selected to run the automation. + */ + resources?: Resource; + /** + * Bot Token + */ + botToken?: string; + /** + * Client Id of the Application + */ + clientId?: string; + /** + * Client Secret of the Application. + */ + clientSecret?: string; + /** + * Signing Secret of the Application. Confirm that each request comes from Slack by + * verifying its unique signature. + */ + signingSecret?: string; + /** + * User Token + */ + userToken?: string; + /** + * Whether the suggested tests should be active or not upon suggestion + * + * Whether the AutoPilot Workflow should be active or not. + */ + active?: boolean; + backfillConfiguration?: BackfillConfiguration; + /** + * Maximum number of events processed at a time (Default 100). + * + * Maximum number of events sent in a batch (Default 100). + * + * Number of entities to process in each batch. + */ + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; + /** + * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property + * Type and are facing errors. Bear in mind that recreating the index will delete your + * DataAssets and a backfill will be needed. + */ + recreateDataAssetsIndex?: boolean; + sendToAdmins?: boolean; + sendToTeams?: boolean; + /** + * Enable automatic performance tuning based on cluster capabilities and database entity + * count + */ + autoTune?: boolean; + /** + * Overrides applied to staged indexes during bulk reindex. Reverted to liveIndexSettings + * before alias swap. Nothing reads from staged indexes, so refresh=-1 and replicas=0 are + * safe. Defaults: refresh=-1, replicas=0, durability=async, syncInterval=30s, + * forceMergeOnPromote=false. + */ + bulkIndexSettings?: BulkIndexOverrides; + /** + * Number of threads to use for reindexing + */ + consumerThreads?: number; + /** + * List of Entities to Reindex + * + * List of entity types to warm up in cache. Use 'all' to warm up all entity types. + */ + entities?: string[]; + /** + * Initial backoff time in milliseconds + */ + initialBackoff?: number; + /** + * Settings applied to staged indexes before alias swap (live serving values). Tune for read + * freshness and HA. Defaults: refresh=1s (near-real-time, required if users/agents + * read-after-write), replicas=1, shards=1, durability=request. + */ + liveIndexSettings?: IndexSettings; + /** + * Override liveIndexSettings for specific entity types. Useful for large or specialized + * entities (e.g. 'container' on instances with 500k+ assets, 'queryCostRecord' for + * high-cardinality time series). Keys are entity type names; values override the global + * liveIndexSettings. + */ + liveIndexSettingsByEntity?: { [key: string]: IndexSettings }; + /** + * Maximum backoff time in milliseconds + */ + maxBackoff?: number; + /** + * Maximum number of concurrent requests to the search index + */ + maxConcurrentRequests?: number; + /** + * Maximum number of retries for a failed request + */ + maxRetries?: number; + /** + * Number of entities per partition for distributed indexing. Smaller values create more + * partitions for better distribution across servers. Range: 1000-50000. + */ + partitionSize?: number; + /** + * Maximum number of events sent in a batch (Default 100). + */ + payLoadSize?: number; + /** + * Number of threads to use for reindexing + */ + producerThreads?: number; + /** + * Queue Size to user internally for reindexing. + */ + queueSize?: number; + /** + * Search index mapping language. + */ + searchIndexMappingLanguage?: SearchIndexMappingLanguage; + /** + * Per-entity-type override for time series max days. Keys are entity type names (e.g. + * testCaseResult, queryCostRecord), values are number of days. Entities not listed here use + * the default Time Series Max Days value. + */ + timeSeriesEntityDays?: { [key: string]: number }; + /** + * Maximum age in days for time series data during reindexing. Default 0 (index all data). + * Set to a positive value like 15 to limit to recent data only. + */ + timeSeriesMaxDays?: number; + /** + * In multi-instance deployments, claim each entity type via Redis SETNX so only one + * instance warms it. Disable to let every instance warm independently (idempotent but + * redundant). + */ + enableDistributedClaim?: boolean; + /** + * Force cache warmup even if another instance is detected (use with caution). + */ + force?: boolean; + /** + * Pre-warm the per-entity bundle cache (tags + certification) so the first read after + * deploy doesn't fan out to the DB. Disable for very large installs. + */ + warmBundles?: boolean; + /** + * Optionally pre-warm common relationship fields in the read bundle cache. Requires Warm + * Read Bundles. This adds extra relationship-table and entity-reference reads during + * warmup, so enable it only when first-read relationship latency matters. + */ + warmRelationships?: boolean; + /** + * Enter the retention period for Activity Threads of type = 'Conversation' records in days + * (e.g., 30 for one month, 60 for two months). + */ + activityThreadsRetentionPeriod?: number; + /** + * Enter the retention period for Audit Log entries in days (e.g., 90 for three months). + */ + auditLogRetentionPeriod?: number; + /** + * Enter the retention period for change event records in days (e.g., 7 for one week, 30 for + * one month). + */ + changeEventRetentionPeriod?: number; + /** + * Enter the retention period for Profile Data in days (e.g., 30 for one month, 60 for two + * months). + */ + profileDataRetentionPeriod?: number; + /** + * Enter the retention period for Test Case Results in days (e.g., 30 for one month, 60 for + * two months). + */ + testCaseResultsRetentionPeriod?: number; + /** + * Service Entity Link for which to trigger the application. + */ + entityLink?: string; + [property: string]: any; +} + +/** + * Action to take on those entities. E.g., propagate description through lineage, auto + * tagging, etc. + * + * Apply Classification Tags to the selected assets. + * + * Remove Classification Tags Action Type + * + * Apply Glossary Terms to the selected assets. + * + * Remove Glossary Terms Action Type + * + * Add domains to the selected assets. + * + * Remove domains from the selected assets. + * + * Apply Tags to the selected assets. + * + * Add a Custom Property to the selected assets. + * + * Remove Owner Action Type + * + * Add an owner to the selected assets. + * + * Add Test Cases to the selected assets. + * + * Remove Test Cases Action Type + * + * Add owners to the selected assets. + * + * Remove Custom Properties Action Type + * + * Add a Data Product to the selected assets. + * + * Remove a Data Product to the selected assets. + * + * Propagate description, tags and glossary terms via lineage + * + * ML Tagging action configuration for external automator. + */ +export interface Action { + /** + * Apply tags to the children of the selected assets that match the criteria. E.g., columns, + * tasks, topic fields,... + * + * Remove tags from the children of the selected assets. E.g., columns, tasks, topic + * fields,... + * + * Apply terms to the children of the selected assets that match the criteria. E.g., + * columns, tasks, topic fields,... + * + * Remove terms from the children of the selected assets. E.g., columns, tasks, topic + * fields,... + * + * Apply the description to the children of the selected assets that match the criteria. + * E.g., columns, tasks, topic fields,... + * + * Remove descriptions from the children of the selected assets. E.g., columns, tasks, topic + * fields,... + * + * Add tests to the selected table columns + * + * Remove tests to the selected table columns + */ + applyToChildren?: string[]; + /** + * Update tags even if they are already defined in the asset. By default, incoming tags are + * merged with the existing ones. + * + * Update terms even if they are already defined in the asset. By default, incoming terms + * are merged with the existing ones. + * + * Update the domains even if they are defined in the asset. By default, we will only apply + * the domains to assets without domains. + * + * Update the description even if they are already defined in the asset. By default, we'll + * only add the descriptions to assets without the description set. + * + * Update the Custom Property even if it is defined in the asset. By default, we will only + * apply the owners to assets without the given Custom Property informed. + * + * Update the tier even if it is defined in the asset. By default, we will only apply the + * tier to assets without tier. + * + * Update the test even if it is defined in the asset. By default, we will only apply the + * test to assets without the existing test already existing. + * + * Update the owners even if it is defined in the asset. By default, we will only apply the + * owners to assets without owner. + * + * Update the Data Product even if the asset belongs to a different Domain. By default, we + * will only add the Data Product if the asset has no Domain, or it belongs to the same + * domain as the Data Product. + * + * Update descriptions, tags and Glossary Terms via lineage even if they are already defined + * in the asset. By default, descriptions are only updated if they are not already defined + * in the asset, and incoming tags are merged with the existing ones. + */ + overwriteMetadata?: boolean; + /** + * Classification Tags to apply (source must be 'Classification') + * + * Classification Tags to remove (source must be 'Classification') + */ + tags?: TierElement[]; + /** + * Application Type + */ + type: ActionType; + /** + * Remove tags from all the children and parent of the selected assets. + * + * Remove terms from all the children and parent of the selected assets. + * + * Remove descriptions from all the children and parent of the selected assets. + */ + applyToAll?: boolean; + /** + * Remove tags by its label type + * + * Remove terms by its label type + */ + labels?: LabelElement[]; + /** + * Glossary Terms to apply + * + * Glossary Terms to remove + */ + terms?: TierElement[]; + /** + * Domains to apply + */ + domains?: EntityReference[]; + /** + * Description to apply + */ + description?: string; + /** + * Owners to apply + * + * Custom Properties keys to remove + */ + customProperties?: any; + /** + * tier to apply + */ + tier?: TierElement; + /** + * Test Cases to apply + */ + testCases?: TestCaseDefinitions[]; + /** + * Remove all test cases + */ + removeAll?: boolean; + /** + * Test Cases to remove + */ + testCaseDefinitions?: string[]; + /** + * Owners to apply + */ + owners?: EntityReference[]; + /** + * Data Products to apply + * + * Data Products to remove + */ + dataProducts?: EntityReference[]; + /** + * Propagate the metadata to columns via column-level lineage. + */ + propagateColumnLevel?: boolean; + /** + * Propagate description through lineage + */ + propagateDescription?: boolean; + /** + * Propagate domains from the parent through lineage + */ + propagateDomains?: boolean; + /** + * Propagate glossary terms through lineage + */ + propagateGlossaryTerms?: boolean; + /** + * Propagate owner from the parent + */ + propagateOwner?: boolean; + /** + * Propagate the metadata to the parents (e.g., tables) via lineage. + */ + propagateParent?: boolean; + /** + * Propagate tags through lineage + */ + propagateTags?: boolean; + /** + * Propagate tier from the parent + */ + propagateTier?: boolean; + /** + * Number of levels to propagate lineage. If not set, it will propagate to all levels. + */ + propagationDepth?: number; + /** + * Mode for calculating propagation depth. 'ROOT' calculates depth from root nodes (sources + * with no parents). 'DATA_ASSET' calculates depth relative to each data asset being + * processed, ensuring each asset only receives metadata from nodes within the specified + * number of hops upstream. + */ + propagationDepthMode?: PropagationDepthMode; + /** + * Determines how the filter selects entities. 'SOURCE' (default): filtered entities push + * their metadata downstream to all discovered entities via lineage. 'TARGET': filtered + * entities receive metadata from upstream lineage. + */ + propagationFilterMode?: PropagationFilterMode; + /** + * List of configurations to stop propagation based on conditions + */ + propagationStopConfigs?: PropagationStopConfig[]; + /** + * Use the optimized propagation algorithm that reduces memory usage and API calls. + * Recommended for large lineage graphs. If set to false, uses the original propagation + * algorithm. Default is true. + */ + useOptimizedPropagation?: boolean; +} + +/** + * Remove tags by its label type + * + * Remove terms by its label type + */ +export enum LabelElement { + Automated = "Automated", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Mode for calculating propagation depth. 'ROOT' calculates depth from root nodes (sources + * with no parents). 'DATA_ASSET' calculates depth relative to each data asset being + * processed, ensuring each asset only receives metadata from nodes within the specified + * number of hops upstream. + */ +export enum PropagationDepthMode { + DataAsset = "DATA_ASSET", + Root = "ROOT", +} + +/** + * Determines how the filter selects entities. 'SOURCE' (default): filtered entities push + * their metadata downstream to all discovered entities via lineage. 'TARGET': filtered + * entities receive metadata from upstream lineage. + */ +export enum PropagationFilterMode { + Source = "SOURCE", + Target = "TARGET", +} + +/** + * Configuration to stop lineage propagation based on conditions + */ +export interface PropagationStopConfig { + /** + * The metadata attribute to check for stopping propagation + */ + metadataAttribute: MetadataAttribute; + /** + * List of attribute values that will stop propagation when any of them is matched + */ + value: Array; +} + +/** + * The metadata attribute to check for stopping propagation + */ +export enum MetadataAttribute { + Description = "description", + Domains = "domains", + GlossaryTerms = "glossaryTerms", + Owner = "owner", + Tags = "tags", + Tier = "tier", +} + +/** + * This schema defines the type for labeling an entity with a Tag. + * + * tier to apply + * + * Domains the asset belongs to. When not set, the asset inherits the domain from the parent + * it belongs to. + * + * This schema defines the EntityReferenceList type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * The ingestion agent responsible for executing the ingestion pipeline. + * + * The processing engine responsible for executing the ingestion pipeline logic. + * + * Link to the service (such as database, messaging, storage services, etc. for which this + * ingestion pipeline ingests the metadata from. + * + * Service to be modified + */ +export interface TagLabel { + /** + * Timestamp when this tag was applied in ISO 8601 format + */ + appliedAt?: Date; + /** + * Who it is that applied this tag (e.g: a bot, AI or a human) + */ + appliedBy?: string; + /** + * Description for the tag label. + * + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this tag. + * + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Link to the tag resource. + * + * Link to the entity resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType?: LabelTypeEnum; + /** + * Additional metadata associated with this tag label, such as recognizer information for + * automatically applied tags. + */ + metadata?: TagLabelMetadata; + /** + * Name of the tag or glossary term. + * + * Name of the entity instance. + */ + name?: string; + /** + * An explanation of why this tag was proposed, specially for autoclassification tags + */ + reason?: string; + /** + * Label is from Tags or Glossary. + */ + source?: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state?: State; + style?: Style; + tagFQN?: string; + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id?: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type?: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ +export enum LabelTypeEnum { + Automated = "Automated", + Derived = "Derived", + Generated = "Generated", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Additional metadata associated with this tag label, such as recognizer information for + * automatically applied tags. + * + * Additional metadata associated with a tag label, including information about how the tag + * was applied. + */ +export interface TagLabelMetadata { + /** + * Epoch time in milliseconds when the certification tag expires + */ + expiryDate?: number; + /** + * Metadata about the recognizer that automatically applied this tag + */ + recognizer?: TagLabelRecognizerMetadata; +} + +/** + * Metadata about the recognizer that automatically applied this tag + * + * Metadata about the recognizer that applied a tag, including scoring and pattern + * information. + */ +export interface TagLabelRecognizerMetadata { + /** + * Details of patterns that matched during recognition + */ + patterns?: PatternMatch[]; + /** + * Unique identifier of the recognizer that applied this tag + */ + recognizerId: string; + /** + * Human-readable name of the recognizer + */ + recognizerName: string; + /** + * Confidence score assigned by the recognizer (0.0 to 1.0) + */ + score: number; + /** + * What the recognizer analyzed to apply this tag + */ + target?: Target; +} + +/** + * Information about a pattern that matched during recognition + */ +export interface PatternMatch { + /** + * Name of the pattern that matched + */ + name: string; + /** + * Regular expression or pattern definition + */ + regex?: string; + /** + * Confidence score for this specific pattern match + */ + score: number; +} + +/** + * What the recognizer analyzed to apply this tag + */ +export enum Target { + ColumnName = "column_name", + Content = "content", +} + +/** + * Label is from Tags or Glossary. + */ +export enum TagSource { + Classification = "Classification", + Glossary = "Glossary", +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = "Confirmed", + Suggested = "Suggested", +} + +/** + * UI Style is used to associate a color code and/or icon to entity to customize the look of + * that entity in UI. + */ +export interface Style { + /** + * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. + */ + color?: string; + /** + * Cover image configuration for the entity. + */ + coverImage?: CoverImage; + /** + * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. + */ + iconURL?: string; +} + +/** + * Cover image configuration for the entity. + * + * Cover image configuration for an entity. This is used to display a banner or header image + * for entities like Domain, Glossary, Data Product, etc. + */ +export interface CoverImage { + /** + * Position of the cover image in CSS background-position format. Supports keywords (top, + * center, bottom) or pixel values (e.g., '20px 30px'). + */ + position?: string; + /** + * URL of the cover image. + */ + url?: string; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + * + * tier to apply + */ +export interface TierElement { + /** + * Timestamp when this tag was applied in ISO 8601 format + */ + appliedAt?: Date; + /** + * Who it is that applied this tag (e.g: a bot, AI or a human) + */ + appliedBy?: string; + /** + * Description for the tag label. + */ + description?: string; + /** + * Display Name that identifies this tag. + */ + displayName?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType: LabelTypeEnum; + /** + * Additional metadata associated with this tag label, such as recognizer information for + * automatically applied tags. + */ + metadata?: TagLabelMetadata; + /** + * Name of the tag or glossary term. + */ + name?: string; + /** + * An explanation of why this tag was proposed, specially for autoclassification tags + */ + reason?: string; + /** + * Label is from Tags or Glossary. + */ + source: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + style?: Style; + tagFQN: string; +} + +/** + * Minimum set of requirements to get a Test Case request ready + */ +export interface TestCaseDefinitions { + /** + * Compute the passed and failed row count for the test case. + */ + computePassedFailedRowCount?: boolean; + parameterValues?: TestCaseParameterValue[]; + /** + * Tags to apply + */ + tags?: TierElement[]; + /** + * Fully qualified name of the test definition. + */ + testDefinition?: string; + /** + * If the test definition supports it, use dynamic assertion to evaluate the test case. + */ + useDynamicAssertion?: boolean; + [property: string]: any; +} + +/** + * This schema defines the parameter values that can be passed for a Test Case. + */ +export interface TestCaseParameterValue { + /** + * name of the parameter. Must match the parameter names in testCaseParameterDefinition + */ + name?: string; + /** + * value to be passed for the Parameters. These are input from Users. We capture this in + * string and convert during the runtime. + */ + value?: string; + [property: string]: any; +} + +/** + * Application Type + * + * Add Tags action type. + * + * Remove Classification Tags Action Type. + * + * Add Terms action type. + * + * Remove Terms Action Type. + * + * Add Domain Action Type. + * + * Remove Domain Action Type + * + * Add Description Action Type. + * + * Add Custom Properties Action Type. + * + * Remove Description Action Type + * + * Add Tier Action Type. + * + * Remove Tier Action Type + * + * Add Test Case Action Type. + * + * Remove Test Case Action Type + * + * Add Owner Action Type. + * + * Remove Owner Action Type + * + * Remove Custom Properties Action Type. + * + * Add Data Products Action Type. + * + * Remove Data Products Action Type. + * + * Lineage propagation action type. + * + * ML PII Tagging action type. + */ +export enum ActionType { + AddCustomPropertiesAction = "AddCustomPropertiesAction", + AddDataProductAction = "AddDataProductAction", + AddDescriptionAction = "AddDescriptionAction", + AddDomainAction = "AddDomainAction", + AddOwnerAction = "AddOwnerAction", + AddTagsAction = "AddTagsAction", + AddTermsAction = "AddTermsAction", + AddTestCaseAction = "AddTestCaseAction", + AddTierAction = "AddTierAction", + LineagePropagationAction = "LineagePropagationAction", + MLTaggingAction = "MLTaggingAction", + RemoveCustomPropertiesAction = "RemoveCustomPropertiesAction", + RemoveDataProductAction = "RemoveDataProductAction", + RemoveDescriptionAction = "RemoveDescriptionAction", + RemoveDomainAction = "RemoveDomainAction", + RemoveOwnerAction = "RemoveOwnerAction", + RemoveTagsAction = "RemoveTagsAction", + RemoveTermsAction = "RemoveTermsAction", + RemoveTestCaseAction = "RemoveTestCaseAction", + RemoveTierAction = "RemoveTierAction", +} + +/** + * Backfill Configuration + */ +export interface BackfillConfiguration { + /** + * Enable Backfill for the configured dates + */ + enabled?: boolean; + /** + * Date for which the backfill will end + */ + endDate?: Date; + /** + * Date from which to start the backfill + */ + startDate?: Date; + [property: string]: any; +} + +/** + * Overrides applied to staged indexes during bulk reindex. Reverted to liveIndexSettings + * before alias swap. Nothing reads from staged indexes, so refresh=-1 and replicas=0 are + * safe. Defaults: refresh=-1, replicas=0, durability=async, syncInterval=30s, + * forceMergeOnPromote=false. + * + * Overrides applied to a staged index DURING bulk reindex for write throughput. Reverted to + * indexSettings before alias swap. Nothing reads from the staged index, so refresh=-1 and + * replicas=0 are safe here. + */ +export interface BulkIndexOverrides { + /** + * Run _forcemerge to 1 segment before swapping the alias. Improves post-reindex query + * performance at the cost of build time. + */ + forceMergeOnPromote?: boolean; + numberOfReplicas?: number; + refreshInterval?: string; + translogDurability?: TranslogDurability; + translogSyncInterval?: string; +} + +/** + * 'request' = fsync per write (durable). 'async' = fsync on interval (faster, can lose + * '. For + * cross-account Glue catalogs, use the AWS account ID. If not provided, defaults to the + * caller's AWS account. + */ + catalogId?: string; + /** + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. + * + * Optional name to give to the database in OpenMetadata. If left blank, we will use 'epic' + * as the database name. + */ + databaseName?: string; + /** + * S3 Staging Directory. Example: s3://postgres/input/ + */ + s3StagingDir?: string; + /** + * Athena workgroup. + */ + workgroup?: string; + /** + * This parameter determines the mode of authentication for connecting to AzureSQL using + * ODBC. If 'Active Directory Password' is selected, you need to provide the password. If + * 'Active Directory Integrated' is selected, password is not required as it uses the + * logged-in user's credentials. This mode is useful for establishing secure and seamless + * connections with AzureSQL. + * + * This parameter determines the mode of authentication for connecting to Azure Synapse + * using ODBC. If 'Active Directory Password' is selected, you need to provide the password. + * If 'Active Directory Integrated' is selected, password is not required as it uses the + * logged-in user's credentials. If 'Active Directory Service Principal' is selected, you + * need to provide clientId, clientSecret and tenantId. This mode is useful for establishing + * secure and seamless connections with Azure Synapse. + */ + authenticationMode?: any[] | boolean | number | null | AuthenticationModeObject | string; + /** + * Database of the data source. This is optional parameter, if you would like to restrict + * the metadata reading to a single database. When left blank, OpenMetadata Ingestion + * attempts to scan all the databases. + * + * Database of the data source. + * + * Initial Redshift database to connect to. If you want to ingest all databases, set + * ingestAllDatabases to true. + * + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. + * + * Optional: Restrict metadata ingestion to a specific namespace (source/space). When left + * blank, all namespaces will be ingested. + * + * Database of the data source. This is the name of your Fabric Warehouse or Lakehouse. This + * is optional parameter, if you would like to restrict the metadata reading to a single + * database. When left blank, OpenMetadata Ingestion attempts to scan all the databases. + */ + database?: string; + /** + * SQLAlchemy driver for AzureSQL. + * + * ODBC driver version in case of pyodbc connection. + */ + driver?: string; + /** + * Ingest data from all databases in Azuresql. You can use databaseFilterPattern on top of + * this. + * + * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of + * this. + * + * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of + * this. + * + * Ingest data from all databases in TimescaleDB. You can use databaseFilterPattern on top + * of this. + * + * Ingest data from all databases in Redshift. You can use databaseFilterPattern on top of + * this. + * + * Ingest data from all databases in Greenplum. You can use databaseFilterPattern on top of + * this. + * + * Ingest data from all databases in Azure Synapse. You can use databaseFilterPattern on top + * of this. + * + * Ingest data from all databases (Warehouses and Lakehouses) in Microsoft Fabric. You can + * use databaseFilterPattern on top of this. + * + * Ingest data from all databases in Informix. You can use databaseFilterPattern on top of + * this. + */ + ingestAllDatabases?: boolean; + /** + * Database Schema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion + * attempts to scan all the schemas. + * + * databaseSchema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata + * Ingestion attempts to scan all the databaseSchema. + * + * Optional name to give to the schema in OpenMetadata. If left blank, we will use default + * as the schema name + * + * IOMETE database to restrict metadata ingestion to (e.g. default, finance_db). This is an + * optional parameter; if left blank, OpenMetadata attempts to scan all databases in the + * catalog. + */ + databaseSchema?: string; + /** + * Clickhouse SQL connection duration. + */ + duration?: number; + /** + * Use HTTPS Protocol for connection with clickhouse + */ + https?: boolean; + /** + * Path to key file for establishing secure connection + */ + keyfile?: string; + /** + * Establish secure connection with clickhouse + */ + secure?: boolean; + /** + * Catalog of the data source(Example: hive_metastore). This is optional parameter, if you + * would like to restrict the metadata reading to a single catalog. When left blank, + * OpenMetadata Ingestion attempts to scan all the catalog. + * + * Presto catalog + * + * Catalog of the data source. + * + * Catalog of the data source (e.g. spark_catalog). This is an optional parameter; if left + * blank, OpenMetadata uses default catalog. + */ + catalog?: string; + /** + * The maximum amount of time (in seconds) to wait for a successful connection to the data + * source. If the connection attempt takes longer than this timeout period, an error will be + * returned. + * + * Connection timeout in seconds. + * + * Timeout in seconds for connecting to MCP servers + */ + connectionTimeout?: number | number; + /** + * Databricks compute resources URL. + */ + httpPath?: string; + /** + * Policy agent configuration for access control extraction. + */ + policyAgentConfig?: PolicyAgentConfig; + /** + * Table name to fetch the query history. + * + * Table name to fetch the query history. When set, this overrides the default + * 'mysql.general_log' (or 'mysql.slow_log' when 'useSlowLogs' is enabled). The custom table + * must expose columns compatible with the selected log path. + */ + queryHistoryTable?: string; + /** + * CLI Driver version to connect to DB2. If not provided, the latest version will be used. + */ + clidriverVersion?: string; + /** + * License to connect to DB2. + */ + license?: string; + /** + * License file name to connect to DB2. + */ + licenseFileName?: string; + /** + * SSL Mode to connect to Informix. Use 'disable' for no SSL, 'require' for encrypted SSL + * without certificate verification, or 'verify-ca' to validate the server certificate + * against the provided CA certificate. + */ + sslMode?: SSLMode; + supportsViewLineageExtraction?: boolean; + /** + * Available sources to fetch the metadata. + * + * Available sources to fetch files. + * + * Available sources to fetch metadata. + */ + configSource?: DeltaLakeConfigurationSource; + /** + * Authentication mode to connect to hive. + * + * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials + * (for Airbyte Cloud) + */ + auth?: Authentication | AuthEnum; + /** + * Authentication options to pass to Hive connector. These options are based on SQLAlchemy. + * + * Authentication options to pass to Impala connector. These options are based on SQLAlchemy. + */ + authOptions?: string; + /** + * If authenticating with Kerberos specify the Kerberos service name + */ + kerberosServiceName?: string; + /** + * Hive Metastore Connection Details + */ + metastoreConnection?: HiveMetastoreConnectionDetails; + /** + * Enable SSL connection to Hive server. When enabled, SSL transport will be used for secure + * communication. + * + * Establish secure connection with Impala + */ + useSSL?: boolean; + /** + * Authentication mode to connect to Impala. + */ + authMechanism?: AuthMechanismEnum; + /** + * Enable SSL/TLS encryption for the MSSQL connection. When enabled, all data transmitted + * between the client and server will be encrypted. + */ + encrypt?: boolean; + /** + * Trust the server certificate without validation. Set to false in production to validate + * server certificates against the certificate authority. + */ + trustServerCertificate?: boolean; + /** + * Use slow logs to extract lineage. + */ + useSlowLogs?: boolean; + /** + * How to run the SQLite database. :memory: by default. + */ + databaseMode?: string; + /** + * This directory will be used to set the LD_LIBRARY_PATH env variable. It is required if + * you need to enable thick connection mode. By default, we bring instant client 19 and + * point to /instantclient. + */ + instantClientDirectory?: string; + /** + * Connect with oracle by either passing service name or database schema name. + */ + oracleConnectionType?: OracleConnectionType; + /** + * Controls how Oracle identifier names (tables, columns, schemas) are stored in + * OpenMetadata. When disabled (default), Oracle's UPPERCASE unquoted identifiers (e.g. + * EMPLOYEES) are not guaranteed to be stored as-is — identifiers with the same letters but + * different case (e.g. unquoted EMPLOYEES and quoted 'employees') will collide into the + * same name. When enabled, names are stored exactly as Oracle persists them, which solves + * same-name collisions between quoted and unquoted identifiers. WARNING: enabling this + * after data has already been ingested with the default setting will change the stored + * names of all existing tables, columns, schemas, and constraints — breaking attached tags, + * descriptions, lineage, data quality tests, and any other metadata associated with those + * entities. If you must switch, soft-delete all previously ingested entities before + * re-ingesting. + */ + preserveIdentifierCase?: boolean; + /** + * Use Oracle DBA_* tables instead of ALL_* tables for metadata ingestion. Requires DBA + * privileges. + */ + useDBATable?: boolean; + /** + * Custom OpenMetadata Classification name for Postgres policy tags. + * + * Custom OpenMetadata Classification name for TimescaleDB policy tags. + */ + classificationName?: string; + /** + * Fully qualified name of the view or table to use for query logs. If not provided, + * defaults to pg_stat_statements. Use this to configure a custom view (e.g., + * my_schema.custom_pg_stat_statements) when direct access to pg_stat_statements is + * restricted. + */ + queryStatementSource?: string; + /** + * Protocol ( Connection Argument ) to connect to Presto. + */ + protocol?: string; + /** + * Verify ( Connection Argument for SSL ) to connect to Presto. + * + * Verify ( Connection Argument for SSL ) to connect to Trino. + */ + verify?: string; + /** + * Salesforce Consumer Key (Client ID) for OAuth 2.0 authentication. This is obtained from + * your Salesforce Connected App configuration. Required along with Consumer Secret for + * OAuth authentication. + */ + consumerKey?: string; + /** + * Salesforce Consumer Secret (Client Secret) for OAuth 2.0 authentication. This is obtained + * from your Salesforce Connected App configuration. Required along with Consumer Key for + * OAuth authentication. + */ + consumerSecret?: string; + /** + * Salesforce Organization ID is the unique identifier for your Salesforce identity + * + * Snowplow BDP Organization ID + * + * Anypoint Platform Organization ID. If not provided, the connector will use the user's + * default organization. + */ + organizationId?: string; + /** + * API version of the Salesforce instance + */ + salesforceApiVersion?: string; + /** + * Domain of Salesforce instance + */ + salesforceDomain?: string; + /** + * Salesforce Security Token for username/password authentication. + */ + securityToken?: string; + /** + * List of Salesforce Object Names to ingest. If specified, only these objects will be + * fetched. Leave empty to fetch all objects (subject to tableFilterPattern). + */ + sobjectNames?: string[]; + /** + * SAP SuccessFactors OData API base URL. For example: https://api4.successfactors.com + */ + baseUrl?: string; + /** + * SAP SuccessFactors Company ID (tenant identifier). Required for all API calls. + */ + companyId?: string; + /** + * PEM-encoded RSA private key used to sign SAML assertions for OAuth2 SAML Bearer flow. + * Required when authType is OAuth2Credentials. + * + * Connection to Snowflake instance via Private Key + */ + privateKey?: string; + /** + * OAuth2 Token endpoint URL. Required when authType is OAuth2Credentials. For example: + * https://api4.successfactors.com/oauth/token + */ + tokenUrl?: string; + /** + * Number of days of ACCESS_HISTORY scanned per query when 'Use Access History for Lineage' + * is enabled. The lineage time window is split into chunks of this many days to keep each + * Snowflake query bounded and avoid client/server timeouts over long windows. Lower this + * value if queries still time out on very busy accounts. + */ + accessHistoryChunkSize?: number; + /** + * If the Snowflake URL is https://xyz1234.us-east-1.gcp.snowflakecomputing.com, then the + * account is xyz1234.us-east-1.gcp + * + * Specifies an account string to override the default account string defined for the + * database user. Accounts are used by the database for workload management and resource + * usage monitoring. + */ + account?: string; + /** + * Full name of the schema where the account usage data is stored. + */ + accountUsageSchema?: string; + /** + * Keep the session alive for long-running scans. + */ + clientSessionKeepAlive?: boolean; + /** + * Cost of credit for the Snowflake account. + */ + creditCost?: number; + /** + * Ingest external and internal stages. + */ + includeStages?: boolean; + /** + * Ingest Snowflake streams as data assets. + */ + includeStreams?: boolean; + /** + * Ingest transient tables alongside permanent ones. + */ + includeTransientTables?: boolean; + /** + * Session query tag used to monitor usage on snowflake. To use a query tag snowflake user + * should have enough privileges to alter the session. + */ + queryTag?: string; + /** + * Snowflake Role. + */ + role?: string; + /** + * Snowflake Passphrase Key used with Private Key + */ + snowflakePrivatekeyPassphrase?: string; + /** + * Snowflake source host for the Snowflake account. + */ + snowflakeSourceHost?: string; + /** + * Use Snowflake's ACCOUNT_USAGE.ACCESS_HISTORY view as the source of query lineage. + * ACCESS_HISTORY provides Snowflake-computed table- and column-level lineage, including for + * queries OpenMetadata cannot parse. Enabled by default; if the configured role cannot read + * ACCESS_HISTORY, ingestion automatically falls back to the legacy query-log parser. + */ + useAccessHistory?: boolean; + /** + * Snowflake warehouse. + */ + warehouse?: string; + /** + * Proxies for the connection to Trino data source + */ + proxies?: { [key: string]: string }; + /** + * Pinot Controller Host and Port of the data source. + */ + pinotControllerHost?: string; + /** + * Bucket Name of the data source. + */ + bucketName?: string; + /** + * Prefix of the data source. + */ + prefix?: string; + /** + * Skip files in cold storage tiers (e.g., S3 Glacier, Azure Archive/Cool/Cold, GCS + * Coldline/Archive). When enabled, only files in hot/standard storage tiers will be + * processed. + */ + skipColdStorage?: boolean; + /** + * Couchbase connection Bucket options. + */ + bucket?: string; + /** + * Hostname of the Couchbase service. + */ + hostport?: string; + /** + * Enable dataflow for ingestion + */ + dataflows?: boolean; + /** + * Custom filter for dataflows + */ + dataflowsCustomFilter?: { [key: string]: any } | string; + /** + * Enable datatables for ingestion + */ + datatables?: boolean; + /** + * Custom filter for datatables + */ + dataTablesCustomFilter?: { [key: string]: any } | string; + /** + * Enable report for ingestion + */ + reports?: boolean; + /** + * Custom filter for reports + */ + reportsCustomFilter?: { [key: string]: any } | string; + /** + * Hostname of SAS Viya deployment. + */ + serverHost?: string; + /** + * Specifies additional data needed by a logon mechanism, such as a secure token, + * Distinguished Name, or a domain/realm name. LOGDATA values are specific to each logon + * mechanism. + */ + logdata?: string; + /** + * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, + * LDAP, KRB5 for Kerberos, or TDNEGO + */ + logmech?: Logmech; + /** + * Specifies the transaction mode for the connection + */ + tmode?: TransactionMode; + /** + * Client SSL/TLS settings. + */ + tls?: SSLTLSSettings; + /** + * HTTP Link for SSAS ACCESS + */ + httpConnection?: string; + /** + * Base URL of the Epic FHIR server + */ + fhirServerUrl?: string; + /** + * FHIR specification version (R4, STU3, DSTU2) + */ + fhirVersion?: FHIRVersion; + /** + * If true, ServiceNow application scopes will be imported as database schemas. Otherwise, a + * single default schema will be used. + */ + includeScopes?: boolean; + /** + * If true, both admin and system tables (sys_* tables) will be fetched. If false, only + * admin tables will be fetched. + */ + includeSystemTables?: boolean; + /** + * BurstIQ customer name for API requests. + */ + biqCustomerName?: string; + /** + * BurstIQ Secure Data Zone (SDZ) name for API requests. + */ + biqSdzName?: string; + /** + * BurstIQ system wallet ID sent as the biq_system_wallet_id header. Required for profiler + * data access. + */ + biqSystemWalletId?: string; + /** + * BurstIQ Keycloak realm name (e.g., 'ems' from https://auth.burstiq.com/realms/ems). + */ + realmName?: string; + /** + * Informix server name as defined in the sqlhosts file or INFORMIXSERVER environment + * variable. + */ + serverName?: string; + /** + * IOMETE lakehouse cluster name to connect to. + */ + cluster?: string; + /** + * IOMETE data plane name. + */ + dataPlane?: string; + /** + * Schema name in HANA where BW/4HANA ABAP metadata tables reside (e.g. SAPHANADB). Check + * your system with: SELECT SCHEMA_NAME FROM SYS.TABLES WHERE TABLE_NAME = 'RSOADSO'. + */ + abapSchema?: string; + /** + * basic.auth.user.info schema registry config property, Client HTTP credentials in the form + * of username:password. + */ + basicAuthUserInfo?: string; + /** + * Kafka bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 + * + * Redpanda bootstrap servers. add them in comma separated values ex: host1:9092,host2:9092 + */ + bootstrapServers?: string; + /** + * Confluent Kafka Consumer Config. From + * https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md + * + * Confluent Redpanda Consumer Config + */ + consumerConfig?: { [key: string]: any }; + /** + * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config + * connection. + */ + consumerConfigSSL?: DbtSSLConfigClass; + /** + * sasl.mechanism Consumer Config property + */ + saslMechanism?: SaslMechanismType; + /** + * sasl.password consumer config property + */ + saslPassword?: string; + /** + * sasl.username consumer config property + */ + saslUsername?: string; + /** + * Confluent Kafka Schema Registry Config. From + * https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient + * + * Confluent Redpanda Schema Registry Config. + */ + schemaRegistryConfig?: { [key: string]: any }; + /** + * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry + * connection. + */ + schemaRegistrySSL?: DbtSSLConfigClass; + /** + * Schema Registry Topic Suffix Name. The suffix to be appended to the topic name to get + * topic schema from registry. + */ + schemaRegistryTopicSuffixName?: string; + /** + * Confluent Kafka Schema Registry URL. + * + * Confluent Redpanda Schema Registry URL. + */ + schemaRegistryURL?: string; + /** + * security.protocol consumer config property + */ + securityProtocol?: KafkaSecurityProtocol; + /** + * Regex to only fetch topics that matches the pattern. + */ + topicFilterPattern?: FilterPattern; + /** + * GCP credentials configuration for authenticating with Pub/Sub. + */ + gcpConfig?: GcpConfigClass; + /** + * Include dead letter topics in metadata extraction. + */ + includeDeadLetterTopics?: boolean; + /** + * Include subscription metadata for each topic. + */ + includeSubscriptions?: boolean; + /** + * GCP Project ID where Pub/Sub topics are located. If not specified, will be read from + * credentials. + */ + projectId?: string; + /** + * Enable fetching schemas from Pub/Sub Schema Registry. + */ + schemaRegistryEnabled?: boolean; + /** + * Connect to a Pub/Sub emulator rather than the production service. + */ + useEmulator?: boolean; + /** + * Enable encryption for the Amundsen Neo4j Connection. + */ + encrypted?: boolean; + /** + * Maximum connection lifetime for the Amundsen Neo4j Connection. + */ + maxConnectionLifeTime?: number; + /** + * Enable SSL validation for the Amundsen Neo4j Connection. + */ + validateSSL?: boolean; + /** + * Maximum number of events sent in a batch (Default 100). + */ + batchSize?: number; + /** + * List of entities that you need to reindex + */ + entities?: string[]; + recreateIndex?: boolean; + runMode?: RunMode; + /** + * Recreate Indexes with updated Language + */ + searchIndexMappingLanguage?: SearchIndexMappingLanguage; + /** + * OpenMetadata Server Authentication Provider. + */ + authProvider?: AuthProvider; + /** + * Cluster name to differentiate OpenMetadata Server instance + */ + clusterName?: string; + /** + * Configuration for Sink Component in the OpenMetadata Ingestion Framework. + */ + elasticsSearch?: ConfigElasticsSearch; + /** + * Validate Openmetadata Server & Client Version. + */ + enableVersionValidation?: boolean; + extraHeaders?: { [key: string]: string }; + /** + * Force the overwriting of any entity during the ingestion. + */ + forceEntityOverwriting?: boolean; + /** + * Include Dashboards for Indexing + */ + includeDashboards?: boolean; + /** + * Include Database Services for Indexing + */ + includeDatabaseServices?: boolean; + /** + * Include Glossary Terms for Indexing + */ + includeGlossaryTerms?: boolean; + /** + * Include Messaging Services for Indexing + */ + includeMessagingServices?: boolean; + /** + * Include MlModels for Indexing + */ + includeMlModels?: boolean; + /** + * Include Pipelines for Indexing + */ + includePipelines?: boolean; + /** + * Include Pipeline Services for Indexing + */ + includePipelineServices?: boolean; + /** + * Include Tags for Policy + */ + includePolicy?: boolean; + /** + * Include Tables for Indexing + */ + includeTables?: boolean; + /** + * Include Teams for Indexing + */ + includeTeams?: boolean; + /** + * Include Topics for Indexing + */ + includeTopics?: boolean; + /** + * Include Users for Indexing + */ + includeUsers?: boolean; + /** + * Limit the number of records for Indexing. + */ + limitRecords?: number; + /** + * Secrets Manager Loader for the Pipeline Service Client. + */ + secretsManagerLoader?: SecretsManagerClientLoader; + /** + * Secrets Manager Provider for OpenMetadata Server. + */ + secretsManagerProvider?: SecretsManagerProvider; + /** + * OpenMetadata Client security configuration. + */ + securityConfig?: OpenMetadataJWTClientConfig; + /** + * If set to true, when creating a service during the ingestion we will store its Service + * Connection. Otherwise, the ingestion will create a bare service without connection + * details. + */ + storeServiceConnection?: boolean; + /** + * Flag to enable Data Insight Extraction + */ + supportsDataInsightExtraction?: boolean; + /** + * Flag to enable ElasticSearch Reindexing Extraction + */ + supportsElasticSearchReindexingExtraction?: boolean; + /** + * service type of the data source. + */ + databaseServiceName?: string[]; + /** + * Name of the Entity Type available in Atlas. + */ + entity_type?: string; + /** + * service type of the messaging source + * + * Name of the Kafka Messaging Service associated with this Firehose Pipeline Service. e.g. + * local_kafka + * + * Name of the Kafka Messaging Service associated with this KafkaConnect Pipeline Service. + * e.g. local_kafka + */ + messagingServiceName?: string[] | string; + /** + * Custom OpenMetadata Classification name for alation tags. + */ + alationTagClassificationName?: string; + /** + * Specifies if hidden datasources should be included while ingesting. + */ + includeHiddenDatasources?: boolean; + /** + * Specifies if undeployed datasources should be included while ingesting. + */ + includeUndeployedDatasources?: boolean; + /** + * Specifies if Dashboards are to be ingested while running the ingestion job. + */ + ingestDashboards?: boolean; + /** + * Specifies if Datasources are to be ingested while running the ingestion job. + */ + ingestDatasources?: boolean; + /** + * Specifies if Domains are to be ingested while running the ingestion job. + */ + ingestDomains?: boolean; + /** + * Specifies if Knowledge Articles are to be ingested while running the ingestion job. + */ + ingestKnowledgeArticles?: boolean; + /** + * Specifies if Users and Groups are to be ingested while running the ingestion job. + */ + ingestUsersAndGroups?: boolean; + datasourceLinks?: { [key: string]: string }; + /** + * Regex to only include/exclude domains that match the pattern. + */ + domainFilterPattern?: FilterPattern; + /** + * Enable enrichment of existing OpenMetadata assets with Collibra metadata (descriptions, + * tags, owners). When enabled, the connector will match Collibra assets to OpenMetadata + * entities and apply metadata without creating new assets. + */ + enableEnrichment?: boolean; + /** + * Regex to only include/exclude glossaries that match the pattern. + */ + glossaryFilterPattern?: FilterPattern; + /** + * Pipeline Service Number Of Status + */ + numberOfStatus?: number; + /** + * Regex exclude pipelines. + * + * Regex to filter MuleSoft applications by name. + * + * Regex to only include/exclude pipelines that matches the pattern. + * + * Regex to only include/exclude Process Chains that match the pattern. + */ + pipelineFilterPattern?: FilterPattern; + /** + * Underlying database connection + * + * Optional. Underlying SSISDB connection. When omitted, the connector runs in file-only + * mode and run history is not extracted. + */ + databaseConnection?: DatabaseConnectionClass; + /** + * Underlying storage connection + */ + packageConnection?: S3Connection | string; + /** + * Fivetran API Secret. + */ + apiSecret?: string; + /** + * Fivetran API Limit For Pagination. + */ + limit?: number; + /** + * URL to the Dagster instance + * + * DBT cloud Access URL. + * + * SFTP server hostname or IP address + */ + host?: string; + /** + * Number of leading segments to remove from asset key paths before resolving to tables. For + * example, if your asset keys follow the pattern 'project/environment/schema/table' but you + * only need 'schema/table', set this to 2. + */ + stripAssetKeyPrefixLength?: number; + /** + * Connection Time Limit Between OM and Dagster Graphql API in second + */ + timeout?: number; + /** + * We support username/password or client certificate authentication + */ + nifiConfig?: NifiCredentialsConfiguration; + /** + * Number of days to look back when fetching lineage data from Databricks system tables + * (system.access.table_lineage and system.access.column_lineage). Default is 90 days. + */ + lineageLookBackDays?: number; + /** + * Spline UI Host & Port. + */ + uiHostPort?: string; + /** + * Event broker configuration. Choose between Kafka and Kinesis. + */ + brokerConfig?: BrokerConfiguration; + /** + * Map OpenLineage dataset namespaces (or prefixes) to OpenMetadata database service names. + * Used when multiple services of the same type exist. Example: 'mysql://cluster-a:3306' -> + * 'mysql-cluster-a'. + */ + namespaceToServiceMapping?: { [key: string]: string }; + /** + * We support username/password or No Authentication + */ + KafkaConnectConfig?: UsernamePasswordAuthentication; + /** + * ID of your DBT cloud account + */ + accountId?: string; + /** + * DBT cloud Metadata API URL. + */ + discoveryAPI?: string; + /** + * List of IDs of your DBT cloud environments separated by comma `,` + */ + environmentIds?: string[]; + /** + * List of IDs of your DBT cloud jobs seperated by comma `,` + */ + jobIds?: string[]; + /** + * Number of runs to fetch from DBT cloud + */ + numberOfRuns?: number; + /** + * List of IDs of your DBT cloud projects seperated by comma `,` + */ + projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; + /** + * The name of your azure data factory. + */ + factory_name?: string; + /** + * The name of your resource group the data factory is associated with. + */ + resource_group_name?: string; + /** + * Number of days in the past to filter pipeline runs. + */ + run_filter_days?: number; + /** + * The azure subscription identifier. + */ + subscription_id?: string; + /** + * Cloud provider where Snowplow is deployed + */ + cloudProvider?: CloudProvider; + /** + * Path to pipeline configuration files for Community deployment + */ + configPath?: string; + /** + * Snowplow Console URL for BDP deployment + */ + consoleUrl?: string; + /** + * Snowplow deployment type (BDP for managed or Community for self-hosted) + */ + deployment?: SnowplowDeployment; + /** + * Anypoint Platform Environment ID. If not provided, the connector will discover all + * accessible environments. + */ + environmentId?: string; + /** + * Azure Active Directory authority URI. Defaults to https://login.microsoftonline.com/ + */ + authorityUri?: string; + /** + * The Microsoft Fabric workspace ID where the pipelines are located. + */ + workspaceId?: string; + /** + * Regex to only fetch MlModels with names matching the pattern. + */ + mlModelFilterPattern?: FilterPattern; + /** + * Mlflow Model registry backend. E.g., + * mysql+pymysql://mlflow:password@localhost:3307/experiments + */ + registryUri?: string; + /** + * Mlflow Experiment tracking URI. E.g., http://localhost:5000 + */ + trackingUri?: string; + /** + * location/region of google cloud project + */ + location?: string; + /** + * Bucket Names of the data source. + */ + bucketNames?: string[]; + /** + * Console EndPoint URL for S3-compatible services + */ + consoleEndpointURL?: string; + /** + * Regex to only fetch containers that matches the pattern. + */ + containerFilterPattern?: FilterPattern; + /** + * Connection Timeout in Seconds + */ + connectionTimeoutSecs?: number; + /** + * Regex to only fetch search indexes that matches the pattern. + */ + searchIndexFilterPattern?: FilterPattern; + /** + * Email to impersonate using domain-wide delegation + */ + delegatedEmail?: string; + /** + * Regex to only include/exclude directories that matches the pattern. + * + * Regex to only include/exclude directories that match the pattern. + */ + directoryFilterPattern?: FilterPattern; + /** + * Specific shared drive ID to connect to + * + * SharePoint drive ID. If not provided, default document library will be used + */ + driveId?: string; + /** + * Regex to only include/exclude files that matches the pattern. + * + * Regex to only include/exclude files that match the pattern. + */ + fileFilterPattern?: FilterPattern; + /** + * Extract metadata only for Google Sheets files + */ + includeGoogleSheets?: boolean; + /** + * Include shared/team drives in metadata extraction + */ + includeTeamDrives?: boolean; + /** + * Regex to only include/exclude spreadsheets that matches the pattern. + */ + spreadsheetFilterPattern?: FilterPattern; + /** + * Regex to only include/exclude worksheets that matches the pattern. + */ + worksheetFilterPattern?: FilterPattern; + /** + * SharePoint site URL + */ + siteUrl?: string; + /** + * When enabled, extract sample data from structured files (CSV, TSV). This is disabled by + * default to avoid performance overhead. + */ + extractSampleData?: boolean; + /** + * SFTP server port number + */ + port?: number; + /** + * List of root directories to scan for files and subdirectories. If not specified, defaults + * to the user's home directory. + */ + rootDirectories?: string[]; + /** + * When enabled, only catalog structured data files (CSV, TSV) that can have schema + * extracted. Non-structured files like images, PDFs, videos, etc. will be skipped. + */ + structuredDataFilesOnly?: boolean; + /** + * Paths to MCP configuration files to scan for server definitions. Supports Claude Desktop + * config, VS Code settings, etc. + */ + configFilePaths?: string[]; + /** + * How to discover MCP servers + */ + discoveryMethod?: DiscoveryMethod; + /** + * Whether to fetch and catalog prompts from MCP servers + */ + fetchPrompts?: boolean; + /** + * Whether to fetch and catalog resources from MCP servers + */ + fetchResources?: boolean; + /** + * Whether to fetch and catalog tools from MCP servers + */ + fetchTools?: boolean; + /** + * Timeout in seconds for MCP server initialization handshake + */ + initializationTimeout?: number; + /** + * URL of MCP registry to query for server discovery (when discoveryMethod is Registry) + */ + registryUrl?: string; + /** + * Regex to only fetch servers with names matching the pattern + */ + serverFilterPattern?: FilterPattern; + /** + * List of MCP servers to connect to directly (when discoveryMethod is DirectConnection) + */ + servers?: MCPServerConfig[]; + [property: string]: any; +} + +/** + * We support username/password or No Authentication + * + * username/password auth + */ +export interface UsernamePasswordAuthentication { + /** + * KafkaConnect password to authenticate to the API. + */ + password?: string; + /** + * KafkaConnect user to authenticate to the API. + */ + username?: string; +} + +/** + * Choose between Basic authentication (for self-hosted) or OAuth 2.0 client credentials + * (for Airbyte Cloud) + * + * Username and password authentication + * + * OAuth 2.0 client credentials authentication for Airbyte Cloud + */ +export interface Authentication { + /** + * Password to connect to Airbyte. + */ + password?: string; + /** + * Username to connect to Airbyte. + */ + username?: string; + /** + * Client ID for the application registered in Airbyte. + */ + clientId?: string; + /** + * Client Secret for the application registered in Airbyte. + */ + clientSecret?: string; +} + +/** + * Authentication mode to connect to hive. + */ +export enum AuthEnum { + Basic = "BASIC", + Custom = "CUSTOM", + Gssapi = "GSSAPI", + Jwt = "JWT", + Kerberos = "KERBEROS", + LDAP = "LDAP", + None = "NONE", + Nosasl = "NOSASL", + Plain = "PLAIN", +} + +/** + * Authentication mode to connect to Impala. + */ +export enum AuthMechanismEnum { + Gssapi = "GSSAPI", + Jwt = "JWT", + LDAP = "LDAP", + Nosasl = "NOSASL", + Plain = "PLAIN", +} + +/** + * Types of methods used to authenticate to the tableau instance + * + * Basic Auth Credentials + * + * Access Token Auth Credentials + * + * Choose Basic Auth (username/password) for on-premise or OAuth 2.0 Client Credentials for + * SAP S/4HANA Cloud. + * + * Username and password credentials for SAP S/4HANA. + * + * OAuth 2.0 client credentials for SAP S/4HANA Cloud. + * + * Choose between different authentication types for Databricks. + * + * Personal Access Token authentication for Databricks. + * + * OAuth2 Machine-to-Machine authentication using Service Principal credentials for + * Databricks. + * + * Azure Active Directory authentication for Azure Databricks workspaces using Service + * Principal. + * + * Choose Auth Config Type. + * + * Common Database Connection Config + * + * IAM Auth Database Connection Config + * + * Azure Database Connection Config + * + * GCP CloudSQL Database Connection Config. Uses the Google Cloud SQL Python Connector. + * + * Choose Auth Configuration Type. + * + * Configuration for connecting to DataStax Astra DB in the cloud. + * + * Choose between Dremio Cloud (SaaS) or Dremio Software (self-hosted) authentication. + * + * Authentication configuration for Dremio Cloud using Personal Access Token (PAT). Dremio + * Cloud is a fully managed SaaS platform. + * + * Authentication configuration for self-hosted Dremio Software using username and password. + * Dremio Software is deployed on-premises or in your own cloud infrastructure. + * + * ThoughtSpot authentication configuration + * + * Types of methods used to authenticate to the alation instance + * + * API Access Token Auth Credentials + * + * Basic Auth Configuration for ElasticSearch + * + * API Key Authentication for ElasticSearch + * + * AWS credentials configs. + * + * AWS credentials required to access the S3 file. + * + * AWS credentials for generating MWAA CLI token. + * + * AWS credentials configuration. + * + * Authentication type to connect to Apache Ranger. + * + * Configuration for connecting to Ranger Basic Auth. + * + * Authentication method: username/password or SSH private key + * + * Username and password authentication for SFTP + * + * SSH private key authentication for SFTP + */ +export interface AuthenticationType { + /** + * Password to access the service. + * + * Password to authenticate with SAP S/4HANA. + * + * Password to connect to source. + * + * Database user password. Leave empty if using IAM database authentication. + * + * Password for the Dremio Software user account. + * + * Elastic Search Password for Login + * + * Ranger password to authenticate to the API. + * + * SFTP password + */ + password?: string; + /** + * Username to access the service. + * + * Username to authenticate with SAP S/4HANA. + * + * Username for authenticating with Dremio Software. This user should have appropriate + * permissions to access metadata. + * + * Elastic Search Username for Login + * + * Ranger user to authenticate to the API. + * + * SFTP username + */ + username?: string; + /** + * Personal Access Token Name. + */ + personalAccessTokenName?: string; + /** + * Personal Access Token Secret. + */ + personalAccessTokenSecret?: string; + /** + * Authentication type identifier. + */ + authType?: AuthType; + /** + * OAuth 2.0 client ID registered in SAP. + * + * Service Principal Application ID created in your Databricks Account Console for OAuth + * Machine-to-Machine authentication. + */ + clientId?: string; + /** + * OAuth 2.0 client secret. + * + * OAuth Secret generated for the Service Principal in Databricks Account Console. Used for + * secure OAuth2 authentication. + */ + clientSecret?: string; + /** + * OAuth 2.0 token endpoint URL (e.g. /sap/bc/security/oauth2/token). + */ + tokenEndpoint?: string; + /** + * Generated Personal Access Token for Databricks workspace authentication. This token is + * created from User Settings -> Developer -> Access Tokens in your Databricks workspace. + */ + token?: string; + /** + * Azure Service Principal Application (client) ID registered in your Azure Active Directory. + */ + azureClientId?: string; + /** + * Azure Service Principal client secret created in Azure AD for authentication. + */ + azureClientSecret?: string; + /** + * Azure Active Directory Tenant ID where your Service Principal is registered. + */ + azureTenantId?: string; + awsConfig?: AWSCredentials; + azureConfig?: AzureCredentials; + /** + * Use GCP IAM for database authentication instead of a password. + */ + enableIamAuth?: boolean; + /** + * GCP credentials to use. If not provided, Application Default Credentials will be used. + */ + gcpConfig?: GcpConfigClass; + /** + * JWT to connect to source. + */ + jwt?: string; + /** + * Configuration for connecting to DataStax Astra DB in the cloud. + */ + cloudConfig?: DataStaxAstraDBConfiguration; + /** + * Personal Access Token for authenticating with Dremio Cloud. Generate this token from your + * Dremio Cloud account settings under Settings -> Personal Access Tokens. + */ + personalAccessToken?: string; + /** + * Dremio Cloud Project ID (required). This unique identifier can be found in your Dremio + * Cloud project URL or project settings. + */ + projectId?: string; + /** + * Dremio Cloud region where your organization is hosted. Choose 'US' for United States + * region or 'EU' for European region. + */ + region?: CloudRegion; + /** + * URL to your self-hosted Dremio Software instance, including protocol and port (e.g., + * http://localhost:9047 or https://dremio.example.com:9047). + */ + hostPort?: string; + /** + * Access Token for the API + */ + accessToken?: string; + /** + * Elastic Search API Key for API Authentication + */ + apiKey?: string; + /** + * Elastic Search API Key ID for API Authentication + */ + apiKeyId?: string; + /** + * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume + * Role + */ + assumeRoleArn?: string; + /** + * An identifier for the assumed role session. Use the role session name to uniquely + * identify a session when the same role is assumed by different principals or for different + * reasons. Required Field in case of Assume Role + */ + assumeRoleSessionName?: string; + /** + * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume + * Role + */ + assumeRoleSourceIdentity?: string; + /** + * AWS Access key ID. + */ + awsAccessKeyId?: string; + /** + * AWS Region + */ + awsRegion?: string; + /** + * AWS Secret Access Key. + */ + awsSecretAccessKey?: string; + /** + * AWS Session Token. + */ + awsSessionToken?: string; + /** + * Enable AWS IAM authentication. When enabled, uses the default credential provider chain + * (environment variables, instance profile, etc.). Defaults to false for backward + * compatibility. + */ + enabled?: boolean; + /** + * EndPoint URL for the AWS + */ + endPointURL?: string; + /** + * The name of a profile to use with the boto session. + */ + profileName?: string; + /** + * SSH private key content in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys. + */ + privateKey?: string; + /** + * Passphrase for the private key (if encrypted) + */ + privateKeyPassphrase?: string; +} + +/** + * Authentication type identifier. + */ +export enum AuthType { + Basic = "basic", + Oauth2 = "oauth2", +} + +/** + * AWS credentials configs. + * + * AWS credentials required to access the S3 file. + * + * AWS credentials for generating MWAA CLI token. + * + * AWS credentials configuration. + */ +export interface AWSCredentials { + /** + * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume + * Role + */ + assumeRoleArn?: string; + /** + * An identifier for the assumed role session. Use the role session name to uniquely + * identify a session when the same role is assumed by different principals or for different + * reasons. Required Field in case of Assume Role + */ + assumeRoleSessionName?: string; + /** + * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume + * Role + */ + assumeRoleSourceIdentity?: string; + /** + * AWS Access key ID. + */ + awsAccessKeyId?: string; + /** + * AWS Region + */ + awsRegion: string; + /** + * AWS Secret Access Key. + */ + awsSecretAccessKey?: string; + /** + * AWS Session Token. + */ + awsSessionToken?: string; + /** + * Enable AWS IAM authentication. When enabled, uses the default credential provider chain + * (environment variables, instance profile, etc.). Defaults to false for backward + * compatibility. + */ + enabled?: boolean; + /** + * EndPoint URL for the AWS + */ + endPointURL?: string; + /** + * The name of a profile to use with the boto session. + */ + profileName?: string; +} + +/** + * Azure Cloud Credentials + * + * Available sources to fetch metadata. + * + * Azure Credentials + */ +export interface AzureCredentials { + /** + * Account Name of your storage account + */ + accountName?: string; + /** + * Your Service Principal App ID (Client ID) + */ + clientId?: string; + /** + * Your Service Principal Password (Client Secret) + */ + clientSecret?: string; + /** + * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default + */ + scopes?: string; + /** + * Tenant ID of your Azure Subscription + */ + tenantId?: string; + /** + * Key Vault Name + */ + vaultName?: string; +} + +/** + * Configuration for connecting to DataStax Astra DB in the cloud. + */ +export interface DataStaxAstraDBConfiguration { + /** + * Timeout in seconds for establishing new connections to Cassandra. + */ + connectTimeout?: number; + /** + * Timeout in seconds for individual Cassandra requests. + */ + requestTimeout?: number; + /** + * File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax + * Astra DB. + */ + secureConnectBundle?: string; + /** + * The Astra DB application token used for authentication. + */ + token?: string; + [property: string]: any; +} + +/** + * GCP credentials configs. + * + * GCP credentials to use. If not provided, Application Default Credentials will be used. + * + * GCP Credentials + * + * GCP credentials configuration for authenticating with Pub/Sub. + * + * GCP credentials configuration. + * + * GCP Credentials for Google Drive API + */ +export interface GcpConfigClass { + /** + * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP + * Credentials Path + */ + gcpConfig: GCPCredentialsConfiguration; + /** + * we enable the authenticated service account to impersonate another service account + */ + gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; +} + +/** + * Dremio Cloud region where your organization is hosted. Choose 'US' for United States + * region or 'EU' for European region. + */ +export enum CloudRegion { + Eu = "EU", + Us = "US", +} + +/** + * Choose how to authenticate with SAP SuccessFactors OData API. + * + * Authentication type to connect to SAP SuccessFactors. + * + * Database Authentication types not requiring config. + */ +export enum NoConfigAuthenticationTypes { + BasicAuth = "BasicAuth", + OAuth2 = "OAuth2", + OAuth2Credentials = "OAuth2Credentials", +} + +/** + * ThoughtSpot authentication configuration + * + * Types of methods used to authenticate to the alation instance + * + * Basic Auth Credentials + * + * API Access Token Auth Credentials + * + * Choose between Connected App (OAuth 2.0) or Basic Authentication. + * + * OAuth 2.0 client credentials authentication for Airbyte Cloud + */ +export interface Authenticationation { + /** + * Password to access the service. + */ + password?: string; + /** + * Username to access the service. + */ + username?: string; + /** + * Access Token for the API + */ + accessToken?: string; + /** + * Client ID for the application registered in Airbyte. + */ + clientId?: string; + /** + * Client Secret for the application registered in Airbyte. + */ + clientSecret?: string; +} + +export interface AuthenticationModeObject { + /** + * Authentication from Connection String for AzureSQL. + * + * Authentication from Connection String for Azure Synapse. + */ + authentication?: AuthenticationEnum; + /** + * Connection Timeout from Connection String for AzureSQL. + * + * Connection Timeout from Connection String for Azure Synapse. + */ + connectionTimeout?: number; + /** + * Encrypt from Connection String for AzureSQL. + * + * Encrypt from Connection String for Azure Synapse. + */ + encrypt?: boolean; + /** + * Trust Server Certificate from Connection String for AzureSQL. + * + * Trust Server Certificate from Connection String for Azure Synapse. + */ + trustServerCertificate?: boolean; + [property: string]: any; +} + +/** + * Authentication from Connection String for AzureSQL. + * + * Authentication from Connection String for Azure Synapse. + */ +export enum AuthenticationEnum { + ActiveDirectoryIntegrated = "ActiveDirectoryIntegrated", + ActiveDirectoryPassword = "ActiveDirectoryPassword", + ActiveDirectoryServicePrincipal = "ActiveDirectoryServicePrincipal", +} + +/** + * Event broker configuration. Choose between Kafka and Kinesis. + * + * Kafka broker configuration for OpenLineage events. + * + * AWS Kinesis Data Streams configuration for OpenLineage events. + */ +export interface BrokerConfiguration { + /** + * Kafka bootstrap servers URL. + */ + brokersUrl?: string; + /** + * Kafka consumer group name. + */ + consumerGroupName?: string; + /** + * Initial Kafka consumer offset. + * + * Initial Kinesis shard iterator type. + */ + consumerOffsets?: InitialConsumerOffsets; + /** + * Max allowed wait time. + * + * Poll interval in seconds. + */ + poolTimeout?: number; + /** + * SASL Configuration details. + */ + saslConfig?: SASLClientConfig; + /** + * Kafka security protocol config. + */ + securityProtocol?: KafkaSecurityProtocol; + /** + * Max allowed inactivity time. + * + * Max inactivity timeout in seconds. + */ + sessionTimeout?: number; + /** + * SSL Configuration details. + */ + sslConfig?: DbtSSLConfigClass; + /** + * Topic from where OpenLineage events will be pulled. + */ + topicName?: string; + /** + * AWS credentials configuration. + */ + awsConfig?: AWSCredentials; + /** + * Kinesis Data Stream name. + */ + streamName?: string; +} + +/** + * Initial Kafka consumer offset. + * + * Initial Kinesis shard iterator type. + */ +export enum InitialConsumerOffsets { + Earliest = "earliest", + InitialConsumerOffsetsLATEST = "LATEST", + Latest = "latest", + TrimHorizon = "TRIM_HORIZON", +} + +/** + * SASL Configuration details. + * + * SASL client configuration. + */ +export interface SASLClientConfig { + /** + * SASL security mechanism + */ + saslMechanism?: SaslMechanismType; + /** + * The SASL authentication password. + */ + saslPassword?: string; + /** + * The SASL authentication username. + */ + saslUsername?: string; +} + +/** + * sasl.mechanism Consumer Config property + * + * SASL Mechanism consumer config property + * + * SASL security mechanism + */ +export enum SaslMechanismType { + Gssapi = "GSSAPI", + Oauthbearer = "OAUTHBEARER", + Plain = "PLAIN", + ScramSHA256 = "SCRAM-SHA-256", + ScramSHA512 = "SCRAM-SHA-512", +} + +/** + * Kafka security protocol config. + * + * security.protocol consumer config property + */ +export enum KafkaSecurityProtocol { + Plaintext = "PLAINTEXT", + SSL = "SSL", + SaslPlaintext = "SASL_PLAINTEXT", + SaslSSL = "SASL_SSL", +} + +/** + * Qlik Authentication Certificate By Values + * + * Qlik Authentication Certificate File Path + */ +export interface QlikCertificatesBy { + sslConfig?: DbtSSLConfigClass; + /** + * Client Certificate + */ + clientCertificate?: string; + /** + * Client Key Certificate. + */ + clientKeyCertificate?: string; + /** + * Root Certificate. + */ + rootCertificate?: string; + [property: string]: any; +} + +/** + * Cloud provider where Snowplow is deployed + */ +export enum CloudProvider { + Aws = "AWS", + Azure = "Azure", + Gcp = "GCP", +} + +/** + * Available sources to fetch the metadata. + * + * Deltalake Metastore configuration. + * + * DeltaLake Storage Connection Config + * + * Available sources to fetch files. + * + * Local config source where no extra information needs to be sent. + * + * Azure Datalake Storage will ingest files in container + * + * DataLake GCS storage will ingest metadata of files + * + * DataLake S3 bucket will ingest metadata of files in bucket + * + * Azure Cloud Credentials + * + * Available sources to fetch metadata. + * + * Azure Credentials + */ +export interface DeltaLakeConfigurationSource { + /** + * pySpark App Name. + */ + appName?: string; + /** + * Metastore connection configuration, depending on your metastore type. + * + * Available sources to fetch files. + */ + connection?: Connection; + /** + * Bucket Name of the data source. + */ + bucketName?: string; + /** + * Prefix of the data source. + */ + prefix?: string; + securityConfig?: DbtSecurityConfigClass; + /** + * Account Name of your storage account + */ + accountName?: string; + /** + * Your Service Principal App ID (Client ID) + */ + clientId?: string; + /** + * Your Service Principal Password (Client Secret) + */ + clientSecret?: string; + /** + * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default + */ + scopes?: string; + /** + * Tenant ID of your Azure Subscription + */ + tenantId?: string; + /** + * Key Vault Name + */ + vaultName?: string; +} + +/** + * Metastore connection configuration, depending on your metastore type. + * + * Available sources to fetch files. + * + * DataLake S3 bucket will ingest metadata of files in bucket + */ +export interface Connection { + /** + * Thrift connection to the metastore service. E.g., localhost:9083 + */ + metastoreHostPort?: string; + /** + * Driver class name for JDBC metastore. The value will be mapped as + * spark.hadoop.javax.jdo.option.ConnectionDriverName sparks property. E.g., + * org.mariadb.jdbc.Driver + */ + driverName?: string; + /** + * Class path to JDBC driver required for JDBC connection. The value will be mapped as + * spark.driver.extraClassPath sparks property. + */ + jdbcDriverClassPath?: string; + /** + * JDBC connection to the metastore database. E.g., jdbc:mysql://localhost:3306/demo_hive + */ + metastoreDb?: string; + /** + * Password to use against metastore database. The value will be mapped as + * spark.hadoop.javax.jdo.option.ConnectionPassword sparks property. + */ + password?: string; + /** + * Username to use against metastore database. The value will be mapped as + * spark.hadoop.javax.jdo.option.ConnectionUserName sparks property. + */ + username?: string; + /** + * Local path for the local file with metastore data. E.g., /tmp/metastore.db + */ + metastoreFilePath?: string; + securityConfig?: AWSCredentials; +} + +/** + * Choose between API or database connection fetch metadata from superset. + * + * Superset API Connection Config + * + * Postgres Database Connection Config + * + * Mysql Database Connection Config + * + * Choose between local file system path (object) or S3 bucket location (object) for Access + * database files. + * + * Local filesystem path to a single Access database file or a directory containing Access + * files. + * + * S3 Connection. + * + * Choose between Database connection or HDB User Store connection. + * + * Sap Hana Database SQL Connection Config + * + * Sap Hana Database HDB User Store Connection Config + * + * Choose between mysql and postgres connection for alation database + * + * Choose between database connection or REST API connection to fetch metadata from + * Airflow. + * + * Airflow REST API Connection Config for connecting via REST API. + * + * Lineage Backend Connection Config + * + * SQLite Database Connection Config + * + * Matillion Auth Configuration + * + * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. + */ +export interface ConfigConnection { + /** + * Password for Superset. + * + * Password to connect to Hana. + * + * Password to connect to SQLite. Blank for in-memory database. + * + * Password to connect to the Matillion. + */ + password?: string; + /** + * Authentication provider for the Superset service. For basic user/password authentication, + * the default value `db` can be used. This parameter is used internally to connect to + * Superset's REST API. + */ + provider?: Provider; + /** + * SSL Configuration details. + */ + sslConfig?: ConnectionSSLConfig; + /** + * Username for Superset. + * + * Username to connect to Postgres. This user should have privileges to read all the + * metadata in Postgres. + * + * Username to connect to MySQL. This user should have privileges to read all the metadata + * in Mysql. + * + * Username to connect to Hana. This user should have privileges to read all the metadata. + * + * Username to connect to SQLite. Blank for in-memory database. + * + * Username to connect to the Matillion. This user should have privileges to read all the + * metadata in Matillion. + */ + username?: string; + /** + * Whether to verify SSL certificates when connecting to the Airflow API. + */ + verifySSL?: boolean | VerifySSL; + /** + * Choose Auth Config Type. + */ + authType?: AuthTypeClass; + /** + * Custom OpenMetadata Classification name for Postgres policy tags. + */ + classificationName?: string; + connectionArguments?: { [key: string]: any }; + connectionOptions?: { [key: string]: string }; + /** + * Database of the data source. This is optional parameter, if you would like to restrict + * the metadata reading to a single database. When left blank, OpenMetadata Ingestion + * attempts to scan all the databases. + * + * Database of the data source. + */ + database?: string; + /** + * Regex to only include/exclude databases that matches the pattern. + */ + databaseFilterPattern?: FilterPattern; + /** + * Host and port of the source service. + * + * Host and port of the MySQL service. For GCP CloudSQL, use the instance connection name in + * the format 'project_id:region:instance_name'. + * + * Host and port of the Hana service. + * + * Host and port of the SQLite service. Blank for in-memory database. + * + * Matillion Host + */ + hostPort?: string; + /** + * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of + * this. + */ + ingestAllDatabases?: boolean; + /** + * Fully qualified name of the view or table to use for query logs. If not provided, + * defaults to pg_stat_statements. Use this to configure a custom view (e.g., + * my_schema.custom_pg_stat_statements) when direct access to pg_stat_statements is + * restricted. + */ + queryStatementSource?: string; + sampleDataStorageConfig?: SampleDataStorageConfig; + /** + * Regex to only include/exclude schemas that matches the pattern. + */ + schemaFilterPattern?: FilterPattern; + /** + * SQLAlchemy driver scheme options. + */ + scheme?: ConnectionScheme; + sslMode?: SSLMode; + /** + * Regex to only include/exclude stored procedures that matches the pattern. + */ + storedProcedureFilterPattern?: FilterPattern; + supportsDatabase?: boolean; + supportsDataDiff?: boolean; + supportsDBTExtraction?: boolean; + supportsLineageExtraction?: boolean; + supportsMetadataExtraction?: boolean; + supportsProfiler?: boolean; + supportsQueryComment?: boolean; + supportsUsageExtraction?: boolean; + /** + * Regex to only include/exclude tables that matches the pattern. + */ + tableFilterPattern?: FilterPattern; + /** + * Service Type + */ + type?: ConnectionType; + /** + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. + */ + databaseName?: string; + /** + * Database Schema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion + * attempts to scan all the schemas. + * + * Database Schema of the data source. This is an optional parameter, if you would like to + * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion + * attempts to scan all the schemas. + */ + databaseSchema?: string; + /** + * Table name to fetch the query history. When set, this overrides the default + * 'mysql.general_log' (or 'mysql.slow_log' when 'useSlowLogs' is enabled). The custom table + * must expose columns compatible with the selected log path. + */ + queryHistoryTable?: string; + /** + * Use slow logs to extract lineage. + */ + useSlowLogs?: boolean; + /** + * Absolute path to the .accdb or .mdb file, or a directory. Supports ~ expansion (e.g. + * ~/data/sales.accdb). All .accdb and .mdb files found recursively in a directory will be + * ingested. + */ + localFilePath?: string; + awsConfig?: AWSCredentials; + /** + * Bucket Names of the data source. + */ + bucketNames?: string[]; + /** + * Console EndPoint URL for S3-compatible services + */ + consoleEndpointURL?: string; + /** + * Regex to only fetch containers that matches the pattern. + */ + containerFilterPattern?: FilterPattern; + /** + * HDB Store User Key generated from the command `hdbuserstore SET + * ` + */ + userKey?: string; + /** + * Airflow REST API version. + */ + apiVersion?: APIVersion; + /** + * Choose an authentication method: Basic Auth (username/password), Access Token, GCP + * Service Account (for Cloud Composer), or AWS Credentials (for MWAA). + */ + authConfig?: AuthenticationConfiguration; + /** + * Regex exclude pipelines. + */ + pipelineFilterPattern?: FilterPattern; + /** + * How to run the SQLite database. :memory: by default. + */ + databaseMode?: string; + supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; +} + +/** + * Airflow REST API version. + * + * Airflow REST API version. Use v1 for Airflow 2.x and v2 for Airflow 3.x. Auto will detect + * the version automatically. + */ +export enum APIVersion { + Auto = "auto", + V1 = "v1", + V2 = "v2", +} + +/** + * Choose an authentication method: Basic Auth (username/password), Access Token, GCP + * Service Account (for Cloud Composer), or AWS Credentials (for MWAA). + * + * Username and password for Airflow API authentication. + * + * Static access token for Airflow API authentication. + * + * GCP credentials for Google Cloud Composer. Supports service account values, credentials + * path, workload identity (external account), and ADC. Tokens are auto-refreshed at + * runtime. + * + * AWS MWAA (Managed Workflows for Apache Airflow) authentication configuration. + */ +export interface AuthenticationConfiguration { + /** + * Password for basic authentication to the Airflow API. + */ + password?: string; + /** + * Username for basic authentication to the Airflow API. + */ + username?: string; + /** + * Static access token for Airflow API authentication. + */ + token?: string; + /** + * GCP credentials configuration. + */ + credentials?: GcpConfigClass; + /** + * MWAA credentials and environment configuration. + */ + mwaaConfig?: MWAAConfiguration; +} + +/** + * MWAA credentials and environment configuration. + */ +export interface MWAAConfiguration { + /** + * AWS credentials for generating MWAA CLI token. + */ + awsConfig: AWSCredentials; + /** + * The name of your MWAA environment. + */ + mwaaEnvironmentName: string; +} + +/** + * Choose Auth Config Type. + * + * Common Database Connection Config + * + * IAM Auth Database Connection Config + * + * Azure Database Connection Config + * + * GCP CloudSQL Database Connection Config. Uses the Google Cloud SQL Python Connector. + */ +export interface AuthTypeClass { + /** + * Password to connect to source. + * + * Database user password. Leave empty if using IAM database authentication. + */ + password?: string; + awsConfig?: AWSCredentials; + azureConfig?: AzureCredentials; + /** + * Use GCP IAM for database authentication instead of a password. + */ + enableIamAuth?: boolean; + /** + * GCP credentials to use. If not provided, Application Default Credentials will be used. + */ + gcpConfig?: GcpConfigClass; +} + +/** + * Authentication provider for the Superset service. For basic user/password authentication, + * the default value `db` can be used. This parameter is used internally to connect to + * Superset's REST API. + */ +export enum Provider { + DB = "db", + LDAP = "ldap", +} + +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + +/** + * Storage config to store sample data + */ +export interface SampleDataStorageConfig { + config?: DataStorageConfig; +} + +/** + * Storage config to store sample data + */ +export interface DataStorageConfig { + /** + * Bucket Name + */ + bucketName?: string; + /** + * Provide the pattern of the path where the generated sample data file needs to be stored. + */ + filePathPattern?: string; + /** + * When this field enabled a single parquet file will be created to store sample data, + * otherwise we will create a new file per day + */ + overwriteData?: boolean; + /** + * Prefix of the data source. + */ + prefix?: string; + storageConfig?: AwsCredentials; + [property: string]: any; +} + +/** + * AWS credentials configs. + * + * AWS credentials required to access the S3 file. + * + * AWS credentials for generating MWAA CLI token. + * + * AWS credentials configuration. + */ +export interface AwsCredentials { + /** + * The Amazon Resource Name (ARN) of the role to assume. Required Field in case of Assume + * Role + */ + assumeRoleArn?: string; + /** + * An identifier for the assumed role session. Use the role session name to uniquely + * identify a session when the same role is assumed by different principals or for different + * reasons. Required Field in case of Assume Role + */ + assumeRoleSessionName?: string; + /** + * The Amazon Resource Name (ARN) of the role to assume. Optional Field in case of Assume + * Role + */ + assumeRoleSourceIdentity?: string; + /** + * AWS Access key ID. + */ + awsAccessKeyId?: string; + /** + * AWS Region + */ + awsRegion?: string; + /** + * AWS Secret Access Key. + */ + awsSecretAccessKey?: string; + /** + * AWS Session Token. + */ + awsSessionToken?: string; + /** + * Enable AWS IAM authentication. When enabled, uses the default credential provider chain + * (environment variables, instance profile, etc.). Defaults to false for backward + * compatibility. + */ + enabled?: boolean; + /** + * EndPoint URL for the AWS + */ + endPointURL?: string; + /** + * The name of a profile to use with the boto session. + */ + profileName?: string; +} + +/** + * SQLAlchemy driver scheme options. + */ +export enum ConnectionScheme { + MysqlPymysql = "mysql+pymysql", + PgspiderPsycopg2 = "pgspider+psycopg2", + PostgresqlPsycopg2 = "postgresql+psycopg2", + SqlitePysqlite = "sqlite+pysqlite", +} + +/** + * Client SSL configuration + * + * OpenMetadata Client configured to validate SSL certificates. + * + * SSL Configuration for OpenMetadata Server + * + * SSL Configuration details. + * + * CA certificate, client certificate, and private key for SSL validation. Required when + * verifySSL is 'validate'. + * + * SSL Configuration details for DB2 connection. Provide CA certificate for server + * validation, and optionally client certificate and key for mutual TLS authentication. + * + * SSL/TLS certificate configuration for client authentication. Provide CA certificate, + * client certificate, and private key for mutual TLS authentication. + * + * SSL Configuration details. Provide the CA certificate to validate the Informix server + * certificate. Paste the PEM content directly or upload the certificate file. + * + * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config + * connection. + * + * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry + * connection. + * + * SSL certificate configuration for validating the server certificate when fetching dbt + * artifacts. + */ +export interface ConnectionSSLConfig { + /** + * The CA certificate used for SSL validation. + */ + caCertificate?: string; + /** + * The SSL certificate used for client authentication. + */ + sslCertificate?: string; + /** + * The private key associated with the SSL certificate. + */ + sslKey?: string; +} + +/** + * SSL Mode to connect to database. + * + * SSL Mode to connect to Informix. Use 'disable' for no SSL, 'require' for encrypted SSL + * without certificate verification, or 'verify-ca' to validate the server certificate + * against the provided CA certificate. + */ +export enum SSLMode { + Allow = "allow", + Disable = "disable", + Prefer = "prefer", + Require = "require", + VerifyCA = "verify-ca", + VerifyFull = "verify-full", +} + +/** + * Service Type + * + * Service type. + * + * S3 service type + */ +export enum ConnectionType { + Backend = "Backend", + MatillionDPC = "MatillionDPC", + MatillionETL = "MatillionETL", + Mysql = "Mysql", + Postgres = "Postgres", + RESTAPI = "RestAPI", + S3 = "S3", + SQLite = "SQLite", +} + +/** + * GCP credentials configs. + * + * GCP credentials to use. If not provided, Application Default Credentials will be used. + * + * GCP Credentials + * + * GCP credentials configuration for authenticating with Pub/Sub. + * + * GCP credentials configuration. + * + * GCP Credentials for Google Drive API + * + * Azure Cloud Credentials + * + * Available sources to fetch metadata. + * + * Azure Credentials + */ +export interface PurpleGCPCredentials { + /** + * We support two ways of authenticating to GCP i.e via GCP Credentials Values or GCP + * Credentials Path + */ + gcpConfig?: GCPCredentialsConfiguration; + /** + * we enable the authenticated service account to impersonate another service account + */ + gcpImpersonateServiceAccount?: GCPImpersonateServiceAccountValues; + /** + * Account Name of your storage account + */ + accountName?: string; + /** + * Your Service Principal App ID (Client ID) + */ + clientId?: string; + /** + * Your Service Principal Password (Client Secret) + */ + clientSecret?: string; + /** + * Scopes to get access token, for e.g. api://6dfX33ab-XXXX-49df-XXXX-3459eX817d3e/.default + */ + scopes?: string; + /** + * Tenant ID of your Azure Subscription + */ + tenantId?: string; + /** + * Key Vault Name + */ + vaultName?: string; +} + +/** + * Underlying database connection + * + * Mssql Database Connection Config + * + * Optional. Underlying SSISDB connection. When omitted, the connector runs in file-only + * mode and run history is not extracted. + */ +export interface DatabaseConnectionClass { + connectionArguments?: { [key: string]: any }; + connectionOptions?: { [key: string]: string }; + /** + * Database of the data source. This is optional parameter, if you would like to restrict + * the metadata reading to a single database. When left blank, OpenMetadata Ingestion + * attempts to scan all the databases. + */ + database: string; + /** + * Regex to only include/exclude databases that matches the pattern. + */ + databaseFilterPattern?: FilterPattern; + /** + * ODBC driver version in case of pyodbc connection. + */ + driver?: string; + /** + * Enable SSL/TLS encryption for the MSSQL connection. When enabled, all data transmitted + * between the client and server will be encrypted. + */ + encrypt?: boolean; + /** + * Host and port of the MSSQL service. + */ + hostPort?: string; + /** + * Ingest data from all databases in Mssql. You can use databaseFilterPattern on top of this. + */ + ingestAllDatabases?: boolean; + /** + * Password to connect to MSSQL. + */ + password?: string; + sampleDataStorageConfig?: SampleDataStorageConfig; + /** + * Regex to only include/exclude schemas that matches the pattern. + */ + schemaFilterPattern?: FilterPattern; + /** + * SQLAlchemy driver scheme options. + */ + scheme?: MssqlScheme; + /** + * SSL/TLS certificate configuration for client authentication. Provide CA certificate, + * client certificate, and private key for mutual TLS authentication. + */ + sslConfig?: DbtSSLConfigClass; + /** + * Regex to only include/exclude stored procedures that matches the pattern. + */ + storedProcedureFilterPattern?: FilterPattern; + supportsDatabase?: boolean; + supportsDataDiff?: boolean; + supportsDBTExtraction?: boolean; + supportsLineageExtraction?: boolean; + supportsMetadataExtraction?: boolean; + supportsProfiler?: boolean; + supportsQueryComment?: boolean; + supportsUsageExtraction?: boolean; + /** + * Regex to only include/exclude tables that matches the pattern. + */ + tableFilterPattern?: FilterPattern; + /** + * Trust the server certificate without validation. Set to false in production to validate + * server certificates against the certificate authority. + */ + trustServerCertificate?: boolean; + /** + * Service Type + */ + type?: MssqlType; + /** + * Username to connect to MSSQL. This user should have privileges to read all the metadata + * in MsSQL. + */ + username?: string; +} + +/** + * SQLAlchemy driver scheme options. + */ +export enum MssqlScheme { + MssqlPymssql = "mssql+pymssql", + MssqlPyodbc = "mssql+pyodbc", + MssqlPytds = "mssql+pytds", +} + +/** + * Service Type + * + * Service type. + */ +export enum MssqlType { + Mssql = "Mssql", +} + +/** + * Snowplow deployment type (BDP for managed or Community for self-hosted) + * + * Snowplow deployment type + */ +export enum SnowplowDeployment { + Bdp = "BDP", + Community = "Community", +} + +/** + * How to discover MCP servers + * + * Method to discover MCP servers + */ +export enum DiscoveryMethod { + ConfigFile = "ConfigFile", + DirectConnection = "DirectConnection", + Registry = "Registry", +} + +/** + * Configuration for Sink Component in the OpenMetadata Ingestion Framework. + */ +export interface ConfigElasticsSearch { + config?: { [key: string]: any }; + /** + * Type of sink component ex: metadata + */ + type: string; +} + +/** + * FHIR specification version (R4, STU3, DSTU2) + */ +export enum FHIRVersion { + Dstu2 = "DSTU2", + R4 = "R4", + Stu3 = "STU3", +} + +/** + * Do not set any credentials. Note that credentials are required to extract .lkml views and + * their lineage. + * + * Credentials for a GitHub repository + * + * Credentials for a BitBucket repository + * + * Credentials for a Gitlab repository + */ +export interface Credentials { + /** + * GitHub instance URL. For GitHub.com, use https://github.com + * + * BitBucket instance URL. For BitBucket Cloud, use https://bitbucket.org + * + * Gitlab instance URL. For Gitlab.com, use https://gitlab.com + */ + gitHostURL?: string; + repositoryName?: string; + repositoryOwner?: string; + token?: string; + /** + * Credentials Type + */ + type?: NoGitCredentialsType; + /** + * Main production branch of the repository. E.g., `main` + */ + branch?: string; +} + +/** + * Credentials Type + * + * GitHub Credentials type + * + * BitBucket Credentials type + * + * Gitlab Credentials type + */ +export enum NoGitCredentialsType { + BitBucket = "BitBucket", + GitHub = "GitHub", + Gitlab = "Gitlab", +} + +/** + * The authentication method that the user uses to sign in. + */ +export enum IdentityType { + Anonymous = "ANONYMOUS", + Iam = "IAM", + Quicksight = "QUICKSIGHT", +} + +/** + * Specifies the logon authentication method. Possible values are TD2 (the default), JWT, + * LDAP, KRB5 for Kerberos, or TDNEGO + */ +export enum Logmech { + Custom = "CUSTOM", + Jwt = "JWT", + Krb5 = "KRB5", + LDAP = "LDAP", + Td2 = "TD2", + Tdnego = "TDNEGO", +} + +/** + * Hive Metastore Connection Details + * + * Postgres Database Connection Config + * + * Mysql Database Connection Config + */ +export interface HiveMetastoreConnectionDetails { + /** + * Choose Auth Config Type. + */ + authType?: AuthTypeClass; + /** + * Custom OpenMetadata Classification name for Postgres policy tags. + */ + classificationName?: string; + connectionArguments?: { [key: string]: any }; + connectionOptions?: { [key: string]: string }; + /** + * Database of the data source. This is optional parameter, if you would like to restrict + * the metadata reading to a single database. When left blank, OpenMetadata Ingestion + * attempts to scan all the databases. + */ + database?: string; + /** + * Regex to only include/exclude databases that matches the pattern. + */ + databaseFilterPattern?: FilterPattern; + /** + * Host and port of the source service. + * + * Host and port of the MySQL service. For GCP CloudSQL, use the instance connection name in + * the format 'project_id:region:instance_name'. + */ + hostPort?: string; + /** + * Ingest data from all databases in Postgres. You can use databaseFilterPattern on top of + * this. + */ + ingestAllDatabases?: boolean; + /** + * Fully qualified name of the view or table to use for query logs. If not provided, + * defaults to pg_stat_statements. Use this to configure a custom view (e.g., + * my_schema.custom_pg_stat_statements) when direct access to pg_stat_statements is + * restricted. + */ + queryStatementSource?: string; + sampleDataStorageConfig?: SampleDataStorageConfig; + /** + * Regex to only include/exclude schemas that matches the pattern. + */ + schemaFilterPattern?: FilterPattern; + /** + * SQLAlchemy driver scheme options. + */ + scheme?: HiveMetastoreConnectionDetailsScheme; + /** + * SSL Configuration details. + */ + sslConfig?: DbtSSLConfigClass; + sslMode?: SSLMode; + /** + * Regex to only include/exclude stored procedures that matches the pattern. + */ + storedProcedureFilterPattern?: FilterPattern; + supportsDatabase?: boolean; + supportsDataDiff?: boolean; + supportsDBTExtraction?: boolean; + supportsLineageExtraction?: boolean; + supportsMetadataExtraction?: boolean; + supportsProfiler?: boolean; + supportsQueryComment?: boolean; + supportsUsageExtraction?: boolean; + /** + * Regex to only include/exclude tables that matches the pattern. + */ + tableFilterPattern?: FilterPattern; + /** + * Service Type + */ + type?: HiveMetastoreConnectionDetailsType; + /** + * Username to connect to Postgres. This user should have privileges to read all the + * metadata in Postgres. + * + * Username to connect to MySQL. This user should have privileges to read all the metadata + * in Mysql. + */ + username?: string; + /** + * Optional name to give to the database in OpenMetadata. If left blank, we will use default + * as the database name. + */ + databaseName?: string; + /** + * Database Schema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single schema. When left blank, OpenMetadata Ingestion + * attempts to scan all the schemas. + */ + databaseSchema?: string; + /** + * Table name to fetch the query history. When set, this overrides the default + * 'mysql.general_log' (or 'mysql.slow_log' when 'useSlowLogs' is enabled). The custom table + * must expose columns compatible with the selected log path. + */ + queryHistoryTable?: string; + /** + * Use slow logs to extract lineage. + */ + useSlowLogs?: boolean; +} + +/** + * SQLAlchemy driver scheme options. + */ +export enum HiveMetastoreConnectionDetailsScheme { + MysqlPymysql = "mysql+pymysql", + PgspiderPsycopg2 = "pgspider+psycopg2", + PostgresqlPsycopg2 = "postgresql+psycopg2", +} + +/** + * Service Type + * + * Service type. + */ +export enum HiveMetastoreConnectionDetailsType { + Mysql = "Mysql", + Postgres = "Postgres", +} + +/** + * We support username/password or client certificate authentication + * + * Configuration for connecting to Nifi Basic Auth. + * + * Configuration for connecting to Nifi Client Certificate Auth. + */ +export interface NifiCredentialsConfiguration { + /** + * Nifi password to authenticate to the API. + */ + password?: string; + /** + * Nifi user to authenticate to the API. + */ + username?: string; + /** + * Boolean marking if we need to verify the SSL certs for Nifi. False by default. + */ + verifySSL?: boolean; + /** + * Path to the root CA certificate + */ + certificateAuthorityPath?: string; + /** + * Path to the client certificate + */ + clientCertificatePath?: string; + /** + * Path to the client key + */ + clientkeyPath?: string; +} + +/** + * OpenAPI Schema source config. Either a URL or a file path must be provided. + * + * Open API Schema URL Connection Config + * + * Open API Schema File Path Connection Config + * + * Open API Schema S3 Connection Config + */ +export interface OpenAPISchemaConnection { + /** + * Open API Schema URL. + */ + openAPISchemaURL?: string; + /** + * Path to a local OpenAPI schema file. + */ + openAPISchemaFilePath?: string; + /** + * AWS credentials required to access the S3 file. + */ + awsCredentials?: AWSCredentials; + /** + * S3 URL of the OpenAPI schema file (JSON or YAML). Example: + * https://bucket-name.s3.amazonaws.com/path/to/openapi_schema.json + */ + openAPISchemaS3URL?: string; +} + +/** + * Connect with oracle by either passing service name or database schema name. + */ +export interface OracleConnectionType { + /** + * databaseSchema of the data source. This is optional parameter, if you would like to + * restrict the metadata reading to a single databaseSchema. When left blank, OpenMetadata + * Ingestion attempts to scan all the databaseSchema. + */ + databaseSchema?: string; + /** + * The Oracle Service name is the TNS alias that you give when you remotely connect to your + * database. + */ + oracleServiceName?: string; + /** + * Pass the full constructed TNS string, e.g., + * (DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=myhost)(PORT=1530)))(CONNECT_DATA=(SID=MYSERVICENAME))). + */ + oracleTNSConnection?: string; + [property: string]: any; +} + +/** + * S3 Connection. + */ +export interface S3Connection { + awsConfig: AWSCredentials; + /** + * Bucket Names of the data source. + */ + bucketNames?: string[]; + connectionArguments?: { [key: string]: any }; + connectionOptions?: { [key: string]: string }; + /** + * Console EndPoint URL for S3-compatible services + */ + consoleEndpointURL?: string; + /** + * Regex to only fetch containers that matches the pattern. + */ + containerFilterPattern?: FilterPattern; + supportsMetadataExtraction?: boolean; + supportsProfiler?: boolean; + /** + * Service Type + */ + type?: S3ConnectionType; +} + +/** + * Service Type + * + * S3 service type + */ +export enum S3ConnectionType { + S3 = "S3", +} + +/** + * Source to get the .pbit files to extract lineage information + * + * Local config source where no extra information needs to be sent. + * + * Azure storage config for pbit files + * + * GCS storage config for pbit files + * + * S3 storage config for pbit files + */ +export interface PowerBIPbitFilesSource { + /** + * Directory path for the pbit files + */ + path?: string; + /** + * pbit File Configuration type + */ + pbitFileConfigType?: PbitFileConfigType; + /** + * Path of the folder where the .pbit files will be unzipped and datamodel schema will be + * extracted + */ + pbitFilesExtractDir?: string; + prefixConfig?: BucketDetails; + securityConfig?: DbtSecurityConfigClass; +} + +/** + * pbit File Configuration type + */ +export enum PbitFileConfigType { + Azure = "azure", + Gcs = "gcs", + Local = "local", + S3 = "s3", +} + +/** + * Details of the bucket where the .pbit files are stored + */ +export interface BucketDetails { + /** + * Name of the bucket where the .pbit files are stored + */ + bucketName?: string; + /** + * Path of the folder where the .pbit files are stored + */ + objectPrefix?: string; +} + +/** + * Policy agent configuration for access control extraction. + */ +export interface PolicyAgentConfig { + /** + * Enable policy agent extraction. + */ + enabled?: boolean; + /** + * Supports column-level access policy extraction. + */ + supportsColumnAccess?: boolean; + /** + * Supports full access policy extraction. + */ + supportsFullAccess?: boolean; + /** + * Supports masked access policy extraction. + */ + supportsMaskedAccess?: boolean; +} + +/** + * This schema publisher run modes. + */ +export enum RunMode { + Batch = "batch", + Stream = "stream", +} + +/** + * SQLAlchemy driver scheme options. + * + * Mongo connection scheme options. + * + * Couchbase driver scheme options. + */ +export enum ConfigScheme { + AwsathenaREST = "awsathena+rest", + Bigquery = "bigquery", + ClickhouseHTTP = "clickhouse+http", + ClickhouseNative = "clickhouse+native", + CockroachdbPsycopg2 = "cockroachdb+psycopg2", + Couchbase = "couchbase", + Databricks = "databricks", + Db2IBMDB = "db2+ibm_db", + Doris = "doris", + Druid = "druid", + ExaWebsocket = "exa+websocket", + Hana = "hana", + Hive = "hive", + HiveHTTP = "hive+http", + HiveHTTPS = "hive+https", + Ibmi = "ibmi", + Impala = "impala", + Impala4 = "impala4", + Informix = "informix", + Mongodb = "mongodb", + MongodbSrv = "mongodb+srv", + MssqlPymssql = "mssql+pymssql", + MssqlPyodbc = "mssql+pyodbc", + MssqlPytds = "mssql+pytds", + MysqlPymysql = "mysql+pymysql", + OracleCxOracle = "oracle+cx_oracle", + PgspiderPsycopg2 = "pgspider+psycopg2", + Pinot = "pinot", + PinotHTTP = "pinot+http", + PinotHTTPS = "pinot+https", + PostgresqlPsycopg2 = "postgresql+psycopg2", + Presto = "presto", + RedshiftPsycopg2 = "redshift+psycopg2", + Snowflake = "snowflake", + SqlitePysqlite = "sqlite+pysqlite", + Teradatasql = "teradatasql", + Trino = "trino", + VerticaVerticaPython = "vertica+vertica_python", +} + +/** + * Configuration for a single MCP server to connect to directly + */ +export interface MCPServerConfig { + /** + * API key for authenticated MCP servers + */ + apiKey?: string; + /** + * Arguments to pass to the command + */ + args?: string[]; + /** + * Command to execute for Stdio transport (e.g., 'npx', 'uvx', 'python') + */ + command?: string; + /** + * Environment variables for the server process + */ + env?: { [key: string]: string }; + /** + * Name to assign to this MCP server + */ + name: string; + transport?: TransportType; + /** + * URL for SSE or StreamableHTTP transport + */ + url?: string; + [property: string]: any; +} + +/** + * MCP transport protocol type + */ +export enum TransportType { + SSE = "SSE", + Stdio = "Stdio", + StreamableHTTP = "StreamableHTTP", +} + +export enum SpaceType { + Data = "Data", + Managed = "Managed", + Personal = "Personal", + Shared = "Shared", +} + +/** + * SSL Configuration for OpenMetadata Server + * + * Client SSL configuration + * + * SSL Configuration details. + * + * CA certificate, client certificate, and private key for SSL validation. Required when + * verifySSL is 'validate'. + * + * SSL Configuration details for DB2 connection. Provide CA certificate for server + * validation, and optionally client certificate and key for mutual TLS authentication. + * + * SSL/TLS certificate configuration for client authentication. Provide CA certificate, + * client certificate, and private key for mutual TLS authentication. + * + * SSL Configuration details. Provide the CA certificate to validate the Informix server + * certificate. Paste the PEM content directly or upload the certificate file. + * + * Consumer Config SSL Config. Configuration for enabling SSL for the Consumer Config + * connection. + * + * Schema Registry SSL Config. Configuration for enabling SSL for the Schema Registry + * connection. + * + * SSL certificate configuration for validating the server certificate when fetching dbt + * artifacts. + * + * OpenMetadata Client configured to validate SSL certificates. + * + * SSL Config + */ +export interface SSLConfigObject { + /** + * The CA certificate used for SSL validation. + */ + caCertificate?: string; + /** + * The SSL certificate used for client authentication. + */ + sslCertificate?: string; + /** + * The private key associated with the SSL certificate. + */ + sslKey?: string; + /** + * SSL Certificates + */ + certificates?: SSLCertificates; + [property: string]: any; +} + +/** + * SSL Certificates + * + * SSL Certificates By Path + * + * SSL Certificates By Values + */ +export interface SSLCertificates { + /** + * CA Certificate Path + */ + caCertPath?: string; + /** + * Client Certificate Path + */ + clientCertPath?: string; + /** + * Private Key Path + */ + privateKeyPath?: string; + /** + * CA Certificate Value + */ + caCertValue?: string; + /** + * Client Certificate Value + */ + clientCertValue?: string; + /** + * Private Key Value + */ + privateKeyValue?: string; + /** + * Staging Directory Path + */ + stagingDir?: string; +} + +/** + * Client SSL/TLS settings. + */ +export enum SSLTLSSettings { + DisableTLS = "disable-tls", + IgnoreCertificate = "ignore-certificate", + ValidateCertificate = "validate-certificate", +} + +/** + * Specifies the transaction mode for the connection + */ +export enum TransactionMode { + ANSI = "ANSI", + Default = "DEFAULT", + Tera = "TERA", +} + +/** + * Type of token to use for authentication + */ +export enum TokenType { + Personal = "personal", + Workspace = "workspace", +} + +/** + * REST API Type + * + * REST API type + * + * Service Type + * + * Looker service type + * + * Metabase service type + * + * PowerBI service type + * + * PowerBIReportServer service type + * + * Redash service type + * + * Superset service type + * + * Tableau service type + * + * Mode service type + * + * Custom dashboard service type + * + * service type + * + * QuickSight service type + * + * Qlik sense service type + * + * Lightdash service type + * + * MicroStrategy service type + * + * Qlik Cloud service type + * + * Sigma service type + * + * ThoughtSpot service type + * + * Grafana service type + * + * Service type. + * + * SAP S/4HANA service type + * + * Custom database service type + * + * Kafka service type + * + * Redpanda service type + * + * Custom messaging service type + * + * Amundsen service type + * + * Metadata to Elastic Search type + * + * OpenMetadata service type + * + * Collibra service type + * + * Custom pipeline service type + * + * SAP BW/4HANA pipeline service type. + * + * Custom Ml model service type + * + * S3 service type + * + * ADLS service type + * + * Gcs service type + * + * Custom storage service type + * + * ElasticSearch Type + * + * ElasticSearch service type + * + * OpenSearch Type + * + * OpenSearch service type + * + * Custom search service type + * + * Apache Ranger service type + * + * Google Drive service type + * + * SharePoint service type + * + * SFTP service type + * + * Custom Drive service type + * + * Service type + */ +export enum PurpleType { + Adls = "ADLS", + Airbyte = "Airbyte", + Airflow = "Airflow", + Alation = "Alation", + AlationSink = "AlationSink", + Amundsen = "Amundsen", + Athena = "Athena", + Atlas = "Atlas", + AzureSQL = "AzureSQL", + BigQuery = "BigQuery", + BigTable = "BigTable", + BurstIQ = "BurstIQ", + Cassandra = "Cassandra", + Clickhouse = "Clickhouse", + Cockroach = "Cockroach", + Collibra = "Collibra", + Couchbase = "Couchbase", + CustomDashboard = "CustomDashboard", + CustomDatabase = "CustomDatabase", + CustomDrive = "CustomDrive", + CustomMessaging = "CustomMessaging", + CustomMlModel = "CustomMlModel", + CustomPipeline = "CustomPipeline", + CustomSearch = "CustomSearch", + CustomStorage = "CustomStorage", + DBTCloud = "DBTCloud", + Dagster = "Dagster", + DataFactory = "DataFactory", + Databricks = "Databricks", + DatabricksPipeline = "DatabricksPipeline", + Datalake = "Datalake", + Db2 = "Db2", + DeltaLake = "DeltaLake", + DomoDashboard = "DomoDashboard", + DomoDatabase = "DomoDatabase", + DomoPipeline = "DomoPipeline", + Doris = "Doris", + Dremio = "Dremio", + Druid = "Druid", + DynamoDB = "DynamoDB", + ElasticSearch = "ElasticSearch", + Epic = "Epic", + Exasol = "Exasol", + Fivetran = "Fivetran", + Flink = "Flink", + Gcs = "GCS", + Glue = "Glue", + GluePipeline = "GluePipeline", + GoogleDrive = "GoogleDrive", + Grafana = "Grafana", + Greenplum = "Greenplum", + Hex = "Hex", + Hive = "Hive", + Impala = "Impala", + Informix = "Informix", + Iomete = "Iomete", + Kafka = "Kafka", + KafkaConnect = "KafkaConnect", + Kinesis = "Kinesis", + KinesisFirehose = "KinesisFirehose", + Lightdash = "Lightdash", + Looker = "Looker", + MCP = "Mcp", + MariaDB = "MariaDB", + Matillion = "Matillion", + Metabase = "Metabase", + MetadataES = "MetadataES", + MicroStrategy = "MicroStrategy", + MicrosoftAccess = "MicrosoftAccess", + MicrosoftFabric = "MicrosoftFabric", + MicrosoftFabricPipeline = "MicrosoftFabricPipeline", + Mlflow = "Mlflow", + Mode = "Mode", + MongoDB = "MongoDB", + Mssql = "Mssql", + Mulesoft = "Mulesoft", + Mysql = "Mysql", + Nifi = "Nifi", + OpenLineage = "OpenLineage", + OpenMetadata = "OpenMetadata", + OpenSearch = "OpenSearch", + Oracle = "Oracle", + PinotDB = "PinotDB", + Postgres = "Postgres", + PowerBI = "PowerBI", + PowerBIReportServer = "PowerBIReportServer", + Presto = "Presto", + PubSub = "PubSub", + QlikCloud = "QlikCloud", + QlikSense = "QlikSense", + QuestDB = "QuestDB", + QuickSight = "QuickSight", + REST = "Rest", + Ranger = "Ranger", + Redash = "Redash", + Redpanda = "Redpanda", + Redshift = "Redshift", + S3 = "S3", + SAS = "SAS", + SFTP = "Sftp", + SQLite = "SQLite", + SageMaker = "SageMaker", + Salesforce = "Salesforce", + SapBw4Hana = "SapBw4Hana", + SapBw4HanaPipeline = "SapBw4HanaPipeline", + SapERP = "SapErp", + SapHana = "SapHana", + SapS4Hana = "SapS4Hana", + SapSuccessFactors = "SapSuccessFactors", + ServiceNow = "ServiceNow", + SharePoint = "SharePoint", + Sigma = "Sigma", + SingleStore = "SingleStore", + Sklearn = "Sklearn", + Snowflake = "Snowflake", + Snowplow = "Snowplow", + Spark = "Spark", + Spline = "Spline", + Ssas = "SSAS", + Ssis = "SSIS", + Ssrs = "Ssrs", + StarRocks = "StarRocks", + Stitch = "Stitch", + Superset = "Superset", + Synapse = "Synapse", + Tableau = "Tableau", + Teradata = "Teradata", + ThoughtSpot = "ThoughtSpot", + Timescale = "Timescale", + Trino = "Trino", + UnityCatalog = "UnityCatalog", + VertexAI = "VertexAI", + Vertica = "Vertica", + Wherescape = "Wherescape", +} + +/** + * Global manifest source. When configured, entries here take precedence over any + * bucket-level openmetadata.json and over defaultManifest for buckets whose containerName + * matches. + * + * No manifest file available. Ingestion would look for bucket-level metadata file instead + * + * Storage Metadata Manifest file path config. + * + * Storage Metadata Manifest file HTTP path config. + * + * Storage Metadata Manifest file S3 path config. + * + * Storage Metadata Manifest file ADLS path config. + * + * Storage Metadata Manifest file GCS path config. + */ +export interface StorageMetadataConfigurationSource { + /** + * Storage Metadata manifest file path to extract locations to ingest from. + */ + manifestFilePath?: string; + /** + * Storage Metadata manifest http file path to extract locations to ingest from. + */ + manifestHttpPath?: string; + prefixConfig?: StorageMetadataBucketDetails; + securityConfig?: DbtSecurityConfigClass; +} + +/** + * Details of the bucket where the storage metadata manifest file is stored + */ +export interface StorageMetadataBucketDetails { + /** + * Name of the top level container where the storage metadata file is stored + */ + containerName: string; + /** + * Path of the folder where the storage metadata file is stored. If the file is at the root, + * you can keep it empty. + */ + objectPrefix?: string; +} + +/** + * Pipeline type + * + * Database Source Config Metadata Pipeline type + * + * Database Source Config Usage Pipeline type + * + * Dashboard Source Config Metadata Pipeline type + * + * Messaging Source Config Metadata Pipeline type + * + * Profiler Source Config Pipeline type + * + * Pipeline Source Config Metadata Pipeline type + * + * MlModel Source Config Metadata Pipeline type + * + * Object Store Source Config Metadata Pipeline type + * + * Storage Service Auto Classification Pipeline type + * + * Drive Source Config Metadata Pipeline type + * + * Search Source Config Metadata Pipeline type + * + * DBT Config Pipeline type + * + * Pipeline Source Config For Application Pipeline type. Nothing is required. + * + * Api Source Config Metadata Pipeline type + * + * Reverse Ingestion Config Pipeline type + * + * MCP Source Config Metadata Pipeline type + * + * Policy Agent Pipeline type + */ +export enum FluffyType { + APIMetadata = "ApiMetadata", + Application = "Application", + AutoClassification = "AutoClassification", + DashboardMetadata = "DashboardMetadata", + DataInsight = "dataInsight", + DatabaseLineage = "DatabaseLineage", + DatabaseMetadata = "DatabaseMetadata", + DatabaseUsage = "DatabaseUsage", + Dbt = "DBT", + DriveMetadata = "DriveMetadata", + MCPMetadata = "McpMetadata", + MessagingMetadata = "MessagingMetadata", + MetadataToElasticSearch = "MetadataToElasticSearch", + MlModelMetadata = "MlModelMetadata", + PipelineMetadata = "PipelineMetadata", + PolicyAgent = "PolicyAgent", + Profiler = "Profiler", + ReverseIngestion = "ReverseIngestion", + SearchMetadata = "SearchMetadata", + StorageMetadata = "StorageMetadata", + TestSuite = "TestSuite", +}