diff --git a/docs/ert/reference/workflows/complete_workflows.rst b/docs/ert/reference/workflows/complete_workflows.rst index 9e4f1d06407..d70dcad7c7f 100644 --- a/docs/ert/reference/workflows/complete_workflows.rst +++ b/docs/ert/reference/workflows/complete_workflows.rst @@ -111,6 +111,10 @@ The :code:`status` field is one of :code:`success`, :code:`failed` or jobs that were stopped because the workflow was cancelled are logged at :code:`INFO` level. +While an experiment is running, the same information is also shown live +in the *Workflows* tab of the run dialog, without having to read the ERT +log. + Workflows hooked in with :code:`HOOK_WORKFLOW` are in addition recorded alongside the experiment they belong to, in :code:`/experiments//workflow_events.jsonl`. That file diff --git a/src/ert/gui/experiments/run_dialog.py b/src/ert/gui/experiments/run_dialog.py index 1a10ca66d63..d44a86cb9b1 100644 --- a/src/ert/gui/experiments/run_dialog.py +++ b/src/ert/gui/experiments/run_dialog.py @@ -69,6 +69,7 @@ RunModelErrorEvent, RunPathCreatedEvent, StartingTotalRunPathCreationEvent, + WorkflowEvent, ) from ert.shared.status.utils import ( byte_with_unit, @@ -83,6 +84,7 @@ RealizationWidget, RunpathProgressWidget, UpdateWidget, + WorkflowLogWidget, ) from .view.disk_space_widget import MountType @@ -386,12 +388,29 @@ def __init__( def is_experiment_done(self) -> bool: return self.flag_experiment_done + def _add_dynamic_tab(self, widget: QWidget, label: str) -> int: + # If the currently selected tab is the most recently added dynamic + # tab, keep following along as new tabs are added; otherwise the + # user has navigated elsewhere and we leave their selection alone. + last_index = self._tab_widget.count() - 1 + follow_new_tab = ( + last_index >= 0 + and not isinstance(self._tab_widget.widget(last_index), WorkflowLogWidget) + and self._tab_widget.currentIndex() == last_index + ) + tab_index = self._tab_widget.addTab(widget, label) + if follow_new_tab: + self._tab_widget.setCurrentIndex(tab_index) + return tab_index + def _current_tab_changed(self, index: int) -> None: widget = self._tab_widget.widget(index) if isinstance(widget, RealizationWidget): widget.refresh_current_selection() - self.fm_step_frame.setHidden(isinstance(widget, UpdateWidget)) + self.fm_step_frame.setHidden( + isinstance(widget, UpdateWidget | WorkflowLogWidget) + ) @Slot(QModelIndex, int, int) def on_snapshot_new_iteration( @@ -413,14 +432,12 @@ def on_snapshot_new_iteration( widget.itemClicked.connect(self._select_real) widget.setProperty("identifier", f"tab-iter-{iteration}") self._select_real(widget._real_list_model.index(0, 0)) - tab_index = self._tab_widget.addTab( + self._add_dynamic_tab( widget, f"Realizations for iteration {iteration}" if not self.is_everest else f"Batch {iteration}...", ) - if self._tab_widget.currentIndex() == self._tab_widget.count() - 2: - self._tab_widget.setCurrentIndex(tab_index) if self.is_everest: self._batch_result_types.append(set()) @@ -461,6 +478,16 @@ def setup_event_monitoring( if rerun_failed_realizations is False: self._snapshot_model.reset() self._tab_widget.clear() + else: + # Other tabs (Realizations/Update) are updated in place for a + # rerun of failed realizations, but the Workflows tab has no + # such per-event update mechanism, so its previous run's rows + # must be cleared explicitly to avoid mixing old and new output. + for i in range(self._tab_widget.count()): + widget = self._tab_widget.widget(i) + if isinstance(widget, WorkflowLogWidget): + widget.clear() + break self._worker_thread = QThread(parent=self) @@ -606,9 +633,7 @@ def _on_event(self, event: object) -> None: self.progress_update_event.emit(status_count, realization_count) case RunModelUpdateBeginEvent(iteration=iteration): widget = UpdateWidget(iteration) - tab_index = self._tab_widget.addTab(widget, f"Update {iteration}") - if self._tab_widget.currentIndex() == self._tab_widget.count() - 2: - self._tab_widget.setCurrentIndex(tab_index) + self._add_dynamic_tab(widget, f"Update {iteration}") widget.begin(event) case RunModelUpdateEndEvent(): self._progress_widget.stop_waiting_progress_bar() @@ -622,6 +647,8 @@ def _on_event(self, event: object) -> None: case RunModelErrorEvent(): self._get_update_widget(event.iteration).error(event) event.write_as_csv(self.output_path) + case WorkflowEvent(): + self._get_or_create_workflow_log_widget().add_event(event) case EverestBatchResultEvent(): batch_types = self._batch_result_types[event.batch] batch_types.add(event.result_type) @@ -660,6 +687,15 @@ def _get_update_widget(self, iteration: int) -> UpdateWidget: return widget raise ValueError("Could not find UpdateWidget") + def _get_or_create_workflow_log_widget(self) -> WorkflowLogWidget: + for i in range(self._tab_widget.count()): + widget = self._tab_widget.widget(i) + if isinstance(widget, WorkflowLogWidget): + return widget + workflow_log_widget = WorkflowLogWidget(self) + self._tab_widget.insertTab(0, workflow_log_widget, "Workflows") + return workflow_log_widget + def update_total_progress( self, progress_value: float, iteration_label: str, iteration: int | None = None ) -> None: diff --git a/src/ert/gui/experiments/view/__init__.py b/src/ert/gui/experiments/view/__init__.py index f85d886abd5..4606d69f4f4 100644 --- a/src/ert/gui/experiments/view/__init__.py +++ b/src/ert/gui/experiments/view/__init__.py @@ -3,6 +3,7 @@ from .realization import RealizationWidget from .runpath_progress_widget import RunpathProgressWidget from .update import UpdateWidget +from .workflow_log import WorkflowLogWidget __all__ = [ "DiskSpaceWidget", @@ -10,4 +11,5 @@ "RealizationWidget", "RunpathProgressWidget", "UpdateWidget", + "WorkflowLogWidget", ] diff --git a/src/ert/gui/experiments/view/workflow_log.py b/src/ert/gui/experiments/view/workflow_log.py new file mode 100644 index 00000000000..2dce986e04e --- /dev/null +++ b/src/ert/gui/experiments/view/workflow_log.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from typing import cast + +from PyQt6.QtCore import Qt +from PyQt6.QtGui import QColor, QFontDatabase +from PyQt6.QtWidgets import ( + QAbstractItemView, + QComboBox, + QHBoxLayout, + QHeaderView, + QLabel, + QPlainTextEdit, + QSplitter, + QTableWidget, + QTableWidgetItem, + QVBoxLayout, + QWidget, +) + +from ert.ensemble_evaluator.state import COLOR_CANCELLED, COLOR_FAILED, COLOR_FINISHED +from ert.run_models.event import WorkflowEvent +from ert.workflow_runner import WorkflowJobStatus + +NO_ITERATION_LABEL = "Pre/post experiment" +NO_OUTPUT_PLACEHOLDER = "(no output)" +_EXPERIMENT_WIDE_HOOKS = frozenset({"PRE_EXPERIMENT", "POST_EXPERIMENT"}) + +_COLUMNS = ("Hook", "Workflow", "Job", "Status", "Time") + + +class WorkflowLogWidget(QWidget): + # Table of workflow job invocations with their captured output. + + def __init__(self, parent: QWidget | None = None) -> None: + super().__init__(parent) + + self._events: dict[int | None, list[WorkflowEvent]] = {} + self._iteration_chosen_by_user = False + + self._iteration_selector = QComboBox(self) + self._iteration_selector.currentIndexChanged.connect(self._on_iteration_changed) + self._iteration_selector.activated.connect(self._on_iteration_chosen_by_user) + + selector_row = QHBoxLayout() + selector_row.setSpacing(6) + selector_row.addWidget(QLabel("Iteration:")) + selector_row.addWidget(self._iteration_selector) + selector_row.addStretch() + + self._table = QTableWidget(0, len(_COLUMNS), self) + self._table.setHorizontalHeaderLabels(_COLUMNS) + vertical_header = self._table.verticalHeader() + assert vertical_header is not None + vertical_header.setVisible(False) + self._table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) + self._table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self._table.setSelectionMode(QAbstractItemView.SelectionMode.SingleSelection) + header = self._table.horizontalHeader() + assert header is not None + header.setSectionResizeMode(QHeaderView.ResizeMode.ResizeToContents) + header.setSectionResizeMode( + _COLUMNS.index("Job"), QHeaderView.ResizeMode.Stretch + ) + self._table.itemSelectionChanged.connect(self._on_row_selected) + + self._stdout_view = self._make_output_view() + self._stderr_view = self._make_output_view() + + detail = QSplitter(Qt.Orientation.Horizontal, self) + detail.addWidget(self._make_labelled_output("Stdout", self._stdout_view)) + detail.addWidget(self._make_labelled_output("Stderr", self._stderr_view)) + + splitter = QSplitter(Qt.Orientation.Vertical, self) + splitter.addWidget(self._table) + splitter.addWidget(detail) + splitter.setStretchFactor(0, 2) + splitter.setStretchFactor(1, 1) + + layout = QVBoxLayout(self) + layout.setContentsMargins(8, 8, 8, 8) + layout.setSpacing(8) + layout.addLayout(selector_row) + layout.addWidget(splitter) + + self._clear_detail() + + def _make_output_view(self) -> QPlainTextEdit: + view = QPlainTextEdit(self) + view.setReadOnly(True) + view.setLineWrapMode(QPlainTextEdit.LineWrapMode.NoWrap) + view.setFont(QFontDatabase.systemFont(QFontDatabase.SystemFont.FixedFont)) + return view + + def _make_labelled_output(self, title: str, view: QPlainTextEdit) -> QWidget: + container = QWidget(self) + layout = QVBoxLayout(container) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(2) + layout.addWidget(QLabel(title, container)) + layout.addWidget(view) + return container + + def add_event(self, event: WorkflowEvent) -> None: + group = self._group_key(event) + is_new_group = group not in self._events + self._events.setdefault(group, []).append(event) + + if is_new_group: + self._rebuild_iteration_selector() + elif group == self._selected_iteration(): + self._append_row(event) + + def _group_key(self, event: WorkflowEvent) -> int | None: + if event.hook in _EXPERIMENT_WIDE_HOOKS: + return None + return event.iteration + + def _selected_iteration(self) -> int | None: + index = self._iteration_selector.currentIndex() + if index < 0: + return None + return cast(int | None, self._iteration_selector.itemData(index)) + + def _rebuild_iteration_selector(self) -> None: + previously_selected = self._selected_iteration() + had_selection = self._iteration_selector.count() > 0 + + iterations: list[int | None] = [] + if None in self._events: + iterations.append(None) + iterations.extend(sorted(i for i in self._events if i is not None)) + + self._iteration_selector.blockSignals(True) + self._iteration_selector.clear() + for iteration in iterations: + label = ( + NO_ITERATION_LABEL if iteration is None else f"Iteration {iteration}" + ) + self._iteration_selector.addItem(label, iteration) + if ( + self._iteration_chosen_by_user + and had_selection + and previously_selected in iterations + ): + self._iteration_selector.setCurrentIndex( + iterations.index(previously_selected) + ) + else: + # Follow the newest iteration until the user picks one themselves. + self._iteration_selector.setCurrentIndex(len(iterations) - 1) + self._iteration_selector.blockSignals(False) + self._on_iteration_changed() + + def _on_iteration_chosen_by_user(self, _index: int) -> None: + self._iteration_chosen_by_user = True + + def _on_iteration_changed(self) -> None: + self._table.clearContents() + self._table.setRowCount(0) + for event in self._events.get(self._selected_iteration(), []): + self._append_row(event) + self._clear_detail() + + def _append_row(self, event: WorkflowEvent) -> None: + row = self._table.rowCount() + self._table.insertRow(row) + + job = event.job_name + if event.arguments: + job += f"({', '.join(event.arguments)})" + status, color = self._status_and_color(event) + + values = ( + event.hook, + event.workflow_name, + job, + status, + event.timestamp.strftime("%H:%M:%S"), + ) + for column, value in enumerate(values): + item = QTableWidgetItem(value) + if column == 3: + item.setBackground(color) + item.setForeground(QColor(0, 0, 0)) + self._table.setItem(row, column, item) + + def _status_and_color(self, event: WorkflowEvent) -> tuple[str, QColor]: + match event.status: + case WorkflowJobStatus.CANCELLED: + return "Cancelled", QColor(*COLOR_CANCELLED) + case WorkflowJobStatus.FAILED: + return "Failed", QColor(*COLOR_FAILED) + case WorkflowJobStatus.SUCCESS: + return "Succeeded", QColor(*COLOR_FINISHED) + + def _on_row_selected(self) -> None: + selection_model = self._table.selectionModel() + assert selection_model is not None + rows = selection_model.selectedRows() + if not rows: + self._clear_detail() + return + events = self._events.get(self._selected_iteration(), []) + row = rows[0].row() + if row >= len(events): + self._clear_detail() + return + event = events[row] + self._stdout_view.setPlainText(event.stdout or NO_OUTPUT_PLACEHOLDER) + self._stderr_view.setPlainText(event.stderr or NO_OUTPUT_PLACEHOLDER) + + def _clear_detail(self) -> None: + self._stdout_view.setPlainText(NO_OUTPUT_PLACEHOLDER) + self._stderr_view.setPlainText(NO_OUTPUT_PLACEHOLDER) + + def clear(self) -> None: + """Discard all displayed workflow events, e.g. before a rerun.""" + self._events = {} + self._iteration_chosen_by_user = False + self._iteration_selector.blockSignals(True) + self._iteration_selector.clear() + self._iteration_selector.blockSignals(False) + self._table.clearContents() + self._table.setRowCount(0) + self._clear_detail() diff --git a/tests/ert/ui_tests/gui/test_workflow_log_tab.py b/tests/ert/ui_tests/gui/test_workflow_log_tab.py new file mode 100644 index 00000000000..dc178f3a896 --- /dev/null +++ b/tests/ert/ui_tests/gui/test_workflow_log_tab.py @@ -0,0 +1,126 @@ +from pathlib import Path +from unittest.mock import Mock + +import pytest +from PyQt6.QtCore import Qt +from PyQt6.QtWidgets import QComboBox, QToolButton +from pytestqt.qtbot import QtBot + +from ert.config import ErtConfig +from ert.gui.experiments import ExperimentPanel, RunDialog +from ert.gui.experiments.ensemble_experiment_panel import EnsembleExperimentPanel +from ert.gui.experiments.view import WorkflowLogWidget +from ert.gui.main import GUILogHandler, _setup_main_window +from ert.run_models import EnsembleExperiment + +_FORWARD_MODEL_SCRIPT = "#!/bin/sh\nexit 0\n" + + +def _write_common_config_files() -> None: + forward_model_script = Path("forward_model.sh") + forward_model_script.write_text(_FORWARD_MODEL_SCRIPT, encoding="utf-8") + forward_model_script.chmod(0o755) + Path("FORWARD_MODEL_JOB").write_text( + "EXECUTABLE forward_model.sh\n", encoding="utf-8" + ) + + +def _run_experiment(qtbot: QtBot, config_file: str) -> RunDialog: + args_mock = Mock() + args_mock.config = config_file + ert_config = ErtConfig.from_file(config_file) + gui = _setup_main_window(ert_config, args_mock, GUILogHandler(), "storage") + qtbot.addWidget(gui) + + experiment_panel = gui.findChild(ExperimentPanel) + assert experiment_panel + simulation_mode_combo = experiment_panel.findChild(QComboBox) + assert simulation_mode_combo + simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) + simulation_settings = gui.findChild(EnsembleExperimentPanel) + assert simulation_settings + simulation_settings._experiment_name_field.setText("workflow_log_experiment") + + run_experiment = experiment_panel.findChild(QToolButton, name="run_experiment") + assert run_experiment + qtbot.mouseClick(run_experiment, Qt.MouseButton.LeftButton) + + qtbot.waitUntil(lambda: gui.findChild(RunDialog) is not None, timeout=10000) + run_dialog = gui.findChild(RunDialog) + assert run_dialog + qtbot.waitUntil(run_dialog.is_experiment_done, timeout=60000) + return run_dialog + + +def _workflow_log_widget(run_dialog: RunDialog) -> WorkflowLogWidget | None: + for i in range(run_dialog._tab_widget.count()): + widget = run_dialog._tab_widget.widget(i) + if isinstance(widget, WorkflowLogWidget): + return widget + return None + + +@pytest.mark.usefixtures("use_tmpdir") +@pytest.mark.timeout(120) +def test_that_a_hooked_workflows_printed_output_is_shown_in_the_workflows_tab( + qtbot: QtBot, +) -> None: + _write_common_config_files() + + workflow_script = Path("printing_workflow.py") + workflow_script.write_text( + "#!/usr/bin/env python3\n" + "import sys\n" + "print('greetings from the workflow')\n" + "print('a warning', file=sys.stderr)\n", + encoding="utf-8", + ) + workflow_script.chmod(0o755) + Path("PRINTING_WORKFLOW_JOB").write_text( + "EXECUTABLE printing_workflow.py\n", encoding="utf-8" + ) + + Path("config.ert").write_text( + """NUM_REALIZATIONS 1 +QUEUE_SYSTEM LOCAL +INSTALL_JOB forward_model FORWARD_MODEL_JOB +FORWARD_MODEL forward_model +LOAD_WORKFLOW_JOB PRINTING_WORKFLOW_JOB PRINTING_WORKFLOW +HOOK_WORKFLOW_JOB printing_job PRINTING_WORKFLOW POST_SIMULATION +""", + encoding="utf-8", + ) + + run_dialog = _run_experiment(qtbot, "config.ert") + + widget = _workflow_log_widget(run_dialog) + assert widget is not None, "no Workflows tab was added to the run dialog" + assert widget._table.rowCount() == 1 + assert widget._table.item(0, 0).text() == "POST_SIMULATION" + assert widget._table.item(0, 2).text() == "PRINTING_WORKFLOW" + assert widget._table.item(0, 3).text() == "Succeeded" + + widget._table.selectRow(0) + assert "greetings from the workflow" in widget._stdout_view.toPlainText() + assert "a warning" in widget._stderr_view.toPlainText() + + +@pytest.mark.usefixtures("use_tmpdir") +@pytest.mark.timeout(120) +def test_that_no_workflows_tab_is_added_when_the_experiment_has_no_workflows( + qtbot: QtBot, +) -> None: + _write_common_config_files() + + Path("config.ert").write_text( + """NUM_REALIZATIONS 1 +QUEUE_SYSTEM LOCAL +INSTALL_JOB forward_model FORWARD_MODEL_JOB +FORWARD_MODEL forward_model +""", + encoding="utf-8", + ) + + run_dialog = _run_experiment(qtbot, "config.ert") + + assert _workflow_log_widget(run_dialog) is None diff --git a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py index 9d47fda4fe2..9beb518c861 100644 --- a/tests/ert/unit_tests/gui/experiments/test_run_dialog.py +++ b/tests/ert/unit_tests/gui/experiments/test_run_dialog.py @@ -1,7 +1,9 @@ import tempfile +from datetime import UTC, datetime from pathlib import Path from queue import SimpleQueue from unittest.mock import MagicMock, Mock, patch +from uuid import uuid4 import pandas as pd import pytest @@ -40,6 +42,7 @@ ) from ert.gui.experiments.view.realization import RealizationWidget from ert.gui.experiments.view.runpath_progress_widget import RunpathProgressWidget +from ert.gui.experiments.view.workflow_log import WorkflowLogWidget from ert.gui.main import GUILogHandler, _setup_main_window from ert.gui.tools.file import FileDialog from ert.run_models import ( @@ -49,11 +52,14 @@ ) from ert.run_models.event import ( FinishedTotalRunPathCreationEvent, + RunModelUpdateBeginEvent, RunPathCreatedEvent, StartingTotalRunPathCreationEvent, + WorkflowEvent, ) from ert.run_models.run_model import RunModel from ert.scheduler.job import Job +from ert.workflow_runner import WorkflowJobStatus from tests.ert.handle_run_path_dialog import handle_run_path_dialog from tests.ert.ui_tests.gui.conftest import wait_for_child from tests.ert.utils import SnapshotBuilder @@ -1305,3 +1311,191 @@ def handle_dialog() -> None: .findChild(QLabel) .text() ) + + +def _stop_event_monitoring(qtbot: QtBot, dialog: RunDialog, queue: SimpleQueue) -> None: + """Let the dialog's queue worker exit so it does not outlive the test.""" + queue.put(EndEvent(failed=False, msg="")) + qtbot.waitUntil(dialog._worker_thread.isFinished, timeout=5000) + + +def _workflow_log_event( + *, + job_name: str = "my_job", + job_index: int = 0, + iteration: int | None = 0, + stdout: str = "hello", +) -> WorkflowEvent: + return WorkflowEvent( + run_id=uuid4(), + hook="POST_SIMULATION", + workflow_name="my_workflow", + job_name=job_name, + job_index=job_index, + arguments=[], + stdout=stdout, + stderr="", + status=WorkflowJobStatus.SUCCESS, + timestamp=datetime.now(tz=UTC), + iteration=iteration, + ) + + +def test_that_the_first_workflow_log_event_adds_a_workflows_tab(qtbot: QtBot) -> None: + queue: SimpleQueue = SimpleQueue() + mock_api = MagicMock() + mock_api.experiment_name = "test" + + dialog = RunDialog("Test", mock_api, queue, MagicMock()) + qtbot.addWidget(dialog) + dialog.setup_event_monitoring() + + assert dialog._tab_widget.count() == 0 + + queue.put(_workflow_log_event(job_name="first", stdout="workflow output")) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 1, timeout=2000) + + assert dialog._tab_widget.tabText(0) == "Workflows" + widget = dialog._tab_widget.widget(0) + assert isinstance(widget, WorkflowLogWidget) + qtbot.waitUntil(lambda: widget._table.rowCount() == 1, timeout=2000) + assert widget._table.item(0, 2).text() == "first" + + _stop_event_monitoring(qtbot, dialog, queue) + + +def test_that_further_workflow_log_events_reuse_the_same_workflows_tab( + qtbot: QtBot, +) -> None: + queue: SimpleQueue = SimpleQueue() + mock_api = MagicMock() + mock_api.experiment_name = "test" + + dialog = RunDialog("Test", mock_api, queue, MagicMock()) + qtbot.addWidget(dialog) + dialog.setup_event_monitoring() + + queue.put(_workflow_log_event(job_name="first")) + queue.put(_workflow_log_event(job_name="second", job_index=1)) + queue.put(_workflow_log_event(job_name="third", job_index=2, iteration=1)) + + widget = None + + def has_all_events() -> bool: + nonlocal widget + if dialog._tab_widget.count() != 1: + return False + widget = dialog._tab_widget.widget(0) + return ( + isinstance(widget, WorkflowLogWidget) + and widget._iteration_selector.count() == 2 + ) + + qtbot.waitUntil(has_all_events, timeout=2000) + assert dialog._tab_widget.count() == 1 + assert widget._table.rowCount() == 1 + assert widget._table.item(0, 2).text() == "third" + + _stop_event_monitoring(qtbot, dialog, queue) + + +def test_that_the_workflows_tab_does_not_steal_focus_from_the_current_tab( + qtbot: QtBot, +) -> None: + queue: SimpleQueue = SimpleQueue() + mock_api = MagicMock() + mock_api.experiment_name = "test" + + dialog = RunDialog("Test", mock_api, queue, MagicMock()) + qtbot.addWidget(dialog) + dialog.setup_event_monitoring() + + queue.put(StartingTotalRunPathCreationEvent(total_runpaths_to_create=1)) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 1, timeout=2000) + runpath_widget = dialog._tab_widget.widget(0) + + queue.put(_workflow_log_event()) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 2, timeout=2000) + + # The Workflows tab is inserted to the left, so the runpath widget + # shifts to index 1, but must remain the current tab. + assert dialog._tab_widget.currentWidget() is runpath_widget + + _stop_event_monitoring(qtbot, dialog, queue) + + +def test_that_new_iteration_and_update_tabs_do_not_steal_focus_from_the_workflows_tab( + qtbot: QtBot, +) -> None: + queue: SimpleQueue = SimpleQueue() + mock_api = MagicMock() + mock_api.experiment_name = "test" + + dialog = RunDialog("Test", mock_api, queue, MagicMock()) + qtbot.addWidget(dialog) + dialog.setup_event_monitoring() + + queue.put(_workflow_log_event()) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 1, timeout=2000) + workflows_widget = dialog._tab_widget.widget(0) + assert isinstance(workflows_widget, WorkflowLogWidget) + + # The user explicitly selects the Workflows tab. + dialog._tab_widget.setCurrentIndex(0) + assert dialog._tab_widget.currentWidget() is workflows_widget + + queue.put( + FullSnapshotEvent( + snapshot=SnapshotBuilder().build(["0"], state.REALIZATION_STATE_UNKNOWN), + iteration_label="Foo", + total_iterations=1, + progress=0.0, + realization_count=1, + status_count={"Unknown": 1}, + iteration=0, + ) + ) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 2, timeout=2000) + assert dialog._tab_widget.currentWidget() is workflows_widget + + queue.put(RunModelUpdateBeginEvent(iteration=0, run_id=uuid4())) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 3, timeout=2000) + assert dialog._tab_widget.currentWidget() is workflows_widget + + _stop_event_monitoring(qtbot, dialog, queue) + + +def test_that_rerunning_failed_realizations_clears_the_workflows_tab( + qtbot: QtBot, +) -> None: + queue: SimpleQueue = SimpleQueue() + mock_api = MagicMock() + mock_api.experiment_name = "test" + + dialog = RunDialog("Test", mock_api, queue, MagicMock()) + qtbot.addWidget(dialog) + dialog.setup_event_monitoring() + + queue.put(_workflow_log_event(job_name="from_first_run")) + qtbot.waitUntil(lambda: dialog._tab_widget.count() == 1, timeout=2000) + workflows_widget = dialog._tab_widget.widget(0) + assert isinstance(workflows_widget, WorkflowLogWidget) + qtbot.waitUntil(lambda: workflows_widget._table.rowCount() == 1, timeout=2000) + + _stop_event_monitoring(qtbot, dialog, queue) + + # Simulate the user clicking "Rerun failed simulations": the Workflows + # tab must not carry over rows from the previous run. + new_queue: SimpleQueue = SimpleQueue() + dialog._event_queue = new_queue + dialog.setup_event_monitoring(rerun_failed_realizations=True) + + assert dialog._tab_widget.widget(0) is workflows_widget + assert workflows_widget._table.rowCount() == 0 + assert workflows_widget._iteration_selector.count() == 0 + + new_queue.put(_workflow_log_event(job_name="from_second_run")) + qtbot.waitUntil(lambda: workflows_widget._table.rowCount() == 1, timeout=2000) + assert workflows_widget._table.item(0, 2).text() == "from_second_run" + + _stop_event_monitoring(qtbot, dialog, new_queue) diff --git a/tests/ert/unit_tests/gui/experiments/view/test_workflow_log_widget.py b/tests/ert/unit_tests/gui/experiments/view/test_workflow_log_widget.py new file mode 100644 index 00000000000..005d01cd232 --- /dev/null +++ b/tests/ert/unit_tests/gui/experiments/view/test_workflow_log_widget.py @@ -0,0 +1,257 @@ +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from ert.gui.experiments.view import WorkflowLogWidget +from ert.gui.experiments.view.workflow_log import ( + NO_ITERATION_LABEL, + NO_OUTPUT_PLACEHOLDER, +) +from ert.run_models.event import WorkflowEvent +from ert.workflow_runner import WorkflowJobStatus + +RUN_ID = uuid4() + + +def make_event( + *, + hook: str = "POST_SIMULATION", + workflow_name: str = "my_workflow", + job_name: str = "my_job", + job_index: int = 0, + arguments: list[str] | None = None, + stdout: str = "hello", + stderr: str = "", + status: WorkflowJobStatus = WorkflowJobStatus.SUCCESS, + iteration: int | None = 0, +) -> WorkflowEvent: + return WorkflowEvent( + run_id=RUN_ID, + hook=hook, + workflow_name=workflow_name, + job_name=job_name, + job_index=job_index, + arguments=arguments or [], + stdout=stdout, + stderr=stderr, + status=status, + timestamp=datetime(2024, 1, 1, 12, 30, 45, tzinfo=UTC), + iteration=iteration, + ) + + +@pytest.fixture +def widget(qtbot): + widget = WorkflowLogWidget() + qtbot.addWidget(widget) + return widget + + +def column_values(widget: WorkflowLogWidget, column: int) -> list[str]: + return [ + widget._table.item(row, column).text() + for row in range(widget._table.rowCount()) + ] + + +def select_row(widget: WorkflowLogWidget, row: int) -> None: + widget._table.selectRow(row) + + +def pick_iteration(widget: WorkflowLogWidget, index: int) -> None: + """Select an iteration the way a user would, via the combo box.""" + widget._iteration_selector.setCurrentIndex(index) + widget._iteration_selector.activated.emit(index) + + +def test_that_workflow_log_widget_starts_with_no_rows(widget): + assert widget._table.rowCount() == 0 + assert widget._iteration_selector.count() == 0 + + +def test_that_each_workflow_log_event_adds_one_table_row(widget): + widget.add_event(make_event(job_name="first")) + widget.add_event(make_event(job_name="second", job_index=1)) + + assert widget._table.rowCount() == 2 + assert column_values(widget, 2) == ["first", "second"] + + +def test_that_job_arguments_are_shown_next_to_the_job_name(widget): + widget.add_event(make_event(job_name="echo", arguments=["a", "b"])) + + assert column_values(widget, 2) == ["echo(a, b)"] + + +def test_that_iteration_selector_lists_every_iteration_seen_in_events(widget): + widget.add_event(make_event(iteration=1)) + widget.add_event(make_event(iteration=0)) + widget.add_event(make_event(iteration=1)) + + labels = [ + widget._iteration_selector.itemText(i) + for i in range(widget._iteration_selector.count()) + ] + assert labels == ["Iteration 0", "Iteration 1"] + + +def test_that_selecting_an_iteration_shows_only_that_iterations_jobs(widget): + widget.add_event(make_event(iteration=0, job_name="job_in_iter_0")) + widget.add_event(make_event(iteration=1, job_name="job_in_iter_1")) + + pick_iteration(widget, 0) + assert column_values(widget, 2) == ["job_in_iter_0"] + + pick_iteration(widget, 1) + + assert column_values(widget, 2) == ["job_in_iter_1"] + + +def test_that_events_arriving_for_the_selected_iteration_are_appended_live(widget): + widget.add_event(make_event(iteration=0, job_name="first")) + widget.add_event(make_event(iteration=1, job_name="other")) + pick_iteration(widget, 1) + + widget.add_event(make_event(iteration=1, job_name="second")) + + assert column_values(widget, 2) == ["other", "second"] + + +def test_that_a_new_iteration_is_selected_automatically_until_the_user_picks_one( + widget, +): + widget.add_event(make_event(iteration=0, job_name="job_in_iter_0")) + + widget.add_event(make_event(iteration=1, job_name="job_in_iter_1")) + + assert widget._iteration_selector.currentText() == "Iteration 1" + assert column_values(widget, 2) == ["job_in_iter_1"] + + +def test_that_a_new_iteration_does_not_override_an_iteration_picked_by_the_user(widget): + widget.add_event(make_event(iteration=0, job_name="job_in_iter_0")) + pick_iteration(widget, 0) + + widget.add_event(make_event(iteration=1, job_name="job_in_iter_1")) + + assert widget._iteration_selector.currentText() == "Iteration 0" + assert column_values(widget, 2) == ["job_in_iter_0"] + + +def test_that_events_without_an_iteration_are_listed_first_in_the_selector(widget): + widget.add_event(make_event(iteration=1, job_name="job_in_iter_1")) + widget.add_event( + make_event(iteration=None, hook="PRE_EXPERIMENT", job_name="early_job") + ) + + assert widget._iteration_selector.itemText(0) == NO_ITERATION_LABEL + assert widget._iteration_selector.itemData(0) is None + + +def test_that_selecting_the_pre_post_experiment_entry_shows_jobs_without_iteration( + widget, +): + widget.add_event(make_event(iteration=1, job_name="job_in_iter_1")) + widget.add_event( + make_event(iteration=None, hook="PRE_EXPERIMENT", job_name="early_job") + ) + + widget._iteration_selector.setCurrentIndex(0) + + assert column_values(widget, 0) == ["PRE_EXPERIMENT"] + assert column_values(widget, 2) == ["early_job"] + + +def test_that_post_experiment_events_are_grouped_as_pre_post_experiment_even_with_an_iteration( # ruff: ignore[line-too-long] + widget, +): + """POST_EXPERIMENT hooks run once for the whole experiment, but by then an + ensemble already exists, so the event carries a real iteration number. It + should still land under "Pre/post experiment", not under that iteration's + tab alongside PRE_SIMULATION/POST_SIMULATION jobs. + """ + widget.add_event( + make_event(iteration=0, hook="POST_SIMULATION", job_name="sim_job") + ) + widget.add_event( + make_event(iteration=0, hook="POST_EXPERIMENT", job_name="teardown_job") + ) + + labels = [ + widget._iteration_selector.itemText(i) + for i in range(widget._iteration_selector.count()) + ] + assert labels == [NO_ITERATION_LABEL, "Iteration 0"] + + widget._iteration_selector.setCurrentIndex(0) + assert column_values(widget, 2) == ["teardown_job"] + + widget._iteration_selector.setCurrentIndex(1) + assert column_values(widget, 2) == ["sim_job"] + + +def test_that_selecting_a_row_shows_that_jobs_stdout_and_stderr(widget): + widget.add_event(make_event(job_name="first", stdout="out 1", stderr="err 1")) + widget.add_event( + make_event(job_name="second", job_index=1, stdout="out 2", stderr="err 2") + ) + + select_row(widget, 1) + + assert widget._stdout_view.toPlainText() == "out 2" + assert widget._stderr_view.toPlainText() == "err 2" + + +def test_that_multiline_output_is_shown_in_full(widget): + output = "line one\nline two\nline three" + widget.add_event(make_event(stdout=output)) + + select_row(widget, 0) + + assert widget._stdout_view.toPlainText() == output + + +def test_that_a_failed_job_is_marked_failed(widget): + widget.add_event(make_event(job_name="ok", status=WorkflowJobStatus.SUCCESS)) + widget.add_event( + make_event(job_name="broken", job_index=1, status=WorkflowJobStatus.FAILED) + ) + + assert column_values(widget, 3) == ["Succeeded", "Failed"] + assert widget._table.item(0, 3).background().color() != ( + widget._table.item(1, 3).background().color() + ) + + +def test_that_a_cancelled_job_is_marked_cancelled_rather_than_succeeded(widget): + widget.add_event(make_event(job_name="ok", status=WorkflowJobStatus.SUCCESS)) + widget.add_event( + make_event(job_name="stopped", job_index=1, status=WorkflowJobStatus.CANCELLED) + ) + + assert column_values(widget, 3) == ["Succeeded", "Cancelled"] + assert widget._table.item(0, 3).background().color() != ( + widget._table.item(1, 3).background().color() + ) + + +def test_that_a_job_with_no_output_shows_a_no_output_placeholder(widget): + widget.add_event(make_event(stdout="", stderr="")) + + select_row(widget, 0) + + assert widget._stdout_view.toPlainText() == NO_OUTPUT_PLACEHOLDER + assert widget._stderr_view.toPlainText() == NO_OUTPUT_PLACEHOLDER + + +def test_that_switching_iteration_clears_the_previously_shown_output(widget): + widget.add_event(make_event(iteration=0, stdout="iteration zero output")) + widget.add_event(make_event(iteration=1, stdout="iteration one output")) + pick_iteration(widget, 0) + select_row(widget, 0) + assert widget._stdout_view.toPlainText() == "iteration zero output" + + pick_iteration(widget, 1) + + assert widget._stdout_view.toPlainText() == NO_OUTPUT_PLACEHOLDER