diff --git a/docs/reference/dynamic_scenarios.rst b/docs/reference/dynamic_scenarios.rst index ad4512912..524570350 100644 --- a/docs/reference/dynamic_scenarios.rst +++ b/docs/reference/dynamic_scenarios.rst @@ -33,12 +33,12 @@ In detail, a single time step of a dynamic simulation is executed according to t If the block executes a :keyword:`require` statement with a false condition, reject the simulation. If it executes :keyword:`terminate` or :keyword:`terminate simulation`, or finishes executing, go to step (e) below to stop the scenario. - e. If the scenario is stopping for one of the reasons above, first recursively stop any sub-scenarios it is running, then revert the effects of any :keyword:`override` statements it executed. + e. If the scenario is stopping for one of the reasons above, save the values of any :keyword:`record final` statements in the scenario, recursively stop any sub-scenarios it is running, then revert the effects of any :keyword:`override` statements it executed. Next, check if any of its :term:`temporal requirements` were not satisfied: if so, reject the simulation. Otherwise, the scenario returns to its parent scenario if it was invoked using :keyword:`do`; if it was the top-level scenario, or if it executed :keyword:`terminate simulation`, we set a flag indicating the top-level scenario has terminated. (We do not terminate immediately since we still need to check monitors in the next step.) -2. Save the values of all :keyword:`record` statements, as well as :keyword:`record initial` statements if it is time step 0. +2. Save the values of all :keyword:`record` statements in currently-running scenarios, as well as :keyword:`record initial` statements for scenarios which have just started. 3. Run each :term:`monitor` instantiated in the currently-running scenarios for one time step (i.e. resume it until it executes :keyword:`wait`). If it executes a :keyword:`require` statement with a false condition, reject the simulation. @@ -66,8 +66,7 @@ In detail, a single time step of a dynamic simulation is executed according to t 9. Update every :term:`dynamic property` of every object to its current value in the simulator. -10. If the simulation is stopping for one of the reasons above, first check if any of the :term:`temporal requirements` of any remaining scenarios were not satisfied: if so, reject the simulation. - Otherwise, save the values of any :keyword:`record final` statements. +10. If the simulation is stopping for one of the reasons above, stop any remaining scenarios as in step (1e) above (including checking :term:`temporal requirements` and saving the values of :keyword:`record final` statements). .. rubric:: Footnotes diff --git a/docs/reference/statements.rst b/docs/reference/statements.rst index aa1de02c0..97a8e5c02 100644 --- a/docs/reference/statements.rst +++ b/docs/reference/statements.rst @@ -285,7 +285,7 @@ The default mutation system adds Gaussian noise to the :prop:`position` and :pro record [initial | final] *value* [as *name*] ---------------------------------------------- Record the value of an expression during each simulation. -The value can be recorded at the start of the simulation (``initial``), at the end of the simulation (``final``), or at every time step (if neither ``initial`` nor ``final`` is specified). +The value can be recorded at the start of the scenario (``initial``), at the end of the scenario (``final``), or at every time step during the scenario (if neither ``initial`` nor ``final`` is specified). The recorded values are available in the ``records`` dictionary of `SimulationResult`: its keys are the given names of the records (or synthesized names if not provided), and the corresponding values are either the value of the recorded expression or a tuple giving its value at each time step as appropriate. For debugging, the records can also be printed out using the :option:`--show-records` command-line option. diff --git a/src/scenic/core/dynamics/scenarios.py b/src/scenic/core/dynamics/scenarios.py index 8aa1021b5..61ee09254 100644 --- a/src/scenic/core/dynamics/scenarios.py +++ b/src/scenic/core/dynamics/scenarios.py @@ -88,6 +88,7 @@ def __init__(self, *args, **kwargs): self._timeLimitInSteps = None # computed at simulation time self._elapsedTime = 0 + self._recordedTime = None self._eventuallySatisfied = None self._overrides = {} @@ -215,10 +216,13 @@ def _start(self): # Prepare recorders simName = veneer.currentSimulation.name + currentTime = veneer.currentSimulation.currentTime globalParams = types.MappingProxyType(veneer._globalParameters) for req in self._recordedExprs: if (recConfig := req.recConfig) and (recorder := recConfig.recorder): - recorder.beginRecording(recConfig, simName, timestep, globalParams) + recorder.beginRecording( + recConfig, simName, timestep, globalParams, currentTime + ) def _step(self): """Execute the (already-started) scenario for one time step. @@ -294,6 +298,15 @@ def _stop(self, reason, quiet=False): assert self._isRunning + if not quiet: + # Record finally-recorded values. + sim = veneer.currentSimulation + for rec in self._recordedFinalExprs: + sim._record(rec.name, rec.evaluate()) + + # Record ordinary `record` statements too if they haven't been already. + self._recordTimeSeries() + # Stop monitors and subscenarios. for monitor in self._monitors: if monitor._isRunning: @@ -356,28 +369,37 @@ def _invokeInner(self, agent, subs): # Check if any sub-scenarios stopped during action execution self._subScenarios = [sub for sub in self._subScenarios if sub._isRunning] - def _evaluateRecordedExprs(self, ty, step): - if ty is RequirementType.record: - place = "_recordedExprs" - elif ty is RequirementType.recordInitial: - place = "_recordedInitialExprs" - elif ty is RequirementType.recordFinal: - place = "_recordedFinalExprs" - else: - assert False, "invalid record type requested" - return self._evaluateRecordedExprsAt(place, step) + def _updateRecords(self): + from scenic.syntax.veneer import currentSimulation + + # _step() was called earlier this time step, so at time step 0 we will + # already have _elapsedTime == 1 + assert self._elapsedTime >= 1 + if self._elapsedTime == 1: + for rec in self._recordedInitialExprs: + currentSimulation._record(rec.name, rec.evaluate()) + + self._recordTimeSeries() - def _evaluateRecordedExprsAt(self, place, step): - values = {} - for rec in getattr(self, place): + for sub in self._subScenarios: + sub._updateRecords() + + def _recordTimeSeries(self): + from scenic.syntax.veneer import currentSimulation + + currentTime = currentSimulation.currentTime + if self._recordedTime == currentTime: + # This time step was already recorded (e.g. the scenario was terminated + # by a behavior after the current state was recorded). + return + + for rec in self._recordedExprs: value = rec.evaluate() - values[rec.name] = value + currentSimulation._recordTimeSeries(rec.name, value) if (recConfig := rec.recConfig) and (recorder := recConfig.recorder): - recorder._record(value, step) - for sub in self._subScenarios: - subvals = sub._evaluateRecordedExprsAt(place, step) - values.update(subvals) - return values + recorder._record(value, currentTime) + + self._recordedTime = currentTime def _runMonitors(self): terminationReason = None diff --git a/src/scenic/core/requirements.py b/src/scenic/core/requirements.py index 9c29b12e8..f816d429b 100644 --- a/src/scenic/core/requirements.py +++ b/src/scenic/core/requirements.py @@ -15,6 +15,7 @@ from scenic.core.errors import InvalidScenarioError from scenic.core.lazy_eval import needsLazyEvaluation from scenic.core.propositions import Atomic, PropositionNode +from scenic.core.utils import DefaultIdentityDict import scenic.syntax.relations as relations @@ -458,6 +459,10 @@ def falsifiedByInner(self, sample): one_time_monitor = self.proposition.create_monitor() return self.closure(sample, one_time_monitor) == rv_ltl.B4.FALSE + def evaluate(self): + # Used only for `terminate when`, etc. defined in setup blocks of subscenarios + return self.closure(DefaultIdentityDict()) + def __str__(self): if self.name: return self.name diff --git a/src/scenic/core/sensors.py b/src/scenic/core/sensors.py index db15f27df..b11f0b7ae 100644 --- a/src/scenic/core/sensors.py +++ b/src/scenic/core/sensors.py @@ -101,7 +101,7 @@ class Recorder: def __init__(self): self._recording = False - def beginRecording(self, config, simulationName, timestep, globalParams): + def beginRecording(self, config, simulationName, timestep, globalParams, currentTime): assert not self._recording self._recording = True self.simulationName = simulationName @@ -121,9 +121,10 @@ def beginRecording(self, config, simulationName, timestep, globalParams): assert val >= 0, val if unit == "steps": assert isinstance(val, int), val - self._delay = val + delay = val else: # unit == "seconds" - self._delay = max(0, math.floor(val / timestep)) + delay = max(0, math.floor(val / timestep)) + self._startTime = currentTime + delay def recordValue(self, value, step): raise NotImplementedError @@ -133,7 +134,8 @@ def endRecording(self, canceled): self._recording = False def _record(self, value, step): - if step >= self._delay and step % self._period == 0: + relativeTime = step - self._startTime + if relativeTime >= 0 and relativeTime % self._period == 0: self.recordValue(np.asarray(value), step) @staticmethod @@ -264,7 +266,7 @@ def videoHandler(path, values, timestep, options): @fileHandler("npz") def npzHandler(path, values, timestep, options): - timesteps, values = zip(*values) + timesteps, values = zip(*values) if values else ([], []) np.savez_compressed(path, timesteps=timesteps, values=values) diff --git a/src/scenic/core/simulators.py b/src/scenic/core/simulators.py index 3e9c0308f..9923e3d63 100644 --- a/src/scenic/core/simulators.py +++ b/src/scenic/core/simulators.py @@ -379,18 +379,11 @@ def __init__( # Run the simulation. terminationType, terminationReason = self._run(dynamicScenario, maxSteps) - # Stop all remaining scenarios. - # (and reject if some 'require eventually' condition was never satisfied) + # Stop all remaining scenarios (and handle their `record final` statements; + # also reject if some `require eventually` condition was never satisfied). for scenario in tuple(reversed(veneer.runningScenarios)): scenario._stop("simulation terminated") - # Record finally-recorded values. - values = dynamicScenario._evaluateRecordedExprs( - RequirementType.recordFinal, self.currentTime - ) - for name, val in values.items(): - self.records[name] = val - # Package up simulation results into a compact object. result = SimulationResult( self.trajectory, @@ -442,7 +435,7 @@ def _run(self, dynamicScenario, maxSteps): ) # Record current state of the simulation - self.recordCurrentState() + self._recordCurrentState() # Run monitors newReason = dynamicScenario._runMonitors() @@ -596,25 +589,18 @@ def createObjectInSimulator(self, obj): """ raise NotImplementedError - def recordCurrentState(self): - dynamicScenario = self.scene.dynamicScenario - records = self.records + def _recordCurrentState(self): + # Record values of `record initial` and `record` statements. + # (calls _record and _recordTimeSeries below) + self.scene.dynamicScenario._updateRecords() - # Record initially-recorded values - step = self.currentTime - if step == 0: - values = dynamicScenario._evaluateRecordedExprs( - RequirementType.recordInitial, step - ) - for name, val in values.items(): - records[name] = val + self.trajectory.append(self.currentState()) - # Record time-series values - values = dynamicScenario._evaluateRecordedExprs(RequirementType.record, step) - for name, val in values.items(): - records[name].append((self.currentTime, val)) + def _record(self, name, value): + self.records[name] = value - self.trajectory.append(self.currentState()) + def _recordTimeSeries(self, name, value): + self.records[name].append((self.currentTime, value)) def replayCanContinue(self): if not self.replaying: diff --git a/src/scenic/syntax/veneer.py b/src/scenic/syntax/veneer.py index b79745030..cb905e5b8 100644 --- a/src/scenic/syntax/veneer.py +++ b/src/scenic/syntax/veneer.py @@ -531,31 +531,27 @@ def executeInRequirement(scenario, boundEgo, values): assert activity == 0 assert not evaluatingRequirement evaluatingRequirement = True - if currentScenario is None: - currentScenario = scenario - clearScenario = True - else: - assert currentScenario is scenario - clearScenario = False - oldEgo = currentScenario._ego - oldObjects = currentScenario._objects - currentScenario._objects = tuple(values[obj] for obj in currentScenario.objects) + with executeInScenario(scenario): + oldEgo = scenario._ego + oldObjects = scenario._objects - if boundEgo: - currentScenario._ego = boundEgo - try: - yield - except RandomControlFlowError as e: - # Such errors should not be possible inside a requirement, since all values - # should have already been sampled: something's gone wrong with our rebinding. - raise RuntimeError("internal error: requirement dependency not sampled") from e - finally: - evaluatingRequirement = False - currentScenario._ego = oldEgo - currentScenario._objects = oldObjects - if clearScenario: - currentScenario = None + scenario._objects = tuple(values[obj] for obj in scenario.objects) + + if boundEgo: + scenario._ego = boundEgo + try: + yield + except RandomControlFlowError as e: + # Such errors should not be possible inside a requirement, since all values + # should have already been sampled: something's gone wrong with our rebinding. + raise AssertionError( + "internal error: requirement dependency not sampled" + ) from e + finally: + evaluatingRequirement = False + scenario._ego = oldEgo + scenario._objects = oldObjects # Dynamic scenarios @@ -837,22 +833,6 @@ def record_final(reqID, value, line, name): makeRequirement(requirements.RequirementType.recordFinal, reqID, value, line, name) -def require_always(reqID, req, line, name): - """Function implementing the 'require always' statement.""" - if not name: - name = f"requirement on line {line}" - makeRequirement(requirements.RequirementType.requireAlways, reqID, req, line, name) - - -def require_eventually(reqID, req, line, name): - """Function implementing the 'require eventually' statement.""" - if not name: - name = f"requirement on line {line}" - makeRequirement( - requirements.RequirementType.requireEventually, reqID, req, line, name - ) - - def terminate_when(reqID, req, line, name): """Function implementing the 'terminate when' statement.""" if not name: @@ -874,9 +854,7 @@ def makeRequirement(ty, reqID, req, line, name, recConfig=None): raise InvalidScenarioError(f'tried to use "{ty.value}" inside a requirement') elif currentBehavior is not None: raise InvalidScenarioError(f'"{ty.value}" inside a behavior on line {line}') - elif currentSimulation is not None: - currentScenario._addDynamicRequirement(ty, req, line, name) - else: # requirement being defined at compile time + else: currentScenario._addRequirement(ty, reqID, req, line, name, 1, recConfig) diff --git a/tests/syntax/test_dynamics.py b/tests/syntax/test_dynamics.py index 4699a2921..11d3ef61d 100644 --- a/tests/syntax/test_dynamics.py +++ b/tests/syntax/test_dynamics.py @@ -2223,6 +2223,7 @@ def test_termination_reason_monitor(): ## Recording +# (see also `test_recording.py`) def test_record(): diff --git a/tests/syntax/test_modular.py b/tests/syntax/test_modular.py index c77a54fce..21f1a3425 100644 --- a/tests/syntax/test_modular.py +++ b/tests/syntax/test_modular.py @@ -572,6 +572,24 @@ def test_subscenario_require_eventually(): assert result is None +def test_subscenario_require_eventually_2(): + """Variant of the above test using `terminate when` instead of `terminate after`.""" + scenario = compileScenic( + """ + scenario Main(): + compose: + do Sub() + wait + scenario Sub(): + ego = new Object + require eventually simulation().currentTime == 2 + terminate when simulation().currentTime == 1 + """ + ) + result = sampleResultOnce(scenario, maxSteps=2) + assert result is None + + def test_subscenario_require_monitor(): """Test that monitors invoked in subscenarios terminate with the subscenario.""" scenario = compileScenic( @@ -595,8 +613,42 @@ def test_subscenario_require_monitor(): assert len(result.trajectory) == 4 +def test_subscenario_record(): + scenario = compileScenic( + """ + scenario Main(): + setup: + record initial simulation().currentTime as mainInitial + record final simulation().currentTime as mainFinal + record simulation().currentTime as mainTime + compose: + wait for 2 steps + do Sub() + wait + scenario Sub(): + ego = new Object + record initial -simulation().currentTime as subInitial + record final -simulation().currentTime as subFinal + record -simulation().currentTime as subNegTime + terminate after 2 steps + """ + ) + result = sampleResult(scenario, maxSteps=5) + records = result.records + assert records["mainInitial"] == 0 + assert records["mainFinal"] == 5 + assert tuple(records["mainTime"]) == ((0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)) + assert records["subInitial"] == -2 + assert records["subFinal"] == -4 + assert tuple(records["subNegTime"]) == ((2, -2), (3, -3), (4, -4)) + + def test_subscenario_terminate_when(): - """Test that 'terminate when' and 'require' are properly handled.""" + """Test that 'terminate when' is properly handled. + + In particular, this catches a bug where `terminate when` in a subscenario was + interpreted as defining a requirement instead of a termination condition. + """ scenario = compileScenic( """ scenario Main(): @@ -605,12 +657,12 @@ def test_subscenario_terminate_when(): wait scenario Sub(): ego = new Object - require eventually simulation().currentTime == 2 terminate when simulation().currentTime == 1 """ ) - result = sampleResultOnce(scenario, maxSteps=2) - assert result is None + result = sampleResultOnce(scenario, maxSteps=3) + assert result is not None + assert len(result.trajectory) == 3 def test_subscenario_terminate_with_parent(): diff --git a/tests/syntax/test_recording.py b/tests/syntax/test_recording.py new file mode 100644 index 000000000..4c03cff60 --- /dev/null +++ b/tests/syntax/test_recording.py @@ -0,0 +1,110 @@ +"""Tests for advanced usages of the `record` statement.""" + +import numpy as np + +from tests.utils import compileScenic, sampleResult + +## Utilities + + +def checkRecordTo(tmp_path, period=None, delay=None, maxSteps=5): + """Helper for testing the `record ... to ...` statement. + + Returns the timesteps at which a value was recorded. + """ + + # Clear out the folder in case the helper is used multiple times in a test + for f in tmp_path.iterdir(): + f.unlink() + + every = f"every {period}" if period else "" + after = f"after {delay}" if delay else "" + scenario = compileScenic( + f""" + record -simulation().currentTime {every} {after} to "value_{{step}}.npy" + record -simulation().currentTime {every} {after} to "series.npz" + """, + params=dict(recordFolder=tmp_path), + ) + result = sampleResult(scenario, maxSteps=maxSteps) + assert result is not None + + recordedTimes = [] + for t in range(maxSteps + 1): + path = tmp_path / f"value_{t}.npy" + if path.exists(): + value = np.load(path) + assert float(value) == -t + recordedTimes.append(t) + + series = np.load(tmp_path / "series.npz") + assert np.array_equal(series["timesteps"], recordedTimes) + assert np.array_equal(series["values"], [-t for t in recordedTimes]) + + return recordedTimes + + +## Recording to files + + +def test_record_to(tmp_path): + times = checkRecordTo(tmp_path) + assert times == [0, 1, 2, 3, 4, 5] + + +def test_record_to_after(tmp_path): + times = checkRecordTo(tmp_path, delay="2 steps") + assert times == [2, 3, 4, 5] + + times = checkRecordTo(tmp_path, delay="3.5 seconds") + assert times == [3, 4, 5] + + times = checkRecordTo(tmp_path, delay="10 steps") + assert times == [] + + +def test_record_to_every(tmp_path): + times = checkRecordTo(tmp_path, period="2 steps") + assert times == [0, 2, 4] + + times = checkRecordTo(tmp_path, period="3.5 seconds") + assert times == [0, 3] + + times = checkRecordTo(tmp_path, period="10 steps") + assert times == [0] + + +def test_record_to_in_subscenario(tmp_path): + scenario = compileScenic( + """ + scenario Main(): + compose: + wait for 2 steps + do Sub(1) + wait for 2 steps + do Sub(2) + wait + scenario Sub(i): + record simulation().currentTime to "value_{step}.npy" + record simulation().currentTime to f"series{i}.npz" + terminate after 2 steps + """, + params=dict(recordFolder=tmp_path), + ) + result = sampleResult(scenario, maxSteps=10) + assert result is not None + + for t in range(11): + path = tmp_path / f"value_{t}.npy" + if 2 <= t <= 4 or 6 <= t <= 8: + value = np.load(path) + assert float(value) == t + else: + assert not path.exists(), t + + series1 = np.load(tmp_path / "series1.npz") + assert np.array_equal(series1["timesteps"], [2, 3, 4]) + assert np.array_equal(series1["values"], [2, 3, 4]) + series2 = np.load(tmp_path / "series2.npz") + assert np.array_equal(series2["timesteps"], [6, 7, 8]) + assert np.array_equal(series2["values"], [6, 7, 8])