Skip to content

Remove references to hydra base, and add several updates #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
knoxsp wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Remove references to hydra base, and add several updates #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11e720c
Remove references to hydra base, and add several updates to work with…
Jul 10, 2025
e96f6e2
Remove domain as an argument
Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 1 addition & 12 deletions hydra_pywr/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from hydra_client.connection import RemoteJSONConnection

from hydra_client.click import hydra_app, make_plugins, write_plugins
from hydra_client.click import hydra_app

from . import runner
from . import exporter
Expand Down Expand Up @@ -132,7 +132,6 @@ def run(obj, scenario_id, template_id, domain, output_frequency, solver, data_di
runner.run_network_scenario(client,
scenario_id,
template_id,
domain,
output_frequency,
data_dir=data_dir)

Expand Down Expand Up @@ -197,16 +196,6 @@ def step_game(obj, scenario_id, child_scenario_ids, filename, attribute_name, in
utils.progress_start_end_dates(client, new_scenario_id)


@cli.command()
@click.pass_obj
@click.argument('docker-image', type=str, default=None)
def register(obj, docker_image):
""" Register the app with the Hydra installation. """
plugins = make_plugins(cli, 'hydra-pywr', docker_image=docker_image)
app_name = docker_image.replace('/', '-').replace(':', '-')
write_plugins(plugins, app_name)


@cli.group()
def template():
pass
Expand Down
16 changes: 13 additions & 3 deletions hydra_pywr/exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,9 +545,19 @@ def build_edges(self):

if hydra_edge["types"][0]["name"].lower() == "slottededge":
for slot in ("src_slot", "dest_slot"):
slot_id = [attr.id for attr in hydra_edge["attributes"] if attr.name == slot][0]
slot_ds = self.get_dataset_by_resource_attr_id(slot_id)
verts.append(slot_ds.value if slot_ds else None)
slot_id = [attr.id for attr in hydra_edge["attributes"] if attr.name == slot]
if not slot_id:
log.warning(f"Edge {hydra_edge['name']} missing slot attribute {slot}")
verts.append(None)
continue
slot_ds = self.get_dataset_by_resource_attr_id(slot_id[0])
if slot_ds:
val = slot_ds.value
if val == "None":
val = None
verts.append(val)
else:
verts.append(None)

edge = PywrEdge(verts)
edges.append(edge)
Expand Down
56 changes: 37 additions & 19 deletions hydra_pywr/importer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

from pywrparser.types.network import PywrNetwork

from hydra_base.lib.objects import JSONObject

import logging
log = logging.getLogger(__name__)
Expand Down Expand Up @@ -58,8 +57,8 @@ def import_json(client, filename, project_id, template_id, network_name, *args,
pnet.add_recorder_references()
pnet.promote_inline_parameters()
pnet.promote_inline_recorders()
pnet.attach_reference_parameters()
pnet.attach_reference_recorders()
# pnet.attach_reference_parameters()
# pnet.attach_reference_recorders()
#pnet.detach_parameters()


Expand Down Expand Up @@ -132,7 +131,13 @@ def get_hydra_attrid_by_name(self, attr_name):
for attr in self.hydra_attributes:
if attr["name"].lower() == attr_name.lower():
return attr["id"]
log.critical(f"Attr {attr_name} not registered")

log.warning(f"Attr {attr_name} not registered. Attempting to register...")

attr = self.hydra.add_attribute(attr={'name': attr_name})
self.hydra_attributes.append(attr)

return attr['id']

def get_next_node_id(self):
self._next_node_id -= 1
Expand All @@ -150,6 +155,8 @@ def get_node_by_name(self, name):
for node in self.hydra_nodes:
if node["name"] == name:
return node
else:
raise ValueError(f"Node with name '{name}' not found in the model.")

def make_hydra_attr(self, name, desc=None):
return { "name": name,
Expand Down Expand Up @@ -205,7 +212,7 @@ def build_hydra_network(self, projection=None, appdata={}):

""" Assemble complete network """
network_name = self.network.metadata.data["title"]
network_description = self.network.metadata.data["description"]
network_description = self.network.metadata.data.get("description", "")
self.network_hydratype = self.get_hydra_network_type()

self.hydra_network = {
Expand Down Expand Up @@ -342,7 +349,9 @@ def make_direct_resource_attr_and_scenario(self, value, attr_name, hydra_datatyp


def make_typed_resource_scenario(self, element, attr_name, local_attr_id):

hydra_datatype = self.lookup_hydra_datatype(element)

dataset = { "name": attr_name,
"type": hydra_datatype,
"value": element.as_json(),
Expand All @@ -361,6 +370,7 @@ def make_typed_resource_scenario(self, element, attr_name, local_attr_id):
def make_network_resource_scenario(self, element, attr_name, local_attr_id):

value = element.data[attr_name]

hydra_datatype = self.lookup_hydra_datatype(value)

dataset = { "name": attr_name,
Expand Down Expand Up @@ -399,6 +409,9 @@ def make_paramrec_resource_scenario(self, element, attr_name, local_attr_id):
def make_resource_scenario(self, element, attr_name, local_attr_id):

value = element.data[attr_name]
if value is None:
return None

hydra_datatype = self.lookup_hydra_datatype(value)

dataset = { "name": attr_name,
Expand All @@ -418,12 +431,18 @@ def make_resource_scenario(self, element, attr_name, local_attr_id):


def lookup_hydra_datatype(self, attr_value):
if isinstance(attr_value, Number):

if attr_value is None:
return None
elif isinstance(attr_value, Number):
return "SCALAR"
elif isinstance(attr_value, list):
return "ARRAY"
elif isinstance(attr_value, dict):
if 'index' in attr_value and 'table' in attr_value:
return "DESCRIPTOR"
Comment on lines 441 to +443

@pmslavin pmslavin Aug 27, 2025

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Certain types of dict datasets are classed as 'descriptor' depending on their keys?

return "DATAFRAME"

elif isinstance(attr_value, str):
return "DESCRIPTOR"
elif isinstance(attr_value, PywrTable):
Expand All @@ -436,7 +455,6 @@ def lookup_hydra_datatype(self, attr_value):
return lookup_parameter_hydra_datatype(attr_value)
elif isinstance(attr_value, PywrRecorder):
return lookup_recorder_hydra_datatype(attr_value)

raise ValueError(f"Unknown data type: '{attr_value}'")


Expand All @@ -456,7 +474,8 @@ def build_hydra_nodes(self):
if ra["attr_id"] == None:
raise ValueError(f"Node '{node.name}' attr '{attr_name}' has invalid attr id: \'{ra['attr_id']}\'")
resource_attributes.append(ra)
resource_scenarios.append(rs)
if rs is not None:
resource_scenarios.append(rs)

hydra_node = {}
hydra_node["resource_type"] = "NODE"
Expand All @@ -471,15 +490,15 @@ def build_hydra_nodes(self):
}]
if "position" in node.data:
proj_data = node.data["position"]
for coords in proj_data.values():
if "geographic" in coords:
coords = coords["geographic"]
elif "schematic" in coords:
coords = coords["schematic"]
if isinstance(coords, list):
x, y = coords[0], coords[1]
elif isinstance(coords, dict):
hydra_node['layout']['geojson'] = coords
if "geographic" in proj_data:
coords = proj_data["geographic"]
elif "schematic" in proj_data:
coords = proj_data["schematic"]

if isinstance(coords, list):
x, y = coords[0], coords[1]
elif isinstance(coords, dict):
hydra_node['layout']['geojson'] = coords
hydra_node["x"] = x
hydra_node["y"] = y
else:
Expand Down Expand Up @@ -563,8 +582,7 @@ def build_parameters_recorders(self):

def add_network_to_hydra(self):
""" Pass network to Hydra"""
network = JSONObject(self.hydra_network)
network_summary = self.hydra.add_network({"net": network})
network_summary = self.hydra.add_network({"net": self.hydra_network})
return network_summary


Expand Down
24 changes: 12 additions & 12 deletions hydra_pywr/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,13 @@ def run_file(filename, domain, output_file):
pfr.run_pywr_model(output_file)


def run_network_scenario(client, scenario_id, template_id, domain,
solver=None, data_dir='/tmp'):
def run_network_scenario(client, scenario_id, template_id,
solver=None, data_dir='/tmp', use_cache=False, **kwargs):
Comment on lines +41 to +42

@pmslavin pmslavin Aug 27, 2025

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do we set use_cache to True when we want to use it? Only the export_json cli command takes a --use-cache arg.


runner = PywrHydraRunner.from_scenario_id(client, scenario_id,
template_id=template_id,
data_dir=data_dir)
data_dir=data_dir,
use_cache=use_cache)
runner.setup(solver=solver)
runner.run_pywr_model()
runner.save_pywr_results()
Expand Down Expand Up @@ -91,7 +92,7 @@ def load_pywr_model_from_file(self, filename, solver=None):
if warnings:
for component, warns in warnings.items():
for warn in warns:
log.info(warn)
log.warning(warn)

if errors:
for component, errs in errors.items():
Expand Down Expand Up @@ -531,21 +532,15 @@ def _add_node_flagged_recorders(self, model):
name = '__{}__:{}'.format(node.name, 'simulated_volume')
NumpyArrayStorageRecorder(model, node, name=name)
else:
import warnings
warnings.warn('Unrecognised node subclass "{}" with name "{}" for timeseries recording. Skipping '
'recording this node.'.format(node.__class__.__name__, node.name),
RuntimeWarning)
log.warning(f'Unrecognised node subclass "{node.__class__.__name__}" with name "{node.name}" for timeseries recording. Skipping recording this node.')

elif flag == 'deficit':
if isinstance(node, Node):
deficit_parameter = DeficitParameter(model, node)
name = '__{}__:{}'.format(node.name, 'simulated_deficit')
NumpyArrayParameterRecorder(model, deficit_parameter, name=name)
else:
import warnings
warnings.warn('Unrecognised node subclass "{}" with name "{}" for deficit recording. Skipping '
'recording this node.'.format(node.__class__.__name__, node.name),
RuntimeWarning)
log.warning(f'Unrecognised node subclass "{node.__class__.__name__}" with name "{node.name}" for deficit recording. Skipping recording this node.')

def _add_parameter_flagged_recorders(self, model):
for parameter_name, flags in self._parameter_recorder_flags.items():
Expand Down Expand Up @@ -774,6 +769,10 @@ def generate_array_recorder_resource_scenarios(self):
except NotImplementedError:
continue

if len(df.columns) == 0:
log.warning(f"Recorder {recorder.name} has no data to save.")
continue

columns = []
for name in df.columns.names:
columns.append([f'{name}: {v}' for v in df.columns.get_level_values(name)])
Expand All @@ -792,6 +791,7 @@ def generate_array_recorder_resource_scenarios(self):
new_col_names.append(colname.split(':')[1].strip())
df.columns = new_col_names


if "__:" in recorder.name:
try:
nodename, attrname = parse_reference_key(recorder.name)
Expand Down