diff --git a/hydra_pywr/cli.py b/hydra_pywr/cli.py index 9a21e0a..a273eb6 100644 --- a/hydra_pywr/cli.py +++ b/hydra_pywr/cli.py @@ -3,7 +3,7 @@ from hydra_client.connection import RemoteJSONConnection -from hydra_client.click import hydra_app, make_plugins, write_plugins +from hydra_client.click import hydra_app from . import runner from . import exporter @@ -132,7 +132,6 @@ def run(obj, scenario_id, template_id, domain, output_frequency, solver, data_di runner.run_network_scenario(client, scenario_id, template_id, - domain, output_frequency, data_dir=data_dir) @@ -197,16 +196,6 @@ def step_game(obj, scenario_id, child_scenario_ids, filename, attribute_name, in utils.progress_start_end_dates(client, new_scenario_id) -@cli.command() -@click.pass_obj -@click.argument('docker-image', type=str, default=None) -def register(obj, docker_image): - """ Register the app with the Hydra installation. """ - plugins = make_plugins(cli, 'hydra-pywr', docker_image=docker_image) - app_name = docker_image.replace('/', '-').replace(':', '-') - write_plugins(plugins, app_name) - - @cli.group() def template(): pass diff --git a/hydra_pywr/exporter.py b/hydra_pywr/exporter.py index 0f160c5..50b4099 100644 --- a/hydra_pywr/exporter.py +++ b/hydra_pywr/exporter.py @@ -545,9 +545,19 @@ def build_edges(self): if hydra_edge["types"][0]["name"].lower() == "slottededge": for slot in ("src_slot", "dest_slot"): - slot_id = [attr.id for attr in hydra_edge["attributes"] if attr.name == slot][0] - slot_ds = self.get_dataset_by_resource_attr_id(slot_id) - verts.append(slot_ds.value if slot_ds else None) + slot_id = [attr.id for attr in hydra_edge["attributes"] if attr.name == slot] + if not slot_id: + log.warning(f"Edge {hydra_edge['name']} missing slot attribute {slot}") + verts.append(None) + continue + slot_ds = self.get_dataset_by_resource_attr_id(slot_id[0]) + if slot_ds: + val = slot_ds.value + if val == "None": + val = None + verts.append(val) + else: + verts.append(None) edge = PywrEdge(verts) edges.append(edge) diff --git a/hydra_pywr/importer.py b/hydra_pywr/importer.py index c26c6e3..f96ac33 100644 --- a/hydra_pywr/importer.py +++ b/hydra_pywr/importer.py @@ -19,7 +19,6 @@ from pywrparser.types.network import PywrNetwork -from hydra_base.lib.objects import JSONObject import logging log = logging.getLogger(__name__) @@ -58,8 +57,8 @@ def import_json(client, filename, project_id, template_id, network_name, *args, pnet.add_recorder_references() pnet.promote_inline_parameters() pnet.promote_inline_recorders() - pnet.attach_reference_parameters() - pnet.attach_reference_recorders() + # pnet.attach_reference_parameters() + # pnet.attach_reference_recorders() #pnet.detach_parameters() @@ -132,7 +131,13 @@ def get_hydra_attrid_by_name(self, attr_name): for attr in self.hydra_attributes: if attr["name"].lower() == attr_name.lower(): return attr["id"] - log.critical(f"Attr {attr_name} not registered") + + log.warning(f"Attr {attr_name} not registered. Attempting to register...") + + attr = self.hydra.add_attribute(attr={'name': attr_name}) + self.hydra_attributes.append(attr) + + return attr['id'] def get_next_node_id(self): self._next_node_id -= 1 @@ -150,6 +155,8 @@ def get_node_by_name(self, name): for node in self.hydra_nodes: if node["name"] == name: return node + else: + raise ValueError(f"Node with name '{name}' not found in the model.") def make_hydra_attr(self, name, desc=None): return { "name": name, @@ -205,7 +212,7 @@ def build_hydra_network(self, projection=None, appdata={}): """ Assemble complete network """ network_name = self.network.metadata.data["title"] - network_description = self.network.metadata.data["description"] + network_description = self.network.metadata.data.get("description", "") self.network_hydratype = self.get_hydra_network_type() self.hydra_network = { @@ -342,7 +349,9 @@ def make_direct_resource_attr_and_scenario(self, value, attr_name, hydra_datatyp def make_typed_resource_scenario(self, element, attr_name, local_attr_id): + hydra_datatype = self.lookup_hydra_datatype(element) + dataset = { "name": attr_name, "type": hydra_datatype, "value": element.as_json(), @@ -361,6 +370,7 @@ def make_typed_resource_scenario(self, element, attr_name, local_attr_id): def make_network_resource_scenario(self, element, attr_name, local_attr_id): value = element.data[attr_name] + hydra_datatype = self.lookup_hydra_datatype(value) dataset = { "name": attr_name, @@ -399,6 +409,9 @@ def make_paramrec_resource_scenario(self, element, attr_name, local_attr_id): def make_resource_scenario(self, element, attr_name, local_attr_id): value = element.data[attr_name] + if value is None: + return None + hydra_datatype = self.lookup_hydra_datatype(value) dataset = { "name": attr_name, @@ -418,12 +431,18 @@ def make_resource_scenario(self, element, attr_name, local_attr_id): def lookup_hydra_datatype(self, attr_value): - if isinstance(attr_value, Number): + + if attr_value is None: + return None + elif isinstance(attr_value, Number): return "SCALAR" elif isinstance(attr_value, list): return "ARRAY" elif isinstance(attr_value, dict): + if 'index' in attr_value and 'table' in attr_value: + return "DESCRIPTOR" return "DATAFRAME" + elif isinstance(attr_value, str): return "DESCRIPTOR" elif isinstance(attr_value, PywrTable): @@ -436,7 +455,6 @@ def lookup_hydra_datatype(self, attr_value): return lookup_parameter_hydra_datatype(attr_value) elif isinstance(attr_value, PywrRecorder): return lookup_recorder_hydra_datatype(attr_value) - raise ValueError(f"Unknown data type: '{attr_value}'") @@ -456,7 +474,8 @@ def build_hydra_nodes(self): if ra["attr_id"] == None: raise ValueError(f"Node '{node.name}' attr '{attr_name}' has invalid attr id: \'{ra['attr_id']}\'") resource_attributes.append(ra) - resource_scenarios.append(rs) + if rs is not None: + resource_scenarios.append(rs) hydra_node = {} hydra_node["resource_type"] = "NODE" @@ -471,15 +490,15 @@ def build_hydra_nodes(self): }] if "position" in node.data: proj_data = node.data["position"] - for coords in proj_data.values(): - if "geographic" in coords: - coords = coords["geographic"] - elif "schematic" in coords: - coords = coords["schematic"] - if isinstance(coords, list): - x, y = coords[0], coords[1] - elif isinstance(coords, dict): - hydra_node['layout']['geojson'] = coords + if "geographic" in proj_data: + coords = proj_data["geographic"] + elif "schematic" in proj_data: + coords = proj_data["schematic"] + + if isinstance(coords, list): + x, y = coords[0], coords[1] + elif isinstance(coords, dict): + hydra_node['layout']['geojson'] = coords hydra_node["x"] = x hydra_node["y"] = y else: @@ -563,8 +582,7 @@ def build_parameters_recorders(self): def add_network_to_hydra(self): """ Pass network to Hydra""" - network = JSONObject(self.hydra_network) - network_summary = self.hydra.add_network({"net": network}) + network_summary = self.hydra.add_network({"net": self.hydra_network}) return network_summary diff --git a/hydra_pywr/runner.py b/hydra_pywr/runner.py index 5d962f0..1a56e16 100644 --- a/hydra_pywr/runner.py +++ b/hydra_pywr/runner.py @@ -38,12 +38,13 @@ def run_file(filename, domain, output_file): pfr.run_pywr_model(output_file) -def run_network_scenario(client, scenario_id, template_id, domain, - solver=None, data_dir='/tmp'): +def run_network_scenario(client, scenario_id, template_id, + solver=None, data_dir='/tmp', use_cache=False, **kwargs): runner = PywrHydraRunner.from_scenario_id(client, scenario_id, template_id=template_id, - data_dir=data_dir) + data_dir=data_dir, + use_cache=use_cache) runner.setup(solver=solver) runner.run_pywr_model() runner.save_pywr_results() @@ -91,7 +92,7 @@ def load_pywr_model_from_file(self, filename, solver=None): if warnings: for component, warns in warnings.items(): for warn in warns: - log.info(warn) + log.warning(warn) if errors: for component, errs in errors.items(): @@ -531,10 +532,7 @@ def _add_node_flagged_recorders(self, model): name = '__{}__:{}'.format(node.name, 'simulated_volume') NumpyArrayStorageRecorder(model, node, name=name) else: - import warnings - warnings.warn('Unrecognised node subclass "{}" with name "{}" for timeseries recording. Skipping ' - 'recording this node.'.format(node.__class__.__name__, node.name), - RuntimeWarning) + log.warning(f'Unrecognised node subclass "{node.__class__.__name__}" with name "{node.name}" for timeseries recording. Skipping recording this node.') elif flag == 'deficit': if isinstance(node, Node): @@ -542,10 +540,7 @@ def _add_node_flagged_recorders(self, model): name = '__{}__:{}'.format(node.name, 'simulated_deficit') NumpyArrayParameterRecorder(model, deficit_parameter, name=name) else: - import warnings - warnings.warn('Unrecognised node subclass "{}" with name "{}" for deficit recording. Skipping ' - 'recording this node.'.format(node.__class__.__name__, node.name), - RuntimeWarning) + log.warning(f'Unrecognised node subclass "{node.__class__.__name__}" with name "{node.name}" for deficit recording. Skipping recording this node.') def _add_parameter_flagged_recorders(self, model): for parameter_name, flags in self._parameter_recorder_flags.items(): @@ -774,6 +769,10 @@ def generate_array_recorder_resource_scenarios(self): except NotImplementedError: continue + if len(df.columns) == 0: + log.warning(f"Recorder {recorder.name} has no data to save.") + continue + columns = [] for name in df.columns.names: columns.append([f'{name}: {v}' for v in df.columns.get_level_values(name)]) @@ -792,6 +791,7 @@ def generate_array_recorder_resource_scenarios(self): new_col_names.append(colname.split(':')[1].strip()) df.columns = new_col_names + if "__:" in recorder.name: try: nodename, attrname = parse_reference_key(recorder.name)