From e706906f738a39f9896303004614c990edeedad4 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 11:48:15 +0200 Subject: [PATCH 01/31] run in subcalcs --- openquake/calculators/classical.py | 42 +++++++- openquake/calculators/tests/logictree_test.py | 5 +- openquake/hazardlib/logictree.py | 81 +++++++++++++++- openquake/hazardlib/lt.py | 9 +- openquake/hazardlib/source_group.py | 26 +++++ openquake/hazardlib/tests/lt_test.py | 97 +++++++++++++++++++ .../expected/hazard_curve-mean-PGA.csv | 4 +- .../qa_tests_data/logictree/case_25/job.ini | 4 +- .../qa_tests_data/logictree/case_25/smlt.xml | 12 +-- 9 files changed, 261 insertions(+), 19 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index ee6825a39232..cc8ab1f5a2e1 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -641,14 +641,46 @@ def _execute(self, sgs, ds): OQ_TASK_NO = os.environ.get('OQ_TASK_NO', '') if OQ_TASK_NO: allargs = [allargs[int(OQ_TASK_NO)]] - if self.few_sites or oq.disagg_by_src: - smap = parallel.Starmap( - classical_disagg, allargs, h5=self.datastore.hdf5) + task_func = (classical_disagg if (self.few_sites or oq.disagg_by_src) + else classical) + smlt = self.full_lt.source_model_lt + # Subcalc mode is triggered by "subcalc" labels in smlt.xml + use_subcalcs = smlt.has_subcalcs and not OQ_TASK_NO + if use_subcalcs: + acc = self._run_subcalcs(allargs, task_func) else: - smap = parallel.Starmap(classical, allargs, h5=self.datastore.hdf5) - acc = smap.reduce(self.agg_dicts, AccumDict(accum=0.)) + smap = parallel.Starmap( + task_func, allargs, h5=self.datastore.hdf5) + acc = smap.reduce(self.agg_dicts, AccumDict(accum=0.)) self._post_execute(acc) + def _run_subcalcs(self, allargs, task_func): + """ + Run one Starmap per subcalc sequentially. + """ + # Map each src_group id to its subcalc label + grp_ids_by_lbl = self.csm.grp_ids_by_subcalc() + label_of_grp = {gid: lbl + for lbl, gids in grp_ids_by_lbl.items() + for gid in gids} + + # Group the task-arg tuples by subcalc label + partitions = {} + for args in allargs: + # strip any tile suffix ("5-2" -> 5) to recover grp_id + gid = int(args[0][0].split('-')[0]) + partitions.setdefault(label_of_grp[gid], []).append(args) + + # Run subcalcs one at a time + acc = AccumDict(accum=0.) + for lbl in sorted(partitions): + logging.info('Subcalc %r: %d tasks', lbl, len(partitions[lbl])) + smap = parallel.Starmap( + task_func, partitions[lbl], h5=self.datastore.hdf5) + acc = smap.reduce(self.agg_dicts, acc) + + return acc + def _post_execute(self, acc): # save the rates and performs some checks oq = self.oqparam diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index 55970b71b0d3..e27fc91647f1 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -503,11 +503,12 @@ def test_case_23_bis(self): def test_case_25(self): # BCHydro-style correlated uncertainties (alt1 + alt2 + alt3) - # sampled to keep the calc fast (highly simplified version) + # run in subcalc mode: three sequential subcalcs grouped by the + # "subcalc" attribute on the top-level sourceModel branches self.run_calc(case_25.__file__, 'job.ini', exports='csv') [got] = export(('hcurves', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hazard_curve-mean-PGA.csv', got) - self.assertEqual(len(self.calc.full_lt.get_realizations()), 50) + self.assertEqual(len(self.calc.full_lt.get_realizations()), 144) def test_case_28(self): # North Africa # MultiPointSource with modify MFD logic tree diff --git a/openquake/hazardlib/logictree.py b/openquake/hazardlib/logictree.py index 0882ee5c0c36..328c1cadd8a6 100644 --- a/openquake/hazardlib/logictree.py +++ b/openquake/hazardlib/logictree.py @@ -503,10 +503,33 @@ def parse_tree(self, tree_node): attach_branches(self) self.source_data = numpy.array(self.source_data, source_dt) unique = numpy.unique(self.source_data['fname']) + self.validate_subcalcs() dt = time.time() - t0 logging.debug('Validated source model logic tree with %d underlying ' 'files in %.2f seconds', len(unique), dt) + def validate_subcalcs(self): + """ + If any top-level sourceModel branch has a "subcalc" label, all + must have one. Subcalc mode is incompatible with sampling. + """ + top = self.branchsets[0] + labelled = [br for br in top.branches if br.subcalc is not None] + if not labelled: + return + if len(labelled) != len(top.branches): + missing = [br.branch_id for br in top.branches + if br.subcalc is None] + raise LogicTreeError( + None, self.filename, + "'subcalc' must be set on every top-level sourceModel " + "branch, or on none. Missing on: %s" % ', '.join(missing)) + if self.num_samples: + raise LogicTreeError( + None, self.filename, + "'subcalc' labels require full enumeration; set " + "number_of_logic_tree_samples = 0 in the job.ini") + @property def utypes(self): """ @@ -514,6 +537,54 @@ def utypes(self): """ return [bs.uncertainty_type for bs in self.branchsets] + @property + def subcalcs(self): + """ + :returns: + dict grouping the top-level "sourceModel" branches by their + "subcalc" attribute. Empty dict if no branch carries a "subcalc" + label. + """ + out = {} + for br in self.branchsets[0].branches: + if br.subcalc is None: + continue + out.setdefault(br.subcalc, []).append(br) + return out + + @property + def subcalc_weights(self): + """ + :returns: + dict giving the summed weight of the top-level sourceModel + branches for each subcalc + """ + return {label: float(sum(br.weight for br in brs)) + for label, brs in self.subcalcs.items()} + + @property + def has_subcalcs(self): + """ + :returns: + True if any top-level sourceModel branch has a + subcalc label + """ + return bool(self.subcalcs) + + def smr_to_subcalc_map(self): + """ + :returns: + dict mapping each source-model realization index to + the subcalc label of its top-level sourceModel branch. + Realizations under an unlabelled top-level branch map + to None. + """ + branch_subcalc = {br.branch_id: br.subcalc + for br in self.branchsets[0].branches} + + return {smr: branch_subcalc.get(rlz.lt_path[0]) + for smr, rlz in enumerate(self)} + def parse_branchset(self, branchset_node, bsno): """ :param branchset_ node: @@ -636,13 +707,21 @@ def parse_branches(self, branchset_node, branchset): raise LogicTreeError( branchnode, self.filename, "branchID '%s' is not unique" % branch_id) + subcalc = branchnode.attrib.get('subcalc') + if (subcalc is not None and + branchset.uncertainty_type != 'sourceModel'): + raise LogicTreeError( + branchnode, self.filename, + "'subcalc' attribute is only allowed on branches of " + "sourceModel branchsets") if value == '': # with logic tree reduction a branch can be empty # see case_68_bis zero_id = branch_id zeros.append(weight) else: - branch = Branch(branch_id, value, weight, bs_id) + branch = Branch(branch_id, value, weight, bs_id, + subcalc=subcalc) self.branches[branch_id] = branch branchset.branches.append(branch) self.shortener[branch_id] = keyno(branch_id, bsno, brno, BASE183) diff --git a/openquake/hazardlib/lt.py b/openquake/hazardlib/lt.py index e40dbc38ff51..39a39689dde7 100644 --- a/openquake/hazardlib/lt.py +++ b/openquake/hazardlib/lt.py @@ -653,13 +653,18 @@ class Branch(object): of ```` child node. :param bs_id: BranchSetID of the branchset to which the branch belongs + :param subcalc: + Optional subcalc label; when set on a top-level "sourceModel" + branch, all downstream branches reachable via "applyToBranches" + chains inherit this label and are grouped into one subcalc """ - def __init__(self, branch_id, value, weight, bs_id=''): + def __init__(self, branch_id, value, weight, bs_id='', subcalc=None): self.branch_id = branch_id self.value = value self.weight = weight self.bs_id = bs_id self.bset = None + self.subcalc = subcalc def is_dummy(self): """ @@ -681,6 +686,8 @@ def is_leaf(self): def to_node(self): attrib = dict(branchID=self.branch_id) + if self.subcalc is not None: + attrib['subcalc'] = self.subcalc nodes = [Node('uncertaintyModel', {}, self.value), Node('uncertaintyWeight', {}, self.weight)] return Node('logicTreeBranch', attrib, None, nodes) diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 8f203e4ecb55..7ddba737468f 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -359,6 +359,32 @@ def get_sources(self, smr=None): srcs.extend(grp) return srcs + def grp_ids_by_subcalc(self): + """ + :returns: + dict grouping src_groups by the subcalc label of the top-level + "sourceModel" branch they trace back to (via "trt_smrs"). + + Raises a ValueError if a src_group spans more than one subcalc. + """ + smlt = self.full_lt.source_model_lt + smr_subcalc = smlt.smr_to_subcalc_map() + out = {} + for grp_id, sg in enumerate(self.src_groups): + labels = set() + for trt_smr in sg.sources[0].trt_smrs: + labels.add(smr_subcalc.get(trt_smr % TWO24)) + labels.discard(None) + if len(labels) > 1: + raise ValueError( + 'src_group %d (%s) spans multiple subcalcs %s; ' + 'cross-subcalc source sharing is not supported' + % (grp_id, sg.trt, sorted(labels))) + if labels: + out.setdefault(labels.pop(), []).append(grp_id) + + return out + def get_trt_smrs(self): """ :returns: an array of trt_smrs (to be stored as an hdf5.vuint32 array) diff --git a/openquake/hazardlib/tests/lt_test.py b/openquake/hazardlib/tests/lt_test.py index 0a300e6e517c..803969ec080f 100644 --- a/openquake/hazardlib/tests/lt_test.py +++ b/openquake/hazardlib/tests/lt_test.py @@ -27,6 +27,8 @@ nrml, lt, sourceconverter, calc, site, valid, contexts) from openquake.hazardlib.calc.hazard_curve import classical from openquake.hazardlib.geo.point import Point +from openquake.hazardlib.logictree import SourceModelLogicTree +from openquake.hazardlib.lt import LogicTreeError CDIR = os.path.dirname(__file__) ae = numpy.testing.assert_equal @@ -676,3 +678,98 @@ def test_mixed_collapsed_apply_uncertainties(self): for mod_src in mod_sg.sources: self.assertEqual(mod_src.mfd.max_mag, 6.5) + +class SubcalcAttributeTestCase(unittest.TestCase): + """ + Tests for the ``subcalc`` attribute on top-level ``sourceModel`` + branches (see :class:`SourceModelLogicTree`). + """ + + SM_ONLY = ''' + + + + + {sm1} + 0.6 + + + {sm2} + 0.4 + + {extra} + +''' + + EXTRA_LSD = ''' + + + 15.0 + 1.0 + + ''' + + SM1 = 'sm1.xml' + SM2 = 'sm2.xml' + + def _parse(self, xml, **kw): + # Parse from a temp file with test_mode=True so referenced + # source-model XMLs are not loaded from disk + path = gettemp(xml, suffix='.xml') + return SourceModelLogicTree(path, test_mode=True, **kw) + + def test_subcalc_on_non_source_model_raises(self): + # A subcalc attribute on a non-sourceModel branch is an error + xml = self.SM_ONLY.format( + sub1=' subcalc="a"', sub2=' subcalc="a"', + sm1=self.SM1, sm2=self.SM2, + extra=self.EXTRA_LSD.format(sub=' subcalc="erroneous"')) + with self.assertRaises(LogicTreeError) as cm: + self._parse(xml) + self.assertIn( + "'subcalc' attribute is only allowed on branches of " + "sourceModel branchsets", str(cm.exception)) + + def test_partial_subcalc_labelling_raises(self): + # Subcalc must be on every top-level branch, or none + xml = self.SM_ONLY.format( + sub1=' subcalc="a"', sub2='', + sm1=self.SM1, sm2=self.SM2, extra='') + with self.assertRaises(LogicTreeError) as cm: + self._parse(xml) + self.assertIn( + "'subcalc' must be set on every top-level sourceModel " + "branch, or on none. Missing on: b2", str(cm.exception)) + + def test_subcalc_grouping_and_weights(self): + # Subcalcs group top-level branches by label and sum weights + xml = self.SM_ONLY.format( + sub1=' subcalc="a"', sub2=' subcalc="b"', + sm1=self.SM1, sm2=self.SM2, extra='') + smlt = self._parse(xml) + self.assertTrue(smlt.has_subcalcs) + self.assertEqual(sorted(smlt.subcalcs), ['a', 'b']) + self.assertAlmostEqual(smlt.subcalc_weights['a'], 0.6) + self.assertAlmostEqual(smlt.subcalc_weights['b'], 0.4) + + def test_sampling_with_subcalcs_raises(self): + # Subcalc labels require full enumeration + xml = self.SM_ONLY.format( + sub1=' subcalc="a"', sub2=' subcalc="b"', + sm1=self.SM1, sm2=self.SM2, extra='') + with self.assertRaises(LogicTreeError) as cm: + self._parse(xml, num_samples=10) + self.assertIn( + "'subcalc' labels require full enumeration; set " + "number_of_logic_tree_samples = 0 in the job.ini", + str(cm.exception)) + + def test_no_subcalc_labels_leaves_smlt_unaffected(self): + # Unlabelled tree has no subcalcs and validates cleanly + xml = self.SM_ONLY.format( + sub1='', sub2='', + sm1=self.SM1, sm2=self.SM2, extra='') + smlt = self._parse(xml) + self.assertFalse(smlt.has_subcalcs) + self.assertEqual(smlt.subcalcs, {}) diff --git a/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv b/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv index f3e3231e33b4..10030b56b523 100644 --- a/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv +++ b/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv @@ -1,3 +1,3 @@ -#,,,,,,,,"generated_by='OpenQuake engine 3.27.0-git4a26d5674f', start_date='2026-07-30T17:01:13', checksum=1689509781, kind='mean', investigation_time=50.0, imt='PGA'" +#,,,,,,,,"generated_by='OpenQuake engine 3.27.0-gitc1e80ac1f2', start_date='2026-07-31T10:19:25', checksum=1641279868, kind='mean', investigation_time=50.0, imt='PGA'" lon,lat,depth,poe-0.0100000,poe-0.0500000,poe-0.1000000,poe-0.2000000,poe-0.5000000,poe-1.0000000 --121.80000,49.10640,0.00000,8.877952E-02,8.286452E-03,1.719700E-03,1.858008E-04,1.861893E-06,2.408214E-09 +-121.80000,49.10640,0.00000,9.328531E-02,8.837306E-03,1.847793E-03,2.059838E-04,2.427462E-06,2.069817E-09 diff --git a/openquake/qa_tests_data/logictree/case_25/job.ini b/openquake/qa_tests_data/logictree/case_25/job.ini index 3871ea7bf5fd..ba4cbecfec47 100644 --- a/openquake/qa_tests_data/logictree/case_25/job.ini +++ b/openquake/qa_tests_data/logictree/case_25/job.ini @@ -1,6 +1,6 @@ [general] -description = BCHydro-style correlated uncertainties (merged alt1 + alt2 + alt3) +description = BCHydro-style correlated uncertainties (subcalc mode: alt1/alt2/alt3) calculation_mode = classical random_seed = 23 @@ -10,7 +10,7 @@ sites = -121.8 49.1064 [logic_tree] -number_of_logic_tree_samples = 50 +number_of_logic_tree_samples = 0 [erf] area_source_discretization = 40.0 diff --git a/openquake/qa_tests_data/logictree/case_25/smlt.xml b/openquake/qa_tests_data/logictree/case_25/smlt.xml index 3d5be9905660..58473f2137cd 100644 --- a/openquake/qa_tests_data/logictree/case_25/smlt.xml +++ b/openquake/qa_tests_data/logictree/case_25/smlt.xml @@ -4,27 +4,27 @@ - + ssm/alt1_NVA.xml 0.469000 - + ssm/alt1_NVA-AB.xml 0.231000 - + ssm/alt2_NVA.xml 0.067000 - + ssm/alt2_NVA-AB.xml 0.033000 - + ssm/alt3_NVA.xml 0.134000 - + ssm/alt3_NVA-AB.xml 0.066000 From ea20261b86124f8e7d19b361fb51d7769e5ef11e Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 12:19:15 +0200 Subject: [PATCH 02/31] fix sampling --- openquake/hazardlib/logictree.py | 110 +++++++++++++-------------- openquake/hazardlib/source_group.py | 11 ++- openquake/hazardlib/tests/lt_test.py | 12 --- 3 files changed, 58 insertions(+), 75 deletions(-) diff --git a/openquake/hazardlib/logictree.py b/openquake/hazardlib/logictree.py index 328c1cadd8a6..5beba035da96 100644 --- a/openquake/hazardlib/logictree.py +++ b/openquake/hazardlib/logictree.py @@ -511,7 +511,7 @@ def parse_tree(self, tree_node): def validate_subcalcs(self): """ If any top-level sourceModel branch has a "subcalc" label, all - must have one. Subcalc mode is incompatible with sampling. + must have one. """ top = self.branchsets[0] labelled = [br for br in top.branches if br.subcalc is not None] @@ -524,11 +524,6 @@ def validate_subcalcs(self): None, self.filename, "'subcalc' must be set on every top-level sourceModel " "branch, or on none. Missing on: %s" % ', '.join(missing)) - if self.num_samples: - raise LogicTreeError( - None, self.filename, - "'subcalc' labels require full enumeration; set " - "number_of_logic_tree_samples = 0 in the job.ini") @property def utypes(self): @@ -571,20 +566,6 @@ def has_subcalcs(self): """ return bool(self.subcalcs) - def smr_to_subcalc_map(self): - """ - :returns: - dict mapping each source-model realization index to - the subcalc label of its top-level sourceModel branch. - Realizations under an unlabelled top-level branch map - to None. - """ - branch_subcalc = {br.branch_id: br.subcalc - for br in self.branchsets[0].branches} - - return {smr: branch_subcalc.get(rlz.lt_path[0]) - for smr, rlz in enumerate(self)} - def parse_branchset(self, branchset_node, bsno): """ :param branchset_ node: @@ -637,6 +618,49 @@ def parse_branchset(self, branchset_node, bsno): "branch '%s' is not yet defined" % branch_id) self.branchsets.append(branchset) + def _check_branch_count(self, bs_id, branches): + maxlen = len(BASE183) + if self.branchID == '' and len(branches) > maxlen: + msg = ('%s: the branchset %s has too many branches (%d > %d)\n' + 'you should split it, see https://docs.openquake.org/' + 'oq-engine/advanced/latest/logic_trees.html') + raise InvalidFile( + msg % (self.filename, bs_id, len(branches), maxlen)) + + def _collect_source_model_files(self, branchnode, value_node, value): + vals = [] # filenames with sources in it + try: + for fname in value_node.text.split(): + if (fname.endswith(('.xml', '.nrml')) + and not self.test_mode): + ok = self.collect_source_model_data( + branchnode['branchID'], fname) + if ok: + vals.append(fname) + except Exception as exc: + raise LogicTreeError( + value_node, self.filename, str(exc)) from exc + if self.branchID and self.branchID not in branchnode['branchID']: + return '' # reduce all branches except branchID + if self.source_id: # only the files containing source_id + srcid = self.source_id.split('@')[0] + return ' '.join(reduce_fnames(vals, srcid)) + return value + + def _validate_branch_metadata(self, branchnode, branchset): + branch_id = branchnode.attrib.get('branchID') + if branch_id in self.branches: + raise LogicTreeError( + branchnode, self.filename, + "branchID '%s' is not unique" % branch_id) + subcalc = branchnode.attrib.get('subcalc') + if subcalc is not None and branchset.uncertainty_type != 'sourceModel': + raise LogicTreeError( + branchnode, self.filename, + "'subcalc' attribute is only allowed on branches of " + "sourceModel branchsets") + return branch_id, subcalc + def parse_branches(self, branchset_node, branchset): """ Create and attach branches at ``branchset_node`` to ``branchset``. @@ -654,21 +678,15 @@ def parse_branches(self, branchset_node, branchset): """ correlated = branchset_node.get('applyToSources') == '*' bs_id = branchset_node['branchSetID'] - weight_sum = 0 branches = branchset_node.nodes if OQ_REDUCE: # only take first branch branches = [branches[0]] branches[0].uncertaintyWeight.text = 1. - values = [] + self._check_branch_count(bs_id, branches) bsno = len(self.branchsets) + weight_sum = 0 + values = [] zeros = [] - maxlen = len(BASE183) - if self.branchID == '' and len(branches) > maxlen: - msg = ('%s: the branchset %s has too many branches (%d > %d)\n' - 'you should split it, see https://docs.openquake.org/' - 'oq-engine/advanced/latest/logic_trees.html') - raise InvalidFile( - msg % (self.filename, bs_id, len(branches), maxlen)) for brno, branchnode in enumerate(branches): weight = ~branchnode.uncertaintyWeight value_node = node_from_elem(branchnode.uncertaintyModel) @@ -684,36 +702,10 @@ def parse_branches(self, branchset_node, branchset): value = parse_uncertainty(branchset.uncertainty_type, value_node, self.filename) if branchset.uncertainty_type in ('sourceModel', 'extendModel'): - vals = [] # filenames with sources in it - try: - for fname in value_node.text.split(): - if (fname.endswith(('.xml', '.nrml')) - and not self.test_mode): - ok = self.collect_source_model_data( - branchnode['branchID'], fname) - if ok: - vals.append(fname) - except Exception as exc: - raise LogicTreeError( - value_node, self.filename, str(exc)) from exc - if (self.branchID and self.branchID not in - branchnode['branchID']): - value = '' # reduce all branches except branchID - elif self.source_id: # only the files containing source_id - srcid = self.source_id.split('@')[0] - value = ' '.join(reduce_fnames(vals, srcid)) - branch_id = branchnode.attrib.get('branchID') - if branch_id in self.branches: - raise LogicTreeError( - branchnode, self.filename, - "branchID '%s' is not unique" % branch_id) - subcalc = branchnode.attrib.get('subcalc') - if (subcalc is not None and - branchset.uncertainty_type != 'sourceModel'): - raise LogicTreeError( - branchnode, self.filename, - "'subcalc' attribute is only allowed on branches of " - "sourceModel branchsets") + value = self._collect_source_model_files( + branchnode, value_node, value) + branch_id, subcalc = self._validate_branch_metadata( + branchnode, branchset) if value == '': # with logic tree reduction a branch can be empty # see case_68_bis diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 7ddba737468f..4c62415fc6cd 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -364,11 +364,15 @@ def grp_ids_by_subcalc(self): :returns: dict grouping src_groups by the subcalc label of the top-level "sourceModel" branch they trace back to (via "trt_smrs"). - + Raises a ValueError if a src_group spans more than one subcalc. """ - smlt = self.full_lt.source_model_lt - smr_subcalc = smlt.smr_to_subcalc_map() + # Build an smr -> subcalc map + branch_subcalc = { + br.branch_id: br.subcalc for br + in self.full_lt.source_model_lt.branchsets[0].branches} + smr_subcalc = {smr: branch_subcalc.get(rlz.lt_path[0]) + for smr, rlz in enumerate(self.full_lt.sm_rlzs)} out = {} for grp_id, sg in enumerate(self.src_groups): labels = set() @@ -382,7 +386,6 @@ def grp_ids_by_subcalc(self): % (grp_id, sg.trt, sorted(labels))) if labels: out.setdefault(labels.pop(), []).append(grp_id) - return out def get_trt_smrs(self): diff --git a/openquake/hazardlib/tests/lt_test.py b/openquake/hazardlib/tests/lt_test.py index 803969ec080f..0d557d6596eb 100644 --- a/openquake/hazardlib/tests/lt_test.py +++ b/openquake/hazardlib/tests/lt_test.py @@ -753,18 +753,6 @@ def test_subcalc_grouping_and_weights(self): self.assertAlmostEqual(smlt.subcalc_weights['a'], 0.6) self.assertAlmostEqual(smlt.subcalc_weights['b'], 0.4) - def test_sampling_with_subcalcs_raises(self): - # Subcalc labels require full enumeration - xml = self.SM_ONLY.format( - sub1=' subcalc="a"', sub2=' subcalc="b"', - sm1=self.SM1, sm2=self.SM2, extra='') - with self.assertRaises(LogicTreeError) as cm: - self._parse(xml, num_samples=10) - self.assertIn( - "'subcalc' labels require full enumeration; set " - "number_of_logic_tree_samples = 0 in the job.ini", - str(cm.exception)) - def test_no_subcalc_labels_leaves_smlt_unaffected(self): # Unlabelled tree has no subcalcs and validates cleanly xml = self.SM_ONLY.format( From a068bb5ae988b5b64709994a11573d10b651668c Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 12:36:05 +0200 Subject: [PATCH 03/31] new QA test instead --- openquake/calculators/tests/logictree_test.py | 33 +++++++++++--- openquake/qa_tests_data/logictree/README.md | 1 + .../logictree/case_24/__init__.py | 0 .../logictree/case_24/gsim_logic_tree.xml | 15 +++++++ .../qa_tests_data/logictree/case_24/job.ini | 38 ++++++++++++++++ .../logictree/case_24/job_nosub.ini | 38 ++++++++++++++++ .../qa_tests_data/logictree/case_24/sm_a.xml | 25 +++++++++++ .../qa_tests_data/logictree/case_24/sm_b.xml | 25 +++++++++++ .../qa_tests_data/logictree/case_24/smlt.xml | 44 +++++++++++++++++++ .../logictree/case_24/smlt_nosub.xml | 43 ++++++++++++++++++ .../expected/hazard_curve-mean-PGA.csv | 4 +- .../qa_tests_data/logictree/case_25/job.ini | 4 +- .../qa_tests_data/logictree/case_25/smlt.xml | 12 ++--- 13 files changed, 265 insertions(+), 17 deletions(-) create mode 100644 openquake/qa_tests_data/logictree/case_24/__init__.py create mode 100644 openquake/qa_tests_data/logictree/case_24/gsim_logic_tree.xml create mode 100644 openquake/qa_tests_data/logictree/case_24/job.ini create mode 100644 openquake/qa_tests_data/logictree/case_24/job_nosub.ini create mode 100644 openquake/qa_tests_data/logictree/case_24/sm_a.xml create mode 100644 openquake/qa_tests_data/logictree/case_24/sm_b.xml create mode 100644 openquake/qa_tests_data/logictree/case_24/smlt.xml create mode 100644 openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index e27fc91647f1..1ffcb3346962 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -32,10 +32,10 @@ from openquake.qa_tests_data.logictree import ( case_01, case_02, case_03, case_04, case_05, case_06, case_07, case_08, case_09, case_10, case_11, case_12, case_13, case_14, case_15, case_16, - case_17, case_18, case_19, case_20, case_21, case_22, case_23, case_25, - case_28, case_29, case_30, case_31, case_32, case_33, case_36, case_39, - case_45, case_46, case_52, case_56, case_58, case_59, case_67, case_68, - case_71, case_73, case_79, case_80, case_83, case_84) + case_17, case_18, case_19, case_20, case_21, case_22, case_23, case_24, + case_25, case_28, case_29, case_30, case_31, case_32, case_33, case_36, + case_39, case_45, case_46, case_52, case_56, case_58, case_59, case_67, + case_68, case_71, case_73, case_79, case_80, case_83, case_84) ae = numpy.testing.assert_equal aac = numpy.testing.assert_allclose @@ -501,14 +501,33 @@ def test_case_23_bis(self): ns = len(self.calc.datastore['source_info']) assert ns == 26 + def test_case_24(self): + # Parity check: subcalc-mode mean hazard curves must match the + # non-subcalc approach for both full enumeration and sampling + + # Full enumeration + self.run_calc(case_24.__file__, 'job.ini') # Using subcalc + sub_full = self.calc.datastore['hcurves-stats'][:] + self.run_calc(case_24.__file__, 'job_nosub.ini') # Regular + nosub_full = self.calc.datastore['hcurves-stats'][:] + aac(sub_full, nosub_full, atol=0, rtol=0) + + # Sampling + self.run_calc(case_24.__file__, 'job.ini', # Using subcalc + number_of_logic_tree_samples='10') + sub_sampled = self.calc.datastore['hcurves-stats'][:] + self.run_calc(case_24.__file__, 'job_nosub.ini', # Regular + number_of_logic_tree_samples='10') + nosub_sampled = self.calc.datastore['hcurves-stats'][:] + aac(sub_sampled, nosub_sampled, atol=0, rtol=0) + def test_case_25(self): # BCHydro-style correlated uncertainties (alt1 + alt2 + alt3) - # run in subcalc mode: three sequential subcalcs grouped by the - # "subcalc" attribute on the top-level sourceModel branches + # sampled to keep the calc fast (highly simplified version) self.run_calc(case_25.__file__, 'job.ini', exports='csv') [got] = export(('hcurves', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/hazard_curve-mean-PGA.csv', got) - self.assertEqual(len(self.calc.full_lt.get_realizations()), 144) + self.assertEqual(len(self.calc.full_lt.get_realizations()), 50) def test_case_28(self): # North Africa # MultiPointSource with modify MFD logic tree diff --git a/openquake/qa_tests_data/logictree/README.md b/openquake/qa_tests_data/logictree/README.md index 7846f8500076..b05529223aea 100644 --- a/openquake/qa_tests_data/logictree/README.md +++ b/openquake/qa_tests_data/logictree/README.md @@ -27,6 +27,7 @@ | case\_22 | Test sigma_model_alatik2015 | | case\_23 | Arctic region and IDL (no bounding box) | | case\_23\_bis | Correlated uncertainties | +| case\_24 | Tests the subcalc feature's parity vs non-subcalc for full-enum + sampling | | case\_28 | Test collapse\_gsim\_logic\_tree | | case\_28\_bis | Test missing z1pt0 | | case\_25 | BC Hydro NVA SSC LT source model LT | diff --git a/openquake/qa_tests_data/logictree/case_24/__init__.py b/openquake/qa_tests_data/logictree/case_24/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openquake/qa_tests_data/logictree/case_24/gsim_logic_tree.xml b/openquake/qa_tests_data/logictree/case_24/gsim_logic_tree.xml new file mode 100644 index 000000000000..0a4537205571 --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/gsim_logic_tree.xml @@ -0,0 +1,15 @@ + + + + + + + BooreAtkinson2008 + 1.0 + + + + + diff --git a/openquake/qa_tests_data/logictree/case_24/job.ini b/openquake/qa_tests_data/logictree/case_24/job.ini new file mode 100644 index 000000000000..ef95c229b0ca --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/job.ini @@ -0,0 +1,38 @@ +[general] + +description = subcalc parity: full-enum + subcalc mode +calculation_mode = classical +random_seed = 42 + +[geometry] + +sites = 0.0 0.0 + +[logic_tree] + +number_of_logic_tree_samples = 0 + +[erf] + +rupture_mesh_spacing = 2.0 +width_of_mfd_bin = 0.5 + +[site_params] + +reference_vs30_type = measured +reference_vs30_value = 760.0 +reference_depth_to_2pt5km_per_sec = 2.5 +reference_depth_to_1pt0km_per_sec = 50.0 + +[calculation] + +source_model_logic_tree_file = smlt.xml +gsim_logic_tree_file = gsim_logic_tree.xml +investigation_time = 50.0 +intensity_measure_types_and_levels = {"PGA": [0.05, 0.1, 0.2, 0.5]} +truncation_level = 3.0 +maximum_distance = 200.0 + +[output] + +mean = true diff --git a/openquake/qa_tests_data/logictree/case_24/job_nosub.ini b/openquake/qa_tests_data/logictree/case_24/job_nosub.ini new file mode 100644 index 000000000000..d8b327c37189 --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/job_nosub.ini @@ -0,0 +1,38 @@ +[general] + +description = subcalc parity: full-enum without subcalc labels +calculation_mode = classical +random_seed = 42 + +[geometry] + +sites = 0.0 0.0 + +[logic_tree] + +number_of_logic_tree_samples = 0 + +[erf] + +rupture_mesh_spacing = 2.0 +width_of_mfd_bin = 0.5 + +[site_params] + +reference_vs30_type = measured +reference_vs30_value = 760.0 +reference_depth_to_2pt5km_per_sec = 2.5 +reference_depth_to_1pt0km_per_sec = 50.0 + +[calculation] + +source_model_logic_tree_file = smlt_nosub.xml +gsim_logic_tree_file = gsim_logic_tree.xml +investigation_time = 50.0 +intensity_measure_types_and_levels = {"PGA": [0.05, 0.1, 0.2, 0.5]} +truncation_level = 3.0 +maximum_distance = 200.0 + +[output] + +mean = true diff --git a/openquake/qa_tests_data/logictree/case_24/sm_a.xml b/openquake/qa_tests_data/logictree/case_24/sm_a.xml new file mode 100644 index 000000000000..d5eb82e2620d --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/sm_a.xml @@ -0,0 +1,25 @@ + + + + + + + + -0.05 0.0 + 0.05 0.0 + + + 90.0 + 0.0 + 10.0 + + PeerMSR + 1.0 + + 0.0 + + + diff --git a/openquake/qa_tests_data/logictree/case_24/sm_b.xml b/openquake/qa_tests_data/logictree/case_24/sm_b.xml new file mode 100644 index 000000000000..71c68c59e546 --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/sm_b.xml @@ -0,0 +1,25 @@ + + + + + + + + -0.03 0.02 + 0.03 0.02 + + + 90.0 + 0.0 + 10.0 + + PeerMSR + 1.0 + + 0.0 + + + diff --git a/openquake/qa_tests_data/logictree/case_24/smlt.xml b/openquake/qa_tests_data/logictree/case_24/smlt.xml new file mode 100644 index 000000000000..5c601ff03870 --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/smlt.xml @@ -0,0 +1,44 @@ + + + + + + + + sm_a.xml + 0.7 + + + sm_b.xml + 0.3 + + + + + + 3.0 1.0 + 0.6 + + + 2.9 1.05 + 0.4 + + + + + + 3.2 0.9 + 0.5 + + + 3.1 0.95 + 0.5 + + + + + diff --git a/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml b/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml new file mode 100644 index 000000000000..08bca4f059ed --- /dev/null +++ b/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml @@ -0,0 +1,43 @@ + + + + + + + + sm_a.xml + 0.7 + + + sm_b.xml + 0.3 + + + + + + 3.0 1.0 + 0.6 + + + 2.9 1.05 + 0.4 + + + + + + 3.2 0.9 + 0.5 + + + 3.1 0.95 + 0.5 + + + + + diff --git a/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv b/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv index 10030b56b523..f3e3231e33b4 100644 --- a/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv +++ b/openquake/qa_tests_data/logictree/case_25/expected/hazard_curve-mean-PGA.csv @@ -1,3 +1,3 @@ -#,,,,,,,,"generated_by='OpenQuake engine 3.27.0-gitc1e80ac1f2', start_date='2026-07-31T10:19:25', checksum=1641279868, kind='mean', investigation_time=50.0, imt='PGA'" +#,,,,,,,,"generated_by='OpenQuake engine 3.27.0-git4a26d5674f', start_date='2026-07-30T17:01:13', checksum=1689509781, kind='mean', investigation_time=50.0, imt='PGA'" lon,lat,depth,poe-0.0100000,poe-0.0500000,poe-0.1000000,poe-0.2000000,poe-0.5000000,poe-1.0000000 --121.80000,49.10640,0.00000,9.328531E-02,8.837306E-03,1.847793E-03,2.059838E-04,2.427462E-06,2.069817E-09 +-121.80000,49.10640,0.00000,8.877952E-02,8.286452E-03,1.719700E-03,1.858008E-04,1.861893E-06,2.408214E-09 diff --git a/openquake/qa_tests_data/logictree/case_25/job.ini b/openquake/qa_tests_data/logictree/case_25/job.ini index ba4cbecfec47..3871ea7bf5fd 100644 --- a/openquake/qa_tests_data/logictree/case_25/job.ini +++ b/openquake/qa_tests_data/logictree/case_25/job.ini @@ -1,6 +1,6 @@ [general] -description = BCHydro-style correlated uncertainties (subcalc mode: alt1/alt2/alt3) +description = BCHydro-style correlated uncertainties (merged alt1 + alt2 + alt3) calculation_mode = classical random_seed = 23 @@ -10,7 +10,7 @@ sites = -121.8 49.1064 [logic_tree] -number_of_logic_tree_samples = 0 +number_of_logic_tree_samples = 50 [erf] area_source_discretization = 40.0 diff --git a/openquake/qa_tests_data/logictree/case_25/smlt.xml b/openquake/qa_tests_data/logictree/case_25/smlt.xml index 58473f2137cd..3d5be9905660 100644 --- a/openquake/qa_tests_data/logictree/case_25/smlt.xml +++ b/openquake/qa_tests_data/logictree/case_25/smlt.xml @@ -4,27 +4,27 @@ - + ssm/alt1_NVA.xml 0.469000 - + ssm/alt1_NVA-AB.xml 0.231000 - + ssm/alt2_NVA.xml 0.067000 - + ssm/alt2_NVA-AB.xml 0.033000 - + ssm/alt3_NVA.xml 0.134000 - + ssm/alt3_NVA-AB.xml 0.066000 From 46c05066c6294b1bad6cea658727b201cd67b417 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 12:38:50 +0200 Subject: [PATCH 04/31] changelog --- debian/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index 905b6ad1f691..0d3e3743c7d3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,8 @@ + [Christopher Brooks] + * Add ability to run a single calculation in subcalcs by specifying + a `subcalc` attribute on each base source. Works in both full enum. + and sampling (tested inside logictree/case24) and unit tests. + [Michele Simionato] * Internal: reduced the hard limit on function length to 79 lines * Fixed the exporters `mag_dst_eps_sig` and `mean_disagg_by_src` for low From 0dde90c487af4df308ecc6f9e81b9e4f43cc5b16 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 12:40:00 +0200 Subject: [PATCH 05/31] clean --- openquake/calculators/classical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index cc8ab1f5a2e1..8d81e44f381d 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -644,7 +644,7 @@ def _execute(self, sgs, ds): task_func = (classical_disagg if (self.few_sites or oq.disagg_by_src) else classical) smlt = self.full_lt.source_model_lt - # Subcalc mode is triggered by "subcalc" labels in smlt.xml + # Subcalc mode is triggered by "subcalc" labels in the SSC LT use_subcalcs = smlt.has_subcalcs and not OQ_TASK_NO if use_subcalcs: acc = self._run_subcalcs(allargs, task_func) From 738b91173a1b6552481be9417589bb91867069b7 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 13:04:54 +0200 Subject: [PATCH 06/31] test quantiles too --- openquake/calculators/tests/logictree_test.py | 5 +++-- openquake/qa_tests_data/logictree/case_24/job.ini | 1 + openquake/qa_tests_data/logictree/case_24/job_nosub.ini | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index 1ffcb3346962..f01c93492438 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -502,8 +502,9 @@ def test_case_23_bis(self): assert ns == 26 def test_case_24(self): - # Parity check: subcalc-mode mean hazard curves must match the - # non-subcalc approach for both full enumeration and sampling + # Parity check: subcalc-mode hazard statistics (mean and + # quantiles) must match the non-subcalc approach for both + # full enumeration and sampling # Full enumeration self.run_calc(case_24.__file__, 'job.ini') # Using subcalc diff --git a/openquake/qa_tests_data/logictree/case_24/job.ini b/openquake/qa_tests_data/logictree/case_24/job.ini index ef95c229b0ca..13ed056871bf 100644 --- a/openquake/qa_tests_data/logictree/case_24/job.ini +++ b/openquake/qa_tests_data/logictree/case_24/job.ini @@ -36,3 +36,4 @@ maximum_distance = 200.0 [output] mean = true +quantiles = 0.05 0.5 0.95 diff --git a/openquake/qa_tests_data/logictree/case_24/job_nosub.ini b/openquake/qa_tests_data/logictree/case_24/job_nosub.ini index d8b327c37189..2728074adb4f 100644 --- a/openquake/qa_tests_data/logictree/case_24/job_nosub.ini +++ b/openquake/qa_tests_data/logictree/case_24/job_nosub.ini @@ -36,3 +36,4 @@ maximum_distance = 200.0 [output] mean = true +quantiles = 0.05 0.5 0.95 From 8e911e508e0c95e318899042c13f8adc218824a1 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 13:47:54 +0200 Subject: [PATCH 07/31] atol --- openquake/calculators/tests/logictree_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index f01c93492438..70a223e07652 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -505,13 +505,15 @@ def test_case_24(self): # Parity check: subcalc-mode hazard statistics (mean and # quantiles) must match the non-subcalc approach for both # full enumeration and sampling + # --> Using a small tolerance because of diff task + # reduction order on remote # Full enumeration self.run_calc(case_24.__file__, 'job.ini') # Using subcalc sub_full = self.calc.datastore['hcurves-stats'][:] self.run_calc(case_24.__file__, 'job_nosub.ini') # Regular nosub_full = self.calc.datastore['hcurves-stats'][:] - aac(sub_full, nosub_full, atol=0, rtol=0) + aac(sub_full, nosub_full, atol=1e-6, rtol=1e-6) # Sampling self.run_calc(case_24.__file__, 'job.ini', # Using subcalc @@ -520,7 +522,7 @@ def test_case_24(self): self.run_calc(case_24.__file__, 'job_nosub.ini', # Regular number_of_logic_tree_samples='10') nosub_sampled = self.calc.datastore['hcurves-stats'][:] - aac(sub_sampled, nosub_sampled, atol=0, rtol=0) + aac(sub_sampled, nosub_sampled, atol=1e-6, rtol=1e-6) def test_case_25(self): # BCHydro-style correlated uncertainties (alt1 + alt2 + alt3) From 9263d11fb63df67aa5179aabef55b99351ed7b35 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 31 Jul 2026 15:31:00 +0200 Subject: [PATCH 08/31] error if not classical or disagg --- openquake/commonlib/readinput.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openquake/commonlib/readinput.py b/openquake/commonlib/readinput.py index 4a287970cbd6..a30724606e34 100644 --- a/openquake/commonlib/readinput.py +++ b/openquake/commonlib/readinput.py @@ -964,6 +964,12 @@ def get_source_model_lt(oqparam): if oqparam.calculation_mode.startswith('scenario'): return logictree.SourceModelLogicTree.fake() smlt = get_smlt(vars(oqparam)) + if smlt.has_subcalcs and oqparam.calculation_mode not in ( + 'classical', 'disaggregation'): + raise InvalidFile( + "%s: 'subcalc' labels are only supported in classical and " + "disaggregation calculations (calculation_mode = %r)" + % (smlt.filename, oqparam.calculation_mode)) for bset in smlt.branchsets: bset.check_duplicates(smlt.filename) srcids = set(smlt.source_data['source']) From bbd9cdd7f8c31387174d6d433cd73be1a1120ae8 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 3 Aug 2026 15:03:26 +0200 Subject: [PATCH 09/31] sequential approach --- debian/changelog | 5 - openquake/calculators/classical.py | 39 +++-- openquake/calculators/tests/logictree_test.py | 32 ++-- openquake/commonlib/oqvalidation.py | 18 +++ openquake/commonlib/readinput.py | 6 - openquake/commonlib/tests/source_test.py | 73 ++++++++++ openquake/hazardlib/logictree.py | 137 +++++------------- openquake/hazardlib/lt.py | 9 +- openquake/hazardlib/source_group.py | 37 +++-- openquake/hazardlib/tests/lt_test.py | 86 ----------- openquake/qa_tests_data/logictree/README.md | 2 +- .../qa_tests_data/logictree/case_24/job.ini | 2 +- .../logictree/case_24/job_nosub.ini | 39 ----- .../qa_tests_data/logictree/case_24/smlt.xml | 7 +- .../logictree/case_24/smlt_nosub.xml | 43 ------ 15 files changed, 183 insertions(+), 352 deletions(-) delete mode 100644 openquake/qa_tests_data/logictree/case_24/job_nosub.ini delete mode 100644 openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml diff --git a/debian/changelog b/debian/changelog index 0d3e3743c7d3..905b6ad1f691 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,3 @@ - [Christopher Brooks] - * Add ability to run a single calculation in subcalcs by specifying - a `subcalc` attribute on each base source. Works in both full enum. - and sampling (tested inside logictree/case24) and unit tests. - [Michele Simionato] * Internal: reduced the hard limit on function length to 79 lines * Fixed the exporters `mag_dst_eps_sig` and `mean_disagg_by_src` for low diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 8d81e44f381d..3d5fcef11889 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -643,40 +643,39 @@ def _execute(self, sgs, ds): allargs = [allargs[int(OQ_TASK_NO)]] task_func = (classical_disagg if (self.few_sites or oq.disagg_by_src) else classical) - smlt = self.full_lt.source_model_lt - # Subcalc mode is triggered by "subcalc" labels in the SSC LT - use_subcalcs = smlt.has_subcalcs and not OQ_TASK_NO - if use_subcalcs: - acc = self._run_subcalcs(allargs, task_func) + if oq.sequential_source_models and not OQ_TASK_NO: + acc = self._run_sequential_source_models(allargs, task_func) else: smap = parallel.Starmap( task_func, allargs, h5=self.datastore.hdf5) acc = smap.reduce(self.agg_dicts, AccumDict(accum=0.)) self._post_execute(acc) - def _run_subcalcs(self, allargs, task_func): + def _run_sequential_source_models(self, allargs, task_func): """ - Run one Starmap per subcalc sequentially. + Run one Starmap per top-level ``sourceModel`` branch sequentially, + so that at most one source model's tasks are in flight at a time. """ - # Map each src_group id to its subcalc label - grp_ids_by_lbl = self.csm.grp_ids_by_subcalc() - label_of_grp = {gid: lbl - for lbl, gids in grp_ids_by_lbl.items() - for gid in gids} - - # Group the task-arg tuples by subcalc label + # Map each src_group id to its top-level sourceModel branch_id + grp_ids_by_smb = self.csm.grp_ids_by_source_model() + smb_of_grp = {gid: smb + for smb, gids in grp_ids_by_smb.items() + for gid in gids} + + # Partition the task-arg tuples by top-level sourceModel branch partitions = {} for args in allargs: - # strip any tile suffix ("5-2" -> 5) to recover grp_id + # Strip any tile suffix ("5-2" -> 5) to recover grp_id gid = int(args[0][0].split('-')[0]) - partitions.setdefault(label_of_grp[gid], []).append(args) + partitions.setdefault(smb_of_grp[gid], []).append(args) - # Run subcalcs one at a time + # Run one source model at a time acc = AccumDict(accum=0.) - for lbl in sorted(partitions): - logging.info('Subcalc %r: %d tasks', lbl, len(partitions[lbl])) + for smb in sorted(partitions): + logging.info('Source model %r: %d tasks', + smb, len(partitions[smb])) smap = parallel.Starmap( - task_func, partitions[lbl], h5=self.datastore.hdf5) + task_func, partitions[smb], h5=self.datastore.hdf5) acc = smap.reduce(self.agg_dicts, acc) return acc diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index 70a223e07652..1eee100f904d 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -502,27 +502,29 @@ def test_case_23_bis(self): assert ns == 26 def test_case_24(self): - # Parity check: subcalc-mode hazard statistics (mean and - # quantiles) must match the non-subcalc approach for both - # full enumeration and sampling - # --> Using a small tolerance because of diff task - # reduction order on remote + # Parity check: with sequential_source_models=true the hazard + # statistics (mean and quantiles) must match the regular + # (all-in-one Starmap) approach for both full enumeration and + # sampling. A small tolerance is used because task reduction + # order can differ across runs # Full enumeration - self.run_calc(case_24.__file__, 'job.ini') # Using subcalc - sub_full = self.calc.datastore['hcurves-stats'][:] - self.run_calc(case_24.__file__, 'job_nosub.ini') # Regular - nosub_full = self.calc.datastore['hcurves-stats'][:] - aac(sub_full, nosub_full, atol=1e-6, rtol=1e-6) + self.run_calc(case_24.__file__, 'job.ini', + sequential_source_models='true') + seq_full = self.calc.datastore['hcurves-stats'][:] + self.run_calc(case_24.__file__, 'job.ini') + reg_full = self.calc.datastore['hcurves-stats'][:] + aac(seq_full, reg_full, atol=1e-6, rtol=1e-6) # Sampling - self.run_calc(case_24.__file__, 'job.ini', # Using subcalc + self.run_calc(case_24.__file__, 'job.ini', + sequential_source_models='true', number_of_logic_tree_samples='10') - sub_sampled = self.calc.datastore['hcurves-stats'][:] - self.run_calc(case_24.__file__, 'job_nosub.ini', # Regular + seq_sampled = self.calc.datastore['hcurves-stats'][:] + self.run_calc(case_24.__file__, 'job.ini', number_of_logic_tree_samples='10') - nosub_sampled = self.calc.datastore['hcurves-stats'][:] - aac(sub_sampled, nosub_sampled, atol=1e-6, rtol=1e-6) + reg_sampled = self.calc.datastore['hcurves-stats'][:] + aac(seq_sampled, reg_sampled, atol=1e-6, rtol=1e-6) def test_case_25(self): # BCHydro-style correlated uncertainties (alt1 + alt2 + alt3) diff --git a/openquake/commonlib/oqvalidation.py b/openquake/commonlib/oqvalidation.py index 0bdf040c8822..b586bd5f9992 100644 --- a/openquake/commonlib/oqvalidation.py +++ b/openquake/commonlib/oqvalidation.py @@ -772,6 +772,14 @@ Example: *ses_seed = 123*. Default: 42 +sequential_source_models: + Flag used in classical and disaggregation calculations to dispatch + tasks one top-level sourceModel branch at a time (one Starmap per + source model, run sequentially). Not compatible with source models + that share sources across top-level branches. + Example: *sequential_source_models = true*. + Default: false + shakemap_id: Used in ShakeMap calculations to download a ShakeMap from the USGS site Example: *shakemap_id = usp000fjta*. @@ -1233,6 +1241,7 @@ class OqParam(valid.ParamSet): ses_per_logic_tree_path = valid.Param( valid.compose(valid.nonzero, valid.positiveint), 1) ses_seed = valid.Param(valid.positiveint, 42) + sequential_source_models = valid.Param(valid.boolean, False) shakemap_id = valid.Param(valid.nice_string, None) # example: shakemap_uri = {'kind': 'usgs_id', 'id': 'XXX'} shakemap_uri = valid.Param(valid.dictionary, {}) @@ -2228,6 +2237,15 @@ def is_valid_disagg_by_src(self): return self.ps_grid_spacing == 0 return True + def is_valid_sequential_source_models(self): + """ + sequential_source_models is only meaningful for classical and + disaggregation calculations + """ + if self.sequential_source_models: + return self.calculation_mode in ('classical', 'disaggregation') + return True + def is_valid_concurrent_tasks(self): """ At most you can use 30_000 tasks diff --git a/openquake/commonlib/readinput.py b/openquake/commonlib/readinput.py index a30724606e34..4a287970cbd6 100644 --- a/openquake/commonlib/readinput.py +++ b/openquake/commonlib/readinput.py @@ -964,12 +964,6 @@ def get_source_model_lt(oqparam): if oqparam.calculation_mode.startswith('scenario'): return logictree.SourceModelLogicTree.fake() smlt = get_smlt(vars(oqparam)) - if smlt.has_subcalcs and oqparam.calculation_mode not in ( - 'classical', 'disaggregation'): - raise InvalidFile( - "%s: 'subcalc' labels are only supported in classical and " - "disaggregation calculations (calculation_mode = %r)" - % (smlt.filename, oqparam.calculation_mode)) for bset in smlt.branchsets: bset.check_duplicates(smlt.filename) srcids = set(smlt.source_data['source']) diff --git a/openquake/commonlib/tests/source_test.py b/openquake/commonlib/tests/source_test.py index 05d21b41887f..793b548778b0 100644 --- a/openquake/commonlib/tests/source_test.py +++ b/openquake/commonlib/tests/source_test.py @@ -17,6 +17,7 @@ # along with OpenQuake. If not, see . import os +import copy import unittest from io import BytesIO @@ -29,7 +30,10 @@ site, geo, mfd, pmf, scalerel, valid, tests as htests) from openquake.hazardlib import source, sourceconverter as s from openquake.hazardlib.tom import PoissonTOM +from openquake.hazardlib.lt import Realization from openquake.hazardlib.logictree import FullLogicTree +from openquake.hazardlib.source_group import CompositeSourceModel, SourceGroup +from openquake.hazardlib.source_reader import sampling_dt from openquake.hazardlib import nrml from openquake.commonlib import tests, readinput @@ -751,3 +755,72 @@ def test_oversampling(self): def tearDown(self): Starmap.shutdown() + + +class SequentialSourcesTestCase(unittest.TestCase): + """ + Tests for the sequential_source_models dispatch including the + grp_ids_by_source_model method, which is used to partition + src_groups by top-level sourceModel branch. + """ + @classmethod + def setUpClass(cls): + # Load a real source, and once per-test have + # their sampling param set to control trt_smrs + conv = s.SourceConverter(investigation_time=50., + rupture_mesh_spacing=1, + complex_fault_mesh_spacing=1, + width_of_mfd_bin=1., + area_source_discretization=1.) + [point_grp, *_] = nrml.to_python(MIXED_SRC_MODEL, conv) + cls.template_src = point_grp[0] + + def _build_csm(self, sm_branch_ids, smrs_per_group): + # sm_branch_ids[i] is the top-level branch_id of smr i; + # smrs_per_group[j] lists the smrs of the j-th src_group + + # One Realization per smr - lt_path[0] is the top-level branch + sm_rlzs = [Realization(bid, 1., i, (bid,)) + for i, bid in enumerate(sm_branch_ids)] + full_lt = object.__new__(FullLogicTree) + full_lt.sm_rlzs = sm_rlzs + + # One SourceGroup per entry - sampling drives trt_smrs + src_groups = [] + for smrs in smrs_per_group: + src = copy.copy(self.template_src) + # Pack given smrs into the sampling array + src.sampling = numpy.array( + [(smr, 1) for smr in smrs], sampling_dt) + # Empty SourceGroup + sg = SourceGroup(self.template_src.tectonic_region_type) + # Store sources + sg.sources = [src] + src_groups.append(sg) + + # Make the CSM + csm = object.__new__(CompositeSourceModel) + csm.full_lt = full_lt + csm.src_groups = src_groups + + return csm + + def test_grp_ids_by_source_model_disjoint(self): + # Two src_groups, each belonging to a distinct top-level + # source model, group cleanly by branch_id + csm = self._build_csm(sm_branch_ids=['sm_a', 'sm_b'], + smrs_per_group=[[0], [1]]) + self.assertEqual(csm.grp_ids_by_source_model(), + {'sm_a': [0], 'sm_b': [1]}) + + def test_grp_ids_by_source_model_shared_raises(self): + # A src_group whose trt_smrs point at smrs from more than + # one top-level sourceModel branch must raise an error + csm = self._build_csm(sm_branch_ids=['sm_a', 'sm_b'], + smrs_per_group=[[0, 1]]) + with self.assertRaises(ValueError) as cm: + csm.grp_ids_by_source_model() + msg = str(cm.exception) + self.assertIn('spans multiple source models', msg) + self.assertIn('sm_a', msg) + self.assertIn('sm_b', msg) diff --git a/openquake/hazardlib/logictree.py b/openquake/hazardlib/logictree.py index 02c5083b58d3..1551dddf4668 100644 --- a/openquake/hazardlib/logictree.py +++ b/openquake/hazardlib/logictree.py @@ -503,28 +503,10 @@ def parse_tree(self, tree_node): attach_branches(self) self.source_data = numpy.array(self.source_data, source_dt) unique = numpy.unique(self.source_data['fname']) - self.validate_subcalcs() dt = time.time() - t0 logging.debug('Validated source model logic tree with %d underlying ' 'files in %.2f seconds', len(unique), dt) - def validate_subcalcs(self): - """ - If any top-level sourceModel branch has a "subcalc" label, all - must have one. - """ - top = self.branchsets[0] - labelled = [br for br in top.branches if br.subcalc is not None] - if not labelled: - return - if len(labelled) != len(top.branches): - missing = [br.branch_id for br in top.branches - if br.subcalc is None] - raise LogicTreeError( - None, self.filename, - "'subcalc' must be set on every top-level sourceModel " - "branch, or on none. Missing on: %s" % ', '.join(missing)) - @property def utypes(self): """ @@ -532,40 +514,6 @@ def utypes(self): """ return [bs.uncertainty_type for bs in self.branchsets] - @property - def subcalcs(self): - """ - :returns: - dict grouping the top-level "sourceModel" branches by their - "subcalc" attribute. Empty dict if no branch carries a "subcalc" - label. - """ - out = {} - for br in self.branchsets[0].branches: - if br.subcalc is None: - continue - out.setdefault(br.subcalc, []).append(br) - return out - - @property - def subcalc_weights(self): - """ - :returns: - dict giving the summed weight of the top-level sourceModel - branches for each subcalc - """ - return {label: float(sum(br.weight for br in brs)) - for label, brs in self.subcalcs.items()} - - @property - def has_subcalcs(self): - """ - :returns: - True if any top-level sourceModel branch has a - subcalc label - """ - return bool(self.subcalcs) - def parse_branchset(self, branchset_node, bsno): """ :param branchset_ node: @@ -618,49 +566,6 @@ def parse_branchset(self, branchset_node, bsno): "branch '%s' is not yet defined" % branch_id) self.branchsets.append(branchset) - def _check_branch_count(self, bs_id, branches): - maxlen = len(BASE183) - if self.branchID == '' and len(branches) > maxlen: - msg = ('%s: the branchset %s has too many branches (%d > %d)\n' - 'you should split it, see https://docs.openquake.org/' - 'oq-engine/advanced/latest/logic_trees.html') - raise InvalidFile( - msg % (self.filename, bs_id, len(branches), maxlen)) - - def _collect_source_model_files(self, branchnode, value_node, value): - vals = [] # filenames with sources in it - try: - for fname in value_node.text.split(): - if (fname.endswith(('.xml', '.nrml')) - and not self.test_mode): - ok = self.collect_source_model_data( - branchnode['branchID'], fname) - if ok: - vals.append(fname) - except Exception as exc: - raise LogicTreeError( - value_node, self.filename, str(exc)) from exc - if self.branchID and self.branchID not in branchnode['branchID']: - return '' # reduce all branches except branchID - if self.source_id: # only the files containing source_id - srcid = self.source_id.split('@')[0] - return ' '.join(reduce_fnames(vals, srcid)) - return value - - def _validate_branch_metadata(self, branchnode, branchset): - branch_id = branchnode.attrib.get('branchID') - if branch_id in self.branches: - raise LogicTreeError( - branchnode, self.filename, - "branchID '%s' is not unique" % branch_id) - subcalc = branchnode.attrib.get('subcalc') - if subcalc is not None and branchset.uncertainty_type != 'sourceModel': - raise LogicTreeError( - branchnode, self.filename, - "'subcalc' attribute is only allowed on branches of " - "sourceModel branchsets") - return branch_id, subcalc - def parse_branches(self, branchset_node, branchset): """ Create and attach branches at ``branchset_node`` to ``branchset``. @@ -678,15 +583,21 @@ def parse_branches(self, branchset_node, branchset): """ correlated = branchset_node.get('applyToSources') == '*' bs_id = branchset_node['branchSetID'] + weight_sum = 0 branches = branchset_node.nodes if OQ_REDUCE: # only take first branch branches = [branches[0]] branches[0].uncertaintyWeight.text = 1. - self._check_branch_count(bs_id, branches) - bsno = len(self.branchsets) - weight_sum = 0 values = [] + bsno = len(self.branchsets) zeros = [] + maxlen = len(BASE183) + if self.branchID == '' and len(branches) > maxlen: + msg = ('%s: the branchset %s has too many branches (%d > %d)\n' + 'you should split it, see https://docs.openquake.org/' + 'oq-engine/advanced/latest/logic_trees.html') + raise InvalidFile( + msg % (self.filename, bs_id, len(branches), maxlen)) for brno, branchnode in enumerate(branches): weight = ~branchnode.uncertaintyWeight value_node = node_from_elem(branchnode.uncertaintyModel) @@ -702,18 +613,36 @@ def parse_branches(self, branchset_node, branchset): value = parse_uncertainty(branchset.uncertainty_type, value_node, self.filename) if branchset.uncertainty_type in ('sourceModel', 'extendModel'): - value = self._collect_source_model_files( - branchnode, value_node, value) - branch_id, subcalc = self._validate_branch_metadata( - branchnode, branchset) + vals = [] # filenames with sources in it + try: + for fname in value_node.text.split(): + if (fname.endswith(('.xml', '.nrml')) + and not self.test_mode): + ok = self.collect_source_model_data( + branchnode['branchID'], fname) + if ok: + vals.append(fname) + except Exception as exc: + raise LogicTreeError( + value_node, self.filename, str(exc)) from exc + if (self.branchID and self.branchID not in + branchnode['branchID']): + value = '' # reduce all branches except branchID + elif self.source_id: # only the files containing source_id + srcid = self.source_id.split('@')[0] + value = ' '.join(reduce_fnames(vals, srcid)) + branch_id = branchnode.attrib.get('branchID') + if branch_id in self.branches: + raise LogicTreeError( + branchnode, self.filename, + "branchID '%s' is not unique" % branch_id) if value == '': # with logic tree reduction a branch can be empty # see case_68_bis zero_id = branch_id zeros.append(weight) else: - branch = Branch(branch_id, value, weight, bs_id, - subcalc=subcalc) + branch = Branch(branch_id, value, weight, bs_id) self.branches[branch_id] = branch branchset.branches.append(branch) self.shortener[branch_id] = keyno(branch_id, bsno, brno, BASE183) diff --git a/openquake/hazardlib/lt.py b/openquake/hazardlib/lt.py index 39a39689dde7..e40dbc38ff51 100644 --- a/openquake/hazardlib/lt.py +++ b/openquake/hazardlib/lt.py @@ -653,18 +653,13 @@ class Branch(object): of ```` child node. :param bs_id: BranchSetID of the branchset to which the branch belongs - :param subcalc: - Optional subcalc label; when set on a top-level "sourceModel" - branch, all downstream branches reachable via "applyToBranches" - chains inherit this label and are grouped into one subcalc """ - def __init__(self, branch_id, value, weight, bs_id='', subcalc=None): + def __init__(self, branch_id, value, weight, bs_id=''): self.branch_id = branch_id self.value = value self.weight = weight self.bs_id = bs_id self.bset = None - self.subcalc = subcalc def is_dummy(self): """ @@ -686,8 +681,6 @@ def is_leaf(self): def to_node(self): attrib = dict(branchID=self.branch_id) - if self.subcalc is not None: - attrib['subcalc'] = self.subcalc nodes = [Node('uncertaintyModel', {}, self.value), Node('uncertaintyWeight', {}, self.weight)] return Node('logicTreeBranch', attrib, None, nodes) diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 4c62415fc6cd..22b54b612098 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -359,33 +359,30 @@ def get_sources(self, smr=None): srcs.extend(grp) return srcs - def grp_ids_by_subcalc(self): + def grp_ids_by_source_model(self): """ :returns: - dict grouping src_groups by the subcalc label of the top-level - "sourceModel" branch they trace back to (via "trt_smrs"). + Dict grouping src_groups by the branch_id of the top-level + sourceModel branch they trace back to (via trt_smrs). - Raises a ValueError if a src_group spans more than one subcalc. + Raises a ValueError if a src_group is shared across more than + one top-level source model, since such groups cannot be + dispatched sequentially by source model. """ - # Build an smr -> subcalc map - branch_subcalc = { - br.branch_id: br.subcalc for br - in self.full_lt.source_model_lt.branchsets[0].branches} - smr_subcalc = {smr: branch_subcalc.get(rlz.lt_path[0]) - for smr, rlz in enumerate(self.full_lt.sm_rlzs)} + # Build an smr -> top-level sourceModel branch_id map + smr_smb = {smr: rlz.lt_path[0] + for smr, rlz in enumerate(self.full_lt.sm_rlzs)} out = {} for grp_id, sg in enumerate(self.src_groups): - labels = set() - for trt_smr in sg.sources[0].trt_smrs: - labels.add(smr_subcalc.get(trt_smr % TWO24)) - labels.discard(None) - if len(labels) > 1: + smbs = {smr_smb[trt_smr % TWO24] + for trt_smr in sg.sources[0].trt_smrs} + if len(smbs) > 1: raise ValueError( - 'src_group %d (%s) spans multiple subcalcs %s; ' - 'cross-subcalc source sharing is not supported' - % (grp_id, sg.trt, sorted(labels))) - if labels: - out.setdefault(labels.pop(), []).append(grp_id) + 'src_group %d (%s) spans multiple source models %s; ' + 'sequential_source_models=true does not support ' + 'sources shared across source models' + % (grp_id, sg.trt, sorted(smbs))) + out.setdefault(smbs.pop(), []).append(grp_id) return out def get_trt_smrs(self): diff --git a/openquake/hazardlib/tests/lt_test.py b/openquake/hazardlib/tests/lt_test.py index 0d557d6596eb..adda2bd29208 100644 --- a/openquake/hazardlib/tests/lt_test.py +++ b/openquake/hazardlib/tests/lt_test.py @@ -27,8 +27,6 @@ nrml, lt, sourceconverter, calc, site, valid, contexts) from openquake.hazardlib.calc.hazard_curve import classical from openquake.hazardlib.geo.point import Point -from openquake.hazardlib.logictree import SourceModelLogicTree -from openquake.hazardlib.lt import LogicTreeError CDIR = os.path.dirname(__file__) ae = numpy.testing.assert_equal @@ -677,87 +675,3 @@ def test_mixed_collapsed_apply_uncertainties(self): # max_mag updated by bs_normal for mod_src in mod_sg.sources: self.assertEqual(mod_src.mfd.max_mag, 6.5) - - -class SubcalcAttributeTestCase(unittest.TestCase): - """ - Tests for the ``subcalc`` attribute on top-level ``sourceModel`` - branches (see :class:`SourceModelLogicTree`). - """ - - SM_ONLY = ''' - - - - - {sm1} - 0.6 - - - {sm2} - 0.4 - - {extra} - -''' - - EXTRA_LSD = ''' - - - 15.0 - 1.0 - - ''' - - SM1 = 'sm1.xml' - SM2 = 'sm2.xml' - - def _parse(self, xml, **kw): - # Parse from a temp file with test_mode=True so referenced - # source-model XMLs are not loaded from disk - path = gettemp(xml, suffix='.xml') - return SourceModelLogicTree(path, test_mode=True, **kw) - - def test_subcalc_on_non_source_model_raises(self): - # A subcalc attribute on a non-sourceModel branch is an error - xml = self.SM_ONLY.format( - sub1=' subcalc="a"', sub2=' subcalc="a"', - sm1=self.SM1, sm2=self.SM2, - extra=self.EXTRA_LSD.format(sub=' subcalc="erroneous"')) - with self.assertRaises(LogicTreeError) as cm: - self._parse(xml) - self.assertIn( - "'subcalc' attribute is only allowed on branches of " - "sourceModel branchsets", str(cm.exception)) - - def test_partial_subcalc_labelling_raises(self): - # Subcalc must be on every top-level branch, or none - xml = self.SM_ONLY.format( - sub1=' subcalc="a"', sub2='', - sm1=self.SM1, sm2=self.SM2, extra='') - with self.assertRaises(LogicTreeError) as cm: - self._parse(xml) - self.assertIn( - "'subcalc' must be set on every top-level sourceModel " - "branch, or on none. Missing on: b2", str(cm.exception)) - - def test_subcalc_grouping_and_weights(self): - # Subcalcs group top-level branches by label and sum weights - xml = self.SM_ONLY.format( - sub1=' subcalc="a"', sub2=' subcalc="b"', - sm1=self.SM1, sm2=self.SM2, extra='') - smlt = self._parse(xml) - self.assertTrue(smlt.has_subcalcs) - self.assertEqual(sorted(smlt.subcalcs), ['a', 'b']) - self.assertAlmostEqual(smlt.subcalc_weights['a'], 0.6) - self.assertAlmostEqual(smlt.subcalc_weights['b'], 0.4) - - def test_no_subcalc_labels_leaves_smlt_unaffected(self): - # Unlabelled tree has no subcalcs and validates cleanly - xml = self.SM_ONLY.format( - sub1='', sub2='', - sm1=self.SM1, sm2=self.SM2, extra='') - smlt = self._parse(xml) - self.assertFalse(smlt.has_subcalcs) - self.assertEqual(smlt.subcalcs, {}) diff --git a/openquake/qa_tests_data/logictree/README.md b/openquake/qa_tests_data/logictree/README.md index b05529223aea..c4b8b1cc629c 100644 --- a/openquake/qa_tests_data/logictree/README.md +++ b/openquake/qa_tests_data/logictree/README.md @@ -27,7 +27,7 @@ | case\_22 | Test sigma_model_alatik2015 | | case\_23 | Arctic region and IDL (no bounding box) | | case\_23\_bis | Correlated uncertainties | -| case\_24 | Tests the subcalc feature's parity vs non-subcalc for full-enum + sampling | +| case\_24 | Tests sequential_source_models parity vs regular for full-enum + sampling | | case\_28 | Test collapse\_gsim\_logic\_tree | | case\_28\_bis | Test missing z1pt0 | | case\_25 | BC Hydro NVA SSC LT source model LT | diff --git a/openquake/qa_tests_data/logictree/case_24/job.ini b/openquake/qa_tests_data/logictree/case_24/job.ini index 13ed056871bf..1b7e7096b8ca 100644 --- a/openquake/qa_tests_data/logictree/case_24/job.ini +++ b/openquake/qa_tests_data/logictree/case_24/job.ini @@ -1,6 +1,6 @@ [general] -description = subcalc parity: full-enum + subcalc mode +description = sequential_source_models parity test calculation_mode = classical random_seed = 42 diff --git a/openquake/qa_tests_data/logictree/case_24/job_nosub.ini b/openquake/qa_tests_data/logictree/case_24/job_nosub.ini deleted file mode 100644 index 2728074adb4f..000000000000 --- a/openquake/qa_tests_data/logictree/case_24/job_nosub.ini +++ /dev/null @@ -1,39 +0,0 @@ -[general] - -description = subcalc parity: full-enum without subcalc labels -calculation_mode = classical -random_seed = 42 - -[geometry] - -sites = 0.0 0.0 - -[logic_tree] - -number_of_logic_tree_samples = 0 - -[erf] - -rupture_mesh_spacing = 2.0 -width_of_mfd_bin = 0.5 - -[site_params] - -reference_vs30_type = measured -reference_vs30_value = 760.0 -reference_depth_to_2pt5km_per_sec = 2.5 -reference_depth_to_1pt0km_per_sec = 50.0 - -[calculation] - -source_model_logic_tree_file = smlt_nosub.xml -gsim_logic_tree_file = gsim_logic_tree.xml -investigation_time = 50.0 -intensity_measure_types_and_levels = {"PGA": [0.05, 0.1, 0.2, 0.5]} -truncation_level = 3.0 -maximum_distance = 200.0 - -[output] - -mean = true -quantiles = 0.05 0.5 0.95 diff --git a/openquake/qa_tests_data/logictree/case_24/smlt.xml b/openquake/qa_tests_data/logictree/case_24/smlt.xml index 5c601ff03870..2539a4cd9001 100644 --- a/openquake/qa_tests_data/logictree/case_24/smlt.xml +++ b/openquake/qa_tests_data/logictree/case_24/smlt.xml @@ -3,14 +3,13 @@ xmlns:gml="http://www.opengis.net/gml"> - + - + sm_a.xml 0.7 - + sm_b.xml 0.3 diff --git a/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml b/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml deleted file mode 100644 index 08bca4f059ed..000000000000 --- a/openquake/qa_tests_data/logictree/case_24/smlt_nosub.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - sm_a.xml - 0.7 - - - sm_b.xml - 0.3 - - - - - - 3.0 1.0 - 0.6 - - - 2.9 1.05 - 0.4 - - - - - - 3.2 0.9 - 0.5 - - - 3.1 0.95 - 0.5 - - - - - From 7b4c5f1ceff322d5445e44f058b3ecb2877454b4 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 3 Aug 2026 16:20:48 +0200 Subject: [PATCH 10/31] cleanup --- openquake/calculators/classical.py | 4 ++-- openquake/commonlib/oqvalidation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 3d5fcef11889..f5d96216169d 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -653,8 +653,8 @@ def _execute(self, sgs, ds): def _run_sequential_source_models(self, allargs, task_func): """ - Run one Starmap per top-level ``sourceModel`` branch sequentially, - so that at most one source model's tasks are in flight at a time. + Run one Starmap per top-level sourceModel branch sequentially, + so that at most one source model's tasks are running at one time. """ # Map each src_group id to its top-level sourceModel branch_id grp_ids_by_smb = self.csm.grp_ids_by_source_model() diff --git a/openquake/commonlib/oqvalidation.py b/openquake/commonlib/oqvalidation.py index b586bd5f9992..9f9f163a94f5 100644 --- a/openquake/commonlib/oqvalidation.py +++ b/openquake/commonlib/oqvalidation.py @@ -2239,7 +2239,7 @@ def is_valid_disagg_by_src(self): def is_valid_sequential_source_models(self): """ - sequential_source_models is only meaningful for classical and + sequential_source_models is only useable in classical and disaggregation calculations """ if self.sequential_source_models: From 33e30b4b30903906013ec1d0156be06aa03a89f9 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 3 Aug 2026 21:38:17 +0200 Subject: [PATCH 11/31] remove sort --- openquake/hazardlib/logictree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openquake/hazardlib/logictree.py b/openquake/hazardlib/logictree.py index 180d6469d2e4..819d9039757e 100644 --- a/openquake/hazardlib/logictree.py +++ b/openquake/hazardlib/logictree.py @@ -916,9 +916,9 @@ def __fromh5__(self, array, attrs): ats = self.bsetdict[bsid].get('applyToSources') atb = self.bsetdict[bsid].get('applyToBranches') if ats: - filters['applyToSources'] = sorted(ats.split()) + filters['applyToSources'] = ats.split() if atb: - filters['applyToBranches'] = sorted(atb.split()) + filters['applyToBranches'] = atb.split() bset = BranchSet(utype, filters, ordinal) bset.id = bsid for no, row in enumerate(rows): From f613b26ba8a9a578368ca7eacfc3e98482c9deb2 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 3 Aug 2026 21:40:50 +0200 Subject: [PATCH 12/31] revert --- openquake/hazardlib/logictree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openquake/hazardlib/logictree.py b/openquake/hazardlib/logictree.py index 819d9039757e..180d6469d2e4 100644 --- a/openquake/hazardlib/logictree.py +++ b/openquake/hazardlib/logictree.py @@ -916,9 +916,9 @@ def __fromh5__(self, array, attrs): ats = self.bsetdict[bsid].get('applyToSources') atb = self.bsetdict[bsid].get('applyToBranches') if ats: - filters['applyToSources'] = ats.split() + filters['applyToSources'] = sorted(ats.split()) if atb: - filters['applyToBranches'] = atb.split() + filters['applyToBranches'] = sorted(atb.split()) bset = BranchSet(utype, filters, ordinal) bset.id = bsid for no, row in enumerate(rows): From 36c7b16f43b2330b012c20a1db5b2715b25e15a4 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 4 Aug 2026 16:16:44 +0200 Subject: [PATCH 13/31] remove dup code --- openquake/hazardlib/source_group.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 22b54b612098..2693a6c47f07 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -34,7 +34,6 @@ U16 = numpy.uint16 TWO16 = 2 ** 16 # 65,536 -TWO16 = 2 ** 16 # 65,536 TWO24 = 2 ** 24 # 16,777,216 TWO30 = 2 ** 30 # 1,073,741,24 TWO32 = 2 ** 32 # 4,294,967,296 From 0822dea62661a382e456accda9062f915b8344d1 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 4 Aug 2026 21:44:56 +0200 Subject: [PATCH 14/31] increase src_grp limit (probably bad idea) --- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- openquake/hazardlib/source_group.py | 8 ++++---- openquake/hazardlib/source_reader.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index f5d96216169d..245dbc1974de 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -453,7 +453,7 @@ def create_rup(self): elif param == 'rup_id': dt = I64 elif param == 'grp_id': - dt = U16 + dt = U32 else: dt = F32 descr.append((param, dt)) diff --git a/openquake/calculators/preclassical.py b/openquake/calculators/preclassical.py index 1625262cb6c0..4b34f609f3ee 100644 --- a/openquake/calculators/preclassical.py +++ b/openquake/calculators/preclassical.py @@ -225,7 +225,7 @@ def store_csm(dstore, csm, sitecol, cmakers): [(_grp_id(blocks[0]), len(cm.gsims), len(tgets), len(blocks), len(cm.gsims) * mb_per_gsim, extra['weight'], extra['codes'], cm.trt) for cm, tgets, blocks, extra in quartets], - [('grp_id', U16), ('gsims', U16), ('tiles', U16), ('blocks', U16), + [('grp_id', U32), ('gsims', U16), ('tiles', U16), ('blocks', U16), ('max_mb', F32), ('weight', F32), ('codes', ' Date: Wed, 5 Aug 2026 11:07:35 +0200 Subject: [PATCH 15/31] clean --- openquake/calculators/classical.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 245dbc1974de..d9ed7836aa34 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -657,27 +657,24 @@ def _run_sequential_source_models(self, allargs, task_func): so that at most one source model's tasks are running at one time. """ # Map each src_group id to its top-level sourceModel branch_id - grp_ids_by_smb = self.csm.grp_ids_by_source_model() - smb_of_grp = {gid: smb - for smb, gids in grp_ids_by_smb.items() - for gid in gids} + smb_of_grp = { + gid: smb + for smb, gids in self.csm.grp_ids_by_source_model().items() + for gid in gids} # Partition the task-arg tuples by top-level sourceModel branch - partitions = {} + partitions = AccumDict(accum=[]) for args in allargs: # Strip any tile suffix ("5-2" -> 5) to recover grp_id gid = int(args[0][0].split('-')[0]) - partitions.setdefault(smb_of_grp[gid], []).append(args) + partitions[smb_of_grp[gid]].append(args) # Run one source model at a time acc = AccumDict(accum=0.) - for smb in sorted(partitions): - logging.info('Source model %r: %d tasks', - smb, len(partitions[smb])) - smap = parallel.Starmap( - task_func, partitions[smb], h5=self.datastore.hdf5) + for smb, part in sorted(partitions.items()): + logging.info('Source model %r: %d tasks', smb, len(part)) + smap = parallel.Starmap(task_func, part, h5=self.datastore.hdf5) acc = smap.reduce(self.agg_dicts, acc) - return acc def _post_execute(self, acc): From 8e15c1356ce763e1ecf69aa1cbe7916b9b94aee7 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 11:15:57 +0200 Subject: [PATCH 16/31] more cleaning --- openquake/calculators/classical.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index d9ed7836aa34..0e08608024db 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -653,16 +653,16 @@ def _execute(self, sgs, ds): def _run_sequential_source_models(self, allargs, task_func): """ - Run one Starmap per top-level sourceModel branch sequentially, - so that at most one source model's tasks are running at one time. + Run one Starmap per sourceModel branch sequentially, so that + at most one source model's tasks are running at one time. """ - # Map each src_group id to its top-level sourceModel branch_id + # Map each src_group id to its sourceModel branch_id smb_of_grp = { gid: smb for smb, gids in self.csm.grp_ids_by_source_model().items() for gid in gids} - # Partition the task-arg tuples by top-level sourceModel branch + # Partition the task-arg tuples by sourceModel branch partitions = AccumDict(accum=[]) for args in allargs: # Strip any tile suffix ("5-2" -> 5) to recover grp_id @@ -675,6 +675,7 @@ def _run_sequential_source_models(self, allargs, task_func): logging.info('Source model %r: %d tasks', smb, len(part)) smap = parallel.Starmap(task_func, part, h5=self.datastore.hdf5) acc = smap.reduce(self.agg_dicts, acc) + return acc def _post_execute(self, acc): From 2a1be44ee4bf7f4433b06a6db4721037f441262d Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 11:33:17 +0200 Subject: [PATCH 17/31] clean --- openquake/calculators/classical.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 0e08608024db..46651d59ebf6 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -654,7 +654,7 @@ def _execute(self, sgs, ds): def _run_sequential_source_models(self, allargs, task_func): """ Run one Starmap per sourceModel branch sequentially, so that - at most one source model's tasks are running at one time. + only a single source model's tasks are running at one time. """ # Map each src_group id to its sourceModel branch_id smb_of_grp = { @@ -669,7 +669,7 @@ def _run_sequential_source_models(self, allargs, task_func): gid = int(args[0][0].split('-')[0]) partitions[smb_of_grp[gid]].append(args) - # Run one source model at a time + # Run one source modl at a time acc = AccumDict(accum=0.) for smb, part in sorted(partitions.items()): logging.info('Source model %r: %d tasks', smb, len(part)) From 04d70e6995049bf64da213929aff4429b5f0e858 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 11:48:30 +0200 Subject: [PATCH 18/31] revert --- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 46651d59ebf6..b14801894acd 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -453,7 +453,7 @@ def create_rup(self): elif param == 'rup_id': dt = I64 elif param == 'grp_id': - dt = U32 + dt = U16 else: dt = F32 descr.append((param, dt)) diff --git a/openquake/calculators/preclassical.py b/openquake/calculators/preclassical.py index 4b34f609f3ee..1625262cb6c0 100644 --- a/openquake/calculators/preclassical.py +++ b/openquake/calculators/preclassical.py @@ -225,7 +225,7 @@ def store_csm(dstore, csm, sitecol, cmakers): [(_grp_id(blocks[0]), len(cm.gsims), len(tgets), len(blocks), len(cm.gsims) * mb_per_gsim, extra['weight'], extra['codes'], cm.trt) for cm, tgets, blocks, extra in quartets], - [('grp_id', U32), ('gsims', U16), ('tiles', U16), ('blocks', U16), + [('grp_id', U16), ('gsims', U16), ('tiles', U16), ('blocks', U16), ('max_mb', F32), ('weight', F32), ('codes', ' Date: Wed, 5 Aug 2026 11:49:20 +0200 Subject: [PATCH 19/31] revert --- openquake/hazardlib/tests/lt_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openquake/hazardlib/tests/lt_test.py b/openquake/hazardlib/tests/lt_test.py index adda2bd29208..0a300e6e517c 100644 --- a/openquake/hazardlib/tests/lt_test.py +++ b/openquake/hazardlib/tests/lt_test.py @@ -675,3 +675,4 @@ def test_mixed_collapsed_apply_uncertainties(self): # max_mag updated by bs_normal for mod_src in mod_sg.sources: self.assertEqual(mod_src.mfd.max_mag, 6.5) + From e41be4a7ffdee8d3e4e60c426dd4cd05bc5e3bcb Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 11:52:07 +0200 Subject: [PATCH 20/31] revert --- openquake/hazardlib/source_group.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 1204c543cf9b..22b54b612098 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -32,7 +32,8 @@ from openquake.hazardlib.valid import basename, fragmentno from openquake.hazardlib.contexts import get_cmakers -U32 = numpy.uint32 +U16 = numpy.uint16 +TWO16 = 2 ** 16 # 65,536 TWO16 = 2 ** 16 # 65,536 TWO24 = 2 ** 24 # 16,777,216 TWO30 = 2 ** 30 # 1,073,741,24 @@ -43,10 +44,10 @@ def _grp_id(blk): # NB: grp_id may by passed instead of a source or a block - if isinstance(blk, (U32, int)): + if isinstance(blk, (U16, int)): return blk src = blk[0] - return src if isinstance(src, U32) else src.grp_id + return src if isinstance(src, U16) else src.grp_id def get_unique(sources): @@ -389,7 +390,7 @@ def get_trt_smrs(self): :returns: an array of trt_smrs (to be stored as an hdf5.vuint32 array) """ keys = [sg.sources[0].trt_smrs for sg in self.src_groups] - assert len(keys) < TWO32, len(keys) + assert len(keys) < TWO16, len(keys) return [numpy.array(trt_smrs, numpy.uint32) for trt_smrs in keys] def get_cmakers(self): From 8a6c61d747b4030fa2d243c0b328e9ce12a14d7e Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 11:54:03 +0200 Subject: [PATCH 21/31] revert --- openquake/hazardlib/source_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openquake/hazardlib/source_reader.py b/openquake/hazardlib/source_reader.py index 7a8bc06b843c..118f1fa94102 100644 --- a/openquake/hazardlib/source_reader.py +++ b/openquake/hazardlib/source_reader.py @@ -44,7 +44,7 @@ source_info_dt = numpy.dtype([ ('source_id', hdf5.vstr), # 0 - ('grp_id', U32), # 1 + ('grp_id', U16), # 1 ('code', (numpy.bytes_, 1)), # 2 ('calc_time', F32), # 3 ('num_ctxs', numpy.uint64), # 4 From d9deb52a5c88d22ac0ddb594c6c34c116ec34797 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 18:14:51 +0200 Subject: [PATCH 22/31] try splitting preclassical too --- openquake/calculators/preclassical.py | 110 ++++++++++++++++++-------- openquake/hazardlib/source_group.py | 20 +++++ 2 files changed, 95 insertions(+), 35 deletions(-) diff --git a/openquake/calculators/preclassical.py b/openquake/calculators/preclassical.py index 1625262cb6c0..0578ff4c9002 100644 --- a/openquake/calculators/preclassical.py +++ b/openquake/calculators/preclassical.py @@ -298,22 +298,16 @@ def populate_csm(self): self.store() logging.info('Building cmakers') trt_smrs = csm.get_trt_smrs() - self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) self.datastore.hdf5.save_vlen('trt_smrs', trt_smrs) + if oq.sequential_source_models: + grp_ids_by_batch = [ + numpy.array( + [sg.sources[0].grp_id for sg in sgs_batch], U32) + for _, _, sgs_batch, _ in csm.iter_source_model_batches()] + self.datastore.hdf5.save_vlen('grp_ids_by_batch', grp_ids_by_batch) sites = csm.sitecol if csm.sitecol else None if sites is None: logging.warning('No sites??') - - L = oq.imtls.size - Gfull = self.full_lt.gfull([cm.trt_smrs for cm in self.cmakers]) - Gt = sum(len(cm.gsims) for cm in self.cmakers) - extra = f'<{Gfull}' if Gt < Gfull else '' - if sites is not None: - nbytes = 4 * len(self.sitecol) * L * Gt - # Gt is known before starting the preclassical - logging.warning(f'Global RateMap of %s ({Gt=}%s)', - general.humansize(nbytes), extra) - if sites and not self.few_sites: # in SAM from 539,831 -> 11,430 sites lowres = sites.lower_res(res=4)[0] # res=4 ~39 km @@ -324,30 +318,13 @@ def populate_csm(self): sf = SourceFilter(sites, oq.maximum_distance) else: sf = SourceFilter(None) - atomic_sources = [] - normal_sources = [] reqv = 'reqv' in oq.inputs if reqv: logging.warning( 'Using equivalent distance approximation and ' 'collapsing hypocenters and nodal planes') - multifaults = [] - cmakers = self.cmakers.to_array() - for sg in csm.src_groups: - for src in sg: - if src.code == b'F': - multifaults.append(src) - if reqv and sg.trt in oq.inputs['reqv']: - if src.source_id not in oq.reqv_ignore_sources: - collapse_nphc(src) - grp_id = sg.sources[0].grp_id - # do nothing for atomic sources except counting the ruptures - if sg.atomic: - # compute weight sequentially - cmakers[grp_id].set_weight(sg, sf) - atomic_sources.extend(sg) - else: - normal_sources.extend(sg) + multifaults = [src for sg in csm.src_groups for src in sg + if src.code == b'F'] if multifaults: with hdf5.File(multifaults[0].hdf5path, 'r') as h5: secparams = h5['secparams'][:] @@ -356,11 +333,75 @@ def populate_csm(self): len(multifaults), general.humansize(secparams.nbytes)) else: secparams = () - self._process(atomic_sources, normal_sources, sf, secparams) + if oq.sequential_source_models: + # Bound preclassical memory by iterating one source model + # at a time; rebuild cmakers at end for post_execute + self._run_batched(sf, secparams, reqv) + self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) + else: + self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) + atomic_sources = [] + normal_sources = [] + cmakers = self.cmakers.to_array() + for sg in csm.src_groups: + for src in sg: + if reqv and sg.trt in oq.inputs['reqv']: + if src.source_id not in oq.reqv_ignore_sources: + collapse_nphc(src) + grp_id = sg.sources[0].grp_id + if sg.atomic: + cmakers[grp_id].set_weight(sg, sf) + atomic_sources.extend(sg) + else: + normal_sources.extend(sg) + self._process(atomic_sources, normal_sources, sf, secparams) + L = oq.imtls.size + Gfull = self.full_lt.gfull([cm.trt_smrs for cm in self.cmakers]) + Gt = sum(len(cm.gsims) for cm in self.cmakers) + extra = f'<{Gfull}' if Gt < Gfull else '' + if sites is not None: + nbytes = 4 * len(self.sitecol) * L * Gt + logging.warning(f'Global RateMap of %s ({Gt=}%s)', + general.humansize(nbytes), extra) allsources = csm.get_sources() self.store_source_info(source_data(allsources)) - def _process(self, atomic_sources, normal_sources, sf, secparams): + def _run_batched(self, sf, secparams, reqv): + """ + Run preclassical per source-model batch when the flag of + sequential_source_models is True. + """ + oq = self.oqparam + csm = self.csm + for batch_id, sm_id, sgs_batch, trt_smrs_batch in ( + csm.iter_source_model_batches()): + logging.info( + 'Preclassical batch %d (source model %r): %d src_groups', + batch_id, sm_id, len(sgs_batch)) + cmakers_batch = get_cmakers(trt_smrs_batch, csm.full_lt, oq) + cmaker_by_grp = { + sg.sources[0].grp_id: cm + for sg, cm in zip(sgs_batch, cmakers_batch.to_array())} + atomic_batch = [] + normal_batch = [] + for sg in sgs_batch: + for src in sg: + if reqv and sg.trt in oq.inputs['reqv']: + if src.source_id not in oq.reqv_ignore_sources: + collapse_nphc(src) + grp_id = sg.sources[0].grp_id + if sg.atomic: + cmaker_by_grp[grp_id].set_weight(sg, sf) + atomic_batch.extend(sg) + else: + normal_batch.extend(sg) + self._process(atomic_batch, normal_batch, sf, secparams, + cmaker_by_grp=cmaker_by_grp) + + def _process(self, atomic_sources, normal_sources, sf, secparams, + cmaker_by_grp=None): + if cmaker_by_grp is None: + cmaker_by_grp = dict(enumerate(self.cmakers.to_array())) # run preclassical in parallel for non-atomic sources if normal_sources: sources_by_key = groupby( @@ -371,10 +412,9 @@ def _process(self, atomic_sources, normal_sources, sf, secparams): # avoid a segfault in macOS self.datastore.swmr_on() smap = parallel.Starmap(preclassical, h5=self.datastore.hdf5) - cmakers = self.cmakers.to_array() num_tasks = len(sources_by_key) for grp_id, srcs in sources_by_key.items(): - cmaker = cmakers[grp_id] + cmaker = cmaker_by_grp[grp_id] cmaker.gsims = list(cmaker.gsims) # reducing data transfer pointlike = [src for src in srcs if hasattr(src, 'nodal_plane_distribution')] diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 22b54b612098..83bdf3ad53ae 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -393,6 +393,26 @@ def get_trt_smrs(self): assert len(keys) < TWO16, len(keys) return [numpy.array(trt_smrs, numpy.uint32) for trt_smrs in keys] + def iter_source_model_batches(self): + """ + Iterate "src_groups" in batches, one per sourceModel branch + """ + # Map sourceModel branch id -> list of global grp_ids + grp_ids_by_sm = self.grp_ids_by_source_model() + + # Sorted iteration makes batch_id constant across runs + for batch_id, sm_branch_id in enumerate(sorted(grp_ids_by_sm)): + grp_ids = grp_ids_by_sm[sm_branch_id] + + # Pick this batch's src_groups by their global grp_ids + src_groups_batch = [self.src_groups[gid] for gid in grp_ids] + + # Per-batch trt_smrs arrays + trt_smrs_batch = [ + numpy.array(sg.sources[0].trt_smrs, numpy.uint32) + for sg in src_groups_batch] + yield batch_id, sm_branch_id, src_groups_batch, trt_smrs_batch + def get_cmakers(self): """ :param oq: the OqParam used to build the CompositeSourceModel From cf65d136c1feebdd9d43b5d9b14d15dd276e083f Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 18:37:51 +0200 Subject: [PATCH 23/31] unit test --- openquake/commonlib/tests/source_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/openquake/commonlib/tests/source_test.py b/openquake/commonlib/tests/source_test.py index 793b548778b0..378320753279 100644 --- a/openquake/commonlib/tests/source_test.py +++ b/openquake/commonlib/tests/source_test.py @@ -824,3 +824,24 @@ def test_grp_ids_by_source_model_shared_raises(self): self.assertIn('spans multiple source models', msg) self.assertIn('sm_a', msg) self.assertIn('sm_b', msg) + + def test_iter_source_model_batches(self): + # Three top-level source models with 4, 2, 5 src_groups + # respectively; batches must partition src_groups cleanly and + # be yielded in sorted sm_branch_id order + csm = self._build_csm( + sm_branch_ids=['smA', 'smB', 'smC'], + smrs_per_group=[[0]]*4 + [[1]]*2 + [[2]]*5) + batches = list(csm.iter_source_model_batches()) + + self.assertEqual(len(batches), 3) + self.assertEqual([b[0] for b in batches], [0, 1, 2]) + self.assertEqual([b[1] for b in batches], ['smA', 'smB', 'smC']) + self.assertEqual([len(b[2]) for b in batches], [4, 2, 5]) + self.assertEqual([len(b[3]) for b in batches], [4, 2, 5]) + # Union of per-batch src_groups recontructs the full list + union = [] + for _, _, sgs, _ in batches: + union.extend(sgs) + self.assertEqual({id(sg) for sg in union}, + {id(sg) for sg in csm.src_groups}) From 78e6fae2746a78928c33cf5620361c48c7435551 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2026 18:47:35 +0200 Subject: [PATCH 24/31] helper func --- openquake/calculators/preclassical.py | 43 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/openquake/calculators/preclassical.py b/openquake/calculators/preclassical.py index 0578ff4c9002..1a15afc84a41 100644 --- a/openquake/calculators/preclassical.py +++ b/openquake/calculators/preclassical.py @@ -335,26 +335,11 @@ def populate_csm(self): secparams = () if oq.sequential_source_models: # Bound preclassical memory by iterating one source model - # at a time; rebuild cmakers at end for post_execute + # at a time and rebuild cmakers at end self._run_batched(sf, secparams, reqv) self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) else: - self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) - atomic_sources = [] - normal_sources = [] - cmakers = self.cmakers.to_array() - for sg in csm.src_groups: - for src in sg: - if reqv and sg.trt in oq.inputs['reqv']: - if src.source_id not in oq.reqv_ignore_sources: - collapse_nphc(src) - grp_id = sg.sources[0].grp_id - if sg.atomic: - cmakers[grp_id].set_weight(sg, sf) - atomic_sources.extend(sg) - else: - normal_sources.extend(sg) - self._process(atomic_sources, normal_sources, sf, secparams) + self._run_regular(trt_smrs, sf, secparams, reqv) L = oq.imtls.size Gfull = self.full_lt.gfull([cm.trt_smrs for cm in self.cmakers]) Gt = sum(len(cm.gsims) for cm in self.cmakers) @@ -398,6 +383,30 @@ def _run_batched(self, sf, secparams, reqv): self._process(atomic_batch, normal_batch, sf, secparams, cmaker_by_grp=cmaker_by_grp) + def _run_regular(self, trt_smrs, sf, secparams, reqv): + """ + Run preclassical in a single pass over all src_groups when + sequential_source_models is False. + """ + oq = self.oqparam + csm = self.csm + self.cmakers = get_cmakers(trt_smrs, csm.full_lt, oq) + atomic_sources = [] + normal_sources = [] + cmakers = self.cmakers.to_array() + for sg in csm.src_groups: + for src in sg: + if reqv and sg.trt in oq.inputs['reqv']: + if src.source_id not in oq.reqv_ignore_sources: + collapse_nphc(src) + grp_id = sg.sources[0].grp_id + if sg.atomic: + cmakers[grp_id].set_weight(sg, sf) + atomic_sources.extend(sg) + else: + normal_sources.extend(sg) + self._process(atomic_sources, normal_sources, sf, secparams) + def _process(self, atomic_sources, normal_sources, sf, secparams, cmaker_by_grp=None): if cmaker_by_grp is None: From 82cb0f0a59cf01f7f668780fc4abc34af13bbbe8 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 6 Aug 2026 07:44:13 +0200 Subject: [PATCH 25/31] widen task_no dtype --- openquake/baselib/performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openquake/baselib/performance.py b/openquake/baselib/performance.py index 72b2211e255f..4f9911a8c89a 100644 --- a/openquake/baselib/performance.py +++ b/openquake/baselib/performance.py @@ -46,7 +46,7 @@ # this is why below I am using ' Date: Wed, 12 Aug 2026 00:57:46 +0200 Subject: [PATCH 26/31] work with extendModel --- openquake/calculators/classical.py | 20 ++++- openquake/calculators/tests/logictree_test.py | 22 ++++++ openquake/commonlib/tests/source_test.py | 73 +++++++++++++------ openquake/hazardlib/source_group.py | 52 +++++++++---- 4 files changed, 127 insertions(+), 40 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index b14801894acd..7fbaa2105b89 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -655,6 +655,15 @@ def _run_sequential_source_models(self, allargs, task_func): """ Run one Starmap per sourceModel branch sequentially, so that only a single source model's tasks are running at one time. + + NOTE: Unless extendModel is being used, the sharing of src_groups + over base source models is not permitted. If extendModel is being + used, then the sharing of src_groups over base source models is + permitted and a "shared" batch is dispatched last. The memory + footprint of this "shared" batch could be similar (or equal) to + that observed in the "regular" (i.e., none-sequential) approach + if extendModel is heavily used in the logic tree (because many + of the src_grps would be piled into this final "shared" batch). """ # Map each src_group id to its sourceModel branch_id smb_of_grp = { @@ -669,9 +678,16 @@ def _run_sequential_source_models(self, allargs, task_func): gid = int(args[0][0].split('-')[0]) partitions[smb_of_grp[gid]].append(args) - # Run one source modl at a time + # Sort per-source model partitions for reproducible + # order with "shared" batch last + per_sm_keys = sorted(k for k in partitions if k is not None) + ordered_keys = per_sm_keys + ( + [None] if None in partitions else []) + + # Run one source model at a time acc = AccumDict(accum=0.) - for smb, part in sorted(partitions.items()): + for smb in ordered_keys: + part = partitions[smb] logging.info('Source model %r: %d tasks', smb, len(part)) smap = parallel.Starmap(task_func, part, h5=self.datastore.hdf5) acc = smap.reduce(self.agg_dicts, acc) diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index 1eee100f904d..262337d26008 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -811,6 +811,28 @@ def test_case_83(self): self.run_calc(case_83.__file__, 'job_expanded_LT.ini') [fname_ex] = export(('hcurves/mean', 'csv'), self.calc.datastore) self.assertEqualFiles(fname_em, fname_ex) + reg_full = self.calc.datastore['hcurves-stats'][:] + + # Check that sequential approach matches regular with + # full enumeration + self.run_calc(case_83.__file__, 'job_extendModel.ini', + sequential_source_models='true') + seq_full = self.calc.datastore['hcurves-stats'][:] + aac(seq_full, reg_full, atol=1e-6, rtol=1e-6) + + # Run regular approach with sampling + self.run_calc(case_83.__file__, 'job_extendModel.ini', + number_of_logic_tree_samples='10') + reg_sampled = self.calc.datastore['hcurves-stats'][:] + + # Check that sequential approach matches regular with + # sampling + self.run_calc(case_83.__file__, 'job_extendModel.ini', + sequential_source_models='true', + number_of_logic_tree_samples='10') + seq_sampled = self.calc.datastore['hcurves-stats'][:] + + aac(seq_sampled, reg_sampled, atol=1e-6, rtol=1e-6) def test_case_83_eb(self): # event based sampling with double extendModel diff --git a/openquake/commonlib/tests/source_test.py b/openquake/commonlib/tests/source_test.py index 378320753279..2864df1ace96 100644 --- a/openquake/commonlib/tests/source_test.py +++ b/openquake/commonlib/tests/source_test.py @@ -30,8 +30,8 @@ site, geo, mfd, pmf, scalerel, valid, tests as htests) from openquake.hazardlib import source, sourceconverter as s from openquake.hazardlib.tom import PoissonTOM -from openquake.hazardlib.lt import Realization -from openquake.hazardlib.logictree import FullLogicTree +from openquake.hazardlib.lt import Realization, BranchSet +from openquake.hazardlib.logictree import FullLogicTree, SourceModelLogicTree from openquake.hazardlib.source_group import CompositeSourceModel, SourceGroup from openquake.hazardlib.source_reader import sampling_dt from openquake.hazardlib import nrml @@ -775,17 +775,23 @@ def setUpClass(cls): [point_grp, *_] = nrml.to_python(MIXED_SRC_MODEL, conv) cls.template_src = point_grp[0] - def _build_csm(self, sm_branch_ids, smrs_per_group): - # sm_branch_ids[i] is the top-level branch_id of smr i; - # smrs_per_group[j] lists the smrs of the j-th src_group + def _build_csm(self, sm_branch_ids, smrs_per_group, has_extend=False): + # sm_branch_ids[i] = branch id of smr i; smrs_per_group[j] = + # smrs of the j-th src_group - # One Realization per smr - lt_path[0] is the top-level branch + # One Realization per smr sm_rlzs = [Realization(bid, 1., i, (bid,)) for i, bid in enumerate(sm_branch_ids)] full_lt = object.__new__(FullLogicTree) full_lt.sm_rlzs = sm_rlzs - # One SourceGroup per entry - sampling drives trt_smrs + # Minimal source_model_lt + utypes = ['sourceModel'] + (['extendModel'] if has_extend else []) + smlt = object.__new__(SourceModelLogicTree) + smlt.branchsets = [BranchSet(ut) for ut in utypes] + full_lt.source_model_lt = smlt + + # One SourceGroup per entry src_groups = [] for smrs in smrs_per_group: src = copy.copy(self.template_src) @@ -813,35 +819,54 @@ def test_grp_ids_by_source_model_disjoint(self): self.assertEqual(csm.grp_ids_by_source_model(), {'sm_a': [0], 'sm_b': [1]}) - def test_grp_ids_by_source_model_shared_raises(self): + def test_grp_ids_by_source_model_shared_raises_without_extend(self): # A src_group whose trt_smrs point at smrs from more than # one top-level sourceModel branch must raise an error + # if extendModel is not used in the logic tree csm = self._build_csm(sm_branch_ids=['sm_a', 'sm_b'], smrs_per_group=[[0, 1]]) with self.assertRaises(ValueError) as cm: csm.grp_ids_by_source_model() - msg = str(cm.exception) - self.assertIn('spans multiple source models', msg) - self.assertIn('sm_a', msg) - self.assertIn('sm_b', msg) + self.assertEqual( + str(cm.exception), + "src_group 0 (Stable Continental Crust) spans multiple " + "source models ['sm_a', 'sm_b']; " + "sequential_source_models=true does not support sources " + "shared across source models outside of extendModel" + ) + + def test_grp_ids_by_source_model_shared_with_extend(self): + # With extendModel the src_grp sharing is permitted and + # it results in a cross-SM group under a key of None + csm = self._build_csm(sm_branch_ids=['sm_a', 'sm_b'], + smrs_per_group=[[0], [1], [0, 1]], + has_extend=True) + self.assertEqual(csm.grp_ids_by_source_model(), + {'sm_a': [0], 'sm_b': [1], None: [2]}) def test_iter_source_model_batches(self): - # Three top-level source models with 4, 2, 5 src_groups - # respectively; batches must partition src_groups cleanly and - # be yielded in sorted sm_branch_id order + # 12 src_groups total across three SMs plus one cross-SM group: + # 4 groups belong to smA only + # 2 groups belong to smB only + # 5 groups belong to smC only + # 1 group spans all three SMs (smrs=[0, 1, 2]) -> shared + # Expected: four batches, per-SM ones sorted (4, 2, 5 groups), + # shared batch last with the single cross-SM group (12 total) csm = self._build_csm( sm_branch_ids=['smA', 'smB', 'smC'], - smrs_per_group=[[0]]*4 + [[1]]*2 + [[2]]*5) + smrs_per_group=[[0]]*4 + [[1]]*2 + [[2]]*5 + [[0, 1, 2]], + has_extend=True) batches = list(csm.iter_source_model_batches()) - self.assertEqual(len(batches), 3) - self.assertEqual([b[0] for b in batches], [0, 1, 2]) - self.assertEqual([b[1] for b in batches], ['smA', 'smB', 'smC']) - self.assertEqual([len(b[2]) for b in batches], [4, 2, 5]) - self.assertEqual([len(b[3]) for b in batches], [4, 2, 5]) - # Union of per-batch src_groups recontructs the full list + self.assertEqual(len(batches), 4) + self.assertEqual([b[0] for b in batches], [0, 1, 2, 3]) + self.assertEqual([b[1] for b in batches], + ['smA', 'smB', 'smC', None]) + self.assertEqual([len(b[2]) for b in batches], [4, 2, 5, 1]) + # Shared batch trt_smrs cover every SM + self.assertEqual(sorted(batches[-1][3][0].tolist()), [0, 1, 2]) + # Batches together cover every src_group exactly once union = [] for _, _, sgs, _ in batches: union.extend(sgs) - self.assertEqual({id(sg) for sg in union}, - {id(sg) for sg in csm.src_groups}) + self.assertEqual(set(union), set(csm.src_groups)) diff --git a/openquake/hazardlib/source_group.py b/openquake/hazardlib/source_group.py index 83bdf3ad53ae..bfeef7e81c4a 100644 --- a/openquake/hazardlib/source_group.py +++ b/openquake/hazardlib/source_group.py @@ -364,25 +364,40 @@ def grp_ids_by_source_model(self): :returns: Dict grouping src_groups by the branch_id of the top-level sourceModel branch they trace back to (via trt_smrs). + + NOTE: When the logic tree uses extendModel, groups whose + trt_smrs span more than one sourceModel branch form a "shared" + cross-source-model batch. Without extendModel such sharing is + treated as an error. + + NOTE: If there is heavy usage of extendModel within a logic + tree, this large batch could be have a memory profile close + to (if not equal to) the "regular" (i.e., none sequential) + approach. + """ + has_extend = any( + bset.uncertainty_type == 'extendModel' + for bset in self.full_lt.source_model_lt.branchsets) - Raises a ValueError if a src_group is shared across more than - one top-level source model, since such groups cannot be - dispatched sequentially by source model. - """ # Build an smr -> top-level sourceModel branch_id map smr_smb = {smr: rlz.lt_path[0] for smr, rlz in enumerate(self.full_lt.sm_rlzs)} - out = {} + out = collections.defaultdict(list) for grp_id, sg in enumerate(self.src_groups): smbs = {smr_smb[trt_smr % TWO24] for trt_smr in sg.sources[0].trt_smrs} if len(smbs) > 1: - raise ValueError( - 'src_group %d (%s) spans multiple source models %s; ' - 'sequential_source_models=true does not support ' - 'sources shared across source models' - % (grp_id, sg.trt, sorted(smbs))) - out.setdefault(smbs.pop(), []).append(grp_id) + if not has_extend: + raise ValueError( + 'src_group %d (%s) spans multiple source ' + 'models %s; sequential_source_models=true ' + 'does not support sources shared across ' + 'source models outside of extendModel' + % (grp_id, sg.trt, sorted(smbs))) + key = None + else: + key = smbs.pop() + out[key].append(grp_id) return out def get_trt_smrs(self): @@ -395,13 +410,22 @@ def get_trt_smrs(self): def iter_source_model_batches(self): """ - Iterate "src_groups" in batches, one per sourceModel branch + Iterate "src_groups" in batches, one per sourceModel branch. + + NOTE: A final "shared batch" (if any) is run which carries the + src_groups whose trt_smrs span multiple sourceModel branches + (i.e., when extendModel has been used - else shared src_groups + are forbidden in the sequential approach). """ # Map sourceModel branch id -> list of global grp_ids grp_ids_by_sm = self.grp_ids_by_source_model() - # Sorted iteration makes batch_id constant across runs - for batch_id, sm_branch_id in enumerate(sorted(grp_ids_by_sm)): + # Per-SM batches sorted for reproducible batch_id, shared last + per_sm_keys = sorted(k for k in grp_ids_by_sm if k is not None) + ordered_keys = per_sm_keys + ( + [None] if None in grp_ids_by_sm else []) + + for batch_id, sm_branch_id in enumerate(ordered_keys): grp_ids = grp_ids_by_sm[sm_branch_id] # Pick this batch's src_groups by their global grp_ids From c29f2ca770df9b8937a6673e8b46b79b983e7e80 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 12 Aug 2026 01:10:16 +0200 Subject: [PATCH 27/31] readme --- openquake/calculators/tests/logictree_test.py | 1 - openquake/qa_tests_data/logictree/README.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/openquake/calculators/tests/logictree_test.py b/openquake/calculators/tests/logictree_test.py index 262337d26008..2c49dbc89efa 100644 --- a/openquake/calculators/tests/logictree_test.py +++ b/openquake/calculators/tests/logictree_test.py @@ -831,7 +831,6 @@ def test_case_83(self): sequential_source_models='true', number_of_logic_tree_samples='10') seq_sampled = self.calc.datastore['hcurves-stats'][:] - aac(seq_sampled, reg_sampled, atol=1e-6, rtol=1e-6) def test_case_83_eb(self): diff --git a/openquake/qa_tests_data/logictree/README.md b/openquake/qa_tests_data/logictree/README.md index c4b8b1cc629c..5ba64587ebf6 100644 --- a/openquake/qa_tests_data/logictree/README.md +++ b/openquake/qa_tests_data/logictree/README.md @@ -52,6 +52,6 @@ | case\_73 | Tests some epistemic uncertainties in a source-specific LT | | case\_79 | Tests disagg\_by\_src with semicolon sources | | case\_80 | Tests areaSourceGeometryAbsolute | -| case\_83 | Tests extendModel and reqv | +| case\_83 | Tests extendModel and reqv + sequential source models with extendModel | | case\_83\_eb | Double extendModel with event based sampling | | case\_84 | Tests maxMagGRRelativeNoMoBalance uncertainty | From f65de2fe9fa4491fec0a322e250c48b2d0617b82 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2026 10:09:24 +0200 Subject: [PATCH 28/31] test - widen dtype of gid to u32 --- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- openquake/hazardlib/map_array.py | 2 +- openquake/hazardlib/source_group.py | 9 ++++----- openquake/hazardlib/source_reader.py | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/openquake/calculators/classical.py b/openquake/calculators/classical.py index 7fbaa2105b89..eecd02afca42 100644 --- a/openquake/calculators/classical.py +++ b/openquake/calculators/classical.py @@ -453,7 +453,7 @@ def create_rup(self): elif param == 'rup_id': dt = I64 elif param == 'grp_id': - dt = U16 + dt = U32 else: dt = F32 descr.append((param, dt)) diff --git a/openquake/calculators/preclassical.py b/openquake/calculators/preclassical.py index 1a15afc84a41..d62e6db5b64b 100644 --- a/openquake/calculators/preclassical.py +++ b/openquake/calculators/preclassical.py @@ -225,7 +225,7 @@ def store_csm(dstore, csm, sitecol, cmakers): [(_grp_id(blocks[0]), len(cm.gsims), len(tgets), len(blocks), len(cm.gsims) * mb_per_gsim, extra['weight'], extra['codes'], cm.trt) for cm, tgets, blocks, extra in quartets], - [('grp_id', U16), ('gsims', U16), ('tiles', U16), ('blocks', U16), + [('grp_id', U32), ('gsims', U16), ('tiles', U16), ('blocks', U16), ('max_mb', F32), ('weight', F32), ('codes', ' Date: Tue, 18 Aug 2026 09:49:08 +0200 Subject: [PATCH 29/31] remove dtype changes --- openquake/baselib/performance.py | 2 +- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- openquake/hazardlib/map_array.py | 2 +- openquake/hazardlib/source_group.py | 9 +++++---- openquake/hazardlib/source_reader.py | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/openquake/baselib/performance.py b/openquake/baselib/performance.py index 4f9911a8c89a..72b2211e255f 100644 --- a/openquake/baselib/performance.py +++ b/openquake/baselib/performance.py @@ -46,7 +46,7 @@ # this is why below I am using ' Date: Tue, 18 Aug 2026 20:36:03 +0200 Subject: [PATCH 30/31] dtypes widened --- openquake/baselib/performance.py | 4 ++-- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- openquake/hazardlib/map_array.py | 2 +- openquake/hazardlib/source_group.py | 9 ++++----- openquake/hazardlib/source_reader.py | 2 +- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/openquake/baselib/performance.py b/openquake/baselib/performance.py index 72b2211e255f..b6c98b2da04c 100644 --- a/openquake/baselib/performance.py +++ b/openquake/baselib/performance.py @@ -46,7 +46,7 @@ # this is why below I am using ' Date: Mon, 31 Aug 2026 09:57:00 +0200 Subject: [PATCH 31/31] revert dtype widen --- openquake/baselib/performance.py | 4 ++-- openquake/calculators/classical.py | 2 +- openquake/calculators/preclassical.py | 2 +- openquake/hazardlib/map_array.py | 2 +- openquake/hazardlib/source_group.py | 9 +++++---- openquake/hazardlib/source_reader.py | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/openquake/baselib/performance.py b/openquake/baselib/performance.py index b6c98b2da04c..72b2211e255f 100644 --- a/openquake/baselib/performance.py +++ b/openquake/baselib/performance.py @@ -46,7 +46,7 @@ # this is why below I am using '