diff --git a/CMakeLists.txt b/CMakeLists.txt index 21ac656880..d26b6b6ac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -402,6 +402,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -866,6 +867,7 @@ if(openPMD_BUILD_TESTING) list(APPEND ${out_list} test/Files_Core/automatic_variable_encoding.cpp test/Files_Core/read_nonexistent_attribute.cpp + test/Files_Core/custom_hierarchy.cpp ) endif() endmacro() diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..6a325e96ab --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,190 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + using CustomHierarchyData = ContainerData; +} // namespace internal + +class CustomHierarchy; + +/* + * This is its own class, so the return value of asContainerOf() is also + * convsertible again. + */ +template +class ConvertibleContainer : public Container +{ + template + friend class ConversibleContainer; + friend class CustomHierarchy; + +protected: + using Container_t = Container; + using Data_t = internal::ContainerData; + static_assert( + std::is_base_of_v); + + using Container_t::Container_t; + +private: + explicit ConvertibleContainer() = default; + +public: + auto asDataset() -> RecordComponent; + // TODO also: asContainerOf() +}; + +namespace traits +{ + template <> + struct DeferredInitPolicy> + { + template + static void call(Container_const_or_not &); + }; + + template <> + struct GenerationPolicy + { + constexpr static bool is_noop = false; + template + void operator()(Container &cont, Iterator &it) + { + auto &writable = it->second.writable(); + + // These should be different + auto child_shared_data = &it->second.Attributable::get(); + auto parent_shared_data = &cont.Attributable::get(); + if (child_shared_data == parent_shared_data) + { + throw std::runtime_error( + "Trying to emplace object as its own child"); + } + + // These might be different, but might also be the same + // + // For an explanation, ref. the documentation of + // Writable::attributable: This is a pointer back to the first + // created Attributable instance linking this Writable. There might + // be multiple Attributable objects linking the same backend + // Writable object when opening multiple "views" on the same backend + // object, e.g. when a scalar Record is at the same time a + // RecordComponent, or when reopening an object as a + // CustomHierarchy. + // + // Since CustomHierarchy performs no memory management by default, + // we must ensure that the backpointer in Writable::attributable + // remains valid when the frontend instance pointed by + // Writable::attributable *is* the CustomHierarchy instance (happens + // when it is the first frontend object created for that backend + // object). + auto backpointer = writable.attributable; + auto emplaced_pointer = it->second.m_attri.get(); + auto *group_metadata = + (**cont.m_attri).m_writable.objectType.requireGroup(); + if (backpointer == emplaced_pointer) + { + group_metadata + ->m_children_managed_as_custom_hierarchy[it->first] = + // NO move!! The iterator must stay alive + std::make_shared(it->second); + } + } + }; +} // namespace traits + +// If we want to break the DeferredInitPolicy logic, we may alternatively derive +// this privately from ConvertibleContainer, and then overload +// access operators to return CustomHierarchy instances +class CustomHierarchy : public ConvertibleContainer +{ + friend class Iteration; + friend class Container; + friend class Attributable; + friend struct traits::DeferredInitPolicy>; + +private: + using Parent_t = ConvertibleContainer; + using Container_t = typename Parent_t::Container_t; + using Data_t = typename Parent_t::Data_t; + +protected: + CustomHierarchy(NoInit); + CustomHierarchy(std::shared_ptr other); + CustomHierarchy(Attributable const &other); + + void read(std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(); + + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; + + // TODO should we automatically read upon returning / instantiating a + // CustomHierarchy object? i.e. upon Attributable::customHierarchy() and + // CustomHierarchy::operator[](). + // + // set max_recursion_depth = 0 for infinite cycling + // recursion depth includes the current object + // recursion will not continue expanding into regions that are already known + // (hence not transitively expand into unknown subregions of known regions) + auto read(size_t max_recursion_depth = 1) -> CustomHierarchy; + + void printRecursively(); + +private: + void printRecursively(std::string indent); +}; +} // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,66 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -121,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 0892627f2d..ccaf21d1ae 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include @@ -122,14 +123,16 @@ namespace internal */ bool allow_reopening_implicitly = false; - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; /** * Cached copy of the key under which this Iteration lives in @@ -483,8 +486,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(Iterator &it) + template + void operator()(Container &, Iterator &it) { it->second.get().m_iterationIndex = it->first; } diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 1ec1ff8d9c..7b13a39784 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -74,8 +74,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(T &it) + template + void operator()(Container &, T &it) { it->second.particlePatches.linkHierarchy(it->second.writable()); } diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 3def700f71..e6419702fc 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -72,38 +72,41 @@ namespace internal RecordComponentData &operator=(RecordComponentData const &) = delete; RecordComponentData &operator=(RecordComponentData &&) = delete; - /** - * Chunk reading/writing requests on the contained dataset. - */ - std::queue m_chunks; + [[nodiscard]] auto constantValue() const -> auto const & + { + return m_dataset_meta->m_constantValue; + } + auto constantValue() -> auto & + { + return m_dataset_meta->m_constantValue; + } + [[nodiscard]] auto isEmpty() const -> auto const & + { + return m_dataset_meta->m_isEmpty; + } + auto isEmpty() -> auto & + { + return m_dataset_meta->m_isEmpty; + } + [[nodiscard]] auto hasBeenExtended() const -> auto const & + { + return m_dataset_meta->m_hasBeenExtended; + } + auto hasBeenExtended() -> auto & + { + return m_dataset_meta->m_hasBeenExtended; + } - void push_chunk(IOTask &&task); - /** - * Stores the value for constant record components. - * Ignored otherwise. - */ - Attribute m_constantValue{-1}; - /** - * True if this component is an empty dataset, i.e. its extent is zero - * in at least one dimension. - * Treated by the openPMD-api as a special case of constant record - * components. - */ - bool m_isEmpty = false; - /** - * User has extended the dataset, but the EXTEND task must yet be - * flushed to the backend - */ - bool m_hasBeenExtended = false; - - void reset() override + [[nodiscard]] auto chunks() const -> auto const & { - BaseRecordComponentData::reset(); - m_chunks = std::queue(); - m_constantValue = -1; - m_isEmpty = false; - m_hasBeenExtended = false; + return m_dataset_meta->m_chunks; } + auto chunks() -> auto & + { + return m_dataset_meta->m_chunks; + } + + void push_chunk(IOTask &&task); }; template class BaseRecordData; @@ -134,6 +137,9 @@ class RecordComponent friend T &internal::makeOwning(T &self, Series); friend class internal::ScientificDefaults; friend class Attributable; + template + friend class ConvertibleContainer; + friend class CustomHierarchy; public: enum class Allocation diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..0e8c157751 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,12 +90,13 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. */ auto &rc = get(); - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { throw error::WrongAPIUsage( "[RecordComponent] Must specify dataset type and extent before " @@ -121,7 +122,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) // restriction // TODO: Add some form of collective ::commitDefinitions() call to // RecordComponents to be called by users before the Span API - if (!written()) + if (!writable().parent || !writable().parent->written) { /* * The openPMD backend might not yet know about this dataset. @@ -129,8 +130,11 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); - Parameter dCreate(rc.m_dataset.value()); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); + } + if (!this->written()) + { + Parameter dCreate(rc.dataset().value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 93dfe333b4..0b8f480097 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -33,6 +33,7 @@ #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" @@ -205,14 +206,18 @@ namespace internal * Detected IO format (backend). */ Format m_format; - /** - * Whether a step is currently active for this iteration. - * Used for group-based iteration layout, see SeriesData.hpp for - * iteration-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; + /** * True if a user opts into lazy parsing. */ @@ -261,7 +266,6 @@ namespace internal struct RankTableData { - Attributable m_attributable; std::variant< NoSourceSpecified, SourceSpecifiedViaJSON, @@ -447,6 +451,11 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + /** + * @return String representing the path to mesh records, relative(!) to + * basePath. + */ + std::optional meshesPathOptional() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -502,6 +511,11 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + /** + * @return String representing the path to particle species, relative(!) to + * basePath. + */ + std::optional particlesPathOptional() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -900,9 +914,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); - void flushRankTable(); + void flushRankTable(FlushLevel, Attributable &attributable); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ @@ -985,8 +997,10 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. + * @param l This operation must only run at flush level write_datasets, + * Noop otherwise. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, @@ -997,6 +1011,9 @@ OPENPMD_private Series &setIterationEncoding_internal( IterationEncoding iterationEncoding, internal::default_or_explicit); + Series &setParticlesPath_internal(std::string const &particlesPath); + Series &setMeshesPath_internal(std::string const &meshesPath); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) diff --git a/include/openPMD/auxiliary/Defer.hpp b/include/openPMD/auxiliary/Defer.hpp index c6bc4e0533..8792a14ee0 100644 --- a/include/openPMD/auxiliary/Defer.hpp +++ b/include/openPMD/auxiliary/Defer.hpp @@ -21,7 +21,8 @@ struct defer_type std::move(functor)(); } - explicit defer_type() = default; + explicit defer_type() : do_run_this(false) + {} struct forwarding_tag {}; @@ -32,7 +33,8 @@ struct defer_type {} template - defer_type(defer_type &&other) : functor{std::move(other.functor)} + defer_type(defer_type &&other) + : functor{std::move(other.functor)}, do_run_this(other.do_run_this) { other.do_run_this = false; } @@ -41,6 +43,7 @@ struct defer_type auto operator=(defer_type &&other) { functor = std::move(other.functor); + do_run_this = other.do_run_this; other.do_run_this = false; } diff --git a/include/openPMD/auxiliary/MonadicOperations.hpp b/include/openPMD/auxiliary/MonadicOperations.hpp new file mode 100644 index 0000000000..c96f682bc8 --- /dev/null +++ b/include/openPMD/auxiliary/MonadicOperations.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace openPMD::auxiliary +{ +template +auto optional_and_then(Optional &&arg, F &&then) + -> std::invoke_result_t())> +{ + if (!arg.has_value()) + { + return std::nullopt; + } + return std::forward(then)(*std::forward(arg)); +} +} // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 8d617aad3f..b395c199e1 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -163,6 +163,9 @@ using ScalarType_t = typename detail::ScalarType::type; template using VectorType_t = std::vector>; +template +using dependent_const = std::conditional_t; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index f05cc8d15b..84d03b833b 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -60,6 +60,7 @@ namespace internal struct HomogenizeExtents; struct ConfigAttribute; class ScientificDefaults; + class AttributableData; class SharedAttributableData { @@ -112,6 +113,8 @@ namespace internal using SharedData_t = std::shared_ptr; using A_MAP = SharedData_t::element_type::A_MAP; + using parent_t = std::shared_ptr; + friend class openPMD::CustomHierarchy; public: AttributableData(); @@ -120,12 +123,24 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline auto asSharedPtrOfAttributable() + -> std::shared_ptr & + { + return *this; + } + + [[nodiscard]] inline auto asSharedPtrOfAttributable() const + -> std::shared_ptr const & + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; // Make copies explicit, only to be used under the conditions described // above - void cloneFrom(AttributableData const &other); + void cloneFrom(parent_t const &other); template T asInternalCopyOf() @@ -249,6 +264,7 @@ class Attributable friend struct internal::HomogenizeExtents; friend struct internal::ConfigAttribute; friend class internal::ScientificDefaults; + friend class CustomHierarchy; protected: // tag for internal constructor @@ -294,6 +310,15 @@ class Attributable */ Attribute getAttribute(std::string const &key) const; + /** Retrieve value of Attribute stored with provided key. + * + * @throw no_such_attribute_error If no Attribute is currently stored with + * the provided key. + * @param key Key (i.e. name) of the Attribute to retrieve value for. + * @return If found, the stored Attribute in Variant form. + */ + std::optional getAttributeOptional(std::string const &key) const; + /** Remove Attribute of provided value both logically and physically. * * @param key Key (i.e. name) of the Attribute to remove. @@ -363,6 +388,9 @@ class Attributable */ void iterationFlush(std::string backendConfig = "{}"); + void customHierarchyFlush( + internal::FlushParams const &, bool managed_as_custom_object); + /** String serialization to describe an Attributable * * This object contains the Series data path as well as the openPMD object @@ -457,6 +485,25 @@ class Attributable [[nodiscard]] OpenpmdStandard openPMDStandard() const; + // TODO: Add parse? parameter, dont parse, parse this object, parse + // recursively + // Alternatively, parse upon deferred initialization? + // CustomHierarchy is ephemeral, and i dont even know what that word + // means. but i want to say that things are created on the spot. if we go to + // the same place in the openPMD file using different paths, the objects + // will be different, except for the backend data stored in + // sharedattributabledata and writable classes. even when later turning a + // customhierarchy object into a recordcomponent, this will not be same + // recordcomponent object as in the usual openpmd hierarchy. + // All data that needs to be shared within instances, is inside + // SharedAttributableData (including Writable). + // If objects are *created* by customhierarchies, the pointer + // writable.attributable will point to that object as the first instance. so + // i guess we will still need to store the objects somewhere and cannot keep + // them fully ephemeral. This job is done by + // GroupMetaData::m_children_managed_as_custom_hierarchy. + auto customHierarchies() -> CustomHierarchy; + // clang-format off OPENPMD_protected // clang-format on @@ -478,7 +525,7 @@ OPENPMD_protected /** @} */ template - void seriesFlush_impl(internal::FlushParams const &); + void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler); void flushAttributes(internal::FlushParams const &); @@ -582,6 +629,8 @@ OPENPMD_protected return (*m_attri)->m_writable; } + void preferCurrentBackpointer() const; + inline void setData(std::shared_ptr attri) { m_attri = std::move(attri); @@ -606,6 +655,34 @@ OPENPMD_protected { return writable().dirtyRecursive; } + void determineUnsetDirty(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + setDirty(false); + break; + // FlushLevel::InternalFlush is only used for directly calling the IO + // handler and should not bother with middle-end state manipulations + case FlushLevel::InternalFlush: + // Used for parsing + if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing) + { + throw error::Internal( + "Parsing procedures should directly unset dirty."); + } + else + { + throw error::Internal( + "Internal flushes should not unset dirty flags."); + } + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + // noop + break; + } + } void setDirty(bool dirty_in) { auto &w = writable(); @@ -730,6 +807,6 @@ Attributable::readVectorFloatingpoint(std::string const &key) const std::is_floating_point::value, "Type of attribute must be floating point"); - return getAttribute(key).get >(); + return getAttribute(key).get>(); } } // namespace openPMD diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 295b12e909..b8940fcf29 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -409,10 +409,12 @@ template auto BaseRecord::emplace(Args &&...args) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().emplace(std::forward(args)...); + auto res = this->syncInsertResult( + this->container_front().emplace(std::forward(args)...)); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + this->container_back(/* verify = */ true).erase(res.first->first); + this->container_front().erase(res.first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; diff --git a/include/openPMD/backend/BaseRecordComponent.hpp b/include/openPMD/backend/BaseRecordComponent.hpp index af23e08468..baef8454f0 100644 --- a/include/openPMD/backend/BaseRecordComponent.hpp +++ b/include/openPMD/backend/BaseRecordComponent.hpp @@ -24,6 +24,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/Error.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Writable.hpp" #include @@ -39,38 +40,58 @@ namespace internal class BaseRecordComponentData : virtual public AttributableData { public: - /** - * The type and extent of the dataset defined by this component. - */ - std::optional m_dataset; - /** - * True if this is defined as a constant record component as specified - * in the openPMD standard. - * If yes, then no heavy-weight dataset is created and the dataset is - * instead defined via light-weight attributes. - */ - bool m_isConstant = false; - /** - * Tracks if there was any write access to the record component. - * Necessary in BaseRecord to track if the scalar component has been - * used and is used by BaseRecord to determine the return value of - * the BaseRecord::scalar() method. - */ - bool m_datasetDefined = false; - BaseRecordComponentData(BaseRecordComponentData const &) = delete; BaseRecordComponentData(BaseRecordComponentData &&) = delete; BaseRecordComponentData & operator=(BaseRecordComponentData const &) = delete; BaseRecordComponentData &operator=(BaseRecordComponentData &&) = delete; + internal::object_type::DatasetMetaData *m_dataset_meta = nullptr; + + void setDatasetDefined() + { + m_dataset_meta = (**this).m_writable.objectType.initDataset(); + } + BaseRecordComponentData() = default; + [[nodiscard]] inline auto dataset() const -> auto const & + { + return m_dataset_meta->m_dataset; + } + inline auto dataset() -> auto & + { + return m_dataset_meta->m_dataset; + } + + [[nodiscard]] inline auto isConstant() const -> auto const & + { + return m_dataset_meta->m_isConstant; + } + inline auto isConstant() -> auto & + { + return m_dataset_meta->m_isConstant; + } + + [[nodiscard]] inline auto datasetDefined() const -> bool + { + return (**this).m_writable.objectType.isDataset(); + } + virtual void reset() { - m_dataset = std::nullopt; - m_isConstant = false; - m_datasetDefined = false; + (**this).m_writable.objectType.initGroup(); + m_dataset_meta = nullptr; + } + + template + void cloneFrom(Arg &&arg) + { + AttributableData::cloneFrom(std::forward(arg)); + if (datasetDefined()) + { + setDatasetDefined(); + } } }; } // namespace internal @@ -172,11 +193,14 @@ class BaseRecordComponent : virtual public Attributable inline Data_t const &get() const { + // cannot call this in the const overload + // setDatasetDefined(*m_baseRecordComponentData); return *m_baseRecordComponentData; } inline Data_t &get() { + setDatasetDefined(*m_baseRecordComponentData); return *m_baseRecordComponentData; } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 839f6452c9..3a0643538d 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -25,6 +25,7 @@ #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/Writable.hpp" #include #include @@ -53,18 +54,41 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = true; - template - void operator()(T &) + template + void operator()(Container &, T &) + {} + }; + + template + struct DeferredInitPolicy + { + static void call(Container_t &) + {} + static void call(Container_t const &) {} }; } // namespace traits +class CustomHierarchy; + namespace internal { class SeriesData; template class EraseStaleEntries; + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + template < typename T, typename T_key = std::string, @@ -105,7 +129,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; @@ -117,10 +141,64 @@ class Container : virtual public Attributable friend class internal::EraseStaleEntries; friend class StatefulIterator; + using Self_t = Container; + friend struct traits::DeferredInitPolicy; + protected: using ContainerData = internal::ContainerData; using InternalContainer = T_container; + using stringify_t = std::conditional_t< + std::is_same_v, + std::string const &, + std::string>; + static auto key_as_string(T_key const &key) -> stringify_t + { + if constexpr (std::is_same_v) + { + return key; + } + else + { + return std::to_string(key); + } + } + + template + struct SynchronizedContainers + { + using front_t = auxiliary::dependent_const; + using back_t = auxiliary::dependent_const< + const_, + internal::object_type::GroupMetaData::children_map_t>; + + front_t *front; + back_t *back; + + template + inline auto for_both(Functor &&f) + { + f(*this->front); + return f(*this->back); + } + + template + inline auto for_both_to_string(Functor &&f) + { + f(*this->front, [](auto key) { return key; }); + if constexpr (std::is_same_v) + { + return f(*this->back, [](auto key) { return key; }); + } + else + { + return f( + *this->back, + static_cast(&std::to_string)); + } + } + }; + std::shared_ptr m_containerData; inline void setData(std::shared_ptr containerData) @@ -129,16 +207,72 @@ class Container : virtual public Attributable Attributable::setData(m_containerData); } - inline InternalContainer const &container() const + inline SynchronizedContainers container() const { +#ifndef NDEBUG + auto size_front = container_front().size(); + auto size_back = container_back(/* verify = */ true).size(); + if (size_front > size_back) + { + throw std::runtime_error( + "Invalid container state: " + std::to_string(size_front) + + " != " + std::to_string(size_back) + "."); + } +#endif + traits::DeferredInitPolicy::call(*this); + return {&container_front(), &container_back(/* verify = */ true)}; + } + + inline SynchronizedContainers container() + { +#ifndef NDEBUG + auto size_front = container_front().size(); + auto size_back = container_back(/* verify = */ true).size(); + if (size_front > size_back) + { + throw std::runtime_error( + "Invalid container state: " + std::to_string(size_front) + + " != " + std::to_string(size_back) + "."); + } +#endif + traits::DeferredInitPolicy::call(*this); + return {&container_front(), &container_back(/* verify = */ true)}; + } + + inline auto container_front() const -> + typename SynchronizedContainers::front_t & + { + traits::DeferredInitPolicy::call(*this); return m_containerData->m_container; } - inline InternalContainer &container() + inline auto container_front() -> + typename SynchronizedContainers::front_t & { + traits::DeferredInitPolicy::call(*this); return m_containerData->m_container; } + inline auto container_back(bool verify) const -> + typename SynchronizedContainers::back_t & + { + if (verify) + { + traits::DeferredInitPolicy::call(*this); + } + return (**m_attri).m_writable.objectType.requireGroup()->m_children; + } + + inline auto container_back(bool verify) -> + typename SynchronizedContainers::back_t & + { + if (verify) + { + traits::DeferredInitPolicy::call(*this); + } + return this->writable().objectType.requireGroup()->m_children; + } + public: using key_type = typename InternalContainer::key_type; using mapped_type = typename InternalContainer::mapped_type; @@ -260,7 +394,8 @@ class Container : virtual public Attributable auto emplace(Args &&...args) -> decltype(InternalContainer().emplace(std::forward(args)...)) { - return container().emplace(std::forward(args)...); + return syncInsertResult( + container_front().emplace(std::forward(args)...)); } template @@ -289,6 +424,34 @@ OPENPMD_protected Container(NoInit); + auto syncInsertResult(std::pair res) + -> std::pair + { + if (res.second) + { + syncInsertResult(res.first); + } + return res; + } + auto syncInsertResult(iterator res) -> iterator + { + auto &cont = container_back(/* verify = */ false); + decltype(auto) key = key_as_string(res->first); + auto it = cont.find(key); + if (it == cont.end()) + { + cont.emplace(key_as_string(res->first), *res->second.m_attri); + } + else + { + // uhhm this might cause edge cases + // backend value is older, so it gets seniority + res->second.m_attri->asSharedPtrOfAttributable() = it->second; + res->second.preferCurrentBackpointer(); + } + return res; + } + public: /* * Need to define these manually due to the virtual inheritance from diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp index de11c91f14..a327dd18fc 100644 --- a/include/openPMD/backend/ContainerImpl.tpp +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -19,7 +19,9 @@ * If not, see . */ +#include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/Writable.hpp" /* * Instantiations in src/backend/Container.cpp @@ -33,101 +35,101 @@ namespace openPMD template auto Container::begin() noexcept -> iterator { - return container().begin(); + return container_front().begin(); } template auto Container::begin() const noexcept -> const_iterator { - return container().begin(); + return container_front().begin(); } template auto Container::cbegin() const noexcept -> const_iterator { - return container().cbegin(); + return container_front().cbegin(); } template auto Container::end() noexcept -> iterator { - return container().end(); + return container_front().end(); } template auto Container::end() const noexcept -> const_iterator { - return container().end(); + return container_front().end(); } template auto Container::cend() const noexcept -> const_iterator { - return container().cend(); + return container_front().cend(); } template auto Container::rbegin() noexcept -> reverse_iterator { - return container().rbegin(); + return container_front().rbegin(); } template auto Container::rbegin() const noexcept -> const_reverse_iterator { - return container().rbegin(); + return container_front().rbegin(); } template auto Container::crbegin() const noexcept -> const_reverse_iterator { - return container().crbegin(); + return container_front().crbegin(); } template auto Container::rend() noexcept -> reverse_iterator { - return container().rend(); + return container_front().rend(); } template auto Container::rend() const noexcept -> const_reverse_iterator { - return container().rend(); + return container_front().rend(); } template auto Container::crend() const noexcept -> const_reverse_iterator { - return container().crend(); + return container_front().crend(); } template auto Container::empty() const noexcept -> bool { - return container().empty(); + return container_front().empty(); } template auto Container::size() const noexcept -> size_type { - return container().size(); + return container_front().size(); } template auto Container::at(key_type const &key) -> mapped_type & { - return container().at(key); + return container_front().at(key); } template auto Container::at(key_type const &key) const -> mapped_type const & { - return container().at(key); + return container_front().at(key); } template auto Container::operator[](key_type const &key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) + auto it = container_front().find(key); + if (it != container_front().end()) return it->second; else { @@ -140,7 +142,9 @@ auto Container::operator[](key_type const &key) T t = T(); t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; + auto inserted_iterator = + syncInsertResult(container_front().insert({key, std::move(t)})) + .first; auto &ret = inserted_iterator->second; if constexpr (std::is_same_v) { @@ -151,7 +155,7 @@ auto Container::operator[](key_type const &key) ret.writable().ownKeyWithinParent = std::to_string(key); } traits::GenerationPolicy gen; - gen(inserted_iterator); + gen(*this, inserted_iterator); return ret; } } @@ -159,8 +163,8 @@ template auto Container::operator[](key_type &&key) -> mapped_type & { - auto it = container().find(key); - if (it != container().end()) + auto it = container_front().find(key); + if (it != container_front().end()) return it->second; else { @@ -173,7 +177,9 @@ auto Container::operator[](key_type &&key) T t = T(); t.linkHierarchy(writable()); - auto inserted_iterator = container().insert({key, std::move(t)}).first; + auto inserted_iterator = + syncInsertResult(container_front().insert({key, std::move(t)})) + .first; auto &ret = inserted_iterator->second; if constexpr (std::is_same_v) { @@ -184,7 +190,7 @@ auto Container::operator[](key_type &&key) ret.writable().ownKeyWithinParent = std::to_string(std::move(key)); } traits::GenerationPolicy gen; - gen(inserted_iterator); + gen(*this, inserted_iterator); return ret; } } @@ -203,49 +209,72 @@ template auto Container::insert(value_type const &value) -> std::pair { - return container().insert(value); + return syncInsertResult(container_front().insert(value)); } template auto Container::insert(value_type &&value) -> std::pair { - return container().insert(value); + return syncInsertResult(container_front().insert(value)); } template auto Container::insert( const_iterator hint, value_type const &value) -> iterator { - return container().insert(hint, value); + return syncInsertResult(container_front().insert(hint, value)); } template auto Container::insert( const_iterator hint, value_type &&value) -> iterator { - return container().insert(hint, value); + return syncInsertResult(container_front().insert(hint, value)); } template auto Container::insert( std::initializer_list ilist) -> void { - container().insert(ilist); + std::vector< + internal::object_type::GroupMetaData::children_map_t::value_type> + internal_insert_list; + internal_insert_list.reserve(ilist.size()); + auto &cont = container_back(/* verify = */ false); + for (auto &v : ilist) + { + decltype(auto) key = key_as_string(v.first); + auto it = cont.find(key); + if (it == cont.end()) + { + internal_insert_list.emplace_back(key, *v.second.m_attri); + } + else + { + // backend value is older, so it gets seniority + v.second.m_attri->asSharedPtrOfAttributable() = it->second; + v.second.preferCurrentBackpointer(); + } + } + container_front().insert(std::move(ilist)); + cont.insert(internal_insert_list.begin(), internal_insert_list.end()); } template auto Container::swap(Container &other) -> void { - container().swap(other.container()); + container_front().swap(other.container_front()); + container_back(/* verify = */ true) + .swap(other.container_back(/* verify = */ true)); } template auto Container::find(key_type const &key) -> iterator { - return container().find(key); + return container_front().find(key); } template auto Container::find(key_type const &key) const -> const_iterator { - return container().find(key); + return container_front().find(key); } /** This returns either 1 if the key is found in the container of 0 if not. @@ -257,7 +286,7 @@ template auto Container::count(key_type const &key) const -> size_type { - return container().count(key); + return container_front().count(key); } /** Checks if there is an element with a key equivalent to an exiting key in @@ -270,7 +299,7 @@ template auto Container::contains(key_type const &key) const -> bool { - return container().find(key) != container().end(); + return container_front().find(key) != container_front().end(); } template @@ -280,15 +309,17 @@ auto Container::erase(key_type const &key) -> size_type throw std::runtime_error( "Can not erase from a container in a read-only Series."); - auto res = container().find(key); - if (res != container().end() && res->second.written()) + auto res = container_front().find(key); + if (res != container_front().end() && res->second.written()) { Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - return container().erase(key); + return container().for_both_to_string([&key](auto &map, auto &&to_string) { + return map.erase(to_string(key)); + }); } template @@ -298,14 +329,15 @@ auto Container::erase(iterator res) -> iterator throw std::runtime_error( "Can not erase from a container in a read-only Series."); - if (res != container().end() && res->second.written()) + if (res != container_front().end() && res->second.written()) { Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); IOHandler()->flush(internal::defaultFlushParams); } - return container().erase(res); + container_back(/* verify = */ true).erase(key_as_string(res->first)); + return container_front().erase(res); } template @@ -315,7 +347,7 @@ auto Container::clear_unchecked() -> void throw std::runtime_error( "Clearing a written container not (yet) implemented."); - container().clear(); + container().for_both([](auto &map) { map.clear(); }); } template @@ -329,7 +361,11 @@ auto Container::flush( IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(flushParams); + customHierarchyFlush(flushParams, /* managed_as_custom_object = */ false); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } } template @@ -421,12 +457,12 @@ namespace internal template EraseStaleEntries::~EraseStaleEntries() { - auto &map = m_originalContainer.container(); + auto map = m_originalContainer.container(); using iterator_t = typename Container_t::InternalContainer::const_iterator; std::vector deleteMe; - deleteMe.reserve(map.size() - m_accessedKeys.size()); - for (iterator_t it = map.begin(); it != map.end(); ++it) + deleteMe.reserve(map.front->size() - m_accessedKeys.size()); + for (iterator_t it = map.front->begin(); it != map.front->end(); ++it) { auto lookup = m_accessedKeys.find(it->first); if (lookup == m_accessedKeys.end()) @@ -436,7 +472,8 @@ namespace internal } for (auto &it : deleteMe) { - map.erase(it); + map.back->erase(it->first); + map.front->erase(it); } } } // namespace internal diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp new file mode 100644 index 0000000000..6cea297bf2 --- /dev/null +++ b/include/openPMD/backend/PerIterationData.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "openPMD/ChunkInfo.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/backend/Attributable.hpp" + +#include + +namespace openPMD::internal +{ +struct NoSourceSpecified +{}; +struct SourceSpecifiedViaJSON +{ + std::string value; +}; +struct SourceSpecifiedManually +{ + std::string value; +}; + +struct RankTableData +{ + Attributable m_attributable; + std::variant< + NoSourceSpecified, + SourceSpecifiedViaJSON, + SourceSpecifiedManually> + m_rankTableSource; + std::optional m_bufferedRead; +}; + +/* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each Iteration. + * Access must be distinguished by iteration encoding. + */ +struct PerIterationData +{ + /** + * Whether a step is currently active for this iteration. + * Used for group-based iteration layout, see SeriesData.hpp for + * iteration-based layout. + * Access via stepStatus() method to automatically select the correct + * one among both flags. + */ + StepStatus m_stepStatus = StepStatus::NoStep; + Attributable m_rankTableAttributable; +}; +} // namespace openPMD::internal diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index a58af82f9e..06db3b3d91 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -20,10 +20,13 @@ */ #pragma once +#include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include +#include #include +#include #include // expose private and protected members for invasive testing @@ -45,6 +48,13 @@ class AbstractIOHandlerImplCommon; template class Span; class Series; +class CustomHierarchy; + +namespace traits +{ + template + struct GenerationPolicy; +} // namespace traits namespace internal { @@ -52,6 +62,144 @@ namespace internal class AttributableData; class SeriesData; class ScientificDefaults; + class BaseRecordComponentData; + class RecordComponentData; +} // namespace internal + +namespace internal::object_type +{ + struct DatasetMetaData + { + /** + * Chunk reading/writing requests on the contained dataset. + */ + std::queue m_chunks; + /** + * The type and extent of the dataset defined by this component. + */ + std::optional m_dataset; + /** + * Stores the value for constant record components. + * Ignored otherwise. + */ + Attribute m_constantValue{-1}; + /** + * True if this is defined as a constant record component as specified + * in the openPMD standard. + * If yes, then no heavy-weight dataset is created and the dataset is + * instead defined via light-weight attributes. + */ + bool m_isConstant = false; + + /** + * True if this component is an empty dataset, i.e. its extent is zero + * in at least one dimension. + * Treated by the openPMD-api as a special case of constant record + * components. + */ + bool m_isEmpty = false; + /** + * User has extended the dataset, but the EXTEND task must yet be + * flushed to the backend + */ + bool m_hasBeenExtended = false; + }; + + struct GroupMetaData + { + // Using shared_ptr in here because + // AttributableData is non-copyable and non-movable, but we need + // movability for map handling + // Disallowing move in AttributableData is only a measure for code + // discipline anyway. + using children_map_t = + std::map>; + children_map_t m_children; + + // Attributable::customHierarchies() creates objects of type + // CustomHierarchy ephemerally on the spot. If that object is the first + // object for its associated SharedAttributableData instance, it must be + // stored somewhere still, because the first instance is back-referenced + // by Writable class (TODO: turn that back-reference into a weak_ptr?). + // Store these objects in the parent to avoid reference cycles. + // Need shared_ptr because size is not yet known and unique_ptr cannot + // be managed by std::map. + using children_object_storage_t = + std::map>; + children_object_storage_t m_children_managed_as_custom_hierarchy; + }; +} // namespace internal::object_type + +namespace internal +{ + struct ObjectType + : std::variant + { + using variant_t = std:: + variant; + using variant_t::variant; + + [[nodiscard]] auto as_base() const -> variant_t const & + { + return *this; + } + auto as_base() -> variant_t & + { + return *this; + } + [[nodiscard]] auto isDataset() const -> bool + { + return std::holds_alternative( + as_base()); + } + [[nodiscard]] auto isGroup() const -> bool + { + return std::holds_alternative( + as_base()); + } + auto initDataset() -> object_type::DatasetMetaData * + { + if (auto res = + std::get_if(&as_base()); + res) + { + return res; + } + return &as_base().emplace(); + } + auto initGroup() -> object_type::GroupMetaData * + { + if (auto res = std::get_if(&as_base()); + res) + { + return res; + } + return &as_base().emplace(); + } + + auto requireGroup() -> object_type::GroupMetaData * + { + if (auto res = std::get_if(&as_base()); + res) + { + return res; + } + throw error::Internal( + "Internal object status: Group type required."); + } + + template + void ifGroup(Functor &&f) + { + std::visit( + auxiliary::overloaded{ + [&f](object_type::GroupMetaData &object_metadata) { + std::forward(f)(object_metadata); + }, + [](object_type::DatasetMetaData &) {}}, + as_base()); + } + }; } // namespace internal namespace detail { @@ -108,6 +256,13 @@ class Writable final friend struct Parameter; friend struct Parameter; friend class internal::ScientificDefaults; + template + friend class ConvertibleContainer; + friend class CustomHierarchy; + template + friend struct traits::GenerationPolicy; + friend class internal::BaseRecordComponentData; + friend class internal::RecordComponentData; private: Writable(internal::AttributableData *); @@ -128,14 +283,15 @@ class Writable final * it. */ template - void seriesFlush(std::string backendConfig = "{}"); + void + seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true); // clang-format off OPENPMD_private // clang-format on template - void seriesFlush(internal::FlushParams const &); + void seriesFlush(internal::FlushParams const &, bool flush_io_handler); /* * These members need to be shared pointers since distinct instances of * Writable may share them. @@ -154,6 +310,7 @@ OPENPMD_private * If multiple Attributables share the same Writable, then the creating one. * (See SharedAttributableData) */ + // TODO turn this into a weak pointer internal::AttributableData *attributable = nullptr; Writable *parent = nullptr; @@ -196,5 +353,7 @@ OPENPMD_private * */ bool written = false; + + internal::ObjectType objectType = internal::object_type::GroupMetaData{}; }; } // namespace openPMD diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 91e5f79195..e5da821699 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -26,6 +26,7 @@ namespace openPMD {} // IWYU pragma: begin_exports +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..9fc76dbbca --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,986 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/auxiliary/Defer.hpp" +#include "openPMD/backend/Container.hpp" + +namespace openPMD +{ +namespace traits +{ + template + void DeferredInitPolicy>::call( + Container_const_or_not &container) + { + // Need to sync backend objects into the CustomHierarchy instance + // Need to be a bit sneaky, we must modify my_container&, but this + // method might be called as const. shared_ptr<>s implement interior + // mutability, so use that here. + + // auto &container_front = container.container_front(); + auto &container_front = container.m_containerData->m_container; + auto &container_back = container.container_back(/* verify = */ false); + + auto size_front = container_front.size(); + auto size_back = container_back.size(); + + if (size_front == size_back) + { + return; + } + else if (size_front > size_back) + { + std::stringstream error; + auto print = [&error](auto const &map) -> std::stringstream & { + if (map.empty()) + { + error << "[]"; + } + else + { + error << '['; + auto it = map.begin(); + error << (it++)->first; + auto end = map.end(); + for (; it != end; ++it) + { + error << ", " << it->first; + } + error << ']'; + } + return error; + }; + error << "CustomHierarchy went into illegal state at '" + << container.myPath().openPMDPath() + << "':\nfront container: "; + print(container_front) << "\nback container: "; + print(container_back) << '\n'; + throw error::Internal(error.str()); + } + + GenerationPolicy gen; + + auto it = container_front.begin(); + auto end = container_front.end(); + for (auto const &[key, attributable] : container_back) + { + if (it == end || it->first != key) + { + // under the invariant that the front container contains no + // elements that are not present in the back container, it now + // points to an entry past the to-be-inserted key + it = container_front.emplace_hint( + it, key, CustomHierarchy(attributable)); + gen(container, it); + } + ++it; + } + } + template void DeferredInitPolicy>::call( + Container &); + template void DeferredInitPolicy>::call( + Container const &); +} // namespace traits + +template <> +auto ConvertibleContainer::asDataset() -> RecordComponent +{ + if (this->written()) + { + if (this->writable().objectType.isGroup()) + { + throw error::WrongAPIUsage( + "Can't cast a group object into a dataset."); + } + RecordComponent res; + res.get().cloneFrom(*this->m_attri); + return res; + } + else + { + if (this->writable().objectType.isDataset() || + access::write(this->IOHandler()->m_frontendAccess) || + this->IOHandler()->m_seriesStatus == + internal::SeriesStatus::Parsing) + { + // this is now a dataset. + this->writable().objectType.initDataset(); + RecordComponent res; + res.get().cloneFrom(*this->m_attri); + return res; + } + else + { + // TODO check if this has valid uses + throw error::WrongAPIUsage( + "Read only: Trying to access a non-written object as a " + "RecordComponent."); + } + } +} + +CustomHierarchy::CustomHierarchy() : ConvertibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} + +CustomHierarchy::CustomHierarchy(NoInit) : ConvertibleContainer(NoInit{}) +{} + +CustomHierarchy::CustomHierarchy( + std::shared_ptr other) + : ConvertibleContainer(NoInit{}) +{ + auto data = std::make_shared(); + data->asSharedPtrOfAttributable() = std::move(other); + setData(std::move(data)); +} + +CustomHierarchy::CustomHierarchy(Attributable const &other) + : CustomHierarchy(other.m_attri->asSharedPtrOfAttributable()) +{} + +auto CustomHierarchy::read(size_t const max_recursion_depth) -> CustomHierarchy +{ + auxiliary::opaque_defer_type reset_parsing_status; + if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing) + { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; + reset_parsing_status = auxiliary::defer([&]() { + IOHandler()->m_seriesStatus = internal::SeriesStatus::Default; + }); + } + if (!writable().written) + { + auto do_throw = [&]() { + throw error::WrongAPIUsage( + "Cannot read contents of CustomHierarchy path '" + + myPath().openPMDPath() + + "', since the backend does not yet / no longer know about this " + "object. Ensure that the object is open (e.g. by opening its " + "containing Iteration or by reading parent paths first)"); + }; + auto parent = writable().parent; + if (!parent) + { + do_throw(); + } + Attributable parent_attributable(NoInit{}); + parent_attributable.setData( + std::shared_ptr{ + parent->attributable, [](auto const *) {}}); + CustomHierarchy parent_(parent_attributable); + parent_.read(1); + if (!writable() /* still not */.written) + { + do_throw(); + } + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Attributable::readAttributes(ReadMode::FullyReread); + + if (writable().objectType.isDataset()) + { + return *this; + } + + auto do_recurse = [this, max_recursion_depth](CustomHierarchy &child) { + switch (max_recursion_depth) + { + case 0: + IOHandler()->flush(internal::defaultFlushParams); + child.read(0); + break; + case 1: + break; + default: + IOHandler()->flush(internal::defaultFlushParams); + child.read(max_recursion_depth - 1); + break; + } + }; + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + + IOHandler()->flush(internal::defaultFlushParams); + + auto &container_back_ = container_back(/* verify = */ true); + for (auto const &path : *pList.paths) + { + if (auto it = container_back_.find(path); + it != container_back_.end() && it->second->m_writable.written) + { + // Is already known + continue; + } + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + subpath.linkHierarchy(this->writable()); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + do_recurse(subpath); + } + + for (auto const &path : *dList.datasets) + { + if (auto it = container_back_.find(path); + it != container_back_.end() && it->second->m_writable.written) + { + // Is already known + continue; + } + Parameter dOpen; + dOpen.name = path; + + auto &subpath = this->operator[](path); + subpath.linkHierarchy(this->writable()); + IOHandler()->enqueue(IOTask(&subpath, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + + subpath.setWritten(false, Attributable::EnqueueAsynchronously::No); + subpath.asDataset().resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + subpath.setWritten(true, Attributable::EnqueueAsynchronously::No); + do_recurse(subpath); + } + + setDirty(false); + IOHandler()->flush(internal::defaultFlushParams); + + return *this; +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error("Use Attributable::customHierarchyFlush instead!"); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} + +void CustomHierarchy::printRecursively() +{ + std::cout << &writable() << " [" << dirty() << "," << dirtyRecursive() + << "] "; + if (empty()) + { + std::cout << "\n"; + return; + } + else + { + std::cout << writable().ownKeyWithinParent << '\n'; + printRecursively(""); + std::cout.flush(); + } +} + +void CustomHierarchy::printRecursively(std::string indent) +{ + auto print_indent = [&indent]() { std::cout << indent; }; + auto it = begin(); + auto end_ = end(); + if (it == end_) + { + return; + } + auto prev = it; + ++it; + auto next_indent = indent + "│ "; + for (; it != end_; prev = it, ++it) + { + std::cout << &prev->second.writable() << " [" << prev->second.dirty() + << "," << prev->second.dirtyRecursive() << "] "; + print_indent(); + std::cout << "├─" << prev->first << '\n'; + prev->second.printRecursively(next_indent); + } + std::cout << &prev->second.writable() << " [" << prev->second.dirty() << "," + << prev->second.dirtyRecursive() << "] "; + print_indent(); + std::cout << "└─" << prev->first << '\n'; + prev->second.printRecursively(indent + " "); +} +} // namespace openPMD + +#if 0 +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex const ®ex, std::vector const &path) +{ + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); +} +} // namespace + +namespace openPMD +{ +namespace internal +{ + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + + MeshesParticlesPath::MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles) + { + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) + { + std::stringstream build_regex; + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) + { + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); + } + setDefaultMeshesParticlesPath(meshes, particles, *this); + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path) const + { + if (isMeshContainer(path)) + { + return ContainedType::Mesh; + } + else if (isParticleContainer(path)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const + { + return anyPathRegexMatches(particleRegex, path); + } + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const + { + return anyPathRegexMatches(meshRegex, path); + } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. + */ + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) + { + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); + } + } +} // namespace internal + +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) +{} + +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter aList; + + Mesh &m = map[mesh_name]; + + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + // Find constant scalar meshes. shape generally required for meshes, + // shape also required for scalars. + // https://github.com/openPMD/openPMD-standard/pull/289 + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; + + Parameter dOpen; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) +{ + Parameter pOpen; + Parameter pList; + + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &data = get(); + auto embeddedMeshes = data.embeddedMeshesWrapped(); + auto embeddedParticles = data.embeddedParticlesWrapped(); + EraseStaleMeshes meshesMap(embeddedMeshes); + EraseStaleParticles particlesMap(embeddedParticles); + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } + break; + } + case internal::ContainedType::Mesh: { + try + { + readNonscalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticleSpecies(particlesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + particlesMap.forget(path); + } + break; + } + } + } + for (auto const &path : *dList.datasets) + { + switch (mpp.determineType(currentPath)) + { + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + break; + } + case internal::ContainedType::Mesh: + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + } + + for (auto const &path : constantComponentsPushback) + { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + } + setDirty(false); +} + +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + if (!dirtyRecursive()) + { + return; + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + for (auto &[name, mesh] : data.embeddedMeshesInternal()) + { + if (!mpp.isMeshContainer(currentPath)) + { + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); + } + mesh.flush(name, flushParams); + } + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) + { + if (!mpp.isParticleContainer(currentPath)) + { + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + ; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); + } + particleSpecies.flush(name, flushParams); + } + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) + { + dataset.flush(name, flushParams, /* set_defaults = */ false); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR +#endif diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 1d181033eb..1d8a325964 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1049,25 +1049,16 @@ void ADIOS2File::flush_impl( drainedUniquePtrPuts.swap(m_uniquePtrPuts); } - if (readOnly(m_mode)) + if (readOnly(m_mode) || flush_level::write_datasets(level)) { - level = FlushLevel::UserFlush; - } - - switch (level) - { - case FlushLevel::UserFlush: performPutGets(*this, eng); m_updateSpans.clear(); m_buffer.clear(); m_alreadyEnqueued.clear(); drainedUniquePtrPuts.clear(); - - break; - - case FlushLevel::InternalFlush: - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: + } + else + { /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid @@ -1084,7 +1075,6 @@ void ADIOS2File::flush_impl( "wrong time."); } m_buffer.clear(); - break; } } @@ -1366,7 +1356,12 @@ static std::vector availableAttributesOrVariablesPrefixed( { if (auxiliary::starts_with(it->first, var)) { - ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + if (it->first.size() > var.size()) + { + ret.emplace_back(auxiliary::replace_first(it->first, var, "")); + } + // else we have just found the prefix itself. + // might happen if prefix == "/" } else { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e8084848d..6f8c36a9eb 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -582,7 +582,7 @@ overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) std::future ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(flushParams.flushLevel); detail::ADIOS2File::ADIOS2FlushParams adios2FlushParams{ flushParams.flushLevel, m_flushTarget}; @@ -803,7 +803,11 @@ void ADIOS2IOHandlerImpl::createPath( /* Sanitize path */ if (!auxiliary::starts_with(parameters.path, '/')) { - path = filePositionToString(setAndGetFilePosition(writable)) + "/" + + auto file_position = + filePositionToString(setAndGetFilePosition(writable)); + path = + (auxiliary::ends_with(file_position, '/') ? file_position + : file_position + "/") + auxiliary::removeSlashes(parameters.path); } else diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 5f2bdeb2f5..564eebba01 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -27,6 +27,29 @@ #include +namespace openPMD +{ +std::ostream &operator<<(std::ostream &os, FlushLevel l) +{ + switch (l) + { + case FlushLevel::UserFlush: + os << "UserFlush"; + break; + case FlushLevel::InternalFlush: + os << "InternalFlush"; + break; + case FlushLevel::SkeletonOnly: + os << "SkeletonOnly"; + break; + case FlushLevel::CreateOrOpenFiles: + os << "CreateOrOpenFiles"; + break; + } + return os; +} +} // namespace openPMD + namespace openPMD::auxiliary { using pair_t = std::pair; diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 4f93ff1a5b..8b8518691d 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,6 +21,8 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -87,10 +89,72 @@ void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const } } -std::future AbstractIOHandlerImpl::flush() +namespace +{ + void verifyFlushType(Operation op, FlushLevel l) + { + auto do_throw = [&](char const *least_flush_level) { + std::stringstream err; + err << "Operation " << op << " is not allowed below flush level " + << least_flush_level << ", but flush level was " << l << "."; + throw error::Internal(err.str()); + }; + switch (op) + { + case Operation::ADVANCE: + case Operation::CREATE_FILE: + case Operation::CHECK_FILE: + case Operation::OPEN_FILE: + case Operation::CLOSE_FILE: + case Operation::DELETE_FILE: + case Operation::DEREGISTER: + case Operation::TOUCH: + case Operation::LIST_ATTS: + case Operation::LIST_PATHS: + case Operation::OPEN_PATH: + case Operation::SET_WRITTEN: + case Operation::CREATE_PATH: + break; + case Operation::CLOSE_PATH: + case Operation::DELETE_PATH: + case Operation::CREATE_DATASET: + case Operation::EXTEND_DATASET: + case Operation::OPEN_DATASET: + case Operation::DELETE_DATASET: + case Operation::LIST_DATASETS: + if (!flush_level::flush_hierarchy(l)) + { + do_throw("SkeletonOnly (hierarchy operations)"); + } + break; + case Operation::GET_BUFFER_VIEW: + case Operation::DELETE_ATT: + case Operation::WRITE_ATT: + case Operation::READ_ATT: + case Operation::READ_ATT_ALLSTEPS: + case Operation::AVAILABLE_CHUNKS: + if (!flush_level::write_attributes(l)) + { + do_throw("InternalFlush (metadata operations)"); + } + break; + case Operation::WRITE_DATASET: + case Operation::READ_DATASET: + if (!flush_level::write_datasets(l)) + { + do_throw("UserFlush (flushpoint operations)"); + } + break; + } + } +} // namespace + +std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; + writeToStderr("\nFLUSHING"); + while (!(*m_handler).m_work.empty()) { IOTask &i = (*m_handler).m_work.front(); @@ -152,6 +216,7 @@ std::future AbstractIOHandlerImpl::flush() ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); createDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::EXTEND_DATASET: { @@ -165,6 +230,7 @@ std::future AbstractIOHandlerImpl::flush() i.writable, "] EXTEND_DATASET"); extendDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::OPEN_FILE: { @@ -221,6 +287,7 @@ std::future AbstractIOHandlerImpl::flush() "] OPEN_DATASET: ", parameter.name); openDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::DELETE_FILE: { @@ -280,6 +347,7 @@ std::future AbstractIOHandlerImpl::flush() ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); writeDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::WRITE_ATT: { @@ -311,6 +379,7 @@ std::future AbstractIOHandlerImpl::flush() ", extent=", [¶meter]() { return vec_as_string(parameter.extent); }); readDataset(i.writable, parameter); + i.writable->objectType.initDataset(); break; } case O::GET_BUFFER_VIEW: { @@ -457,11 +526,17 @@ std::future AbstractIOHandlerImpl::flush() auto ¶meter = deref_dynamic_cast>( i.parameter.get()); writeToStderr( - "[", i.writable->parent, "->", i.writable, "] SET_WRITTEN"); + "[", + i.writable->parent, + "->", + i.writable, + "] SET_WRITTEN ", + parameter.target_status ? "true" : "false"); setWritten(i.writable, parameter); break; } } + verifyFlushType(i.operation, l); } catch (...) { @@ -506,6 +581,7 @@ std::future AbstractIOHandlerImpl::flush() } (*m_handler).m_work.pop(); } + writeToStderr("FLUSHED\n"); return std::future(); } diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 714a69b9d3..3096b2297c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -3562,7 +3562,7 @@ auto HDF5IOHandlerImpl::requireFile( std::future HDF5IOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(params.flushLevel); if (params.backendConfig.json().contains("hdf5")) { diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 26010ead52..af12ff766c 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -35,6 +35,97 @@ Writable *getWritable(Attributable *a) return &a->writable(); } +std::ostream &operator<<(std::ostream &os, Operation op) +{ + switch (op) + { + case Operation::CREATE_FILE: + os << "CREATE_FILE"; + break; + case Operation::CHECK_FILE: + os << "CHECK_FILE"; + break; + case Operation::OPEN_FILE: + os << "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + os << "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + os << "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + os << "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + os << "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + os << "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + os << "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + os << "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + os << "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + os << "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + os << "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + os << "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + os << "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + os << "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + os << "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + os << "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + os << "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + os << "WRITE_ATT"; + break; + case Operation::READ_ATT: + os << "READ_ATT"; + break; + case Operation::READ_ATT_ALLSTEPS: + os << "READ_ATT_ALLSTEPS"; + break; + case Operation::LIST_ATTS: + os << "LIST_ATTS"; + break; + case Operation::ADVANCE: + os << "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + os << "AVAILABLE_CHUNKS"; + break; + case Operation::DEREGISTER: + os << "DEREGISTER"; + break; + case Operation::TOUCH: + os << "TOUCH"; + break; + case Operation::SET_WRITTEN: + os << "SET_WRITTEN"; + break; + } + return os; +} template <> void AbstractParameter::warnUnusedParameters( json::TracingJSON &config, diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index c531aabb00..9af9a06728 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -53,8 +53,8 @@ JSONIOHandler::JSONIOHandler( {} #endif -std::future JSONIOHandler::flush(internal::ParsedFlushParams &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams ¶ms) { - return m_impl.flush(); + return m_impl.flush(params); } } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 7647746d74..0660d1fe41 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -444,9 +445,9 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; -std::future JSONIOHandlerImpl::flush() +std::future JSONIOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - AbstractIOHandlerImpl::flush(); + AbstractIOHandlerImpl::flush(params.flushLevel); if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) { throw error::Internal( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 67a28c51bc..71e07fdd03 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -280,16 +280,6 @@ void Iteration::flushFileBased( fCreate.name = filename; IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = s.get().m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - s.get() - .m_rankTable.m_attributable.get() - .m_writable.abstractFilePosition.reset(); - s.flushRankTable(); - /* create basePath */ Parameter pCreate; pCreate.path = auxiliary::replace_first(s.basePath(), "%T/", ""); @@ -306,15 +296,16 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - switch (flushParams.flushLevel) + auto &rankTableAttributable = + get().m_perIterationData.m_rankTableAttributable; + if (!rankTableAttributable.written()) + { + s.flushRankTable(flushParams.flushLevel, rankTableAttributable); + } + + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -329,15 +320,9 @@ void Iteration::flushGroupBased( IOHandler()->enqueue(IOTask(this, pCreate)); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -352,18 +337,14 @@ void Iteration::flushVariableBased( IOHandler()->enqueue(IOTask(this, pOpen)); } - switch (flushParams.flushLevel) + if (!flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - flush(flushParams); - break; } - if (!written()) + flush(flushParams); + + if (!written() && flush_level::write_datasets(flushParams.flushLevel)) { /* create iteration path */ Parameter pOpen; @@ -389,13 +370,16 @@ void Iteration::flush(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); + + customHierarchyFlush(flushParams, /* managed_as_custom_object = */ false); + if (access::readOnly(IOHandler()->m_frontendAccess)) { for (auto &m : meshes) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); - setDirty(false); + determineUnsetDirty(flushParams.flushLevel); } else { @@ -403,16 +387,28 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath")) + auto set_and_get_mp_path = + [&](char const *attrName, + char const *defaultVal, + Series &(Series::*set)(std::string const &)) -> std::string { + if (s.containsAttribute(attrName)) + { + return s.getAttribute(attrName).get(); + } + else { - s.setMeshesPath("meshes/"); - s.flushMeshesPath(); + (s.*set)(defaultVal); + return defaultVal; } + }; + + if (!meshes.empty() || s.containsAttribute("meshesPath")) + { + auto meshesPath = set_and_get_mp_path( + "meshesPath", "meshes/", &Series::setMeshesPath); if (meshes.dirtyRecursive()) { - meshes.flush(s.meshesPath(), flushParams); + meshes.flush(meshesPath, flushParams); for (auto &m : meshes) { m.second.flush(m.first, flushParams); @@ -426,14 +422,11 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath")) - { - s.setParticlesPath("particles/"); - s.flushParticlesPath(); - } + auto particlesPath = set_and_get_mp_path( + "particlesPath", "particles/", &Series::setParticlesPath); if (particles.dirtyRecursive()) { - particles.flush(s.particlesPath(), flushParams); + particles.flush(particlesPath, flushParams); for (auto &species : particles) { species.second.flush(species.first, flushParams); @@ -447,11 +440,12 @@ void Iteration::flush(internal::FlushParams const &flushParams) flushAttributes(flushParams); } + if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); + determineUnsetDirty(flushParams.flushLevel); + meshes.determineUnsetDirty(flushParams.flushLevel); + particles.determineUnsetDirty(flushParams.flushLevel); } } @@ -577,7 +571,7 @@ void Iteration::read_impl(std::string const &groupPath) std::cerr << "Cannot read meshes in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - meshes.container().clear(); + meshes.container().for_both([](auto &map) { map.clear(); }); } } meshes.setDirty(false); @@ -593,7 +587,7 @@ void Iteration::read_impl(std::string const &groupPath) std::cerr << "Cannot read particles in iteration " << groupPath << " and will skip them due to read error:\n" << err.what() << std::endl; - particles.container().clear(); + particles.container().for_both([](auto &map) { map.clear(); }); } } particles.setDirty(false); @@ -655,7 +649,7 @@ void Iteration::readMeshes(std::string const &meshesPath) MeshRecordComponent &mrc = m; IOHandler()->enqueue(IOTask(&mrc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; + mrc.get().isConstant() = true; } try { @@ -785,7 +779,7 @@ auto Iteration::beginStep( } else { - series.get().m_stepStatus = StepStatus::DuringStep; + series.get().m_perIterationData.m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -888,10 +882,10 @@ StepStatus Iteration::getStepStatus() { using IE = IterationEncoding; case IE::fileBased: - return get().m_stepStatus; + return get().m_perIterationData.m_stepStatus; case IE::groupBased: case IE::variableBased: - return s.get().m_stepStatus; + return s.get().m_perIterationData.m_stepStatus; default: throw std::runtime_error("[Iteration] unreachable"); } @@ -904,11 +898,11 @@ void Iteration::setStepStatus(StepStatus status) { using IE = IterationEncoding; case IE::fileBased: - get().m_stepStatus = status; + get().m_perIterationData.m_stepStatus = status; break; case IE::groupBased: case IE::variableBased: - s.get().m_stepStatus = status; + s.get().m_perIterationData.m_stepStatus = status; break; default: throw std::runtime_error("[Iteration] unreachable"); @@ -920,6 +914,26 @@ void Iteration::linkHierarchy(Writable &w) Attributable::linkHierarchy(w); meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); + get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); + + auto &container_back = + this->writable().objectType.requireGroup()->m_children; + auto s = retrieveSeries(); + auto link_mp = [&](auto &meshes_or_particles, + std::optional const &mp_path, + char const *default_) { + if (mp_path) + { + container_back[auxiliary::replace_all_nonrecursively( + *mp_path, "/", "")] = *meshes_or_particles.m_attri; + } + else + { + container_back[default_] = *meshes_or_particles.m_attri; + } + }; + link_mp(meshes, s.meshesPathOptional(), "meshes"); + link_mp(particles, s.particlesPathOptional(), "particles"); } void Iteration::runDeferredParseAccess() diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 8d8227f23d..f49df18e13 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -446,15 +446,18 @@ void Mesh::flush_impl( if (access::readOnly(IOHandler()->m_frontendAccess)) { auto &m = get(); - if (m.m_datasetDefined) + if (m.datasetDefined()) { T_RecordComponent::flush(SCALAR, flushParams); } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // TODO why is there no unsetDirty operation here? } else { @@ -470,6 +473,9 @@ void Mesh::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) { comp.second.parent() = &this->writable(); @@ -485,6 +491,8 @@ void Mesh::flush_impl( } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } @@ -497,7 +505,6 @@ void Mesh::read() { internal::HomogenizeExtents homogenizeExtents( IOHandler()->m_verify_homogeneous_extents); - internal::EraseStaleEntries map{*this}; if (scalar()) { @@ -506,6 +513,7 @@ void Mesh::read() } else { + internal::EraseStaleEntries map{*this}; Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); IOHandler()->flush(internal::defaultFlushParams); @@ -516,7 +524,7 @@ void Mesh::read() MeshRecordComponent &rc = map[component]; pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; try { rc.read(); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index e27208e596..1803a576ad 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -61,7 +61,8 @@ void ParticlePatches::read() std::cerr << "Cannot read patch record '" << record_name << "' due to read error and will skip it:" << err.what() << std::endl; - this->container().erase(record_name); + this->container().for_both( + [&record_name](auto &map) { map.erase(record_name); }); } } @@ -113,7 +114,8 @@ void ParticlePatches::read() << "Cannot read record component '" << component_name << "' in particle patch and will skip it due to read error:\n" << err.what() << std::endl; - Container::container().erase(component_name); + Container::container().for_both( + [&component_name](auto &map) { map.erase(component_name); }); } } setDirty(false); diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8a2b9b58f7..aab92f249e 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -39,6 +39,8 @@ void ParticleSpecies::visitHierarchy(HierarchyVisitor &v, bool recursive) ParticleSpecies::ParticleSpecies() { particlePatches.writable().ownKeyWithinParent = "particlePatches"; + container_back(/* verify = */ true)["particlePatches"] = + *particlePatches.m_attri; } void ParticleSpecies::read() @@ -91,7 +93,7 @@ void ParticleSpecies::read() RecordComponent &rc = r; IOHandler()->enqueue(IOTask(&rc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; } internal::HomogenizeExtents recordExtents; try @@ -113,9 +115,11 @@ void ParticleSpecies::read() if (!hasParticlePatches) { - auto &container = particlePatches.container(); - container.erase("numParticles"); - container.erase("numParticlesOffset"); + auto container = particlePatches.container(); + container.for_both([](auto &map_) { + map_.erase("numParticles"); + map_.erase("numParticlesOffset"); + }); particlePatches.setDirty(false); } @@ -178,6 +182,8 @@ void ParticleSpecies::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &record : *this) record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) @@ -197,11 +203,8 @@ void ParticleSpecies::flush( patch.second.flush(patch.first, flushParams); } } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - particlePatches.setDirty(false); - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); + particlePatches.determineUnsetDirty(flushParams.flushLevel); } void ParticleSpecies::scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) diff --git a/src/Record.cpp b/src/Record.cpp index f99f26f015..6651f406b0 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -71,9 +71,12 @@ void Record::flush_impl( } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } + // TODO why is there no unsetDirty operation here? } else { @@ -89,6 +92,9 @@ void Record::flush_impl( Parameter pCreate; pCreate.path = name; IOHandler()->enqueue(IOTask(this, pCreate)); + + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) { comp.second.parent() = getWritable(this); @@ -98,13 +104,14 @@ void Record::flush_impl( } else { - if (scalar()) { T_RecordComponent::flush(name, flushParams); } else { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } @@ -151,7 +158,7 @@ auto Record::read() -> internal::HomogenizeExtents RecordComponent &rc = (*this)[component]; pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); - rc.get().m_isConstant = true; + rc.get().isConstant() = true; try { rc.read(); @@ -161,7 +168,8 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->container().for_both( + [&component](auto &map) { map.erase(component); }); continue; } check_extent(rc); @@ -190,7 +198,8 @@ auto Record::read() -> internal::HomogenizeExtents std::cerr << "Cannot read record component '" << component << "' and will skip it due to read error:\n" << err.what() << std::endl; - this->container().erase(component); + this->container().for_both( + [&component](auto &map) { map.erase(component); }); continue; } check_extent(rc); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ec254d3b99..1e62f0241f 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -22,6 +22,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -71,7 +72,7 @@ namespace internal } #endif a.setDirtyRecursive(true); - m_chunks.push(std::move(task)); + chunks().push(std::move(task)); } static constexpr char const *note_on_deactivating_this_check = R"( @@ -220,7 +221,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) auto &rc = get(); if (written()) { - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { throw error::Internal( "Internal control flow error: Written record component must " @@ -228,14 +229,14 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) } if (d.dtype == Datatype::UNDEFINED) { - d.dtype = rc.m_dataset.value().dtype; + d.dtype = rc.dataset().value().dtype; } - else if (d.dtype != rc.m_dataset.value().dtype) + else if (d.dtype != rc.dataset().value().dtype) { throw std::runtime_error( "Cannot change the datatype of a dataset."); } - rc.m_hasBeenExtended = true; + rc.hasBeenExtended() = true; } if (d.extent.empty()) @@ -256,26 +257,26 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) } else { - rc.m_dataset = std::move(d); + rc.dataset() = std::move(d); setDirty(true); return *this; } } - rc.m_isEmpty = false; + rc.isEmpty() = false; if (written()) { - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { throw error::Internal( "Internal control flow error: Written record component must " "have defined datatype and extent."); } - rc.m_dataset.value().extend(std::move(d.extent)); + rc.dataset().value().extend(std::move(d.extent)); } else { - rc.m_dataset = std::move(d); + rc.dataset() = std::move(d); } setDirty(true); @@ -285,9 +286,9 @@ RecordComponent &RecordComponent::resetDataset(Dataset d) uint8_t RecordComponent::getDimensionality() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - return rc.m_dataset.value().rank; + return rc.dataset().value().rank; } else { @@ -298,9 +299,9 @@ uint8_t RecordComponent::getDimensionality() const Extent RecordComponent::getExtent() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - return rc.m_dataset.value().extent; + return rc.dataset().value().extent; } else { @@ -337,7 +338,7 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d) auto &rc = get(); if (written()) { - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { throw error::Internal( "Internal control flow error: Written record component must " @@ -352,37 +353,37 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d) } if (d.dtype == Datatype::UNDEFINED) { - d.dtype = rc.m_dataset.value().dtype; + d.dtype = rc.dataset().value().dtype; } - else if (d.dtype != rc.m_dataset.value().dtype) + else if (d.dtype != rc.dataset().value().dtype) { throw std::runtime_error( "Cannot change the datatype of a dataset."); } - rc.m_dataset.value().extend(std::move(d.extent)); - rc.m_hasBeenExtended = true; + rc.dataset().value().extend(std::move(d.extent)); + rc.hasBeenExtended() = true; } else { - rc.m_dataset = std::move(d); + rc.dataset() = std::move(d); } - if (rc.m_dataset.value().extent.size() == 0) + if (rc.dataset().value().extent.size() == 0) throw std::runtime_error("Dataset extent must be at least 1D."); - rc.m_isEmpty = true; + rc.isEmpty() = true; setDirty(true); if (!written()) { switchType>( - rc.m_dataset.value().dtype, *this); + rc.dataset().value().dtype, *this); } return *this; } bool RecordComponent::empty() const { - return get().m_isEmpty; + return get().isEmpty(); } void RecordComponent::visitHierarchy(HierarchyVisitor &v, bool) @@ -404,10 +405,12 @@ void RecordComponent::flush( } if (access::readOnly(IOHandler()->m_frontendAccess)) { - while (!rc.m_chunks.empty()) + // TODO: if we are going to allow custom hierarchies on constant + // components (technically possible), the flush would go here + while (!rc.chunks().empty()) { - IOHandler()->enqueue(rc.m_chunks.front()); - rc.m_chunks.pop(); + IOHandler()->enqueue(rc.chunks().front()); + rc.chunks().pop(); } } else @@ -415,11 +418,11 @@ void RecordComponent::flush( /* * This catches when a user forgets to use resetDataset. */ - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { // The check for !written() is technically not needed, just // defensive programming against internal bugs that go on us. - if (!written() && rc.m_chunks.empty() && !rc.m_isConstant) + if (!written() && rc.chunks().empty() && !rc.isConstant()) { // No data written yet, just accessed the object so far without // doing anything @@ -452,8 +455,8 @@ void RecordComponent::flush( IOHandler()->enqueue(IOTask(this, pCreate)); Parameter aWrite; aWrite.name = "value"; - aWrite.dtype = rc.m_constantValue.dtype; - aWrite.m_resource = rc.m_constantValue.getAny(); + aWrite.dtype = rc.constantValue().dtype; + aWrite.m_resource = rc.constantValue().getAny(); if (isVBased) { aWrite.changesOverSteps = Parameter< @@ -477,13 +480,13 @@ void RecordComponent::flush( else { Parameter dCreate( - rc.m_dataset.value()); + rc.dataset().value()); dCreate.name = name; IOHandler()->enqueue(IOTask(this, dCreate)); } } - if (rc.m_hasBeenExtended) + if (rc.hasBeenExtended()) { if (constant()) { @@ -511,24 +514,26 @@ void RecordComponent::flush( else { Parameter pExtend( - rc.m_dataset.value().extent); + rc.dataset().value().extent); IOHandler()->enqueue(IOTask(this, std::move(pExtend))); - rc.m_hasBeenExtended = false; + rc.hasBeenExtended() = false; } } - while (!rc.m_chunks.empty()) + while (!rc.chunks().empty()) { - IOHandler()->enqueue(rc.m_chunks.front()); - rc.m_chunks.pop(); + IOHandler()->enqueue(rc.chunks().front()); + rc.chunks().pop(); } + // TODO maybe guard against custom hierarchies on datasets? + // they can be created upon constant components however.. + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + flushAttributes(flushParams); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void RecordComponent::read() @@ -739,8 +744,8 @@ RecordComponent &RecordComponent::makeConstant(T value) auto &rc = get(); - rc.m_constantValue = Attribute(value); - rc.m_isConstant = true; + rc.constantValue() = Attribute(value); + rc.isConstant() = true; return *this; } @@ -886,7 +891,7 @@ void RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) std::optional val = switchNonVectorType>( - /* dt = */ getDatatype(), rc.m_constantValue); + /* dt = */ getDatatype(), rc.constantValue()); if (val.has_value()) { diff --git a/src/Series.cpp b/src/Series.cpp index 855178c18d..e80caacfea 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -36,6 +36,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/MonadicOperations.hpp" #include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" @@ -272,6 +273,13 @@ std::string Series::meshesPath() const return getAttribute("meshesPath").get(); } +std::optional Series::meshesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("meshesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setMeshesPath(std::string const &mp) { auto &series = get(); @@ -285,12 +293,7 @@ Series &Series::setMeshesPath(std::string const &mp) "A files meshesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(mp, '/')) - setAttribute("meshesPath", mp); - else - setAttribute("meshesPath", mp + "/"); - setDirty(true); - return *this; + return setMeshesPath_internal(mp); } std::vector Series::availableDatasets() @@ -339,10 +342,11 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } if (iterationEncoding() == IterationEncoding::fileBased) { - std::cerr << "[Series] Use rank table in file-based iteration encoding " - "at your own risk. Make sure to have an iteration open " - "before calling this." - << std::endl; + std::cerr + << "[Series] Use rank table in file-based iteration encoding " + "at your own risk. Make sure to have the first iteration open " + "before calling this." + << std::endl; if (iterations.empty()) { return {}; @@ -355,6 +359,18 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) IOHandler()->enqueue(IOTask(this, openFile)); #endif } + Attributable &attributable = + iterationEncoding() == IterationEncoding::fileBased + /* + * Only second class support for file encoding. We indiscriminately use + * the first Iteration for this operation. It is on the user to ensure + * that this Iteration is actually open. The warning printed above + * informs about this. + */ + ? iterations.begin() + ->second.get() + .m_perIterationData.m_rankTableAttributable + : series.m_perIterationData.m_rankTableAttributable; auto datasets = availableDatasets(); if (std::find(datasets.begin(), datasets.end(), "rankTable") == datasets.end()) @@ -364,7 +380,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Parameter openDataset; openDataset.name = "rankTable"; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); + IOHandler()->enqueue(IOTask(&attributable, openDataset)); IOHandler()->flush(internal::defaultFlushParams); if (openDataset.extent->size() != 2) @@ -394,7 +410,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = [&openDataset, this, &get, &rankTable]() { + auto doReadDataset = [&openDataset, this, &get, &attributable]() { Parameter readDataset; // read the whole thing readDataset.offset.resize(2); @@ -404,8 +420,8 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) readDataset.dtype = Datatype::CHAR; readDataset.data = get; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::defaultFlushParams); + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); }; #if openPMD_HAVE_MPI @@ -464,8 +480,12 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable() +void Series::flushRankTable(FlushLevel l, Attributable &attributable) { + if (!flush_level::global_flushpoint(l)) + { + return; + } auto &series = get(); auto &rankTable = series.m_rankTable; auto maybeMyRankInfo = std::visit( @@ -508,8 +528,8 @@ void Series::flushRankTable() int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = [&size, &maxSize, &rankTable, this]() { - if (rankTable.m_attributable.written()) + auto createRankTable = [&size, &maxSize, this, &attributable]() { + if (attributable.written()) { return; } @@ -518,19 +538,17 @@ void Series::flushRankTable() param.name = "rankTable"; param.dtype = Datatype::CHAR; param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(param))); + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); }; - auto writeDataset = [&rank, &maxSize, this, &rankTable]( + auto writeDataset = [&rank, &maxSize, this, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; chunk.offset = {uint64_t(rank), 0}; chunk.extent = {num_lines, maxSize}; chunk.data = std::move(put); - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(chunk))); + IOHandler()->enqueue(IOTask(&attributable, std::move(chunk))); }; #if openPMD_HAVE_MPI @@ -574,8 +592,7 @@ void Series::flushRankTable() // Must ensure that the Writable is consistently set to written on all // ranks - series.m_rankTable.m_attributable.setWritten( - true, EnqueueAsynchronously::OnlyAsync); + attributable.setWritten(true, EnqueueAsynchronously::OnlyAsync); return; } #endif @@ -596,6 +613,13 @@ std::string Series::particlesPath() const return getAttribute("particlesPath").get(); } +std::optional Series::particlesPathOptional() const +{ + return auxiliary::optional_and_then( + getAttributeOptional("particlesPath"), + [](Attribute const &attr) { return attr.getOptional(); }); +} + Series &Series::setParticlesPath(std::string const &pp) { auto &series = get(); @@ -609,12 +633,7 @@ Series &Series::setParticlesPath(std::string const &pp) "A files particlesPath can not (yet) be changed after it has been " "written."); - if (auxiliary::ends_with(pp, '/')) - setAttribute("particlesPath", pp); - else - setAttribute("particlesPath", pp + "/"); - setDirty(true); - return *this; + return setParticlesPath_internal(pp); } std::string Series::author() const @@ -981,7 +1000,8 @@ void Series::init( std::make_unique(parsed_directory, at)); auto &series = get(); series.iterations.linkHierarchy(writable()); - series.m_rankTable.m_attributable.linkHierarchy(writable()); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy( + writable()); series.m_deferred_initialization = [called_this_already = false, filepath, @@ -1207,7 +1227,11 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; - series.m_rankTable.m_attributable.linkHierarchy(writable); + series->m_writable.ownKeyWithinParent = "ROOT"; + auto &container_back = + series->m_writable.objectType.requireGroup()->m_children; + container_back["data"] = *series.iterations.m_attri; + series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; @@ -1469,6 +1493,12 @@ void Series::flushFileBased( case IO::HasBeenOpened: // continue below it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } break; } @@ -1494,6 +1524,12 @@ void Series::flushFileBased( case Access::APPEND_RANDOM_ACCESS: case Access::APPEND_LINEAR: { bool allDirty = dirty(); + // In flush level SkeletonOnly, we might need to set some attributes + // (especially: particlesPath, meshesPath), but cannot flush them yet + // (as writing attributes is only permissible at higher flush levels). + // This flag records if the Series became dirty during this flush. If + // yes, we set the Series back to dirty at the end of flushing. + bool hasBecomeDirty = false; for (auto it = begin; it != end; ++it) { // Phase 1 @@ -1522,6 +1558,12 @@ void Series::flushFileBased( it->second.flushFileBased(filename, it->first, flushParams); + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } + series.iterations.flush( auxiliary::replace_first(basePath(), "%T/", ""), flushParams); @@ -1543,11 +1585,30 @@ void Series::flushFileBased( } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes + * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but + * for some reason, this leads to hanging parallel tests..? */ + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) + { + if (allDirty && !dirty()) + { + throw error::Internal( + "Flush mode SkeletonOnly must not unset dirty flags."); + } + hasBecomeDirty |= + flushParams.flushLevel == FlushLevel::SkeletonOnly && + !allDirty && dirty(); + } setDirty(allDirty); } - setDirty(false); - + if (!hasBecomeDirty) + { + determineUnsetDirty(flushParams.flushLevel); + } + else + { + setDirty(true); + } // Phase 3 if (flushIOHandler) { @@ -1587,7 +1648,14 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } + it->second.flush(flushParams); + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } break; } @@ -1600,6 +1668,12 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + // Phase 3 Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); @@ -1632,8 +1706,13 @@ void Series::flushGorVBased( Parameter fCreate; fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); + } - flushRankTable(); + if (!series.m_perIterationData.m_rankTableAttributable.written()) + { + flushRankTable( + flushParams.flushLevel, + series.m_perIterationData.m_rankTableAttributable); } series.iterations.flush( @@ -1664,6 +1743,13 @@ void Series::flushGorVBased( throw std::runtime_error( "[Series] Internal control flow error"); } + + if (it == begin) + { + customHierarchyFlush( + flushParams, /* unset_dirty = */ false); + } + break; case IO::RemainsClosed: break; @@ -1678,6 +1764,12 @@ void Series::flushGorVBased( } } + if (begin == end) + { + customHierarchyFlush( + flushParams, /* managed_as_custom_object = */ false); + } + flushAttributes(flushParams); Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); @@ -1688,26 +1780,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { @@ -1918,7 +1990,10 @@ void Series::readFileBased( for (auto index : unparseableIterations) { - series.iterations.container().erase(index); + series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { + map.erase(to_string(index)); + }); } if (padding > 0) @@ -2254,7 +2329,10 @@ creating new iterations. std::cerr << "Cannot read iteration '" << index << "' and will skip it due to read error:\n" << err.what() << std::endl; - series.iterations.container().erase(index); + series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { + map.erase(to_string(index)); + }); return {err}; } i.get().m_closed = internal::CloseStatus::Open; @@ -2285,7 +2363,17 @@ creating new iterations. readableIterations.reserve(pList.paths->size()); for (auto const &it : *pList.paths) { - IterationIndex_t index = std::stoull(it); + IterationIndex_t index; + try + { + index = std::stoull(it); + } + catch (std::exception const &e) + { + std::cerr << "[Warning] Could not parse '" << it + << "' as an Iteration index. Will skip." << std::endl; + continue; + } if (read_only_this_single_iteration.has_value() && index != *read_only_this_single_iteration) { @@ -2416,7 +2504,6 @@ creating new iterations. void Series::readBase() { - auto &series = get(); Parameter aRead; aRead.name = "openPMD"; @@ -2502,16 +2589,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.meshes.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setMeshesPath(val.value()); - - for (auto &it : series.iterations) - it.second.meshes.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setMeshesPath_internal(val.value()); } else throw error::ReadError( @@ -2544,16 +2624,9 @@ void Series::readBase() .getOptional(); val.has_value()) { - /* allow setting the meshes path after completed IO */ - for (auto &it : series.iterations) - it.second.particles.setWritten( - false, Attributable::EnqueueAsynchronously::No); - - setParticlesPath(val.value()); - - for (auto &it : series.iterations) - it.second.particles.setWritten( - true, Attributable::EnqueueAsynchronously::No); + /* use internal api to allow setting the meshes path after completed + * IO */ + setParticlesPath_internal(val.value()); } else throw error::ReadError( @@ -2700,7 +2773,7 @@ AdvanceStatus Series::advance( if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2806,7 +2879,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2829,8 +2902,12 @@ AdvanceStatus Series::advance(AdvanceMode mode) return *param.status; } -void Series::flushStep(bool doFlush) +void Series::flushStep(bool doFlush, FlushLevel l) { + if (!flush_level::write_datasets(l)) + { + return; + } auto &series = get(); if (!series.m_currentlyActiveIterations.empty() && access::write(IOHandler()->m_frontendAccess)) @@ -2913,6 +2990,25 @@ Series &Series::setIterationEncoding_internal( return *this; } +Series &Series::setParticlesPath_internal(std::string const &pp) +{ + if (auxiliary::ends_with(pp, '/')) + setAttribute("particlesPath", pp); + else + setAttribute("particlesPath", pp + "/"); + setDirty(true); + return *this; +} + +Series &Series::setMeshesPath_internal(std::string const &mp) +{ + if (auxiliary::ends_with(mp, '/')) + setAttribute("meshesPath", mp); + else + setAttribute("meshesPath", mp + "/"); + setDirty(true); + return *this; +} auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { @@ -3334,12 +3430,12 @@ namespace internal */ if (impl.iterationEncoding() != IterationEncoding::fileBased) { - impl.flushStep(/* doFlush = */ true); + impl.flushStep(/* doFlush = */ true, FlushLevel::UserFlush); } } // Not strictly necessary, but clear the map of iterations // This releases the openPMD hierarchy - iterations.container().clear(); + iterations.container().for_both([](auto &map) { map.clear(); }); // Release the IO Handler if (IOHandler) { diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index ef899e4207..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -49,7 +49,7 @@ StringMatrix collectStringsAsMatrixTo( 1, MPI_INT, destRank, - MPI_COMM_WORLD); + communicator); int maxLength = std::accumulate( recvcounts.begin(), recvcounts.end(), 0, [](int a, int b) { return std::max(a, b); @@ -78,7 +78,7 @@ StringMatrix collectStringsAsMatrixTo( displs.data(), MPI_CHAR, destRank, - MPI_COMM_WORLD); + communicator); return res; } @@ -95,7 +95,7 @@ std::vector distributeStringsToAllRanks( int *displs = new int[size]; MPI_Allgather( - &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, MPI_COMM_WORLD); + &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); char *namesBuffer; { @@ -116,7 +116,7 @@ std::vector distributeStringsToAllRanks( sizesBuffer, displs, MPI_CHAR, - MPI_COMM_WORLD); + communicator); std::vector hostnames(size); for (int i = 0; i < size; ++i) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d19fa31a00..53cc30143e 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -19,8 +19,10 @@ * If not, see . */ #include "openPMD/backend/Attributable.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/ParticleSpecies.hpp" #include "openPMD/RecordComponent.hpp" @@ -29,6 +31,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/HierarchyVisitorImpl.hpp" +#include "openPMD/backend/Writable.hpp" #include #include @@ -53,12 +56,10 @@ namespace internal : SharedData_t({raw_ptr, [](auto const *) {}}) {} - void AttributableData::cloneFrom(AttributableData const &other) + void AttributableData::cloneFrom(parent_t const &other) { - using parent_t = std::shared_ptr; static_cast(*this) = static_cast(other); } - } // namespace internal Attributable::Attributable() @@ -105,13 +106,25 @@ bool Attributable::setAttribute(std::string const &key, Attribute attribute) } Attribute Attributable::getAttribute(std::string const &key) const +{ + auto attribute = getAttributeOptional(key); + if (attribute.has_value()) + { + return *attribute; + } + + throw no_such_attribute_error(key); +} + +std::optional +Attributable::getAttributeOptional(std::string const &key) const { auto &attri = get(); auto it = attri.m_attributes.find(key); if (it != attri.m_attributes.cend()) return it->second; - throw no_such_attribute_error(key); + return std::nullopt; } bool Attributable::deleteAttribute(std::string const &key) @@ -179,6 +192,76 @@ void Attributable::iterationFlush(std::string backendConfig) std::move(backendConfig)); } +void Attributable::customHierarchyFlush( + internal::FlushParams const &flushParams, bool managed_as_custom_object) +{ + // customHierarchies().printRecursively(); + if (!dirtyRecursive()) + { + return; + } + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + if (managed_as_custom_object && + access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + auto &data = get(); + Parameter pCreate; + + data.m_writable.objectType.ifGroup([&](auto &group_metadata) { + for (auto &[name, subpath_pointer] : + group_metadata.m_children_managed_as_custom_hierarchy) + { + auto &subpath = *subpath_pointer; + auto backpointer = subpath.writable().attributable; + auto casted_backpointer = + dynamic_cast(backpointer); + if (!casted_backpointer) + { + throw error::Internal( + "SharedAttributableData::m_children_managed_as_custom_" + "hierarchy contained " + "an object that should be flushed conventionally."); + } + if (subpath.writable().objectType.isGroup()) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + subpath.customHierarchyFlush(flushParams, true); + } + else if (subpath.writable().objectType.isDataset()) + { + subpath.asDataset().flush(name, flushParams); + } + else + { + throw std::runtime_error("Unreachable!"); + } + } + }); + + if (managed_as_custom_object && + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } + // customHierarchies().printRecursively(); +} + Series Attributable::retrieveSeries() const { Writable const *findSeries = &writable(); @@ -343,27 +426,37 @@ OpenpmdStandard Attributable::openPMDStandard() const return IOHandler()->m_standard; } +auto Attributable::customHierarchies() -> CustomHierarchy +{ + // No need to emplace this in + // SharedAttributableData::m_children_managed_as_custom_hierarchy. Only + // those instances of CustomHierarchy need to be emplaced that do not have a + // counter-object inside the openPMD hierarchy keeping it alive, e.g. + // children created or read by the returned instance outside the openPMD + // hierarchy. + return CustomHierarchy{*this}; +} + template -void Attributable::seriesFlush_impl(internal::FlushParams const &flushParams) +void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler) { - writable().seriesFlush(flushParams); + writable().seriesFlush(flushParams, flush_io_handler); } -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - switch (flushParams.flushLevel) + if (access::readOnly(IOHandler()->m_frontendAccess)) + { + throw std::runtime_error("Control flow error"); + } + if (!flush_level::write_attributes(flushParams.flushLevel)) { - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - // pass - break; } if (dirty()) { @@ -377,10 +470,7 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void Attributable::readAttributes(ReadMode mode) @@ -592,6 +682,61 @@ void Attributable::readAttributes(ReadMode mode) setDirty(false); } +void Attributable::preferCurrentBackpointer() const +{ + /* + * This is called when reopening some object as a specific type (e.g. + * RecordComponent) that had originally been opened generically as + * CustomHierarchy already. In this case, the specific type's pointer should + * be preferred for Writable::attributable as it has more information. + */ + auto this_as_custom_hierarchy = dynamic_cast(this); + if (this_as_custom_hierarchy) + { + return; + } + + auto &shareddata = **m_attri; + auto &w = shareddata.m_writable; + + auto backpointer_as_custom_hierarchy = + dynamic_cast(w.attributable); + if (!backpointer_as_custom_hierarchy) + { + return; + } + + // Now: + // !this_as_custom_hierarchy && backpointer_as_custom_hierarchy + + w.attributable = m_attri.get(); + + // Now: + // !this_as_custom_hierarchy && !backpointer_as_custom_hierarchy + + if (!w.parent) + { + throw error::Internal( + "CustomHierarchy object was created without parent. Why?"); + } + + // dont manage this as a customhierarchy instance + auto count_of_erased_elements = + (*w.parent->attributable) + ->m_writable.objectType.requireGroup() + ->m_children_managed_as_custom_hierarchy.erase( + w.ownKeyWithinParent); + if (count_of_erased_elements != 1) + { + throw error::Internal( + "Unexpected state: Expected to erase 1 element from internal " + "object storage, found " + + std::to_string(count_of_erased_elements) + " instead."); + } + + // std::cout << "REWIRED '" << myPath().openPMDPath() << "'." << std::endl; +} + void Attributable::setWritten(bool val, EnqueueAsynchronously ea) { switch (ea) diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index c4eab2318f..70fdff4a48 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/backend/scientific_defaults/ConfigAttribute.hpp" @@ -197,7 +198,7 @@ namespace internal template auto BaseRecord::begin() -> iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -210,7 +211,7 @@ auto BaseRecord::begin() -> iterator template auto BaseRecord::begin() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -223,7 +224,7 @@ auto BaseRecord::begin() const -> const_iterator template auto BaseRecord::cbegin() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ false); } @@ -236,7 +237,7 @@ auto BaseRecord::cbegin() const -> const_iterator template auto BaseRecord::end() -> iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -249,7 +250,7 @@ auto BaseRecord::end() -> iterator template auto BaseRecord::end() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -262,7 +263,7 @@ auto BaseRecord::end() const -> const_iterator template auto BaseRecord::cend() const -> const_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeIterator(/* is_end = */ true); } @@ -275,78 +276,78 @@ auto BaseRecord::cend() const -> const_iterator template auto BaseRecord::rbegin() -> reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(this->container_front().rbegin()); } } template auto BaseRecord::rbegin() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().rbegin()); + return makeReverseIterator(this->container_front().rbegin()); } } template auto BaseRecord::crbegin() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ false); } else { - return makeReverseIterator(this->container().crbegin()); + return makeReverseIterator(this->container_front().crbegin()); } } template auto BaseRecord::rend() -> reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(this->container_front().rend()); } } template auto BaseRecord::rend() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().rend()); + return makeReverseIterator(this->container_front().rend()); } } template auto BaseRecord::crend() const -> const_reverse_iterator { - if (get().m_datasetDefined) + if (get().datasetDefined()) { return makeReverseIterator(/* is_end = */ true); } else { - return makeReverseIterator(this->container().crend()); + return makeReverseIterator(this->container_front().crend()); } } @@ -455,7 +456,7 @@ auto BaseRecord::at(key_type const &key) const -> mapped_type const & bool const keyScalar = (key == RecordComponent::SCALAR); if (keyScalar) { - if (!get().m_datasetDefined) + if (!get().datasetDefined()) { throw std::out_of_range( "[at()] Requested scalar entry from non-scalar record."); @@ -509,7 +510,7 @@ template auto BaseRecord::find(key_type const &key) -> iterator { auto &r = get(); - if (r.m_datasetDefined) + if (r.datasetDefined()) { if (key == RecordComponent::SCALAR) { @@ -526,7 +527,7 @@ auto BaseRecord::find(key_type const &key) -> iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(this->container_front().find(key)); } } @@ -534,7 +535,7 @@ template auto BaseRecord::find(key_type const &key) const -> const_iterator { auto &r = get(); - if (r.m_datasetDefined) + if (r.datasetDefined()) { if (key == RecordComponent::SCALAR) { @@ -551,7 +552,7 @@ auto BaseRecord::find(key_type const &key) const -> const_iterator } else { - return makeIterator(r.m_container.find(key)); + return makeIterator(this->container_front().find(key)); } } @@ -560,7 +561,7 @@ auto BaseRecord::count(key_type const &key) const -> size_type { if (key == RecordComponent::SCALAR) { - return get().m_datasetDefined ? 1 : 0; + return get().datasetDefined() ? 1 : 0; } else { @@ -625,10 +626,12 @@ auto BaseRecord::insert(value_type const &value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(value); + auto res = this->syncInsertResult(this->container_front().insert(value)); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + // this->container().erase(res.first); + this->container_front().erase(res.first); + this->container_back(/* verify = */ true).erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -638,10 +641,12 @@ template auto BaseRecord::insert(value_type &&value) -> std::pair { detail::verifyNonscalar(this); - auto res = this->container().insert(std::move(value)); + auto res = this->syncInsertResult( + this->container_front().insert(std::move(value))); if (res.first->first == RecordComponent::SCALAR) { - this->container().erase(res.first); + this->container_front().erase(res.first); + this->container_back(/* verify = */ true).erase(res.first->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return {makeIterator(std::move(res.first)), res.second}; @@ -658,13 +663,15 @@ auto BaseRecord::insert(const_iterator hint, value_type const &value) [this](typename const_iterator::Right) { return static_cast const *>(this) ->container() - .begin(); + .front->begin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, value); + auto res = this->syncInsertResult( + this->container_front().insert(base_hint, value)); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + this->container_front().erase(res); + this->container_back(/* verify = */ true).erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -681,13 +688,15 @@ auto BaseRecord::insert(const_iterator hint, value_type &&value) [this](typename const_iterator::Right) { return static_cast const *>(this) ->container() - .begin(); + .front->begin(); }}, hint.m_iterator); - auto res = this->container().insert(base_hint, std::move(value)); + auto res = this->syncInsertResult( + this->container_front().insert(base_hint, std::move(value))); if (res->first == RecordComponent::SCALAR) { - this->container().erase(res); + this->container_front().erase(res); + this->container_back(/* verify = */ true).erase(res->first); throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); } return makeIterator(res); @@ -717,7 +726,28 @@ template auto BaseRecord::insert(std::initializer_list ilist) -> void { detail::verifyNonscalar(this); - this->container().insert(std::move(ilist)); + std::vector< + internal::object_type::GroupMetaData::children_map_t::value_type> + internal_insert_list; + internal_insert_list.reserve(ilist.size()); + auto &cont = this->container_back(/* verify = */ false); + for (auto &v : ilist) + { + decltype(auto) key = this->key_as_string(v.first); + auto it = cont.find(key); + if (it == cont.end()) + { + internal_insert_list.emplace_back(key, *v.second.m_attri); + } + else + { + // backend value is older, so it gets seniority + v.second.m_attri->asSharedPtrOfAttributable() = it->second; + v.second.preferCurrentBackpointer(); + } + } + this->container_front().insert(std::move(ilist)); + cont.insert(internal_insert_list.begin(), internal_insert_list.end()); /* * We skip this check as it changes the runtime of this call from * O(last-first) to O(container().size()). @@ -737,7 +767,9 @@ auto BaseRecord::swap(BaseRecord &other) noexcept -> void { detail::verifyNonscalar(this); detail::verifyNonscalar(&other); - this->container().swap(other.container()); + this->container_front().swap(other.container_front()); + this->container_back(/* verify = */ true) + .swap(other.container_back(/* verify = */ true)); } template @@ -798,10 +830,7 @@ inline void BaseRecord::flush( } this->flush_impl(name, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - this->setDirty(false); - } + this->determineUnsetDirty(flushParams.flushLevel); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 21a7352850..3005702066 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -38,13 +38,13 @@ BaseRecordComponent &BaseRecordComponent::resetDatatype(Datatype d) "written."); auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - rc.m_dataset.value().dtype = d; + rc.dataset().value().dtype = d; } else { - rc.m_dataset = Dataset{d, {1}}; + rc.dataset() = Dataset{d, {1}}; } setDirty(true); return *this; @@ -53,9 +53,9 @@ BaseRecordComponent &BaseRecordComponent::resetDatatype(Datatype d) Datatype BaseRecordComponent::getDatatype() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - return rc.m_dataset.value().dtype; + return rc.dataset().value().dtype; } else { @@ -65,15 +65,16 @@ Datatype BaseRecordComponent::getDatatype() const bool BaseRecordComponent::constant() const { - return get().m_isConstant; + auto &data = get(); + return data.datasetDefined() && data.isConstant(); } std::optional BaseRecordComponent::joinedDimension() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - return rc.m_dataset.value().joinedDimension(); + return rc.dataset().value().joinedDimension(); } else { @@ -84,14 +85,14 @@ std::optional BaseRecordComponent::joinedDimension() const ChunkTable BaseRecordComponent::availableChunks() { auto &rc = get(); - if (rc.m_isConstant) + if (rc.isConstant()) { - if (!rc.m_dataset.has_value()) + if (!rc.dataset().has_value()) { return ChunkTable{}; } - Offset offset(rc.m_dataset.value().extent.size(), 0); - return ChunkTable{{std::move(offset), rc.m_dataset.value().extent}}; + Offset offset(rc.dataset().value().extent.size(), 0); + return ChunkTable{{std::move(offset), rc.dataset().value().extent}}; } if (auto iteration_data = containingIteration().first; iteration_data.has_value()) @@ -122,12 +123,12 @@ BaseRecordComponent::BaseRecordComponent(NoInit) : Attributable(NoInit()) void BaseRecordComponent::setDatasetDefined( internal::BaseRecordComponentData &data) { - data.m_datasetDefined = true; + data.setDatasetDefined(); } bool BaseRecordComponent::datasetDefined() const { auto &data = get(); - return data.m_datasetDefined; + return data.datasetDefined(); } } // namespace openPMD diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp index fc0785d2c1..c39ffc3e91 100644 --- a/src/backend/Container.cpp +++ b/src/backend/Container.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/ContainerImpl.tpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -42,6 +43,7 @@ OPENPMD_INSTANTIATE(PatchRecord) OPENPMD_INSTANTIATE(PatchRecordComponent) OPENPMD_INSTANTIATE(Record) OPENPMD_INSTANTIATE(RecordComponent) +OPENPMD_INSTANTIATE(CustomHierarchy) OPENPMD_INSTANTIATE(Iteration OPENPMD_COMMA Iteration::IterationIndex_t) #undef OPENPMD_INSTANTIATE #undef OPENPMD_COMMA diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 740f44cc51..dbbdcd402c 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -61,19 +61,16 @@ void PatchRecord::flush_impl( } if (!this->scalar()) { - if (IOHandler()->m_frontendAccess != Access::READ_ONLY) - Container::flush( - path, flushParams); // warning (clang-tidy-10): - // bugprone-parent-virtual-call + Container::flush( + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call + for (auto &comp : *this) comp.second.flush(comp.first, flushParams); } else T_RecordComponent::flush(path, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void PatchRecord::read() @@ -103,7 +100,8 @@ void PatchRecord::read() << component_name << "' and will skip it due to read error:" << err.what() << std::endl; - this->container().erase(component_name); + this->container().for_both( + [&component_name](auto &map) { map.erase(component_name); }); } } diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 5c35d918fe..fdf3975e79 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -44,9 +44,9 @@ uint8_t PatchRecordComponent::getDimensionality() const Extent PatchRecordComponent::getExtent() const { auto &rc = get(); - if (rc.m_dataset.has_value()) + if (rc.dataset().has_value()) { - return rc.m_dataset.value().extent; + return rc.dataset().value().extent; } else { diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ea6e56b9c5..54e2a60b4a 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -52,16 +52,20 @@ Writable::~Writable() } template -void Writable::seriesFlush(std::string backendConfig) +void Writable::seriesFlush(std::string backendConfig, bool flush_io_handler) { seriesFlush( - internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}); + internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}, + flush_io_handler); } -template void Writable::seriesFlush(std::string backendConfig); -template void Writable::seriesFlush(std::string backendConfig); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); template -void Writable::seriesFlush(internal::FlushParams const &flushParams) +void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler) { Attributable impl; impl.setData({attributable, [](auto const *) {}}); @@ -103,10 +107,10 @@ void Writable::seriesFlush(internal::FlushParams const &flushParams) return {series.iterations.begin(), series.iterations.end()}; } }(); - series.flush_impl(begin, end, flushParams); + series.flush_impl(begin, end, flushParams, flush_io_handler); } -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); } // namespace openPMD diff --git a/src/snapshots/StatefulIterator.cpp b/src/snapshots/StatefulIterator.cpp index 2a7f994873..57bdf82d32 100644 --- a/src/snapshots/StatefulIterator.cpp +++ b/src/snapshots/StatefulIterator.cpp @@ -657,8 +657,10 @@ std::optional StatefulIterator::loopBody(Seek const &seek) else if ( series.IOHandler()->m_frontendAccess == Access::READ_LINEAR) { - data.series.iterations.container().erase( - *maybe_current_iteration); + data.series.iterations.container().for_both_to_string( + [&maybe_current_iteration](auto &map, auto &&to_string) { + map.erase(to_string(*maybe_current_iteration)); + }); } } } @@ -853,7 +855,8 @@ void StatefulIterator::deactivateDeadIteration(iteration_index_t index) } break; } - data.series.iterations.container().erase(index); + data.series.iterations.container().for_both_to_string( + [index](auto &map, auto &&to_string) { map.erase(to_string(index)); }); } StatefulIterator &StatefulIterator::operator++() diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 821907b38e..a46669b0b6 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1128,8 +1128,8 @@ TEST_CASE("wrapper_test", "[core]") o.flush(); REQUIRE(all_data.get()[0] == value); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[4].meshes["E"]["y"].get().m_chunks.empty()); - REQUIRE(mrc2.get().m_chunks.empty()); + REQUIRE(o.iterations[4].meshes["E"]["y"].get().chunks().empty()); + REQUIRE(mrc2.get().chunks().empty()); #endif MeshRecordComponent mrc3 = o.iterations[5].meshes["E"]["y"]; @@ -1144,13 +1144,13 @@ TEST_CASE("wrapper_test", "[core]") std::shared_ptr storeData = std::make_shared(44); o.iterations[5].meshes["E"]["y"].storeChunk(storeData, {0}, {1}); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[5].meshes["E"]["y"].get().m_chunks.size() == 1); - REQUIRE(mrc3.get().m_chunks.size() == 1); + REQUIRE(o.iterations[5].meshes["E"]["y"].get().chunks().size() == 1); + REQUIRE(mrc3.get().chunks().size() == 1); #endif o.flush(); #if openPMD_USE_INVASIVE_TESTS - REQUIRE(o.iterations[5].meshes["E"]["y"].get().m_chunks.empty()); - REQUIRE(mrc3.get().m_chunks.empty()); + REQUIRE(o.iterations[5].meshes["E"]["y"].get().chunks().empty()); + REQUIRE(mrc3.get().chunks().empty()); #endif o.iterations[6] @@ -1188,8 +1188,9 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.empty()); - REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.empty()); + .chunks() + .empty()); + REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().chunks().empty()); #endif pp["numParticles"][RecordComponent::SCALAR].store(idx, val); #if openPMD_USE_INVASIVE_TESTS @@ -1198,9 +1199,10 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.size() == 1); + .chunks() + .size() == 1); REQUIRE( - pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.size() == 1); + pp["numParticles"][RecordComponent::SCALAR].get().chunks().size() == 1); #endif std::stringstream u64str; u64str << determineDatatype(); @@ -1222,9 +1224,10 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.size() == 2); + .chunks() + .size() == 2); REQUIRE( - pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.size() == 2); + pp["numParticles"][RecordComponent::SCALAR].get().chunks().size() == 2); #endif o.flush(); #if openPMD_USE_INVASIVE_TESTS @@ -1232,8 +1235,9 @@ TEST_CASE("wrapper_test", "[core]") .particles["electrons"] .particlePatches["numParticles"][RecordComponent::SCALAR] .get() - .m_chunks.empty()); - REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().m_chunks.empty()); + .chunks() + .empty()); + REQUIRE(pp["numParticles"][RecordComponent::SCALAR].get().chunks().empty()); #endif } @@ -1288,7 +1292,7 @@ TEST_CASE("use_count_test", "[core]") REQUIRE( std::get>( static_cast *>( - pprc.get().m_chunks.front().parameter.get()) + pprc.get().chunks().front().parameter.get()) ->data.as_variant()) .use_count() == 1); #endif @@ -1769,6 +1773,11 @@ TEST_CASE("read_nonexistent_attribute", "[core]") read_nonexistent_attribute::read_nonexistent_attribute(); } +TEST_CASE("custom_hierarchy", "[core]") +{ + custom_hierarchy::custom_hierarchy(); +} + TEST_CASE("unique_ptr", "[core]") { auto stdptr = std::make_unique(5); diff --git a/test/Files_Core/CoreTests.hpp b/test/Files_Core/CoreTests.hpp index f769beeb99..1f2d7072d9 100644 --- a/test/Files_Core/CoreTests.hpp +++ b/test/Files_Core/CoreTests.hpp @@ -29,3 +29,8 @@ namespace read_nonexistent_attribute { auto read_nonexistent_attribute() -> void; } + +namespace custom_hierarchy +{ +auto custom_hierarchy() -> void; +} diff --git a/test/Files_Core/custom_hierarchy.cpp b/test/Files_Core/custom_hierarchy.cpp new file mode 100644 index 0000000000..2253b47f6b --- /dev/null +++ b/test/Files_Core/custom_hierarchy.cpp @@ -0,0 +1,169 @@ + +/* Copyright 2026 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IterationEncoding.hpp" +#include "openPMD/version.hpp" +#define OPENPMD_private public: +#define OPENPMD_protected public: + +#include "CoreTests.hpp" + +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/IOTask.hpp" + +#include +namespace custom_hierarchy +{ +using namespace openPMD; + +void write( + std::string const &filename, + std::string const &json_params, + IterationEncoding) +{ + Series series(filename, Access::CREATE_LINEAR, json_params); + auto add_custom_hierarchy = [](auto &&attr) { + attr.customHierarchies()["rabimmel"].setAttribute("rabammel", "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + add_custom_hierarchy(series); + add_custom_hierarchy(series.snapshots()); + add_custom_hierarchy(iteration); + + auto Ex = iteration.meshes["E"]["x"]; + Ex.resetDataset({Datatype::INT, {3}, R"( + resizable = true + hdf5.dataset.chunks = "auto")"}); + std::vector Exdata{1, 2, 3}; + Ex.storeChunk(Exdata, {0}, {3}); + + auto Ex_ = iteration.customHierarchies()["meshes"]["E"]["x"].asDataset(); + REQUIRE(Ex_.getDatatype() == Ex.getDatatype()); + REQUIRE(Ex_.getExtent() == Ex.getExtent()); + iteration.seriesFlush(); + + // Test resizing + Ex_.resetDataset({{6}}); + Ex.storeChunk(Exdata, {3}, {3}); + + auto fully_custom_dataset = + iteration.customHierarchies()["fully"]["custom"]["dataset"].asDataset(); + fully_custom_dataset.resetDataset({Datatype::INT, {3}}); + // fully_custom_dataset.setComment( + // "ALS PRIMA BALLERINA ALS WEIB ALS FEMME FATALE"); + fully_custom_dataset.storeChunk(Exdata, {0}, {3}); + iteration.close(); +} + +void read( + std::string const &filename, + std::string const &json_params, + IterationEncoding) +{ + Series series(filename, Access::READ_LINEAR, json_params); + auto require_custom_hierarchy = [](auto &&attr) { + CustomHierarchy ch = attr.customHierarchies(); + REQUIRE(ch.find("rabimmel") == ch.end()); + ch.read(0); + REQUIRE( + ch["rabimmel"].getAttribute("rabammel").get() == + "rabumm"); + }; + auto iteration = series.snapshots()[0]; + + require_custom_hierarchy(series); + require_custom_hierarchy(series.snapshots()); + require_custom_hierarchy(iteration); + + auto Ex_data = iteration.customHierarchies()["meshes"]["E"]["x"] + .asDataset() + .loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 3; ++i) + { + REQUIRE(Ex_data.get()[i] == (int)i + 1); + REQUIRE(Ex_data.get()[i + 3] == (int)i + 1); + } + + auto custom_data = + iteration.customHierarchies()["fully"]["custom"]["dataset"] + .asDataset() + .loadChunk(); + iteration.seriesFlush(); + for (size_t i = 0; i < 3; ++i) + { + REQUIRE(custom_data.get()[i] == (int)i + 1); + } + iteration.close(); +} + +struct test_config +{ + char const *filename; + char const *json_params; + IterationEncoding encoding; +}; + +void custom_hierarchy() +{ + test_config configs[] = { + {"groupbased.%E", + R"({"iteration_encoding": "group_based"})", + IterationEncoding::groupBased}, + {"filebased_%T.%E", + R"({"iteration_encoding": "file_based"})", + IterationEncoding::fileBased}, + {"variablebased.%E", + R"({"iteration_encoding": "variable_based"})", + IterationEncoding::variableBased}}; + + auto const &variants = getVariants(); + for (auto const &backend : {"adios2", "hdf5", "json", "toml"}) + { + if (!variants.at(backend)) + { + continue; + } + for (auto const &[filename, json_params, encoding] : configs) + { + auto json_params_ = json::merge( + json_params, + std::string( + R"( + { + "adios2": { + "engine": { + "type": "file" + } + }, + "backend": ")") + + backend + R"("})"); + auto filename_ = std::string("../samples/custom_hierarchy/") + + backend + "/" + filename; + write(filename_, json_params_, encoding); + read(filename_, json_params_, encoding); + } + } +} +} // namespace custom_hierarchy diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..d32d31253b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -37,13 +37,32 @@ #include #if !openPMD_HAVE_MPI -TEST_CASE("none", "[parallel]") +#define PARALLEL_TEST_CASE(name, tags) TEST_CASE(#name, tags) + +PARALLEL_TEST_CASE(none, "[parallel]") {} #else #include +#define PARALLEL_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + MPI_Barrier(MPI_COMM_WORLD); \ + int rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ + if (rank == 0) \ + { \ + std::cout << "\nStarting test '" << #name << "'.\n" << std::endl; \ + } \ + MPI_Barrier(MPI_COMM_WORLD); \ + openPMD_parallel_##name(); \ + MPI_Barrier(MPI_COMM_WORLD); \ + } \ + static void openPMD_parallel_##name() + #if openPMD_HAVE_ADIOS2 #include #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) @@ -80,7 +99,7 @@ TEST_CASE("none", "[parallel]") using namespace openPMD; -TEST_CASE("parallel_multi_series_test", "[parallel]") +PARALLEL_TEST_CASE(parallel_multi_series_test, "[parallel]") { std::list allSeries; @@ -223,7 +242,7 @@ void write_test_zero_extent( #endif #if openPMD_HAVE_HDF5 && openPMD_HAVE_MPI -TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(git_hdf5_sample_content_test, "[parallel][hdf5]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -307,7 +326,7 @@ TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") } } -TEST_CASE("hdf5_write_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test, "[parallel][hdf5]") { int mpi_s{-1}; int mpi_r{-1}; @@ -377,13 +396,13 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]") o.flush("hdf5.independent_stores = false"); } -TEST_CASE("hdf5_write_test_zero_extent", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_zero_extent, "[parallel][hdf5]") { write_test_zero_extent(false, "h5", true, true); write_test_zero_extent(true, "h5", true, true); } -TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_chunk, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -397,7 +416,7 @@ TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") REQUIRE(true); } -TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_declare, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -413,7 +432,7 @@ TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") #else -TEST_CASE("no_parallel_hdf5", "[parallel][hdf5]") +PARALLEL_TEST_CASE(no_parallel_hdf5, "[parallel][hdf5]") { REQUIRE(true); } @@ -495,7 +514,7 @@ void available_chunks_test(std::string const &file_ending) } } -TEST_CASE("available_chunks_test", "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -550,14 +569,14 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) } } -TEST_CASE("extend_dataset", "[parallel]") +PARALLEL_TEST_CASE(extend_dataset, "[parallel]") { extendDataset("bp", R"({"backend": "adios2"})"); } #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("adios_write_test", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", @@ -645,25 +664,25 @@ TEST_CASE("adios_write_test", "[parallel][adios]") } } -TEST_CASE("adios_write_test_zero_extent", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_zero_extent, "[parallel][adios]") { write_test_zero_extent(false, "bp", true, true); write_test_zero_extent(true, "bp", true, true); } -TEST_CASE("adios_write_test_skip_chunk", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_chunk, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, true); write_test_zero_extent(true, "bp", false, true); } -TEST_CASE("adios_write_test_skip_declare", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_declare, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, false); write_test_zero_extent(true, "bp", false, false); } -TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios2][bp3]") +PARALLEL_TEST_CASE(hzdr_adios_sample_content_test, "[parallel][adios2][bp3]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -743,7 +762,7 @@ void write_4D_test(std::string const &file_ending) o.flush(); } -TEST_CASE("write_4D_test", "[parallel]") +PARALLEL_TEST_CASE(write_4D_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -776,7 +795,7 @@ void write_makeconst_some(std::string const &file_ending) E_x.makeConstant(42); } -TEST_CASE("write_makeconst_some", "[parallel]") +PARALLEL_TEST_CASE(write_makeconst_some, "[parallel]") { for (auto const &t : getBackends()) { @@ -883,7 +902,7 @@ void close_iteration_test(std::string const &file_ending) } } -TEST_CASE("close_iteration_test", "[parallel]") +PARALLEL_TEST_CASE(close_iteration_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -1002,7 +1021,7 @@ void file_based_write_read(std::string const &file_ending) } } -TEST_CASE("file_based_write_read", "[parallel]") +PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { for (auto const &t : getBackends()) { @@ -1181,7 +1200,7 @@ void hipace_like_write(std::string const &file_ending) } } -TEST_CASE("hipace_like_write", "[parallel]") +PARALLEL_TEST_CASE(hipace_like_write, "[parallel]") { for (auto const &t : getBackends()) { @@ -1191,7 +1210,7 @@ TEST_CASE("hipace_like_write", "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("independent_write_with_collective_flush", "[parallel]") +PARALLEL_TEST_CASE(independent_write_with_collective_flush, "[parallel]") { Series write( "../samples/independent_write_with_collective_flush.bp5", @@ -1225,7 +1244,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") #endif #if openPMD_HAVE_MPI -TEST_CASE("unavailable_backend", "[core][parallel]") +PARALLEL_TEST_CASE(unavailable_backend, "[core][parallel]") { #if !openPMD_HAVE_ADIOS2 { @@ -1373,7 +1392,7 @@ void adios2_streaming(bool variableBasedLayout) } } -TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") +PARALLEL_TEST_CASE(adios2_streaming, "[pseudoserial][adios2]") { #if HAS_ADIOS_2_9 adios2_streaming(true); @@ -1381,7 +1400,7 @@ TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") adios2_streaming(false); } -TEST_CASE("parallel_adios2_json_config", "[parallel][adios2]") +PARALLEL_TEST_CASE(parallel_adios2_json_config, "[parallel][adios2]") { int size{-1}; int rank{-1}; @@ -1592,7 +1611,7 @@ void adios2_ssc() } } -TEST_CASE("adios2_ssc", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_ssc, "[parallel][adios2]") { adios2_ssc(); } @@ -1918,7 +1937,7 @@ void append_mode( #endif } -TEST_CASE("append_mode", "[serial]") +PARALLEL_TEST_CASE(append_mode, "[serial]") { for (auto const &t : testedFileExtensions()) { @@ -2121,7 +2140,7 @@ void joined_dim(std::string const &ext) } } -TEST_CASE("joined_dim", "[parallel]") +PARALLEL_TEST_CASE(joined_dim, "[parallel]") { #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -2146,7 +2165,7 @@ TEST_CASE("joined_dim", "[parallel]") #if openPMD_HAVE_ADIOS2_BP5 // Parallel version of the same test from SerialIOTest.cpp -TEST_CASE("adios2_flush_via_step") +PARALLEL_TEST_CASE(adios2_flush_via_step, "[parallel]") { int size_i(0), rank_i(0); MPI_Comm_rank(MPI_COMM_WORLD, &rank_i); @@ -2253,12 +2272,12 @@ TEST_CASE("adios2_flush_via_step") } #endif -TEST_CASE("read_variablebased_randomaccess") +PARALLEL_TEST_CASE(read_variablebased_randomaccess, "[parallel]") { read_variablebased_randomaccess::read_variablebased_randomaccess(); } -TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +PARALLEL_TEST_CASE(iterate_nonstreaming_series, "[parallel][adios2]") { iterate_nonstreaming_series::iterate_nonstreaming_series(); } @@ -2719,14 +2738,14 @@ void run_test() } } // namespace adios2_chunk_distribution -TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_chunk_distribution, "[parallel][adios2]") { adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI #if openPMD_HAVE_MPI -TEST_CASE("bug_1655_bp5_writer_hangup", "[parallel]") +PARALLEL_TEST_CASE(bug_1655_bp5_writer_hangup, "[parallel]") { bug_1655_bp5_writer_hangup::bug_1655_bp5_writer_hangup(); }