Custom hierarchies – other API concept, for now without extended meshes and particles path

CMakeLists.txt

@@ -402,6 +402,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake)
 set(CORE_SOURCE
         src/config.cpp
         src/ChunkInfo.cpp
+        src/CustomHierarchy.cpp
         src/Dataset.cpp
         src/Datatype.cpp
         src/Error.cpp
@@ -866,6 +867,7 @@ if(openPMD_BUILD_TESTING)
             list(APPEND ${out_list}
                 test/Files_Core/automatic_variable_encoding.cpp
                 test/Files_Core/read_nonexistent_attribute.cpp
+                test/Files_Core/custom_hierarchy.cpp
             )
         endif()
     endmacro()

include/openPMD/CustomHierarchy.hpp

@@ -0,0 +1,190 @@
+/* Copyright 2023 Franz Poeschel
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "openPMD/Error.hpp"
+#include "openPMD/IO/AbstractIOHandler.hpp"
+#include "openPMD/Mesh.hpp"
+#include "openPMD/ParticleSpecies.hpp"
+#include "openPMD/RecordComponent.hpp"
+#include "openPMD/backend/Attributable.hpp"
+#include "openPMD/backend/Container.hpp"
+#include "openPMD/backend/Writable.hpp"
+
+#include <stdexcept>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+namespace openPMD
+{
+class CustomHierarchy;
+namespace internal
+{
+    using CustomHierarchyData = ContainerData<Attributable>;
+} // namespace internal
+
+class CustomHierarchy;
+
+/*
+ * This is its own class, so the return value of asContainerOf() is also
+ * convsertible again.
+ */
+template <typename MappedType>
+class ConvertibleContainer : public Container<MappedType>
+{
+    template <typename>
+    friend class ConversibleContainer;
+    friend class CustomHierarchy;
+
+protected:
+    using Container_t = Container<MappedType>;
+    using Data_t = internal::ContainerData<MappedType>;
+    static_assert(
+        std::is_base_of_v<typename Container_t::ContainerData, Data_t>);
+
+    using Container_t::Container_t;
+
+private:
+    explicit ConvertibleContainer() = default;
+
+public:
+    auto asDataset() -> RecordComponent;
+    // TODO also: asContainerOf()
+};
+
+namespace traits
+{
+    template <>
+    struct DeferredInitPolicy<Container<CustomHierarchy>>
+    {
+        template <typename Container_const_or_not>
+        static void call(Container_const_or_not &);
+    };
+
+    template <>
+    struct GenerationPolicy<CustomHierarchy>
+    {
+        constexpr static bool is_noop = false;
+        template <typename Container, typename Iterator>
+        void operator()(Container &cont, Iterator &it)
+        {
+            auto &writable = it->second.writable();
+
+            // These should be different
+            auto child_shared_data = &it->second.Attributable::get();
+            auto parent_shared_data = &cont.Attributable::get();
+            if (child_shared_data == parent_shared_data)
+            {
+                throw std::runtime_error(
+                    "Trying to emplace object as its own child");
+            }
+
+            // These might be different, but might also be the same
+            //
+            // For an explanation, ref. the documentation of
+            // Writable::attributable: This is a pointer back to the first
+            // created Attributable instance linking this Writable. There might
+            // be multiple Attributable objects linking the same backend
+            // Writable object when opening multiple "views" on the same backend
+            // object, e.g. when a scalar Record is at the same time a
+            // RecordComponent, or when reopening an object as a
+            // CustomHierarchy.
+            //
+            // Since CustomHierarchy performs no memory management by default,
+            // we must ensure that the backpointer in Writable::attributable
+            // remains valid when the frontend instance pointed by
+            // Writable::attributable *is* the CustomHierarchy instance (happens
+            // when it is the first frontend object created for that backend
+            // object).
+            auto backpointer = writable.attributable;
+            auto emplaced_pointer = it->second.m_attri.get();
+            auto *group_metadata =
+                (**cont.m_attri).m_writable.objectType.requireGroup();
+            if (backpointer == emplaced_pointer)
+            {
+                group_metadata
+                    ->m_children_managed_as_custom_hierarchy[it->first] =
+                    // NO move!! The iterator must stay alive
+                    std::make_shared<CustomHierarchy>(it->second);
+            }
+        }
+    };
+} // namespace traits
+
+// If we want to break the DeferredInitPolicy logic, we may alternatively derive
+// this privately from ConvertibleContainer<Attributable>, and then overload
+// access operators to return CustomHierarchy instances
+class CustomHierarchy : public ConvertibleContainer<CustomHierarchy>
+{
+    friend class Iteration;
+    friend class Container<CustomHierarchy>;
+    friend class Attributable;
+    friend struct traits::DeferredInitPolicy<Container<CustomHierarchy>>;
+
+private:
+    using Parent_t = ConvertibleContainer<CustomHierarchy>;
+    using Container_t = typename Parent_t::Container_t;
+    using Data_t = typename Parent_t::Data_t;
+
+protected:
+    CustomHierarchy(NoInit);
+    CustomHierarchy(std::shared_ptr<internal::SharedAttributableData> other);
+    CustomHierarchy(Attributable const &other);
+
+    void read(std::vector<std::string> &currentPath);
+
+    void flush_internal(
+        internal::FlushParams const &, std::vector<std::string> currentPath);
+    void flush(std::string const &path, internal::FlushParams const &) override;
+
+    /**
+     * @brief Link with parent.
+     *
+     * @param w The Writable representing the parent.
+     */
+    void linkHierarchy(Writable &w) override;
+
+public:
+    CustomHierarchy();
+
+    CustomHierarchy(CustomHierarchy const &other) = default;
+    CustomHierarchy(CustomHierarchy &&other) = default;
+
+    CustomHierarchy &operator=(CustomHierarchy const &) = default;
+    CustomHierarchy &operator=(CustomHierarchy &&) = default;
+
+    // TODO should we automatically read upon returning / instantiating a
+    // CustomHierarchy object? i.e. upon Attributable::customHierarchy() and
+    // CustomHierarchy::operator[]().
+    //
+    // set max_recursion_depth = 0 for infinite cycling
+    // recursion depth includes the current object
+    // recursion will not continue expanding into regions that are already known
+    // (hence not transitively expand into unknown subregions of known regions)
+    auto read(size_t max_recursion_depth = 1) -> CustomHierarchy;
+
+    void printRecursively();
+
+private:
+    void printRecursively(std::string indent);
+};
+} // namespace openPMD

include/openPMD/IO/AbstractIOHandler.hpp

@@ -26,6 +26,7 @@
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 #include "openPMD/version.hpp"
+#include <ostream>
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -81,6 +82,66 @@ enum class FlushLevel
     CreateOrOpenFiles
 };
 
+std::ostream &operator<<(std::ostream &, FlushLevel);
+
+namespace flush_level
+{
+    inline constexpr auto global_flushpoint(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    // same as global_flushpoint for now, but we will soon introduce
+    // immediate_flush
+    inline constexpr auto write_datasets(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto write_attributes(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+            return true;
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto flush_hierarchy(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+            return true;
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+} // namespace flush_level
+
 enum class OpenpmdStandard
 {
     v_1_0_0,
@@ -121,6 +182,7 @@ namespace internal
      * To be used for reading
      */
     FlushParams const defaultFlushParams{};
+    FlushParams const publicFlush{FlushLevel::UserFlush};
 
     struct ParsedFlushParams;
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -39,7 +39,7 @@ class AbstractIOHandlerImpl
 
     virtual ~AbstractIOHandlerImpl() = default;
 
-    std::future<void> flush();
+    std::future<void> flush(FlushLevel);
 
     /**
      * Close the file corresponding with the writable and release file handles.

include/openPMD/IO/IOTask.hpp

@@ -35,6 +35,7 @@
 #include <cstddef>
 #include <memory>
 #include <optional>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <variant>
@@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
 }; // note: if you change the enum members here, please update
    // docs/source/dev/design.rst
 
+std::ostream &operator<<(std::ostream &os, Operation op);
+
 namespace internal
 {
     /*

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
 
-    std::future<void> flush();
+    std::future<void> flush(internal::ParsedFlushParams &params);
 
 private:
 #if openPMD_HAVE_MPI

include/openPMD/Iteration.hpp

@@ -28,6 +28,7 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
 #include "openPMD/backend/HierarchyVisitor.hpp"
+#include "openPMD/backend/PerIterationData.hpp"
 #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <cstdint>
@@ -122,14 +123,16 @@ namespace internal
          */
         bool allow_reopening_implicitly = false;
 
-        /**
-         * Whether a step is currently active for this iteration.
-         * Used for file-based iteration layout, see Series.hpp for
-         * group-based layout.
-         * Access via stepStatus() method to automatically select the correct
-         * one among both flags.
+        /*
+         * This stores data items that are:
+         *
+         * 1. global in group and variable encodings
+         * 2. per-iteration in file encoding
+         *
+         * The struct is stored as part of the Series and as part of each
+         * Iteration. Access must be distinguished by iteration encoding.
          */
-        StepStatus m_stepStatus = StepStatus::NoStep;
+        PerIterationData m_perIterationData;
 
         /**
          * Cached copy of the key under which this Iteration lives in
@@ -483,8 +486,8 @@ namespace traits
     struct GenerationPolicy<Iteration>
     {
         constexpr static bool is_noop = false;
-        template <typename Iterator>
-        void operator()(Iterator &it)
+        template <typename Container, typename Iterator>
+        void operator()(Container &, Iterator &it)
         {
             it->second.get().m_iterationIndex = it->first;
         }

include/openPMD/ParticleSpecies.hpp

@@ -74,8 +74,8 @@ namespace traits
     struct GenerationPolicy<ParticleSpecies>
     {
         constexpr static bool is_noop = false;
-        template <typename T>
-        void operator()(T &it)
+        template <typename Container, typename T>
+        void operator()(Container &, T &it)
         {
             it->second.particlePatches.linkHierarchy(it->second.writable());
         }