Parallel HDF5: Try fixing strict collective requirements of HDF5 >= 2.0

include/openPMD/IO/AbstractIOHandler.hpp

@@ -26,6 +26,7 @@
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 #include "openPMD/version.hpp"
+#include <ostream>
 
 #if openPMD_HAVE_MPI
 #include <mpi.h>
@@ -81,6 +82,66 @@ enum class FlushLevel
     CreateOrOpenFiles
 };
 
+std::ostream &operator<<(std::ostream &, FlushLevel);
+
+namespace flush_level
+{
+    inline constexpr auto global_flushpoint(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    // same as global_flushpoint for now, but we will soon introduce
+    // immediate_flush
+    inline constexpr auto write_datasets(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            return true;
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto write_attributes(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+            return true;
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+    inline constexpr auto flush_hierarchy(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+        case FlushLevel::InternalFlush:
+        case FlushLevel::SkeletonOnly:
+            return true;
+        case FlushLevel::CreateOrOpenFiles:
+            return false;
+        }
+        return false; // unreachable
+    }
+} // namespace flush_level
+
 enum class OpenpmdStandard
 {
     v_1_0_0,
@@ -121,6 +182,7 @@ namespace internal
      * To be used for reading
      */
     FlushParams const defaultFlushParams{};
+    FlushParams const publicFlush{FlushLevel::UserFlush};
 
     struct ParsedFlushParams;
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -39,7 +39,7 @@ class AbstractIOHandlerImpl
 
     virtual ~AbstractIOHandlerImpl() = default;
 
-    std::future<void> flush();
+    std::future<void> flush(FlushLevel);
 
     /**
      * Close the file corresponding with the writable and release file handles.

include/openPMD/IO/IOTask.hpp

@@ -35,6 +35,7 @@
 #include <cstddef>
 #include <memory>
 #include <optional>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <variant>
@@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
 }; // note: if you change the enum members here, please update
    // docs/source/dev/design.rst
 
+std::ostream &operator<<(std::ostream &os, Operation op);
+
 namespace internal
 {
     /*

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     void touch(Writable *, Parameter<Operation::TOUCH> const &) override;
 
-    std::future<void> flush();
+    std::future<void> flush(internal::ParsedFlushParams &params);
 
 private:
 #if openPMD_HAVE_MPI

include/openPMD/Iteration.hpp

@@ -28,6 +28,7 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
 #include "openPMD/backend/HierarchyVisitor.hpp"
+#include "openPMD/backend/PerIterationData.hpp"
 #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp"
 
 #include <cstdint>
@@ -122,14 +123,16 @@ namespace internal
          */
         bool allow_reopening_implicitly = false;
 
-        /**
-         * Whether a step is currently active for this iteration.
-         * Used for file-based iteration layout, see Series.hpp for
-         * group-based layout.
-         * Access via stepStatus() method to automatically select the correct
-         * one among both flags.
+        /*
+         * This stores data items that are:
+         *
+         * 1. global in group and variable encodings
+         * 2. per-iteration in file encoding
+         *
+         * The struct is stored as part of the Series and as part of each
+         * Iteration. Access must be distinguished by iteration encoding.
          */
-        StepStatus m_stepStatus = StepStatus::NoStep;
+        PerIterationData m_perIterationData;
 
         /**
          * Cached copy of the key under which this Iteration lives in

include/openPMD/RecordComponent.tpp

@@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
     {
         size *= ext;
     }
+
     /*
      * Flushing the skeleton does not create datasets,
      * so we might need to do it now.
@@ -121,15 +122,18 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
         // restriction
         // TODO: Add some form of collective ::commitDefinitions() call to
         // RecordComponents to be called by users before the Span API
-        if (!written())
+        if (!writable().parent || !writable().parent->written)
         {
             /*
              * The openPMD backend might not yet know about this dataset.
              * Flush the openPMD hierarchy to the backend without flushing any
              * actual data yet.
              */
             seriesFlush_impl</* flush_entire_series = */ false>(
-                {FlushLevel::SkeletonOnly});
+                {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false);
+        }
+        if (!this->written())
+        {
             Parameter<Operation::CREATE_DATASET> dCreate(rc.m_dataset.value());
             dCreate.name = Attributable::get().m_writable.ownKeyWithinParent;
             IOHandler()->enqueue(IOTask(this, dCreate));

include/openPMD/Series.hpp

@@ -33,6 +33,7 @@
 #include "openPMD/backend/Container.hpp"
 #include "openPMD/backend/HierarchyVisitor.hpp"
 #include "openPMD/backend/ParsePreference.hpp"
+#include "openPMD/backend/PerIterationData.hpp"
 #include "openPMD/config.hpp"
 #include "openPMD/snapshots/Snapshots.hpp"
 #include "openPMD/version.hpp"
@@ -205,14 +206,18 @@ namespace internal
          * Detected IO format (backend).
          */
         Format m_format;
-        /**
-         *  Whether a step is currently active for this iteration.
-         * Used for group-based iteration layout, see SeriesData.hpp for
-         * iteration-based layout.
-         * Access via stepStatus() method to automatically select the correct
-         * one among both flags.
+
+        /*
+         * This stores data items that are:
+         *
+         * 1. global in group and variable encodings
+         * 2. per-iteration in file encoding
+         *
+         * The struct is stored as part of the Series and as part of each
+         * Iteration. Access must be distinguished by iteration encoding.
          */
-        StepStatus m_stepStatus = StepStatus::NoStep;
+        PerIterationData m_perIterationData;
+
         /**
          * True if a user opts into lazy parsing.
          */
@@ -261,7 +266,6 @@ namespace internal
 
         struct RankTableData
         {
-            Attributable m_attributable;
             std::variant<
                 NoSourceSpecified,
                 SourceSpecifiedViaJSON,
@@ -900,9 +904,7 @@ OPENPMD_private
         iterations_iterator end,
         internal::FlushParams const &flushParams,
         bool flushIOHandler = true);
-    void flushMeshesPath();
-    void flushParticlesPath();
-    void flushRankTable();
+    void flushRankTable(FlushLevel, Attributable &attributable);
     /* Parameter `read_only_this_single_iteration` used for reopening an
      * Iteration after closing it.
      */
@@ -985,8 +987,10 @@ OPENPMD_private
      *        least one step was written.
      *
      * @param doFlush If true, flush the IO handler.
+     * @param l This operation must only run at flush level write_datasets,
+     *          Noop otherwise.
      */
-    void flushStep(bool doFlush);
+    void flushStep(bool doFlush, FlushLevel l);
 
     /*
      * setIterationEncoding() should only be called by users of our public API,

include/openPMD/backend/Attributable.hpp

@@ -478,7 +478,7 @@ OPENPMD_protected
     /** @} */
 
     template <bool flush_entire_series>
-    void seriesFlush_impl(internal::FlushParams const &);
+    void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler);
 
     void flushAttributes(internal::FlushParams const &);
 
@@ -606,6 +606,34 @@ OPENPMD_protected
     {
         return writable().dirtyRecursive;
     }
+    void determineUnsetDirty(FlushLevel fl)
+    {
+        switch (fl)
+        {
+        case FlushLevel::UserFlush:
+            setDirty(false);
+            break;
+        // FlushLevel::InternalFlush is only used for directly calling the IO
+        // handler and should not bother with middle-end state manipulations
+        case FlushLevel::InternalFlush:
+            // Used for parsing
+            if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing)
+            {
+                throw error::Internal(
+                    "Parsing procedures should directly unset dirty.");
+            }
+            else
+            {
+                throw error::Internal(
+                    "Internal flushes should not unset dirty flags.");
+            }
+            break;
+        case FlushLevel::SkeletonOnly:
+        case FlushLevel::CreateOrOpenFiles:
+            // noop
+            break;
+        }
+    }
     void setDirty(bool dirty_in)
     {
         auto &w = writable();

include/openPMD/backend/PerIterationData.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "openPMD/ChunkInfo.hpp"
+#include "openPMD/Streaming.hpp"
+#include "openPMD/backend/Attributable.hpp"
+
+#include <variant>
+
+namespace openPMD::internal
+{
+struct NoSourceSpecified
+{};
+struct SourceSpecifiedViaJSON
+{
+    std::string value;
+};
+struct SourceSpecifiedManually
+{
+    std::string value;
+};
+
+struct RankTableData
+{
+    Attributable m_attributable;
+    std::variant<
+        NoSourceSpecified,
+        SourceSpecifiedViaJSON,
+        SourceSpecifiedManually>
+        m_rankTableSource;
+    std::optional<chunk_assignment::RankMeta> m_bufferedRead;
+};
+
+/*
+ * This stores data items that are:
+ *
+ * 1. global in group and variable encodings
+ * 2. per-iteration in file encoding
+ *
+ * The struct is stored as part of the Series and as part of each Iteration.
+ * Access must be distinguished by iteration encoding.
+ */
+struct PerIterationData
+{
+    /**
+     *  Whether a step is currently active for this iteration.
+     * Used for group-based iteration layout, see SeriesData.hpp for
+     * iteration-based layout.
+     * Access via stepStatus() method to automatically select the correct
+     * one among both flags.
+     */
+    StepStatus m_stepStatus = StepStatus::NoStep;
+    Attributable m_rankTableAttributable;
+};
+} // namespace openPMD::internal

include/openPMD/backend/Writable.hpp

@@ -128,14 +128,15 @@ class Writable final
      * it.
      */
     template <bool flush_entire_series>
-    void seriesFlush(std::string backendConfig = "{}");
+    void
+    seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true);
 
     // clang-format off
 OPENPMD_private
     // clang-format on
 
     template <bool flush_entire_series>
-    void seriesFlush(internal::FlushParams const &);
+    void seriesFlush(internal::FlushParams const &, bool flush_io_handler);
     /*
      * These members need to be shared pointers since distinct instances of
      * Writable may share them.

src/IO/ADIOS/ADIOS2File.cpp

@@ -1049,25 +1049,16 @@ void ADIOS2File::flush_impl(
         drainedUniquePtrPuts.swap(m_uniquePtrPuts);
     }
 
-    if (readOnly(m_mode))
+    if (readOnly(m_mode) || flush_level::write_datasets(level))
     {
-        level = FlushLevel::UserFlush;
-    }
-
-    switch (level)
-    {
-    case FlushLevel::UserFlush:
         performPutGets(*this, eng);
         m_updateSpans.clear();
         m_buffer.clear();
         m_alreadyEnqueued.clear();
         drainedUniquePtrPuts.clear();
-
-        break;
-
-    case FlushLevel::InternalFlush:
-    case FlushLevel::SkeletonOnly:
-    case FlushLevel::CreateOrOpenFiles:
+    }
+    else
+    {
         /*
          * Tasks have been given to ADIOS2, but we don't flush them
          * yet. So, move everything to m_alreadyEnqueued to avoid
@@ -1084,7 +1075,6 @@ void ADIOS2File::flush_impl(
                 "wrong time.");
         }
         m_buffer.clear();
-        break;
     }
 }