diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,66 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -121,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 0892627f2d..0a3c1dfcb0 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include @@ -122,14 +123,16 @@ namespace internal */ bool allow_reopening_implicitly = false; - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; /** * Cached copy of the key under which this Iteration lives in diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..951a520cfb 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. @@ -121,7 +122,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) // restriction // TODO: Add some form of collective ::commitDefinitions() call to // RecordComponents to be called by users before the Span API - if (!written()) + if (!writable().parent || !writable().parent->written) { /* * The openPMD backend might not yet know about this dataset. @@ -129,7 +130,10 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); + } + if (!this->written()) + { Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 93dfe333b4..3f640122cc 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -33,6 +33,7 @@ #include "openPMD/backend/Container.hpp" #include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" @@ -205,14 +206,18 @@ namespace internal * Detected IO format (backend). */ Format m_format; - /** - * Whether a step is currently active for this iteration. - * Used for group-based iteration layout, see SeriesData.hpp for - * iteration-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; + /** * True if a user opts into lazy parsing. */ @@ -261,7 +266,6 @@ namespace internal struct RankTableData { - Attributable m_attributable; std::variant< NoSourceSpecified, SourceSpecifiedViaJSON, @@ -900,9 +904,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); - void flushRankTable(); + void flushRankTable(FlushLevel, Attributable &attributable); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ @@ -985,8 +987,10 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. + * @param l This operation must only run at flush level write_datasets, + * Noop otherwise. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index f05cc8d15b..bcd09d966b 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -478,7 +478,7 @@ OPENPMD_protected /** @} */ template - void seriesFlush_impl(internal::FlushParams const &); + void seriesFlush_impl(internal::FlushParams const &, bool flush_io_handler); void flushAttributes(internal::FlushParams const &); @@ -606,6 +606,34 @@ OPENPMD_protected { return writable().dirtyRecursive; } + void determineUnsetDirty(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + setDirty(false); + break; + // FlushLevel::InternalFlush is only used for directly calling the IO + // handler and should not bother with middle-end state manipulations + case FlushLevel::InternalFlush: + // Used for parsing + if (IOHandler()->m_seriesStatus == internal::SeriesStatus::Parsing) + { + throw error::Internal( + "Parsing procedures should directly unset dirty."); + } + else + { + throw error::Internal( + "Internal flushes should not unset dirty flags."); + } + break; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + // noop + break; + } + } void setDirty(bool dirty_in) { auto &w = writable(); diff --git a/include/openPMD/backend/PerIterationData.hpp b/include/openPMD/backend/PerIterationData.hpp new file mode 100644 index 0000000000..6cea297bf2 --- /dev/null +++ b/include/openPMD/backend/PerIterationData.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include "openPMD/ChunkInfo.hpp" +#include "openPMD/Streaming.hpp" +#include "openPMD/backend/Attributable.hpp" + +#include + +namespace openPMD::internal +{ +struct NoSourceSpecified +{}; +struct SourceSpecifiedViaJSON +{ + std::string value; +}; +struct SourceSpecifiedManually +{ + std::string value; +}; + +struct RankTableData +{ + Attributable m_attributable; + std::variant< + NoSourceSpecified, + SourceSpecifiedViaJSON, + SourceSpecifiedManually> + m_rankTableSource; + std::optional m_bufferedRead; +}; + +/* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each Iteration. + * Access must be distinguished by iteration encoding. + */ +struct PerIterationData +{ + /** + * Whether a step is currently active for this iteration. + * Used for group-based iteration layout, see SeriesData.hpp for + * iteration-based layout. + * Access via stepStatus() method to automatically select the correct + * one among both flags. + */ + StepStatus m_stepStatus = StepStatus::NoStep; + Attributable m_rankTableAttributable; +}; +} // namespace openPMD::internal diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index a58af82f9e..3623d0b39c 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -128,14 +128,15 @@ class Writable final * it. */ template - void seriesFlush(std::string backendConfig = "{}"); + void + seriesFlush(std::string backendConfig = "{}", bool flush_io_handler = true); // clang-format off OPENPMD_private // clang-format on template - void seriesFlush(internal::FlushParams const &); + void seriesFlush(internal::FlushParams const &, bool flush_io_handler); /* * These members need to be shared pointers since distinct instances of * Writable may share them. diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 1d181033eb..0260836136 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -1049,25 +1049,16 @@ void ADIOS2File::flush_impl( drainedUniquePtrPuts.swap(m_uniquePtrPuts); } - if (readOnly(m_mode)) + if (readOnly(m_mode) || flush_level::write_datasets(level)) { - level = FlushLevel::UserFlush; - } - - switch (level) - { - case FlushLevel::UserFlush: performPutGets(*this, eng); m_updateSpans.clear(); m_buffer.clear(); m_alreadyEnqueued.clear(); drainedUniquePtrPuts.clear(); - - break; - - case FlushLevel::InternalFlush: - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: + } + else + { /* * Tasks have been given to ADIOS2, but we don't flush them * yet. So, move everything to m_alreadyEnqueued to avoid @@ -1084,7 +1075,6 @@ void ADIOS2File::flush_impl( "wrong time."); } m_buffer.clear(); - break; } } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2e8084848d..036b4f6098 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -582,7 +582,7 @@ overrideFlushTarget(FlushTarget &inplace, FlushTarget new_val) std::future ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(flushParams.flushLevel); detail::ADIOS2File::ADIOS2FlushParams adios2FlushParams{ flushParams.flushLevel, m_flushTarget}; diff --git a/src/IO/AbstractIOHandler.cpp b/src/IO/AbstractIOHandler.cpp index 5f2bdeb2f5..564eebba01 100644 --- a/src/IO/AbstractIOHandler.cpp +++ b/src/IO/AbstractIOHandler.cpp @@ -27,6 +27,29 @@ #include +namespace openPMD +{ +std::ostream &operator<<(std::ostream &os, FlushLevel l) +{ + switch (l) + { + case FlushLevel::UserFlush: + os << "UserFlush"; + break; + case FlushLevel::InternalFlush: + os << "InternalFlush"; + break; + case FlushLevel::SkeletonOnly: + os << "SkeletonOnly"; + break; + case FlushLevel::CreateOrOpenFiles: + os << "CreateOrOpenFiles"; + break; + } + return os; +} +} // namespace openPMD + namespace openPMD::auxiliary { using pair_t = std::pair; diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 4f93ff1a5b..1ce45c5aa5 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -21,6 +21,8 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -87,10 +89,72 @@ void AbstractIOHandlerImpl::writeToStderr([[maybe_unused]] Args &&...args) const } } -std::future AbstractIOHandlerImpl::flush() +namespace +{ + void verifyFlushType(Operation op, FlushLevel l) + { + auto do_throw = [&](char const *least_flush_level) { + std::stringstream err; + err << "Operation " << op << " is not allowed below flush level " + << least_flush_level << ", but flush level was " << l << "."; + throw error::Internal(err.str()); + }; + switch (op) + { + case Operation::ADVANCE: + case Operation::CREATE_FILE: + case Operation::CHECK_FILE: + case Operation::OPEN_FILE: + case Operation::CLOSE_FILE: + case Operation::DELETE_FILE: + case Operation::DEREGISTER: + case Operation::TOUCH: + case Operation::LIST_ATTS: + case Operation::LIST_PATHS: + case Operation::OPEN_PATH: + case Operation::SET_WRITTEN: + case Operation::CREATE_PATH: + break; + case Operation::CLOSE_PATH: + case Operation::DELETE_PATH: + case Operation::CREATE_DATASET: + case Operation::EXTEND_DATASET: + case Operation::OPEN_DATASET: + case Operation::DELETE_DATASET: + case Operation::LIST_DATASETS: + if (!flush_level::flush_hierarchy(l)) + { + do_throw("SkeletonOnly (hierarchy operations)"); + } + break; + case Operation::GET_BUFFER_VIEW: + case Operation::DELETE_ATT: + case Operation::WRITE_ATT: + case Operation::READ_ATT: + case Operation::READ_ATT_ALLSTEPS: + case Operation::AVAILABLE_CHUNKS: + if (!flush_level::write_attributes(l)) + { + do_throw("InternalFlush (metadata operations)"); + } + break; + case Operation::WRITE_DATASET: + case Operation::READ_DATASET: + if (!flush_level::write_datasets(l)) + { + do_throw("UserFlush (flushpoint operations)"); + } + break; + } + } +} // namespace + +std::future AbstractIOHandlerImpl::flush(FlushLevel l) { using namespace auxiliary; + writeToStderr("\nFLUSHING"); + while (!(*m_handler).m_work.empty()) { IOTask &i = (*m_handler).m_work.front(); @@ -457,11 +521,17 @@ std::future AbstractIOHandlerImpl::flush() auto ¶meter = deref_dynamic_cast>( i.parameter.get()); writeToStderr( - "[", i.writable->parent, "->", i.writable, "] SET_WRITTEN"); + "[", + i.writable->parent, + "->", + i.writable, + "] SET_WRITTEN ", + parameter.target_status ? "true" : "false"); setWritten(i.writable, parameter); break; } } + verifyFlushType(i.operation, l); } catch (...) { @@ -506,6 +576,7 @@ std::future AbstractIOHandlerImpl::flush() } (*m_handler).m_work.pop(); } + writeToStderr("FLUSHED\n"); return std::future(); } diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 714a69b9d3..3096b2297c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -3562,7 +3562,7 @@ auto HDF5IOHandlerImpl::requireFile( std::future HDF5IOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - auto res = AbstractIOHandlerImpl::flush(); + auto res = AbstractIOHandlerImpl::flush(params.flushLevel); if (params.backendConfig.json().contains("hdf5")) { diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 26010ead52..af12ff766c 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -35,6 +35,97 @@ Writable *getWritable(Attributable *a) return &a->writable(); } +std::ostream &operator<<(std::ostream &os, Operation op) +{ + switch (op) + { + case Operation::CREATE_FILE: + os << "CREATE_FILE"; + break; + case Operation::CHECK_FILE: + os << "CHECK_FILE"; + break; + case Operation::OPEN_FILE: + os << "OPEN_FILE"; + break; + case Operation::CLOSE_FILE: + os << "CLOSE_FILE"; + break; + case Operation::DELETE_FILE: + os << "DELETE_FILE"; + break; + case Operation::CREATE_PATH: + os << "CREATE_PATH"; + break; + case Operation::CLOSE_PATH: + os << "CLOSE_PATH"; + break; + case Operation::OPEN_PATH: + os << "OPEN_PATH"; + break; + case Operation::DELETE_PATH: + os << "DELETE_PATH"; + break; + case Operation::LIST_PATHS: + os << "LIST_PATHS"; + break; + case Operation::CREATE_DATASET: + os << "CREATE_DATASET"; + break; + case Operation::EXTEND_DATASET: + os << "EXTEND_DATASET"; + break; + case Operation::OPEN_DATASET: + os << "OPEN_DATASET"; + break; + case Operation::DELETE_DATASET: + os << "DELETE_DATASET"; + break; + case Operation::WRITE_DATASET: + os << "WRITE_DATASET"; + break; + case Operation::READ_DATASET: + os << "READ_DATASET"; + break; + case Operation::LIST_DATASETS: + os << "LIST_DATASETS"; + break; + case Operation::GET_BUFFER_VIEW: + os << "GET_BUFFER_VIEW"; + break; + case Operation::DELETE_ATT: + os << "DELETE_ATT"; + break; + case Operation::WRITE_ATT: + os << "WRITE_ATT"; + break; + case Operation::READ_ATT: + os << "READ_ATT"; + break; + case Operation::READ_ATT_ALLSTEPS: + os << "READ_ATT_ALLSTEPS"; + break; + case Operation::LIST_ATTS: + os << "LIST_ATTS"; + break; + case Operation::ADVANCE: + os << "ADVANCE"; + break; + case Operation::AVAILABLE_CHUNKS: + os << "AVAILABLE_CHUNKS"; + break; + case Operation::DEREGISTER: + os << "DEREGISTER"; + break; + case Operation::TOUCH: + os << "TOUCH"; + break; + case Operation::SET_WRITTEN: + os << "SET_WRITTEN"; + break; + } + return os; +} template <> void AbstractParameter::warnUnusedParameters( json::TracingJSON &config, diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index c531aabb00..9af9a06728 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -53,8 +53,8 @@ JSONIOHandler::JSONIOHandler( {} #endif -std::future JSONIOHandler::flush(internal::ParsedFlushParams &) +std::future JSONIOHandler::flush(internal::ParsedFlushParams ¶ms) { - return m_impl.flush(); + return m_impl.flush(params); } } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 7647746d74..0660d1fe41 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/FlushParametersInternal.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -444,9 +445,9 @@ void JSONIOHandlerImpl::init(openPMD::json::TracingJSON config) JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; -std::future JSONIOHandlerImpl::flush() +std::future JSONIOHandlerImpl::flush(internal::ParsedFlushParams ¶ms) { - AbstractIOHandlerImpl::flush(); + AbstractIOHandlerImpl::flush(params.flushLevel); if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) { throw error::Internal( diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 67a28c51bc..7e708ca4e4 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -280,16 +280,6 @@ void Iteration::flushFileBased( fCreate.name = filename; IOHandler()->enqueue(IOTask(&s.writable(), fCreate)); - /* - * If it was written before, then in the context of another iteration. - */ - auto &attr = s.get().m_rankTable.m_attributable; - attr.setWritten(false, Attributable::EnqueueAsynchronously::Both); - s.get() - .m_rankTable.m_attributable.get() - .m_writable.abstractFilePosition.reset(); - s.flushRankTable(); - /* create basePath */ Parameter pCreate; pCreate.path = auxiliary::replace_first(s.basePath(), "%T/", ""); @@ -306,15 +296,16 @@ void Iteration::flushFileBased( s.openIteration(i, *this); } - switch (flushParams.flushLevel) + auto &rankTableAttributable = + get().m_perIterationData.m_rankTableAttributable; + if (!rankTableAttributable.written()) + { + s.flushRankTable(flushParams.flushLevel, rankTableAttributable); + } + + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -329,15 +320,9 @@ void Iteration::flushGroupBased( IOHandler()->enqueue(IOTask(this, pCreate)); } - switch (flushParams.flushLevel) + if (flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: - break; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: flush(flushParams); - break; } } @@ -352,18 +337,14 @@ void Iteration::flushVariableBased( IOHandler()->enqueue(IOTask(this, pOpen)); } - switch (flushParams.flushLevel) + if (!flush_level::flush_hierarchy(flushParams.flushLevel)) { - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::SkeletonOnly: - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - flush(flushParams); - break; } - if (!written()) + flush(flushParams); + + if (!written() && flush_level::write_datasets(flushParams.flushLevel)) { /* create iteration path */ Parameter pOpen; @@ -395,7 +376,7 @@ void Iteration::flush(internal::FlushParams const &flushParams) m.second.flush(m.first, flushParams); for (auto &species : particles) species.second.flush(species.first, flushParams); - setDirty(false); + determineUnsetDirty(flushParams.flushLevel); } else { @@ -403,16 +384,28 @@ void Iteration::flush(internal::FlushParams const &flushParams) * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); - if (!meshes.empty() || s.containsAttribute("meshesPath")) - { - if (!s.containsAttribute("meshesPath")) + auto set_and_get_mp_path = + [&](char const *attrName, + char const *defaultVal, + Series &(Series::*set)(std::string const &)) -> std::string { + if (s.containsAttribute(attrName)) + { + return s.getAttribute(attrName).get(); + } + else { - s.setMeshesPath("meshes/"); - s.flushMeshesPath(); + (s.*set)(defaultVal); + return defaultVal; } + }; + + if (!meshes.empty() || s.containsAttribute("meshesPath")) + { + auto meshesPath = set_and_get_mp_path( + "meshesPath", "meshes/", &Series::setMeshesPath); if (meshes.dirtyRecursive()) { - meshes.flush(s.meshesPath(), flushParams); + meshes.flush(meshesPath, flushParams); for (auto &m : meshes) { m.second.flush(m.first, flushParams); @@ -426,14 +419,11 @@ void Iteration::flush(internal::FlushParams const &flushParams) if (!particles.empty() || s.containsAttribute("particlesPath")) { - if (!s.containsAttribute("particlesPath")) - { - s.setParticlesPath("particles/"); - s.flushParticlesPath(); - } + auto particlesPath = set_and_get_mp_path( + "particlesPath", "particles/", &Series::setParticlesPath); if (particles.dirtyRecursive()) { - particles.flush(s.particlesPath(), flushParams); + particles.flush(particlesPath, flushParams); for (auto &species : particles) { species.second.flush(species.first, flushParams); @@ -449,9 +439,9 @@ void Iteration::flush(internal::FlushParams const &flushParams) } if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); + determineUnsetDirty(flushParams.flushLevel); + meshes.determineUnsetDirty(flushParams.flushLevel); + particles.determineUnsetDirty(flushParams.flushLevel); } } @@ -785,7 +775,7 @@ auto Iteration::beginStep( } else { - series.get().m_stepStatus = StepStatus::DuringStep; + series.get().m_perIterationData.m_stepStatus = StepStatus::DuringStep; status = series.advance(AdvanceMode::BEGINSTEP); } @@ -888,10 +878,10 @@ StepStatus Iteration::getStepStatus() { using IE = IterationEncoding; case IE::fileBased: - return get().m_stepStatus; + return get().m_perIterationData.m_stepStatus; case IE::groupBased: case IE::variableBased: - return s.get().m_stepStatus; + return s.get().m_perIterationData.m_stepStatus; default: throw std::runtime_error("[Iteration] unreachable"); } @@ -904,11 +894,11 @@ void Iteration::setStepStatus(StepStatus status) { using IE = IterationEncoding; case IE::fileBased: - get().m_stepStatus = status; + get().m_perIterationData.m_stepStatus = status; break; case IE::groupBased: case IE::variableBased: - s.get().m_stepStatus = status; + s.get().m_perIterationData.m_stepStatus = status; break; default: throw std::runtime_error("[Iteration] unreachable"); @@ -920,6 +910,7 @@ void Iteration::linkHierarchy(Writable &w) Attributable::linkHierarchy(w); meshes.linkHierarchy(this->writable()); particles.linkHierarchy(this->writable()); + get().m_perIterationData.m_rankTableAttributable.linkHierarchy(*w.parent); } void Iteration::runDeferredParseAccess() diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8a2b9b58f7..718d3e847a 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -197,11 +197,8 @@ void ParticleSpecies::flush( patch.second.flush(patch.first, flushParams); } } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - particlePatches.setDirty(false); - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); + particlePatches.determineUnsetDirty(flushParams.flushLevel); } void ParticleSpecies::scientificDefaults_impl( internal::WriteOrRead, OpenpmdStandard) diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ec254d3b99..52b7c236c3 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -22,6 +22,7 @@ #include "openPMD/Dataset.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/Format.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/Environment.hpp" @@ -525,10 +526,7 @@ void RecordComponent::flush( flushAttributes(flushParams); } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void RecordComponent::read() diff --git a/src/Series.cpp b/src/Series.cpp index 855178c18d..99fe227582 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -339,10 +339,11 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } if (iterationEncoding() == IterationEncoding::fileBased) { - std::cerr << "[Series] Use rank table in file-based iteration encoding " - "at your own risk. Make sure to have an iteration open " - "before calling this." - << std::endl; + std::cerr + << "[Series] Use rank table in file-based iteration encoding " + "at your own risk. Make sure to have the first iteration open " + "before calling this." + << std::endl; if (iterations.empty()) { return {}; @@ -355,6 +356,18 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) IOHandler()->enqueue(IOTask(this, openFile)); #endif } + Attributable &attributable = + iterationEncoding() == IterationEncoding::fileBased + /* + * Only second class support for file encoding. We indiscriminately use + * the first Iteration for this operation. It is on the user to ensure + * that this Iteration is actually open. The warning printed above + * informs about this. + */ + ? iterations.begin() + ->second.get() + .m_perIterationData.m_rankTableAttributable + : series.m_perIterationData.m_rankTableAttributable; auto datasets = availableDatasets(); if (std::find(datasets.begin(), datasets.end(), "rankTable") == datasets.end()) @@ -364,7 +377,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) } Parameter openDataset; openDataset.name = "rankTable"; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, openDataset)); + IOHandler()->enqueue(IOTask(&attributable, openDataset)); IOHandler()->flush(internal::defaultFlushParams); if (openDataset.extent->size() != 2) @@ -394,7 +407,7 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) new char[writerRanks * lineWidth], [](char const *ptr) { delete[] ptr; }}; - auto doReadDataset = [&openDataset, this, &get, &rankTable]() { + auto doReadDataset = [&openDataset, this, &get, &attributable]() { Parameter readDataset; // read the whole thing readDataset.offset.resize(2); @@ -404,8 +417,8 @@ chunk_assignment::RankMeta Series::rankTable([[maybe_unused]] bool collective) readDataset.dtype = Datatype::CHAR; readDataset.data = get; - IOHandler()->enqueue(IOTask(&rankTable.m_attributable, readDataset)); - IOHandler()->flush(internal::defaultFlushParams); + IOHandler()->enqueue(IOTask(&attributable, readDataset)); + IOHandler()->flush(internal::publicFlush); }; #if openPMD_HAVE_MPI @@ -464,8 +477,12 @@ Series &Series::setRankTable(const std::string &myRankInfo) return *this; } -void Series::flushRankTable() +void Series::flushRankTable(FlushLevel l, Attributable &attributable) { + if (!flush_level::global_flushpoint(l)) + { + return; + } auto &series = get(); auto &rankTable = series.m_rankTable; auto maybeMyRankInfo = std::visit( @@ -508,8 +525,8 @@ void Series::flushRankTable() int rank{0}, size{1}; unsigned long long maxSize = mySize; - auto createRankTable = [&size, &maxSize, &rankTable, this]() { - if (rankTable.m_attributable.written()) + auto createRankTable = [&size, &maxSize, this, &attributable]() { + if (attributable.written()) { return; } @@ -518,19 +535,17 @@ void Series::flushRankTable() param.name = "rankTable"; param.dtype = Datatype::CHAR; param.extent = {uint64_t(size), uint64_t(maxSize)}; - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(param))); + IOHandler()->enqueue(IOTask(&attributable, std::move(param))); }; - auto writeDataset = [&rank, &maxSize, this, &rankTable]( + auto writeDataset = [&rank, &maxSize, this, &attributable]( std::shared_ptr put, size_t num_lines = 1) { Parameter chunk; chunk.dtype = Datatype::CHAR; chunk.offset = {uint64_t(rank), 0}; chunk.extent = {num_lines, maxSize}; chunk.data = std::move(put); - IOHandler()->enqueue( - IOTask(&rankTable.m_attributable, std::move(chunk))); + IOHandler()->enqueue(IOTask(&attributable, std::move(chunk))); }; #if openPMD_HAVE_MPI @@ -574,8 +589,7 @@ void Series::flushRankTable() // Must ensure that the Writable is consistently set to written on all // ranks - series.m_rankTable.m_attributable.setWritten( - true, EnqueueAsynchronously::OnlyAsync); + attributable.setWritten(true, EnqueueAsynchronously::OnlyAsync); return; } #endif @@ -981,7 +995,8 @@ void Series::init( std::make_unique(parsed_directory, at)); auto &series = get(); series.iterations.linkHierarchy(writable()); - series.m_rankTable.m_attributable.linkHierarchy(writable()); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy( + writable()); series.m_deferred_initialization = [called_this_already = false, filepath, @@ -1207,7 +1222,7 @@ void Series::initSeries( series.iterations.linkHierarchy(writable); series.iterations.writable().ownKeyWithinParent = "data"; - series.m_rankTable.m_attributable.linkHierarchy(writable); + series.m_perIterationData.m_rankTableAttributable.linkHierarchy(writable); series.m_name = input->name; @@ -1494,6 +1509,12 @@ void Series::flushFileBased( case Access::APPEND_RANDOM_ACCESS: case Access::APPEND_LINEAR: { bool allDirty = dirty(); + // In flush level SkeletonOnly, we might need to set some attributes + // (especially: particlesPath, meshesPath), but cannot flush them yet + // (as writing attributes is only permissible at higher flush levels). + // This flag records if the Series became dirty during this flush. If + // yes, we set the Series back to dirty at the end of flushing. + bool hasBecomeDirty = false; for (auto it = begin; it != end; ++it) { // Phase 1 @@ -1543,11 +1564,30 @@ void Series::flushFileBased( } /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes + * TODO: Ideally, we would skip this in SkeletonOnly flush mode, but + * for some reason, this leads to hanging parallel tests..? */ + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) + { + if (allDirty && !dirty()) + { + throw error::Internal( + "Flush mode SkeletonOnly must not unset dirty flags."); + } + hasBecomeDirty |= + flushParams.flushLevel == FlushLevel::SkeletonOnly && + !allDirty && dirty(); + } setDirty(allDirty); } - setDirty(false); - + if (!hasBecomeDirty) + { + determineUnsetDirty(flushParams.flushLevel); + } + else + { + setDirty(true); + } // Phase 3 if (flushIOHandler) { @@ -1632,8 +1672,13 @@ void Series::flushGorVBased( Parameter fCreate; fCreate.name = series.m_name; IOHandler()->enqueue(IOTask(this, fCreate)); + } - flushRankTable(); + if (!series.m_perIterationData.m_rankTableAttributable.written()) + { + flushRankTable( + flushParams.flushLevel, + series.m_perIterationData.m_rankTableAttributable); } series.iterations.flush( @@ -1688,26 +1733,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { @@ -2700,7 +2725,7 @@ AdvanceStatus Series::advance( if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2806,7 +2831,7 @@ AdvanceStatus Series::advance(AdvanceMode mode) if (mode == AdvanceMode::ENDSTEP) { - flushStep(/* doFlush = */ false); + flushStep(/* doFlush = */ false, FlushLevel::UserFlush); } Parameter param; @@ -2829,8 +2854,12 @@ AdvanceStatus Series::advance(AdvanceMode mode) return *param.status; } -void Series::flushStep(bool doFlush) +void Series::flushStep(bool doFlush, FlushLevel l) { + if (!flush_level::write_datasets(l)) + { + return; + } auto &series = get(); if (!series.m_currentlyActiveIterations.empty() && access::write(IOHandler()->m_frontendAccess)) @@ -3334,7 +3363,7 @@ namespace internal */ if (impl.iterationEncoding() != IterationEncoding::fileBased) { - impl.flushStep(/* doFlush = */ true); + impl.flushStep(/* doFlush = */ true, FlushLevel::UserFlush); } } // Not strictly necessary, but clear the map of iterations diff --git a/src/auxiliary/Mpi.cpp b/src/auxiliary/Mpi.cpp index ef899e4207..5e8379ff41 100644 --- a/src/auxiliary/Mpi.cpp +++ b/src/auxiliary/Mpi.cpp @@ -49,7 +49,7 @@ StringMatrix collectStringsAsMatrixTo( 1, MPI_INT, destRank, - MPI_COMM_WORLD); + communicator); int maxLength = std::accumulate( recvcounts.begin(), recvcounts.end(), 0, [](int a, int b) { return std::max(a, b); @@ -78,7 +78,7 @@ StringMatrix collectStringsAsMatrixTo( displs.data(), MPI_CHAR, destRank, - MPI_COMM_WORLD); + communicator); return res; } @@ -95,7 +95,7 @@ std::vector distributeStringsToAllRanks( int *displs = new int[size]; MPI_Allgather( - &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, MPI_COMM_WORLD); + &sendLength, 1, MPI_INT, sizesBuffer, 1, MPI_INT, communicator); char *namesBuffer; { @@ -116,7 +116,7 @@ std::vector distributeStringsToAllRanks( sizesBuffer, displs, MPI_CHAR, - MPI_COMM_WORLD); + communicator); std::vector hostnames(size); for (int i = 0; i < size; ++i) diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index d19fa31a00..16f69c6cbb 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -344,26 +344,21 @@ OpenpmdStandard Attributable::openPMDStandard() const } template -void Attributable::seriesFlush_impl(internal::FlushParams const &flushParams) +void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler) { - writable().seriesFlush(flushParams); + writable().seriesFlush(flushParams, flush_io_handler); } -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); -template void -Attributable::seriesFlush_impl(internal::FlushParams const &flushParams); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Attributable::seriesFlush_impl( + internal::FlushParams const &flushParams, bool flush_io_handler); void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - switch (flushParams.flushLevel) + if (!flush_level::write_attributes(flushParams.flushLevel)) { - case FlushLevel::SkeletonOnly: - case FlushLevel::CreateOrOpenFiles: return; - case FlushLevel::InternalFlush: - case FlushLevel::UserFlush: - // pass - break; } if (dirty()) { @@ -377,10 +372,7 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp index c4eab2318f..89777af8d6 100644 --- a/src/backend/BaseRecord.cpp +++ b/src/backend/BaseRecord.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" #include "openPMD/backend/PatchRecordComponent.hpp" #include "openPMD/backend/scientific_defaults/ConfigAttribute.hpp" @@ -798,10 +799,7 @@ inline void BaseRecord::flush( } this->flush_impl(name, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - this->setDirty(false); - } + this->determineUnsetDirty(flushParams.flushLevel); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 740f44cc51..7d68b16035 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -70,10 +70,7 @@ void PatchRecord::flush_impl( } else T_RecordComponent::flush(path, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + determineUnsetDirty(flushParams.flushLevel); } void PatchRecord::read() diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index ea6e56b9c5..54e2a60b4a 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -52,16 +52,20 @@ Writable::~Writable() } template -void Writable::seriesFlush(std::string backendConfig) +void Writable::seriesFlush(std::string backendConfig, bool flush_io_handler) { seriesFlush( - internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}); + internal::FlushParams{FlushLevel::UserFlush, std::move(backendConfig)}, + flush_io_handler); } -template void Writable::seriesFlush(std::string backendConfig); -template void Writable::seriesFlush(std::string backendConfig); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); +template void +Writable::seriesFlush(std::string backendConfig, bool flush_io_handler); template -void Writable::seriesFlush(internal::FlushParams const &flushParams) +void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler) { Attributable impl; impl.setData({attributable, [](auto const *) {}}); @@ -103,10 +107,10 @@ void Writable::seriesFlush(internal::FlushParams const &flushParams) return {series.iterations.begin(), series.iterations.end()}; } }(); - series.flush_impl(begin, end, flushParams); + series.flush_impl(begin, end, flushParams, flush_io_handler); } -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); -template void -Writable::seriesFlush(internal::FlushParams const &flushParams); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); +template void Writable::seriesFlush( + internal::FlushParams const &flushParams, bool flush_io_handler); } // namespace openPMD diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9c28f52945..d32d31253b 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -37,13 +37,32 @@ #include #if !openPMD_HAVE_MPI -TEST_CASE("none", "[parallel]") +#define PARALLEL_TEST_CASE(name, tags) TEST_CASE(#name, tags) + +PARALLEL_TEST_CASE(none, "[parallel]") {} #else #include +#define PARALLEL_TEST_CASE(name, tags) \ + static void openPMD_parallel_##name(); \ + TEST_CASE(#name, tags) \ + { \ + MPI_Barrier(MPI_COMM_WORLD); \ + int rank; \ + MPI_Comm_rank(MPI_COMM_WORLD, &rank); \ + if (rank == 0) \ + { \ + std::cout << "\nStarting test '" << #name << "'.\n" << std::endl; \ + } \ + MPI_Barrier(MPI_COMM_WORLD); \ + openPMD_parallel_##name(); \ + MPI_Barrier(MPI_COMM_WORLD); \ + } \ + static void openPMD_parallel_##name() + #if openPMD_HAVE_ADIOS2 #include #define HAS_ADIOS_2_8 (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208) @@ -80,7 +99,7 @@ TEST_CASE("none", "[parallel]") using namespace openPMD; -TEST_CASE("parallel_multi_series_test", "[parallel]") +PARALLEL_TEST_CASE(parallel_multi_series_test, "[parallel]") { std::list allSeries; @@ -223,7 +242,7 @@ void write_test_zero_extent( #endif #if openPMD_HAVE_HDF5 && openPMD_HAVE_MPI -TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(git_hdf5_sample_content_test, "[parallel][hdf5]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -307,7 +326,7 @@ TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") } } -TEST_CASE("hdf5_write_test", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test, "[parallel][hdf5]") { int mpi_s{-1}; int mpi_r{-1}; @@ -377,13 +396,13 @@ TEST_CASE("hdf5_write_test", "[parallel][hdf5]") o.flush("hdf5.independent_stores = false"); } -TEST_CASE("hdf5_write_test_zero_extent", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_zero_extent, "[parallel][hdf5]") { write_test_zero_extent(false, "h5", true, true); write_test_zero_extent(true, "h5", true, true); } -TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_chunk, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -397,7 +416,7 @@ TEST_CASE("hdf5_write_test_skip_chunk", "[parallel][hdf5]") REQUIRE(true); } -TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") +PARALLEL_TEST_CASE(hdf5_write_test_skip_declare, "[parallel][hdf5]") { //! @todo add via JSON option instead of environment read auto const hdf5_collective = @@ -413,7 +432,7 @@ TEST_CASE("hdf5_write_test_skip_declare", "[parallel][hdf5]") #else -TEST_CASE("no_parallel_hdf5", "[parallel][hdf5]") +PARALLEL_TEST_CASE(no_parallel_hdf5, "[parallel][hdf5]") { REQUIRE(true); } @@ -495,7 +514,7 @@ void available_chunks_test(std::string const &file_ending) } } -TEST_CASE("available_chunks_test", "[parallel][adios]") +PARALLEL_TEST_CASE(available_chunks_test, "[parallel][adios]") { available_chunks_test("bp"); } @@ -550,14 +569,14 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) } } -TEST_CASE("extend_dataset", "[parallel]") +PARALLEL_TEST_CASE(extend_dataset, "[parallel]") { extendDataset("bp", R"({"backend": "adios2"})"); } #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("adios_write_test", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test, "[parallel][adios]") { Series o = Series( "../samples/parallel_write.bp", @@ -645,25 +664,25 @@ TEST_CASE("adios_write_test", "[parallel][adios]") } } -TEST_CASE("adios_write_test_zero_extent", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_zero_extent, "[parallel][adios]") { write_test_zero_extent(false, "bp", true, true); write_test_zero_extent(true, "bp", true, true); } -TEST_CASE("adios_write_test_skip_chunk", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_chunk, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, true); write_test_zero_extent(true, "bp", false, true); } -TEST_CASE("adios_write_test_skip_declare", "[parallel][adios]") +PARALLEL_TEST_CASE(adios_write_test_skip_declare, "[parallel][adios]") { write_test_zero_extent(false, "bp", false, false); write_test_zero_extent(true, "bp", false, false); } -TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios2][bp3]") +PARALLEL_TEST_CASE(hzdr_adios_sample_content_test, "[parallel][adios2][bp3]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -743,7 +762,7 @@ void write_4D_test(std::string const &file_ending) o.flush(); } -TEST_CASE("write_4D_test", "[parallel]") +PARALLEL_TEST_CASE(write_4D_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -776,7 +795,7 @@ void write_makeconst_some(std::string const &file_ending) E_x.makeConstant(42); } -TEST_CASE("write_makeconst_some", "[parallel]") +PARALLEL_TEST_CASE(write_makeconst_some, "[parallel]") { for (auto const &t : getBackends()) { @@ -883,7 +902,7 @@ void close_iteration_test(std::string const &file_ending) } } -TEST_CASE("close_iteration_test", "[parallel]") +PARALLEL_TEST_CASE(close_iteration_test, "[parallel]") { for (auto const &t : getBackends()) { @@ -1002,7 +1021,7 @@ void file_based_write_read(std::string const &file_ending) } } -TEST_CASE("file_based_write_read", "[parallel]") +PARALLEL_TEST_CASE(file_based_write_read, "[parallel]") { for (auto const &t : getBackends()) { @@ -1181,7 +1200,7 @@ void hipace_like_write(std::string const &file_ending) } } -TEST_CASE("hipace_like_write", "[parallel]") +PARALLEL_TEST_CASE(hipace_like_write, "[parallel]") { for (auto const &t : getBackends()) { @@ -1191,7 +1210,7 @@ TEST_CASE("hipace_like_write", "[parallel]") #endif #if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI -TEST_CASE("independent_write_with_collective_flush", "[parallel]") +PARALLEL_TEST_CASE(independent_write_with_collective_flush, "[parallel]") { Series write( "../samples/independent_write_with_collective_flush.bp5", @@ -1225,7 +1244,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]") #endif #if openPMD_HAVE_MPI -TEST_CASE("unavailable_backend", "[core][parallel]") +PARALLEL_TEST_CASE(unavailable_backend, "[core][parallel]") { #if !openPMD_HAVE_ADIOS2 { @@ -1373,7 +1392,7 @@ void adios2_streaming(bool variableBasedLayout) } } -TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") +PARALLEL_TEST_CASE(adios2_streaming, "[pseudoserial][adios2]") { #if HAS_ADIOS_2_9 adios2_streaming(true); @@ -1381,7 +1400,7 @@ TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") adios2_streaming(false); } -TEST_CASE("parallel_adios2_json_config", "[parallel][adios2]") +PARALLEL_TEST_CASE(parallel_adios2_json_config, "[parallel][adios2]") { int size{-1}; int rank{-1}; @@ -1592,7 +1611,7 @@ void adios2_ssc() } } -TEST_CASE("adios2_ssc", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_ssc, "[parallel][adios2]") { adios2_ssc(); } @@ -1918,7 +1937,7 @@ void append_mode( #endif } -TEST_CASE("append_mode", "[serial]") +PARALLEL_TEST_CASE(append_mode, "[serial]") { for (auto const &t : testedFileExtensions()) { @@ -2121,7 +2140,7 @@ void joined_dim(std::string const &ext) } } -TEST_CASE("joined_dim", "[parallel]") +PARALLEL_TEST_CASE(joined_dim, "[parallel]") { #if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ @@ -2146,7 +2165,7 @@ TEST_CASE("joined_dim", "[parallel]") #if openPMD_HAVE_ADIOS2_BP5 // Parallel version of the same test from SerialIOTest.cpp -TEST_CASE("adios2_flush_via_step") +PARALLEL_TEST_CASE(adios2_flush_via_step, "[parallel]") { int size_i(0), rank_i(0); MPI_Comm_rank(MPI_COMM_WORLD, &rank_i); @@ -2253,12 +2272,12 @@ TEST_CASE("adios2_flush_via_step") } #endif -TEST_CASE("read_variablebased_randomaccess") +PARALLEL_TEST_CASE(read_variablebased_randomaccess, "[parallel]") { read_variablebased_randomaccess::read_variablebased_randomaccess(); } -TEST_CASE("iterate_nonstreaming_series", "[serial][adios2]") +PARALLEL_TEST_CASE(iterate_nonstreaming_series, "[parallel][adios2]") { iterate_nonstreaming_series::iterate_nonstreaming_series(); } @@ -2719,14 +2738,14 @@ void run_test() } } // namespace adios2_chunk_distribution -TEST_CASE("adios2_chunk_distribution", "[parallel][adios2]") +PARALLEL_TEST_CASE(adios2_chunk_distribution, "[parallel][adios2]") { adios2_chunk_distribution::run_test(); } #endif // openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI #if openPMD_HAVE_MPI -TEST_CASE("bug_1655_bp5_writer_hangup", "[parallel]") +PARALLEL_TEST_CASE(bug_1655_bp5_writer_hangup, "[parallel]") { bug_1655_bp5_writer_hangup::bug_1655_bp5_writer_hangup(); }