From 04c15bced5e6e808b4c64920966ab6792b98b35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Micha=C3=ABl=20Celerier?= Date: Wed, 21 Jan 2026 01:05:49 -0500 Subject: [PATCH] wip: experiment with bw64 --- .gitmodules | 6 + 3rdparty/3rdparty.cmake | 1 + src/plugins/score-plugin-media/CMakeLists.txt | 14 + .../Media/Sound/BW64/AdmData.hpp | 74 +++++ .../Media/Sound/BW64/Bw64AdmReader.cpp | 273 ++++++++++++++++++ .../Media/Sound/BW64/Bw64AdmReader.hpp | 38 +++ .../Media/Sound/BW64/Bw64Drop.cpp | 172 +++++++++++ .../Media/Sound/BW64/Bw64Drop.hpp | 33 +++ .../score-plugin-media/score_plugin_media.cpp | 7 +- 9 files changed, 617 insertions(+), 1 deletion(-) create mode 100644 src/plugins/score-plugin-media/Media/Sound/BW64/AdmData.hpp create mode 100644 src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.cpp create mode 100644 src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.hpp create mode 100644 src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.cpp create mode 100644 src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.hpp diff --git a/.gitmodules b/.gitmodules index 78d08e5235..5c621dabb4 100755 --- a/.gitmodules +++ b/.gitmodules @@ -112,3 +112,9 @@ [submodule "3rdparty/xtl"] path = 3rdparty/xtl url = https://github.com/xtensor-stack/xtl +[submodule "3rdparty/libadm"] + path = 3rdparty/libadm + url = https://github.com/ebu/libadm +[submodule "3rdparty/libbw64"] + path = 3rdparty/libbw64 + url = https://github.com/ebu/libbw64 diff --git a/3rdparty/3rdparty.cmake b/3rdparty/3rdparty.cmake index 53694ce380..31de540d32 100644 --- a/3rdparty/3rdparty.cmake +++ b/3rdparty/3rdparty.cmake @@ -18,6 +18,7 @@ function(restore_var VAR) endif() endfunction() +include(3rdparty/bw64.cmake) include(3rdparty/dspfilters.cmake) include(3rdparty/eigen.cmake) include(3rdparty/gamma.cmake) diff --git a/src/plugins/score-plugin-media/CMakeLists.txt b/src/plugins/score-plugin-media/CMakeLists.txt index 15d081c8ba..599d598c03 100644 --- a/src/plugins/score-plugin-media/CMakeLists.txt +++ b/src/plugins/score-plugin-media/CMakeLists.txt @@ -35,6 +35,10 @@ set(HDRS ${HDRS} "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/SoundComponent.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/SoundLibraryHandler.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/BW64/AdmData.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/BW64/Bw64AdmReader.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/BW64/Bw64Drop.hpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/Model.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/Presenter.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/View.hpp" @@ -98,6 +102,9 @@ set(SRCS "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/Drop/SoundDrop.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/SoundComponent.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/BW64/Bw64AdmReader.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Sound/BW64/Bw64Drop.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/Model.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/Presenter.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/Media/Effect/Settings/View.cpp" @@ -151,8 +158,15 @@ score_generate_command_list_file(${PROJECT_NAME} "${HDRS}") target_link_libraries(${PROJECT_NAME} PUBLIC ${QT_PREFIX}::Core ${QT_PREFIX}::Widgets score_lib_base score_plugin_engine score_plugin_library score_plugin_audio score_plugin_dataflow + score_plugin_automation score_plugin_curve ) +# BW64/ADM support for object-based audio +if(TARGET bw64 AND TARGET adm) + target_link_libraries(${PROJECT_NAME} PUBLIC bw64 adm) + target_compile_definitions(${PROJECT_NAME} PUBLIC SCORE_HAS_BW64_ADM) +endif() + ### FFMPEG ### if(EMSCRIPTEN) target_include_directories(${PROJECT_NAME} PUBLIC ${OSSIA_SDK}/ffmpeg/include) diff --git a/src/plugins/score-plugin-media/Media/Sound/BW64/AdmData.hpp b/src/plugins/score-plugin-media/Media/Sound/BW64/AdmData.hpp new file mode 100644 index 0000000000..eedd8be273 --- /dev/null +++ b/src/plugins/score-plugin-media/Media/Sound/BW64/AdmData.hpp @@ -0,0 +1,74 @@ +#pragma once +#include + +#include + +#include + +namespace Media::BW64 +{ + +struct AdmAutomationPoint +{ + double time; // normalized 0-1 + double value; +}; + +struct AdmAutomation +{ + enum Parameter + { + X, + Y, + Z, + Gain, + Width, + Height, + Depth, + Diffuse + }; + + Parameter param{X}; + QString name; + double minValue{-1.0}; + double maxValue{1.0}; + std::vector points; +}; + +struct AdmAudioObject +{ + QString name; + int audioChannelIndex{0}; // Channel index in the WAV file (0-based) + std::vector automations; +}; + +struct Bw64AdmData +{ + QString filePath; + TimeVal duration; + int sampleRate{48000}; + int totalChannels{0}; + std::vector objects; + + bool isValid() const noexcept { return !objects.empty() && duration > TimeVal::zero(); } +}; + +// Convert ADM spherical coordinates (azimuth, elevation, distance) to cartesian (x, y, z) +// ADM convention: +// azimuth: 0° = front, +90° = left, -90° = right (degrees) +// elevation: 0° = horizon, +90° = above, -90° = below (degrees) +// distance: 0 to 1 (normalized) +inline void +sphericalToCartesian(double azimuth, double elevation, double distance, double& x, double& y, double& z) +{ + constexpr double deg_to_rad = 3.14159265358979323846 / 180.0; + const double az_rad = azimuth * deg_to_rad; + const double el_rad = elevation * deg_to_rad; + + // ADM/ossia convention: +Y is front, +X is left, +Z is up + x = std::sin(az_rad) * std::cos(el_rad) * distance; + y = std::cos(az_rad) * std::cos(el_rad) * distance; + z = std::sin(el_rad) * distance; +} + +} diff --git a/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.cpp b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.cpp new file mode 100644 index 0000000000..7b02b740a1 --- /dev/null +++ b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.cpp @@ -0,0 +1,273 @@ +#include "Bw64AdmReader.hpp" + +#if defined(SCORE_HAS_BW64_ADM) +#include +#include +#include + +#include + +#include + +namespace Media::BW64 +{ + +bool Bw64AdmReader::isBw64WithAdm(const QString& path) +{ + try + { + auto reader = bw64::readFile(path.toStdString()); + return reader->axmlChunk() != nullptr && reader->chnaChunk() != nullptr; + } + catch(...) + { + return false; + } +} + +// Helper to extract time in seconds from ADM time types +static double admTimeToSeconds(const adm::Time& time) +{ + auto ns = time.asNanoseconds(); + return static_cast(ns.get()) / 1e9; +} + +// Helper to get duration in seconds +static double admDurationToSeconds(const adm::Duration& dur) +{ + auto ns = dur.asNanoseconds(); + return static_cast(ns.get()) / 1e9; +} + +std::optional Bw64AdmReader::parse(const QString& path) +{ + try + { + // Open BW64 file + auto reader = bw64::readFile(path.toStdString()); + + auto axmlChunk = reader->axmlChunk(); + auto chnaChunk = reader->chnaChunk(); + + if(!axmlChunk || !chnaChunk) + { + qDebug() << "BW64 file missing axml or chna chunk:" << path; + return std::nullopt; + } + + // Parse ADM XML + std::istringstream xmlStream(axmlChunk->data()); + auto document = adm::parseXml(xmlStream); + + if(!document) + { + qDebug() << "Failed to parse ADM XML in:" << path; + return std::nullopt; + } + + // Calculate total duration from audio data + const double totalDurationSec + = static_cast(reader->numberOfFrames()) / reader->sampleRate(); + + Bw64AdmData result; + result.filePath = path; + result.sampleRate = reader->sampleRate(); + result.totalChannels = reader->channels(); + result.duration = TimeVal::fromMsecs(totalDurationSec * 1000.0); + + // Build a map from AudioTrackUID to channel index using chna chunk + std::map trackUidToChannel; + for(const auto& audioId : chnaChunk->audioIds()) + { + // trackIndex is 1-based in BW64, convert to 0-based + std::string uid = audioId.uid(); + // Trim whitespace + uid.erase(uid.find_last_not_of(' ') + 1); + trackUidToChannel[uid] = audioId.trackIndex() - 1; + } + + // Process each AudioObject in the document + for(const auto& audioObject : document->getElements()) + { + AdmAudioObject obj; + obj.name = QString::fromStdString( + audioObject->get().get()); + + // Get the AudioTrackUIDs referenced by this object + auto trackUids = audioObject->getReferences(); + if(trackUids.empty()) + continue; + + // Use the first track UID to determine the channel + auto firstTrackUid = trackUids.front(); + std::string uidStr = adm::formatId(firstTrackUid->get()); + auto channelIt = trackUidToChannel.find(uidStr); + if(channelIt != trackUidToChannel.end()) + { + obj.audioChannelIndex = channelIt->second; + } + else + { + // Try to find by iterating + for(const auto& [uid, ch] : trackUidToChannel) + { + if(uid.find(uidStr) != std::string::npos + || uidStr.find(uid) != std::string::npos) + { + obj.audioChannelIndex = ch; + break; + } + } + } + + // Get the AudioChannelFormat to access AudioBlockFormats + auto packFormats = audioObject->getReferences(); + for(const auto& packFormat : packFormats) + { + auto channelFormats = packFormat->getReferences(); + for(const auto& channelFormat : channelFormats) + { + // Check if this is an Objects type (has position automation) + auto typeDescriptor = channelFormat->get(); + if(typeDescriptor != adm::TypeDefinition::OBJECTS) + continue; + + // Get all AudioBlockFormatObjects + auto blockFormats + = channelFormat->getElements(); + + if(blockFormats.empty()) + continue; + + // Prepare automation containers + std::vector xPoints, yPoints, zPoints; + std::vector gainPoints; + std::vector widthPoints, heightPoints, depthPoints; + std::vector diffusePoints; + + for(const auto& block : blockFormats) + { + // Get timing + double startTime = 0.0; + if(block.has()) + { + startTime = admTimeToSeconds(block.get().get()); + } + + // Normalize time to 0-1 + double normalizedTime + = (totalDurationSec > 0) ? (startTime / totalDurationSec) : 0.0; + normalizedTime = std::clamp(normalizedTime, 0.0, 1.0); + + // Extract position + double x = 0, y = 0, z = 0; + + if(block.has()) + { + auto pos = block.get(); + x = pos.get().get(); + y = pos.get().get(); + z = pos.has() ? pos.get().get() : 0.0; + } + else if(block.has()) + { + auto pos = block.get(); + double azimuth = pos.get().get(); + double elevation = pos.get().get(); + double distance = pos.has() ? pos.get().get() : 1.0; + sphericalToCartesian(azimuth, elevation, distance, x, y, z); + } + + xPoints.push_back({normalizedTime, x}); + yPoints.push_back({normalizedTime, y}); + zPoints.push_back({normalizedTime, z}); + + // Extract gain + if(block.has()) + { + auto gain = block.get(); + double gainValue = gain.isLinear() + ? gain.asLinear() + : std::pow(10.0, gain.asDb() / 20.0); + gainPoints.push_back({normalizedTime, gainValue}); + } + + // Extract width/height/depth (extent) + if(!block.isDefault()) + { + widthPoints.push_back({normalizedTime, block.get().get()}); + } + if(!block.isDefault()) + { + heightPoints.push_back({normalizedTime, block.get().get()}); + } + if(!block.isDefault()) + { + depthPoints.push_back({normalizedTime, block.get().get()}); + } + + // Extract diffuse + if(!block.isDefault()) + { + diffusePoints.push_back( + {normalizedTime, block.get().get()}); + } + } + + // Create automation objects for parameters that have data + auto addAutomation = [&](AdmAutomation::Parameter param, + const QString& suffix, + std::vector& points, + double minVal, double maxVal) { + if(points.size() >= 2) + { + AdmAutomation autom; + autom.param = param; + autom.name = obj.name + " " + suffix; + autom.minValue = minVal; + autom.maxValue = maxVal; + autom.points = std::move(points); + obj.automations.push_back(std::move(autom)); + } + }; + + addAutomation(AdmAutomation::X, "X", xPoints, -1.0, 1.0); + addAutomation(AdmAutomation::Y, "Y", yPoints, -1.0, 1.0); + addAutomation(AdmAutomation::Z, "Z", zPoints, -1.0, 1.0); + addAutomation(AdmAutomation::Gain, "Gain", gainPoints, 0.0, 1.0); + addAutomation(AdmAutomation::Width, "Width", widthPoints, 0.0, 360.0); + addAutomation(AdmAutomation::Height, "Height", heightPoints, 0.0, 360.0); + addAutomation(AdmAutomation::Depth, "Depth", depthPoints, 0.0, 360.0); + addAutomation(AdmAutomation::Diffuse, "Diffuse", diffusePoints, 0.0, 1.0); + } + } + + // Only add objects that have automations or valid channel mapping + if(!obj.automations.empty() || obj.audioChannelIndex >= 0) + { + result.objects.push_back(std::move(obj)); + } + } + + if(result.objects.empty()) + { + qDebug() << "No ADM audio objects found in:" << path; + return std::nullopt; + } + + return result; + } + catch(const std::exception& e) + { + qDebug() << "Error parsing BW64/ADM file:" << path << "-" << e.what(); + return std::nullopt; + } + catch(...) + { + qDebug() << "Unknown error parsing BW64/ADM file:" << path; + return std::nullopt; + } +} + +} +#endif // SCORE_HAS_BW64_ADM diff --git a/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.hpp b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.hpp new file mode 100644 index 0000000000..7c7a94c4a0 --- /dev/null +++ b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64AdmReader.hpp @@ -0,0 +1,38 @@ +#pragma once +#include + +#include + +#include + +namespace Media::BW64 +{ + +/** + * @brief Reader for BW64 files with ADM (Audio Definition Model) metadata + * + * BW64 is an extended Broadcast Wave Format that can contain ADM metadata + * for object-based 3D audio. The ADM data includes: + * - Audio objects with their associated audio channels + * - Position automation (azimuth/elevation/distance or x/y/z over time) + * - Gain and other parameter automation + */ +class Bw64AdmReader +{ +public: + /** + * @brief Check if a file is a BW64/WAV file with ADM metadata + * @param path Path to the audio file + * @return true if the file contains axml and chna chunks + */ + static bool isBw64WithAdm(const QString& path); + + /** + * @brief Parse a BW64 file and extract ADM automation data + * @param path Path to the BW64 file + * @return Parsed ADM data, or nullopt if parsing failed + */ + static std::optional parse(const QString& path); +}; + +} diff --git a/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.cpp b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.cpp new file mode 100644 index 0000000000..bd36355711 --- /dev/null +++ b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.cpp @@ -0,0 +1,172 @@ +#include "Bw64Drop.hpp" + +#if defined(SCORE_HAS_BW64_ADM) +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +namespace Media::BW64 +{ + +QSet DropHandler::mimeTypes() const noexcept +{ + return {}; +} + +QSet DropHandler::fileExtensions() const noexcept +{ + // We handle .wav files that have ADM metadata + return {"wav", "bwf"}; +} + +// Convert AdmAutomation points to curve segments +static std::vector +automationToCurveSegments(const AdmAutomation& autom) +{ + std::vector segments; + + if(autom.points.size() < 2) + return segments; + + const double range = autom.maxValue - autom.minValue; + if(range <= 0) + return segments; + + // Filter and deduplicate points + std::vector> filteredPoints; + filteredPoints.reserve(autom.points.size()); + + for(const auto& pt : autom.points) + { + double x = std::clamp(pt.time, 0.0, 1.0); + double y = std::clamp((pt.value - autom.minValue) / range, 0.0, 1.0); + + // Skip if same x as previous point (keep the last value at each time) + if(!filteredPoints.empty() && filteredPoints.back().first >= x) + { + filteredPoints.back().second = y; + } + else + { + filteredPoints.emplace_back(x, y); + } + } + + if(filteredPoints.size() < 2) + return segments; + + // Create IDs for all segments + std::vector> ids; + ids.reserve(filteredPoints.size() - 1); + + for(std::size_t i = 0; i < filteredPoints.size() - 1; i++) + { + ids.push_back(Curve::getSegmentId(ids)); + } + + // Create segments + for(std::size_t i = 0; i < filteredPoints.size() - 1; i++) + { + const auto& pt1 = filteredPoints[i]; + const auto& pt2 = filteredPoints[i + 1]; + + Curve::SegmentData seg; + seg.id = ids[i]; + seg.start = Curve::Point{pt1.first, pt1.second}; + seg.end = Curve::Point{pt2.first, pt2.second}; + + // Link segments + if(i > 0) + seg.previous = ids[i - 1]; + if(i < filteredPoints.size() - 2) + seg.following = ids[i + 1]; + + // Use linear segments + seg.type = Metadata::get(); + seg.specificSegmentData = QVariant::fromValue(Curve::LinearSegmentData{}); + + segments.push_back(std::move(seg)); + } + + return segments; +} + +void DropHandler::dropPath( + std::vector& vec, const score::FilePath& filename, + const score::DocumentContext& ctx) const noexcept +{ + // First check if this is a BW64 file with ADM metadata + if(!Bw64AdmReader::isBw64WithAdm(filename.absolute)) + { + // Not a BW64/ADM file, let the regular sound handler deal with it + return; + } + + // Parse the ADM data + auto admData = Bw64AdmReader::parse(filename.absolute); + if(!admData || !admData->isValid()) + { + return; + } + + // Create processes for each ADM audio object + for(const auto& obj : admData->objects) + { + // Create Sound process for this object's audio channel + { + Process::ProcessDropHandler::ProcessDrop p; + p.creation.key = Metadata::get(); + p.creation.prettyName = obj.name; + p.duration = admData->duration; + + const int channelIndex = obj.audioChannelIndex; + p.setup = [f = score::relativizeFilePath(filename.absolute, ctx), channelIndex, + &ctx](Process::ProcessModel& m, score::Dispatcher& disp) { + auto& proc = static_cast(m); + // Set the audio file with specific channel/stream + disp.submit(new Media::ChangeAudioFile{proc, std::move(f), ctx}); + // Note: For mono extraction from multichannel, we'd need additional + // support in the Sound process. For now, this loads the full file. + }; + vec.push_back(std::move(p)); + } + + // Create Automation processes for each parameter + for(const auto& autom : obj.automations) + { + Process::ProcessDropHandler::ProcessDrop p; + p.creation.key = Metadata::get(); + p.creation.prettyName = autom.name; + p.duration = admData->duration; + + p.setup = [autom](Process::ProcessModel& m, score::Dispatcher& disp) { + auto& automProc = static_cast(m); + + // Set min/max values + disp.submit(new Automation::SetMin{automProc, autom.minValue}); + disp.submit(new Automation::SetMax{automProc, autom.maxValue}); + + // Convert automation points to curve segments + auto segments = automationToCurveSegments(autom); + if(!segments.empty()) + { + disp.submit( + new Curve::UpdateCurve{automProc.curve(), std::move(segments)}); + } + }; + vec.push_back(std::move(p)); + } + } +} + +} +#endif // SCORE_HAS_BW64_ADM diff --git a/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.hpp b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.hpp new file mode 100644 index 0000000000..a82b959bef --- /dev/null +++ b/src/plugins/score-plugin-media/Media/Sound/BW64/Bw64Drop.hpp @@ -0,0 +1,33 @@ +#pragma once + +#if defined(SCORE_HAS_BW64_ADM) +#include + +namespace Media::BW64 +{ + +/** + * @brief Drop handler for BW64 files with ADM metadata + * + * When a BW64 file with ADM metadata is dropped: + * - Creates a Sound process for each ADM audio object (mono channel) + * - Creates Automation processes for X, Y, Z position curves + * - Creates Automation processes for Gain and other parameters + * + * With Shift held, each ADM object group gets its own interval. + */ +class DropHandler final : public Process::ProcessDropHandler +{ + SCORE_CONCRETE("a7e3f2d1-8c4b-4f5e-9a1d-6b2c3e4f5a6b") + +public: + QSet mimeTypes() const noexcept override; + QSet fileExtensions() const noexcept override; + + void dropPath( + std::vector& drops, const score::FilePath& filename, + const score::DocumentContext& ctx) const noexcept override; +}; + +} +#endif // SCORE_HAS_BW64_ADM diff --git a/src/plugins/score-plugin-media/score_plugin_media.cpp b/src/plugins/score-plugin-media/score_plugin_media.cpp index a922907fa5..b06eb461a7 100644 --- a/src/plugins/score-plugin-media/score_plugin_media.cpp +++ b/src/plugins/score-plugin-media/score_plugin_media.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -119,7 +120,11 @@ std::vector score_plugin_media::factories( FW, - FW, + FW, FW, FW, FW