From 194bf43b44c323a67a727f14975d732f4ef26b07 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Wed, 17 Jun 2026 12:20:22 +0200 Subject: [PATCH 1/3] =?UTF-8?q?feat(mongo):=20MongoDB=20driver=20SDAM=20co?= =?UTF-8?q?re=20=E2=80=94=20topology=20discovery,=20health=20monitoring,?= =?UTF-8?q?=20primary=20write-routing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks the MongoDB driver onto a Server Discovery and Monitoring (SDAM) foundation: single-pass topology discovery, background health monitoring, writes routed to the primary, per-query read preferences, and read-preference tag server selection. Adds the wire/codec and topology infrastructure (serverdescription, monitor, wire, wireversion, compression, commands, clustertime) and reworks connection/client/collection/database/cursor onto it. Base of a stacked series: client sessions, multi-document transactions and retryable writes follow in the next PR, and the self-contained MongoDB 8 features (change streams, client bulkWrite, GridFS, CSFLE, Stable API, mongodb+srv, load-balancer mode) as PRs on top of that. Closes #2845, #2847, #2848, #2849, #2851 --- .gitignore | 9 + mongodb/vibe/db/mongo/client.d | 524 +++++++- mongodb/vibe/db/mongo/collection.d | 459 +------ mongodb/vibe/db/mongo/connection.d | 1099 +++++++---------- mongodb/vibe/db/mongo/cursor.d | 91 +- mongodb/vibe/db/mongo/database.d | 96 +- mongodb/vibe/db/mongo/impl/clustertime.d | 93 ++ mongodb/vibe/db/mongo/impl/commands.d | 456 +++++++ mongodb/vibe/db/mongo/impl/compression.d | 207 ++++ mongodb/vibe/db/mongo/impl/crud.d | 154 ++- mongodb/vibe/db/mongo/impl/index.d | 22 + .../vibe/db/mongo/impl/serverdescription.d | 480 +++++++ mongodb/vibe/db/mongo/impl/wire.d | 292 +++++ mongodb/vibe/db/mongo/impl/wireversion.d | 348 ++++++ mongodb/vibe/db/mongo/mongo.d | 43 + mongodb/vibe/db/mongo/monitor.d | 791 ++++++++++++ mongodb/vibe/db/mongo/settings.d | 272 +++- mongodb/vibe/db/mongo/topology.d | 1018 +++++++++++++-- tests/mongodb/_connection/source/app.d | 9 + tests/mongodb/_health-monitor/dub.json | 8 + tests/mongodb/_health-monitor/run.sh | 101 ++ tests/mongodb/_health-monitor/source/app.d | 100 ++ tests/mongodb/_replica-set/run.sh | 44 +- tests/mongodb/_replica-set/source/app.d | 185 ++- tests/mongodb/compression-reconnect/dub.json | 7 + .../compression-reconnect/source/app.d | 148 +++ tests/mongodb/connection-quarantine/dub.json | 7 + .../connection-quarantine/source/app.d | 273 ++++ tests/mongodb/cursor/source/app.d | 8 +- 29 files changed, 5959 insertions(+), 1385 deletions(-) create mode 100644 mongodb/vibe/db/mongo/impl/clustertime.d create mode 100644 mongodb/vibe/db/mongo/impl/commands.d create mode 100644 mongodb/vibe/db/mongo/impl/compression.d create mode 100644 mongodb/vibe/db/mongo/impl/serverdescription.d create mode 100644 mongodb/vibe/db/mongo/impl/wire.d create mode 100644 mongodb/vibe/db/mongo/impl/wireversion.d create mode 100644 mongodb/vibe/db/mongo/monitor.d create mode 100644 tests/mongodb/_health-monitor/dub.json create mode 100755 tests/mongodb/_health-monitor/run.sh create mode 100644 tests/mongodb/_health-monitor/source/app.d create mode 100644 tests/mongodb/compression-reconnect/dub.json create mode 100644 tests/mongodb/compression-reconnect/source/app.d create mode 100644 tests/mongodb/connection-quarantine/dub.json create mode 100644 tests/mongodb/connection-quarantine/source/app.d diff --git a/.gitignore b/.gitignore index a0506eeaf3..bfcf7e5220 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,12 @@ tests/mongodb/_replica-set/replica-set-test tests/mongodb/_replica-set/log* tests/mongodb/readconcern/readconcern-test tests/mongodb/dead-connections/dead-connections-test +tests/mongodb/_health-monitor/health-monitor-test +tests/mongodb/change-stream/change-stream-test +tests/mongodb/loadbalanced/lb-cluster +tests/mongodb/stable-api/stable-api-test +tests/mongodb/session-timeout/session-timeout-test +tests/mongodb/dead-connections/ready +tests/mongodb/transactions/transactions-test +tests/mongodb/connection-quarantine/connection-quarantine-test +tests/mongodb/compression-reconnect/compression-reconnect-test diff --git a/mongodb/vibe/db/mongo/client.d b/mongodb/vibe/db/mongo/client.d index 560a6ddfba..c066507cf2 100644 --- a/mongodb/vibe/db/mongo/client.d +++ b/mongodb/vibe/db/mongo/client.d @@ -13,12 +13,19 @@ public import vibe.db.mongo.database; import vibe.core.connectionpool; import vibe.core.log; +import vibe.core.sync : LocalManualEvent, createManualEvent; import vibe.db.mongo.connection; import vibe.db.mongo.settings; import vibe.db.mongo.topology; +import vibe.db.mongo.monitor; +import vibe.db.mongo.impl.crud; +import vibe.db.mongo.impl.wireversion : WireVersion; +import vibe.data.bson; +import core.time : Duration, seconds, msecs, MonoTime; import std.conv; import std.exception : enforce; +import std.typecons : Nullable; /** Represents a connection to a MongoDB server. @@ -30,11 +37,24 @@ import std.exception : enforce; final class MongoClient { @safe: + // Concurrency contract (HARD): a MongoClient is single-thread / single-event-loop. + // It is safe to share across fibers of ONE thread, but it must NOT be shared across + // OS threads. The connection pools, m_topologyChanged + // (a LocalManualEvent), and the bool flags below are all thread-local and + // unsynchronised; only an event loop on the owning thread may touch them. The + // AtomicTopology wrapper exists solely to give a consistent intra-thread snapshot + // of the topology across a yield point (publish/load is one atomic swap); it is NOT + // a license for cross-thread sharing and does not make the rest of this state safe + // to mutate from another thread. For one client per thread, use a thread-local + // instance (e.g. scopedMongoDB) rather than passing one client between threads. private { - ConnectionPool!MongoConnection m_connections; + ConnectionPool!MongoConnection[string] m_connectionPools; MongoClientSettings m_settings; - TopologyDescription m_topology; + AtomicTopology m_topology; + LocalManualEvent m_topologyChanged; bool m_discoveryInProgress; + + MonitorRegistry m_monitors; } package this(string host, ushort port) @@ -64,16 +84,47 @@ final class MongoClient { package this(MongoClientSettings settings) { m_settings = settings; - - discoverTopology(); - - m_connections = new ConnectionPool!MongoConnection( - &createConnection, - settings.maxConnections - ); - - // force a connection to cause an exception for wrong URLs - lockConnection(); + m_topologyChanged = createManualEvent(); + + // discoverTopology()/lockConnection() throw on an unreachable or invalid + // deployment. The throw must propagate, but destroying a live + // m_topologyChanged (LocalManualEvent) during the unwind segfaults in + // vibe-core (releaseRef -> disposeGCSafe outside an event-loop context). + // LocalManualEvent.init's destructor is a no-op (m_waiter is null), so we + // move it back to .init before rethrowing: the field dtor that then runs + // during unwind is harmless. The freshly-allocated waiter is leaked, but + // only on the failure path where the process is throwing out of the ctor. + try + { + // discoverTopology() runs full SDAM discovery. + discoverTopology(); + + // The monitor registry is always constructed so its call sites + // (handleStaleCommandError / stopMonitoring / activeMonitorCount, and + // resolveHost's requestAllChecks) operate on a real object, never a null. It + // is built BEFORE lockConnection() so server selection during that connect + // cannot dereference a null registry. + ServerProber prober = (MongoHost host) @safe => probeServer(m_settings, host); + m_monitors = new MonitorRegistry(prober, &onMonitorResult, + m_settings.heartbeatFrequencyMS.msecs, m_settings.minHeartbeatFrequencyMS.msecs); + + // force a connection to cause an exception for wrong URLs + lockConnection(); + + // Start the monitors only after the connection succeeds, so a ctor failure + // does not leak background monitor tasks. + m_monitors.reconcileWith(m_topology.load().allKnownHosts()); + } + catch (Exception e) + { + import std.algorithm.mutation : moveEmplace; + // moveEmplace overwrites the live m_topologyChanged with .init WITHOUT + // running its (crashing) destructor first, so the field dtor that runs + // during the rethrow unwind sees a null waiter and is a no-op. + LocalManualEvent harmless; + () @trusted { moveEmplace(harmless, m_topologyChanged); }(); + throw e; + } } /// Returns the read preference configured for this client. @@ -82,6 +133,12 @@ final class MongoClient { return m_settings.readPreference; } + /// Returns the ordered read-preference tag sets configured for this client. + @property string[string][] readPreferenceTags() + { + return m_settings.readPreferenceTags; + } + /// Returns the read concern configured for this client. ReadConcern readConcern() const { @@ -92,14 +149,15 @@ final class MongoClient { */ void cleanupConnections() { - m_connections.removeUnused((conn) nothrow @safe { - try conn.disconnect(); - catch (Exception e) { - logWarn("Error thrown during MongoDB connection close: %s", e.msg); - try () @trusted { logDebug("Full error: %s", e.toString()); } (); - catch (Exception e) {} - } - }); + foreach (pool; m_connectionPools.byValue) + pool.removeUnused((conn) nothrow @safe { + try conn.disconnect(); + catch (Exception e) { + logWarn("Error thrown during MongoDB connection close: %s", e.msg); + try () @trusted { logDebug("Full error: %s", e.toString()); } (); + catch (Exception e) {} + } + }); } /** @@ -149,8 +207,6 @@ final class MongoClient { return MongoDatabase(this, dbName); } - - /** Return a handle to all databases of the server. @@ -171,54 +227,164 @@ final class MongoClient { return ret; } + /// Locks a connection to the server chosen by the configured read preference. package LockedConnection!MongoConnection lockConnection() { + return lockConnectionResolving(false, m_settings.readPreference); + } + + /// Locks a connection to the server chosen by an explicit per-query read preference. + package LockedConnection!MongoConnection lockConnection(ReadPreference pref) + { + return lockConnectionResolving(false, pref); + } + + /// Locks a connection to the primary. Used for write operations, which must + /// always go to the primary regardless of the configured read preference. + package LockedConnection!MongoConnection lockConnectionToPrimary() + { + return lockConnectionResolving(true, ReadPreference.primary); + } + + /// Resolves the host a read should target, retrying once after re-discovery. + package MongoHost resolveHostForRead(ReadPreference pref) + { + try { + return resolveHost(false, pref); + } catch (Exception e) { + logWarn("Read host resolution failed: %s — re-discovering topology", e.msg); + } + + discoverTopology(); + return resolveHost(false, pref); + } + + private LockedConnection!MongoConnection lockConnectionResolving(bool toPrimary, ReadPreference pref) + { + try { + return lockConnectionToHost(resolveHost(toPrimary, pref)); + } catch (Exception e) { + logWarn("Connection acquisition failed: %s — re-discovering topology", e.msg); + } + + discoverTopology(); + return lockConnectionToHost(resolveHost(toPrimary, pref)); + } + + /// Selects a target host, blocking up to `serverSelectionTimeoutMS` for one to appear. + private MongoHost resolveHost(bool toPrimary, ReadPreference pref) + { + auto deadline = MonoTime.currTime + m_settings.serverSelectionTimeoutMS.msecs; + + while (true) + { + // Read the counter before the snapshot so a concurrent update is not missed. + auto topologyVersion = m_topologyChanged.emitCount; + + auto topology = m_topology.load(); + auto selected = selectTarget(topology, toPrimary, pref, + m_settings.localThresholdMS, m_settings.maxStalenessSeconds, + m_settings.readPreferenceTags); + if (!selected.isNull) + return selected.get; + + m_monitors.requestAllChecks(); + + auto remaining = deadline - MonoTime.currTime; + if (remaining <= Duration.zero) + break; + + m_topologyChanged.wait(remaining, topologyVersion); + } + + throw new MongoDriverException(toPrimary + ? "No primary server available for write" + : "No suitable server found for read preference"); + } + + /// On a stale-topology command error, marks the host failed and re-checks it. + private void handleStaleCommandError(MongoHost host, MongoServerErrorCode code) @safe nothrow + { + if (!isStaleTopologyError(code)) + return; + + try { + m_topology.publish(applyFailed(m_topology.load(), host)); + m_topologyChanged.emit(); + m_monitors.requestCheck(host); + } catch (Exception) {} + } + + /// Locks a pooled connection for a specific host (e.g. a cursor re-locking its pinned host). + package LockedConnection!MongoConnection lockConnectionToHost(MongoHost host) + { + auto pool = poolFor(host); + foreach (_; 0 .. 100) { - auto conn = m_connections.lockConnection(); + auto conn = pool.lockConnection(); if (conn.alive) return conn; - m_connections.remove(conn.__conn); + pool.remove(conn.__conn); logDiagnostic("Evicted dead MongoDB connection from pool"); } throw new MongoDriverException("Failed to acquire a live connection after evicting 100 dead connections"); } - package MongoHost getSelectedHost() + /// Drops the connection pools for hosts no longer in `desiredHosts`, disconnecting + /// their idle connections first. Without this, a host that leaves the replica set + /// keeps its pool (and idle sockets) forever, and maxConnections becomes a per-host + /// rather than a global bound. Connections still checked out are closed when the + /// in-flight operation returns them to the (now unreferenced) pool. + private void pruneStalePools(MongoHost[] desiredHosts) @safe { - auto selected = selectServer(m_topology, m_settings.readPreference, m_settings.localThresholdMS, m_settings.maxStalenessSeconds); - enforce!MongoDriverException(!selected.isNull, "No suitable server found for read preference"); + import std.algorithm : map; + import std.array : array; - return selected.get; + auto desiredKeys = desiredHosts.map!(h => hostKey(h)).array; + foreach (key; poolKeysToPrune(m_connectionPools.keys, desiredKeys)) + { + m_connectionPools[key].removeUnused((conn) nothrow @safe { + try conn.disconnect(); + catch (Exception e) { + logWarn("Error closing MongoDB connection for pruned host %s: %s", key, e.msg); + try () @trusted { logDebug("Full error: %s", e.toString()); } (); + catch (Exception e) {} + } + }); + m_connectionPools.remove(key); + } } - private MongoConnection createConnection() @safe + private ConnectionPool!MongoConnection poolFor(MongoHost host) { - auto targetHost = getSelectedHost(); - auto ret = new MongoConnection(m_settings); + auto key = hostKey(host); - try { - ret.connectToHost(targetHost); - return ret; - } catch (Exception e) { - () @trusted { destroy(ret); } (); + if (auto existing = key in m_connectionPools) + return *existing; - logWarn("Connection to %s:%s failed: %s — re-discovering topology", - targetHost.name, targetHost.port, e.msg); - } + auto pool = new ConnectionPool!MongoConnection( + () @safe => createConnectionToHost(host), + m_settings.maxConnections + ); + m_connectionPools[key] = pool; - discoverTopology(); - targetHost = getSelectedHost(); + return pool; + } + + private MongoConnection createConnectionToHost(MongoHost host) @safe + { + auto ret = new MongoConnection(m_settings); + ret.onCommandError(&handleStaleCommandError); - ret = new MongoConnection(m_settings); try { - ret.connectToHost(targetHost); - } catch (Exception e2) { + ret.connectToHost(host); + } catch (Exception e) { () @trusted { destroy(ret); } (); - throw e2; + throw e; } return ret; @@ -237,21 +403,39 @@ final class MongoClient { TopologyDescription newTopology; newTopology.type = initialTopologyType(); + // Seed the configured replica-set name so update()'s setName guard enforces it on + // every probe — including the monitor path, which does not call matchesReplicaSet. + newTopology.setName = m_settings.replicaSet; newTopology.seedCount = cast(uint) m_settings.hosts.length; + // Feed the configured heartbeat into the maxStaleness formula (was hardcoded to 10s). + newTopology.heartbeatFrequencyMS = m_settings.heartbeatFrequencyMS; Exception lastException; + MongoHost[] attempted = m_settings.hosts.dup; foreach (host; m_settings.hosts) { probeAndUpdate(newTopology, host, lastException); } - foreach (host; newTopology.allKnownHosts()) { - if (newTopology.servers.canFind!(s => s.host == host)) - continue; + // A newly discovered host may itself report further hosts we don't know + // about yet, so keep probing until a full pass turns up nothing new. + for (bool foundNew = true; foundNew; ) { + foundNew = false; - probeAndUpdate(newTopology, host, lastException); + foreach (host; newTopology.allKnownHosts()) { + if (attempted.canFind(host)) + continue; + + attempted ~= host; + foundNew = true; + probeAndUpdate(newTopology, host, lastException); + } } - auto selected = selectServer(newTopology, m_settings.readPreference, m_settings.localThresholdMS, m_settings.maxStalenessSeconds); + // Select with the configured readPreferenceTags so discovery's suitability check + // matches runtime selection (resolveHost). Otherwise a tag set matching no server + // passes discovery, then fails (or null-derefs) later in lockConnection(). + auto selected = selectServer(newTopology, m_settings.readPreference, m_settings.localThresholdMS, + m_settings.maxStalenessSeconds, m_settings.readPreferenceTags); if (selected.isNull) { throw lastException !is null @@ -259,7 +443,14 @@ final class MongoClient { : new MongoDriverException("No suitable server found during topology discovery"); } - m_topology = newTopology; + publishTopology(newTopology); + } + + /// Publishes a new topology snapshot and notifies waiters. + private void publishTopology(TopologyDescription topology) + { + m_topology.publish(topology); + m_topologyChanged.emit(); } private void probeAndUpdate(ref TopologyDescription topology, MongoHost host, ref Exception lastException) @@ -285,4 +476,235 @@ final class MongoClient { return TopologyType.unknown; } + + /// Publishes a monitor's probe result as a new snapshot and reconciles the monitor set. + private void onMonitorResult(MongoHost host, Nullable!ServerDescription desc, Duration rtt) + { + auto current = m_topology.load(); + + TopologyDescription next; + if (desc.isNull) + next = applyFailed(current, host); + else + { + auto folded = desc.get; + folded.roundTripTime = cast(float) foldRtt(current, host, rtt); + next = applyDescription(current, host, folded); + } + publishTopology(next); + + auto knownHosts = m_topology.load().allKnownHosts(); + m_monitors.reconcileWith(knownHosts); + pruneStalePools(knownHosts); + } + + /// Folds this probe's measured `rtt` into the host's running RTT average (EWMA). The + /// first sample for a host (no prior probed average) seeds the average with the raw + /// measurement; later samples decay the old average per the SDAM alpha=0.2 formula. + private double foldRtt(ref const TopologyDescription current, MongoHost host, Duration rtt) @safe + { + auto sample = rtt.total!"usecs" / 1_000_000.0; + foreach (ref s; current.servers) + { + if (s.host == host && s.description.roundTripTime > 0) + return ewmaRtt(s.description.roundTripTime, sample, false); + } + return ewmaRtt(0.0, sample, true); + } + + /// Stops all background server monitors. Call before discarding the client. + void stopMonitoring() + { + m_monitors.stopAll(); + } + + /// Tears the client all the way down: stops the background monitors, disconnects + /// the idle pooled connections, and drops every connection pool. Call before + /// discarding a client to release its background tasks and sockets. cleanupConnections + /// needs a live pool, so it runs before the pools are dropped. Connections still + /// checked out by an in-flight operation are closed when that operation returns them. + void close() + { + stopMonitoring(); + cleanupConnections(); + m_connectionPools = null; + } + + /// Number of background server monitors currently running. + size_t activeMonitorCount() const @property + { + return m_monitors.length; + } + + /// Number of per-host connection pools the client currently holds. + size_t connectionPoolCount() const @property + { + return m_connectionPools.length; + } +} + +/** + Owns a MongoClient and closes it when it leaves scope. + + `scopedMongoDB` returns this handle so a client created in a local or thread-local + scope is cleaned up deterministically: the destructor calls `MongoClient.close`, + which stops the background monitors (breaking the monitor-task -> client reference + cycle that would otherwise keep the client reachable for the lifetime of the + process) and drains the connection pools. + + The handle is move-only and forwards every `MongoClient` member through `alias this`: + --- + auto client = scopedMongoDB("127.0.0.1"); + auto users = client.getCollection("myapp.users"); + --- + + To keep a raw, manually-managed `MongoClient` alive beyond the handle's scope (for + example to store it in a long-lived object), call `release()` to take ownership; you + are then responsible for calling `close()` before discarding it. +*/ +struct MongoClientHandle { +@safe: + private MongoClient m_client; + private void delegate() @safe m_stop; + + @disable this(this); + + /// Wraps `client`, closing it (stop monitors, drain pools) when the + /// handle is destroyed. + package this(MongoClient client) + { + m_client = client; + m_stop = &client.close; + } + + /// Test seam: wraps `client` with an explicit cleanup action run on destruction. + package this(MongoClient client, void delegate() @safe stop) + { + m_client = client; + m_stop = stop; + } + + /// Closes the owned client unless ownership was released or moved away. + ~this() + { + if (m_stop is null) + return; + + auto stop = m_stop; + m_stop = null; + stop(); + } + + /// The owned client. Every `MongoClient` member is also reachable directly on the handle. + @property inout(MongoClient) client() inout { return m_client; } + alias client this; + + /// Relinquishes ownership without closing the client; the caller takes over the client's + /// lifetime and must call `close()` before discarding it. + MongoClient release() + { + m_stop = null; + auto c = m_client; + m_client = null; + return c; + } +} + +/// SDAM exponentially-weighted moving average of a server's round-trip time. +/// +/// `sample` is the latest measured RTT, `prev` the running average; `first` seeds the +/// average with the raw sample on the very first measurement. Subsequent samples fold in +/// with alpha=0.2 per the SDAM spec: newAvg = alpha*sample + (1-alpha)*prev. +double ewmaRtt(double prev, double sample, bool first) @safe pure nothrow @nogc +{ + enum double alpha = 0.2; + return first ? sample : alpha * sample + (1.0 - alpha) * prev; +} + +/// ewmaRtt seeds on the first sample and folds later samples with alpha 0.2 +unittest +{ + import std.math : isClose; + + // the first measurement seeds the average with the raw sample (prev is ignored) + assert(ewmaRtt(0.0, 0.040, true) == 0.040, + "the first RTT sample seeds the moving average"); + + // a later sample folds in: 0.2*0.020 + 0.8*0.040 = 0.036 + assert(isClose(ewmaRtt(0.040, 0.020, false), 0.036), + "a later sample is weighted 0.2 against the 0.8-weighted running average"); + + // a steady sample equal to the average leaves it unchanged + assert(isClose(ewmaRtt(0.030, 0.030, false), 0.030), + "a sample equal to the running average leaves it unchanged"); +} + +/// The pool keys to prune: every currently-pooled host key absent from the desired set. +/// +/// `desiredKeys` is the host-key set of the current topology; `pooledKeys` is the set of +/// per-host connection pools the client holds. A key in `pooledKeys` but not in +/// `desiredKeys` belongs to a host that left the deployment, so its pool (and its idle +/// sockets) must be dropped. +string[] poolKeysToPrune(string[] pooledKeys, string[] desiredKeys) @safe pure nothrow +{ + import std.algorithm : canFind, filter; + import std.array : array; + return pooledKeys.filter!(k => !desiredKeys.canFind(k)).array; +} + +/// poolKeysToPrune drops pools for hosts no longer in the topology and keeps the rest +unittest +{ + // a removed host's pool key is pruned; a still-present one is kept; no spurious keys are invented + auto toPrune = poolKeysToPrune(["a:27017", "b:27017", "c:27017"], ["a:27017", "c:27017"]); + assert(toPrune == ["b:27017"], "only the pool whose host left the topology is pruned"); + + assert(poolKeysToPrune(["a:27017"], ["a:27017"]).length == 0, + "a host still in the topology keeps its pool"); + assert(poolKeysToPrune([], ["a:27017"]).length == 0, + "a newly-desired host with no pool yet produces nothing to prune"); +} + +/// MongoClientHandle runs its stop action exactly once when it leaves scope. +unittest +{ + int stops; + { + auto handle = MongoClientHandle(null, () @safe { stops++; }); + } // ~this runs here + + assert(stops == 1, "leaving scope runs the stop action exactly once"); +} + +/// release() relinquishes ownership so the destructor does not stop monitoring. +unittest +{ + int stops; + MongoClient raw; + { + auto handle = MongoClientHandle(null, () @safe { stops++; }); + raw = handle.release(); + } // ~this runs here, but ownership was released + + assert(stops == 0, "release suppresses the stop action"); + assert(raw is null, "release hands back the owned client"); +} + +/// Moving a handle transfers ownership: the stop action runs once, from the destination only. +unittest +{ + import std.algorithm.mutation : move; + + int stops; + { + auto src = MongoClientHandle(null, () @safe { stops++; }); + { + auto dst = move(src); + assert(stops == 0, "moving the handle does not run the stop action"); + } // dst ~this runs here + + assert(stops == 1, "the move destination runs the stop action exactly once"); + } // src ~this runs here on the moved-from handle + + assert(stops == 1, "the moved-from source does not run the stop action again"); } diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index f654600333..3d8c545bdf 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -13,9 +13,12 @@ public import vibe.db.mongo.flags; public import vibe.db.mongo.impl.index; public import vibe.db.mongo.impl.crud; +public import vibe.db.mongo.impl.wireversion; import vibe.core.log; import vibe.db.mongo.client; +import vibe.db.mongo.impl.commands : splitNamespace, buildDeleteCommand, buildUpdateCommand, buildCountPipeline, buildAggregateCommand; +import vibe.db.mongo.settings : ReadPreference; import core.time; import std.algorithm : among, countUntil, find, findSplit; @@ -52,9 +55,10 @@ struct MongoCollection { auto dotidx = fullPath.indexOf('.'); assert(dotidx > 0, "The collection name passed to MongoCollection must be of the form \"dbname.collectionname\"."); + auto ns = splitNamespace(fullPath); m_fullPath = fullPath; - m_db = m_client.getDatabase(fullPath[0 .. dotidx]); - m_name = fullPath[dotidx+1 .. $]; + m_db = m_client.getDatabase(ns.database); + m_name = ns.collection; m_readConcern = m_db.readConcern; } @@ -108,7 +112,7 @@ struct MongoCollection { void update(T, U)(T selector, U update, UpdateFlags flags = UpdateFlags.None) { assert(m_client !is null, "Updating uninitialized MongoCollection."); - auto conn = m_client.lockConnection(); + auto conn = m_client.lockConnectionToPrimary(); ubyte[256] selector_buf = void, update_buf = void; conn.update(m_fullPath, flags, serializeToBson(selector, selector_buf), serializeToBson(update, update_buf)); } @@ -128,7 +132,7 @@ struct MongoCollection { void insert(T)(T document_or_documents, InsertFlags flags = InsertFlags.None) { assert(m_client !is null, "Inserting into uninitialized MongoCollection."); - auto conn = m_client.lockConnection(); + auto conn = m_client.lockConnectionToPrimary(); Bson[] docs; Bson bdocs = () @trusted { return serializeToBson(document_or_documents); } (); if( bdocs.type == Bson.Type.Array ) docs = cast(Bson[])bdocs; @@ -155,15 +159,16 @@ struct MongoCollection { InsertOneResult res; if ("_id" !in doc.get!(Bson[string])) { - doc["_id"] = Bson(res.insertedId = BsonObjectID.generate); + res.insertedId = Bson(BsonObjectID.generate); + doc["_id"] = res.insertedId; } cmd["documents"] = Bson([doc]); - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); foreach (string k, v; serializeToBson(options).byKeyValue) cmd[k] = v; - database.runCommandChecked(cmd).handleWriteResult(res); + database.runWriteCommandChecked(cmd).handleWriteResult(res); return res; } @@ -187,13 +192,13 @@ struct MongoCollection { } } cmd["documents"] = Bson(arr); - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); foreach (string k, v; serializeToBson(options).byKeyValue) cmd[k] = v; auto res = InsertManyResult(insertedIds); - database.runCommandChecked(cmd).handleWriteResult!"insertedCount"(res); + database.runWriteCommandChecked(cmd).handleWriteResult!"insertedCount"(res); return res; } @@ -223,7 +228,7 @@ struct MongoCollection { @safe if (!is(T == DeleteOptions)) { - return deleteImpl([filter], options); + return deleteImpl([filter], options, null); } /** @@ -236,7 +241,7 @@ struct MongoCollection { */ DeleteResult deleteAll(DeleteOptions options = DeleteOptions.init) @safe { - return deleteImpl([Bson.emptyObject], options); + return deleteImpl([Bson.emptyObject], options, null); } /// Implementation helper. It's possible to set custom delete limits with @@ -245,36 +250,17 @@ struct MongoCollection { @safe { assert(m_client !is null, "Querying uninitialized MongoCollection."); - alias FieldsMovedIntoChildren = AliasSeq!("limit", "collation", "hint"); - - Bson cmd = Bson.emptyObject; // empty object because order is important - cmd["delete"] = Bson(m_name); - - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); - auto optionsBson = serializeToBson(options); - foreach (string k, v; optionsBson.byKeyValue) - if (!k.among!FieldsMovedIntoChildren) - cmd[k] = v; - Bson[] deletesBson = new Bson[queries.length]; + Bson[] queryBsons = new Bson[queries.length]; foreach (i, q; queries) - { - auto deleteBson = Bson.emptyObject; - deleteBson["q"] = serializeToBson(q); - foreach (string k, v; optionsBson.byKeyValue) - if (k.among!FieldsMovedIntoChildren) - deleteBson[k] = v; - if (i < limits.length) - deleteBson["limit"] = Bson(limits[i]); - else - deleteBson["limit"] = Bson(0); - deletesBson[i] = deleteBson; - } - cmd["deletes"] = Bson(deletesBson); + queryBsons[i] = serializeToBson(q); + + Bson cmd = buildDeleteCommand(m_name, queryBsons, serializeToBson(options), limits); DeleteResult res; - database.runCommandChecked(cmd).handleWriteResult!"deletedCount"(res); + database.runWriteCommandChecked(cmd).handleWriteResult!"deletedCount"(res); return res; } @@ -371,27 +357,15 @@ struct MongoCollection { { assert(m_client !is null, "Querying uninitialized MongoCollection."); - alias FieldsMovedIntoChildren = AliasSeq!("arrayFilters", - "collation", - "hint", - "upsert"); - - Bson cmd = Bson.emptyObject; // empty object because order is important - cmd["update"] = Bson(m_name); - - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); - auto optionsBson = serializeToBson(options); - foreach (string k, v; optionsBson.byKeyValue) - if (!k.among!FieldsMovedIntoChildren) - cmd[k] = v; - Bson[] updatesBson = new Bson[queries.length]; + Bson[] queryBsons = new Bson[queries.length]; + Bson[] documentBsons = new Bson[queries.length]; + Bson[] perUpdateOptionBsons = new Bson[queries.length]; foreach (i, q; queries) { - auto updateBson = Bson.emptyObject; - auto qbson = serializeToBson(q); - updateBson["q"] = qbson; + queryBsons[i] = serializeToBson(q); auto ubson = serializeToBson(documents[i]); if (mustBeDocument) { @@ -429,17 +403,13 @@ struct MongoCollection { ~ "(this update call would otherwise replace the entire matched object with the passed in update object)"); } } - updateBson["u"] = ubson; - foreach (string k, v; optionsBson.byKeyValue) - if (k.among!FieldsMovedIntoChildren) - updateBson[k] = v; - foreach (string k, v; perUpdateOptions[i].byKeyValue) - updateBson[k] = v; - updatesBson[i] = updateBson; + documentBsons[i] = ubson; + perUpdateOptionBsons[i] = serializeToBson(perUpdateOptions[i]); } - cmd["updates"] = Bson(updatesBson); - auto res = database.runCommandChecked(cmd); + Bson cmd = buildUpdateCommand(m_name, queryBsons, documentBsons, perUpdateOptionBsons, serializeToBson(options)); + + auto res = database.runWriteCommandChecked(cmd); auto ret = UpdateResult( res["n"].to!long, res["nModified"].to!long, @@ -451,7 +421,7 @@ struct MongoCollection { ret.upsertedIds.length = upserted.length; foreach (i, upsert; upserted) { - ret.upsertedIds[i] = upsert["_id"].get!BsonObjectID; + ret.upsertedIds[i] = upsert["_id"]; } } return ret; @@ -663,7 +633,7 @@ struct MongoCollection { void remove(T)(T selector, DeleteFlags flags = DeleteFlags.None) { assert(m_client !is null, "Removing from uninitialized MongoCollection."); - auto conn = m_client.lockConnection(); + auto conn = m_client.lockConnectionToPrimary(); ubyte[256] selector_buf = void; conn.delete_(m_fullPath, flags, serializeToBson(selector, selector_buf)); } @@ -699,7 +669,7 @@ struct MongoCollection { cmd.query = query; cmd.update = update; cmd.fields = returnFieldSelector; - auto ret = database.runCommandChecked(cmd); + auto ret = database.runWriteCommandChecked(cmd); return ret["value"]; } @@ -738,7 +708,7 @@ struct MongoCollection { cmd[key] = value; return 0; }); - auto ret = database.runCommandChecked(cmd); + auto ret = database.runWriteCommandChecked(cmd); return ret["value"]; } @@ -759,7 +729,8 @@ struct MongoCollection { return countImpl!T(query); } - private ulong countImpl(T)(T query, Nullable!ReadConcern readConcern = Nullable!ReadConcern.init) + private ulong countImpl(T)(T query, Nullable!ReadConcern readConcern = Nullable!ReadConcern.init, + Nullable!ReadPreference readPreference = Nullable!ReadPreference.init) { Bson cmd = Bson.emptyObject; cmd["count"] = m_name; @@ -771,7 +742,7 @@ struct MongoCollection { cmd["readConcern"] = serializeToBson(m_readConcern); } - auto reply = database.runCommandChecked(cmd); + auto reply = database.runCommandChecked(cmd, __FUNCTION__, __FILE__, __LINE__, false, readPreference); switch (reply["n"].type) with (Bson.Type) { default: assert(false, "Unsupported data type in BSON reply for COUNT"); case double_: return cast(ulong)reply["n"].get!double; // v2.x @@ -793,16 +764,7 @@ struct MongoCollection { */ ulong countDocuments(T)(T filter, CountOptions options = CountOptions.init) { - // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#count-api-details - Bson[] pipeline = [Bson(["$match": serializeToBson(filter)])]; - if (!options.skip.isNull) - pipeline ~= Bson(["$skip": Bson(options.skip.get)]); - if (!options.limit.isNull) - pipeline ~= Bson(["$limit": Bson(options.limit.get)]); - pipeline ~= Bson(["$group": Bson([ - "_id": Bson(1), - "n": Bson(["$sum": Bson(1)]) - ])]); + Bson[] pipeline = buildCountPipeline(serializeToBson(filter), options); AggregateOptions aggOptions; foreach (i, field; options.tupleof) { @@ -836,6 +798,7 @@ struct MongoCollection { AggregateOptions aggOptions; aggOptions.maxTimeMS = options.maxTimeMS; aggOptions.readConcern = options.readConcern; + aggOptions.readPreference = options.readPreference; try { auto reply = aggregate(pipeline, aggOptions).front; @@ -846,7 +809,7 @@ struct MongoCollection { return 0; } } else { - return countImpl(null, options.readConcern); + return countImpl(null, options.readConcern, options.readPreference); } } @@ -888,24 +851,13 @@ struct MongoCollection { assert(m_client !is null, "Querying uninitialized MongoCollection."); applyDefaultReadConcern(options); - Bson cmd = Bson.emptyObject; // empty object because order is important - cmd["aggregate"] = Bson(m_name); - cmd["$db"] = Bson(m_db.name); - cmd["pipeline"] = serializeToBson(pipeline); MongoConnection conn = m_client.lockConnection(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); - foreach (string k, v; serializeToBson(options).byKeyValue) - { - // spec recommends to omit cursor field when explain is true - if (!options.explain.isNull && options.explain.get && k == "cursor") - continue; - cmd[k] = v; - } - return MongoCursor!R(m_client, cmd, - !options.batchSize.isNull ? options.batchSize.get : 0, - !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs - : !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs - : Duration.max); + + auto pref = options.readPreference.isNull ? m_client.readPreference : options.readPreference.get; + auto result = buildAggregateCommand(m_name, m_db.name, serializeToBson(pipeline), options, pref, m_client.readPreferenceTags); + + return MongoCursor!R(m_client, result.command, result.batchSize, result.getMoreMaxTime, Nullable!ReadPreference(pref)); } /// Example taken from the MongoDB documentation @@ -970,7 +922,7 @@ struct MongoCollection { import std.algorithm : map; - auto res = m_db.runCommandChecked(cmd); + auto res = m_db.runCommandChecked(cmd, __FUNCTION__, __FILE__, __LINE__, false, options.readPreference); static if (is(R == Bson)) return res["values"].byValue; else return res["values"].byValue.map!(b => deserializeBson!R(b)); } @@ -1050,7 +1002,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = name; - database.runCommandChecked(cmd); + database.runWriteCommandChecked(cmd); } /// ditto @@ -1098,14 +1050,14 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = "*"; - database.runCommandChecked(cmd); + database.runWriteCommandChecked(cmd); } /// Unofficial API extension, more efficient multi-index removal on /// MongoDB 4.2+ void dropIndexes(string[] names, DropIndexOptions options = DropIndexOptions.init) @safe { - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); if (conn.description.satisfiesVersion(WireVersion.v42)) { static struct CMD { string dropIndexes; @@ -1115,7 +1067,7 @@ struct MongoCollection { CMD cmd; cmd.dropIndexes = m_name; cmd.index = names; - database.runCommandChecked(cmd); + database.runWriteCommandChecked(cmd); } else { foreach (name; names) dropIndex(name); @@ -1198,7 +1150,7 @@ struct MongoCollection { @safe { string[] keys = new string[models.length]; - MongoConnection conn = m_client.lockConnection(); + MongoConnection conn = m_client.lockConnectionToPrimary(); if (conn.description.satisfiesVersion(WireVersion.v26)) { Bson cmd = Bson.emptyObject; cmd["createIndexes"] = m_name; @@ -1214,7 +1166,7 @@ struct MongoCollection { indexes ~= index; } cmd["indexes"] = Bson(indexes); - database.runCommandChecked(cmd); + database.runWriteCommandChecked(cmd); } else { foreach (model; models) { // trusted to support old compilers which think opt_dup has @@ -1276,7 +1228,7 @@ struct MongoCollection { CMD cmd; cmd.drop = m_name; - auto reply = database.runCommandUnchecked(cmd); + auto reply = database.runWriteCommandUnchecked(cmd); if (reply["ok"].get!double != 1.0) { auto code = reply["code"].opt!int(0); if (code != 26) // NamespaceNotFound @@ -1454,304 +1406,3 @@ struct CursorInitArguments { @embedNullable Nullable!int batchSize; } -/// UDA to unset a nullable field if the server wire version doesn't at least -/// match the given version. (inclusive) -/// -/// Use with $(LREF enforceWireVersionConstraints) -struct MinWireVersion -{ - /// - WireVersion v; -} - -/// ditto -MinWireVersion since(WireVersion v) @safe { return MinWireVersion(v); } - -/// UDA to warn when a nullable field is set and the server wire version matches -/// the given version. (inclusive) -/// -/// Use with $(LREF enforceWireVersionConstraints) -struct DeprecatedSinceWireVersion -{ - /// - WireVersion v; -} - -/// ditto -DeprecatedSinceWireVersion deprecatedSince(WireVersion v) @safe { return DeprecatedSinceWireVersion(v); } - -/// UDA to throw a MongoException when a nullable field is set and the server -/// wire version doesn't match the version. (inclusive) -/// -/// Use with $(LREF enforceWireVersionConstraints) -struct ErrorBeforeWireVersion -{ - /// - WireVersion v; -} - -/// ditto -ErrorBeforeWireVersion errorBefore(WireVersion v) @safe { return ErrorBeforeWireVersion(v); } - -/// UDA to unset a nullable field if the server wire version is newer than the -/// given version. (inclusive) -/// -/// Use with $(LREF enforceWireVersionConstraints) -struct MaxWireVersion -{ - /// - WireVersion v; -} -/// ditto -MaxWireVersion until(WireVersion v) @safe { return MaxWireVersion(v); } - -/// Unsets nullable fields not matching the server version as defined per UDAs. -void enforceWireVersionConstraints(T)(ref T field, int serverVersion, - string file = __FILE__, size_t line = __LINE__) -@safe { - import std.traits : getUDAs; - - string exception; - - foreach (i, ref v; field.tupleof) { - enum minV = getUDAs!(field.tupleof[i], MinWireVersion); - enum maxV = getUDAs!(field.tupleof[i], MaxWireVersion); - enum deprecateV = getUDAs!(field.tupleof[i], DeprecatedSinceWireVersion); - enum errorV = getUDAs!(field.tupleof[i], ErrorBeforeWireVersion); - - static foreach (depr; deprecateV) - if (serverVersion >= depr.v && !v.isNull) - logInfo("User-set field '%s' is deprecated since MongoDB %s (from %s:%s)", - T.tupleof[i].stringof, depr.v, file, line); - - static foreach (err; errorV) - if (serverVersion < err.v && !v.isNull) - exception ~= format("User-set field '%s' is not supported before MongoDB %s\n", - T.tupleof[i].stringof, err.v); - - static foreach (min; minV) - if (serverVersion < min.v) - v.nullify(); - - static foreach (max; maxV) - if (serverVersion > max.v) - v.nullify(); - } - - if (exception.length) - throw new MongoException(exception ~ "from " ~ file ~ ":" ~ line.to!string); -} - -version (unittest) -{ - struct SinceUntilCmd - { - @embedNullable @since(WireVersion.v34) - Nullable!int a; - - @embedNullable @until(WireVersion.v30) - Nullable!int b; - } - - struct ErrorBeforeCmd - { - @embedNullable @errorBefore(WireVersion.v44) - Nullable!int field; - } - - struct DeprecatedCmd - { - @embedNullable @deprecatedSince(WireVersion.v40) - Nullable!int oldField; - } - - struct CombinedCmd - { - @embedNullable @errorBefore(WireVersion.v44) - Nullable!bool allowDiskUse; - - @embedNullable @since(WireVersion.v32) - Nullable!long maxAwaitTimeMS; - - @embedNullable @deprecatedSince(WireVersion.v40) - Nullable!long maxScan; - } - - struct SinceDeprecatedCmd - { - @embedNullable @since(WireVersion.v32) - Nullable!long maxAwaitTimeMS; - - @embedNullable @deprecatedSince(WireVersion.v40) - Nullable!long maxScan; - } -} - -/// @since nullifies field when server version is below minimum -@safe unittest -{ - SinceUntilCmd cmd; - cmd.a = 1; - cmd.b = 2; - - auto test = cmd; - enforceWireVersionConstraints(test, WireVersion.v30); - assert(test.a.isNull); - assert(!test.b.isNull); -} - -/// @until nullifies field when server version exceeds maximum -@safe unittest -{ - SinceUntilCmd cmd; - cmd.a = 1; - cmd.b = 2; - - auto test = cmd; - enforceWireVersionConstraints(test, WireVersion.v32); - assert(test.a.isNull); - assert(test.b.isNull); -} - -/// @since preserves field when server version meets minimum -@safe unittest -{ - SinceUntilCmd cmd; - cmd.a = 1; - cmd.b = 2; - - auto test = cmd; - enforceWireVersionConstraints(test, WireVersion.v34); - assert(!test.a.isNull); - assert(test.b.isNull); -} - -/// @errorBefore throws when field is set and server version is below threshold -@safe unittest -{ - ErrorBeforeCmd cmd; - cmd.field = 42; - try { - enforceWireVersionConstraints(cmd, WireVersion.v40); - assert(false, "Should have thrown"); - } catch (MongoException e) { - // expected - } -} - -/// @errorBefore does not throw when field is set and server version is at threshold -@safe unittest -{ - ErrorBeforeCmd cmd; - cmd.field = 42; - enforceWireVersionConstraints(cmd, WireVersion.v44); - assert(!cmd.field.isNull); -} - -/// @errorBefore does not throw when field is set and server version is above threshold -@safe unittest -{ - ErrorBeforeCmd cmd; - cmd.field = 42; - enforceWireVersionConstraints(cmd, WireVersion.v60); - assert(!cmd.field.isNull); -} - -/// @errorBefore does not throw when field is not set -@safe unittest -{ - ErrorBeforeCmd cmd; - enforceWireVersionConstraints(cmd, WireVersion.v30); - assert(cmd.field.isNull); -} - -/// @deprecatedSince preserves field and only logs at deprecated version -@safe unittest -{ - DeprecatedCmd cmd; - cmd.oldField = 10; - enforceWireVersionConstraints(cmd, WireVersion.v40); - assert(!cmd.oldField.isNull); - assert(cmd.oldField.get == 10); -} - -/// @deprecatedSince preserves field above deprecated version -@safe unittest -{ - DeprecatedCmd cmd; - cmd.oldField = 10; - enforceWireVersionConstraints(cmd, WireVersion.v60); - assert(!cmd.oldField.isNull); -} - -/// @deprecatedSince preserves field below deprecated version without warning -@safe unittest -{ - DeprecatedCmd cmd; - cmd.oldField = 10; - enforceWireVersionConstraints(cmd, WireVersion.v36); - assert(!cmd.oldField.isNull); -} - -/// @deprecatedSince does nothing when field is not set -@safe unittest -{ - DeprecatedCmd cmd; - enforceWireVersionConstraints(cmd, WireVersion.v60); - assert(cmd.oldField.isNull); -} - -/// Combined UDAs: @errorBefore throws while @since and @deprecatedSince still apply -@safe unittest -{ - CombinedCmd cmd; - cmd.allowDiskUse = true; - cmd.maxAwaitTimeMS = 5000; - cmd.maxScan = 100; - - auto t1 = cmd; - try { - enforceWireVersionConstraints(t1, WireVersion.v30); - assert(false, "Should have thrown due to errorBefore(v44)"); - } catch (MongoException e) { - // expected - } -} - -/// Combined UDAs: all fields valid at v44, @deprecatedSince only logs -@safe unittest -{ - CombinedCmd cmd; - cmd.allowDiskUse = true; - cmd.maxAwaitTimeMS = 5000; - cmd.maxScan = 100; - - enforceWireVersionConstraints(cmd, WireVersion.v44); - assert(!cmd.allowDiskUse.isNull); - assert(!cmd.maxAwaitTimeMS.isNull); - assert(!cmd.maxScan.isNull); -} - -/// Combined UDAs: @since nullifies field below minimum while others are independent -@safe unittest -{ - SinceDeprecatedCmd cmd; - cmd.maxAwaitTimeMS = 5000; - cmd.maxScan = 100; - - enforceWireVersionConstraints(cmd, WireVersion.v30); - assert(cmd.maxAwaitTimeMS.isNull); - assert(!cmd.maxScan.isNull); -} - -/// Combined UDAs: @since preserves field at sufficient version -@safe unittest -{ - SinceDeprecatedCmd cmd; - cmd.maxAwaitTimeMS = 5000; - cmd.maxScan = 100; - - enforceWireVersionConstraints(cmd, WireVersion.v34); - assert(!cmd.maxAwaitTimeMS.isNull); - assert(!cmd.maxScan.isNull); -} diff --git a/mongodb/vibe/db/mongo/connection.d b/mongodb/vibe/db/mongo/connection.d index 3bc93685e9..34c31989ed 100644 --- a/mongodb/vibe/db/mongo/connection.d +++ b/mongodb/vibe/db/mongo/connection.d @@ -11,12 +11,18 @@ module vibe.db.mongo.connection; // debug = VibeVerboseMongo; public import vibe.data.bson; +public import vibe.db.mongo.impl.wireversion; +public import vibe.db.mongo.impl.serverdescription; import vibe.core.core : vibeVersionString; import vibe.core.log; import vibe.core.net; import vibe.data.bson; import vibe.db.mongo.flags; +import vibe.db.mongo.impl.compression; +import vibe.db.mongo.impl.clustertime; +import vibe.db.mongo.impl.wire; +import vibe.db.mongo.monitor : MongoServerErrorCode; import vibe.db.mongo.settings; import vibe.db.mongo.topology; import vibe.inet.webform; @@ -63,6 +69,28 @@ class MongoException : Exception { super(message, file, line, next); } + + /// Server-reported error labels (e.g. "TransientTransactionError"). + string[] errorLabels; + + /// Server-reported error code as a MongoServerErrorCode (none when there is no error). + MongoServerErrorCode code; + + /// Whether the given server error label is present. + bool hasErrorLabel(string label) const + { + import std.algorithm : canFind; + return errorLabels.canFind(label); + } +} + +/// A MongoException carries server error labels and reports a present one via hasErrorLabel. +unittest +{ + auto e = new MongoException("transient failure"); + e.errorLabels = ["TransientTransactionError"]; + assert(e.hasErrorLabel("TransientTransactionError") == true, + "hasErrorLabel must return true for an attached label"); } /** @@ -78,6 +106,12 @@ class MongoDriverException : MongoException { super(message, file, line, next); } + + this(string message, MongoServerErrorCode code, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + this.code = code; + } } /** @@ -121,6 +155,244 @@ class MongoAuthException : MongoException { super(message, file, line, next); } + + this(string message, MongoServerErrorCode code, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + this.code = code; + } +} + +/** + * Thrown when the contacted mongo node is no longer primary (e.g. step down). + * + * Carries the server-reported error code (e.g. 10107). + */ +class MongoStepDownException : MongoDriverException +{ +@safe: + + this(string message, MongoServerErrorCode code, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + this.code = code; + } +} + +unittest +{ + auto stepDown = new MongoStepDownException("not primary", MongoServerErrorCode.notWritablePrimary); + assert(stepDown.code == MongoServerErrorCode.notWritablePrimary, "expected stored code notWritablePrimary"); + assert(cast(MongoDriverException)stepDown !is null, + "MongoStepDownException must be catchable as MongoDriverException"); +} + +/** + * Thrown when a connection-level (network) failure interrupts an operation, + * e.g. a socket error or a dropped connection. Retryable for writes with + * session support and for idempotent reads. + */ +class MongoNetworkException : MongoDriverException +{ +@safe: + + this(string message, string file = __FILE__, size_t line = __LINE__, Throwable next = null) + { + super(message, file, line, next); + } +} + +/// MongoNetworkException is a MongoDriverException subclass +unittest +{ + auto networkFailure = new MongoNetworkException("connection reset"); + assert(cast(MongoDriverException) networkFailure !is null, + "MongoNetworkException must be catchable as a MongoDriverException"); +} + +/// MongoDriverException can carry a server error code +unittest +{ + assert(new MongoDriverException("x", MongoServerErrorCode.networkTimeout).code == MongoServerErrorCode.networkTimeout, + "MongoDriverException(message, code) carries the code"); +} + +/// asNetworkError passes a MongoException through unchanged +unittest +{ + auto mongo = new MongoDriverException("boom"); + assert(asNetworkError(mongo) is mongo, "a MongoException must pass through unchanged"); +} + +/// asNetworkError wraps a non-Mongo exception as a MongoNetworkException +unittest +{ + auto raw = new Exception("socket reset"); + auto wrapped = cast(MongoNetworkException) asNetworkError(raw); + assert(wrapped !is null, "a non-Mongo exception becomes a MongoNetworkException"); + assert(wrapped.next is raw, "the original exception is preserved as the cause"); +} + +/// Builds the exception for a non-ok command response: a `MongoStepDownException` for +/// stale-topology codes, otherwise the generic `FallbackException`. In both cases the +/// returned exception carries the server `code`. +Exception commandFailureException(FallbackException = MongoDriverException)( + string message, MongoServerErrorCode code, string[] errorLabels = null) @safe +{ + import vibe.db.mongo.monitor : isStaleTopologyError; + + MongoException e; + if (isStaleTopologyError(code)) + e = new MongoStepDownException(message, code); + else + e = new FallbackException(message, code); + + e.errorLabels = errorLabels; + return e; +} + +unittest +{ + auto e = commandFailureException("primary stepped down", MongoServerErrorCode.notWritablePrimary); + assert(cast(MongoStepDownException) e !is null, + "stale code notWritablePrimary must yield a MongoStepDownException"); + assert((cast(MongoStepDownException) e).code == MongoServerErrorCode.notWritablePrimary, + "step-down exception must carry the server code notWritablePrimary"); +} + +unittest +{ + auto e = commandFailureException("duplicate key", MongoServerErrorCode.duplicateKey); + assert(cast(MongoStepDownException) e is null, + "a non-stale code must not be classified as a step-down"); + assert(cast(MongoDriverException) e !is null, + "a non-stale command failure stays a generic MongoDriverException"); +} + +/// A non-stale command failure carries its server error code on the MongoException base. +unittest +{ + auto e = commandFailureException("network timeout", MongoServerErrorCode.networkTimeout); + assert((cast(MongoException) e).code == MongoServerErrorCode.networkTimeout, + "a non-stale command failure must carry its server error code"); +} + +/// commandFailureException attaches the server-reported error labels to the exception +unittest +{ + auto e = commandFailureException("transient failure", MongoServerErrorCode.duplicateKey, + ["TransientTransactionError"]); + assert((cast(MongoException) e).hasErrorLabel("TransientTransactionError"), + "commandFailureException must attach the reply's error labels so hasErrorLabel works"); +} + +/// Extracts the server-reported `errorLabels` array from a command reply +/// (e.g. ["TransientTransactionError"]); an empty array when none are present. +string[] parseErrorLabels(Bson reply) @safe +{ + return reply["errorLabels"].opt!(Bson[]).map!(b => b.get!string).array; +} + +/// parseErrorLabels extracts the errorLabels array from a command reply +unittest +{ + auto reply = Bson(["errorLabels": Bson([Bson("TransientTransactionError"), Bson("RetryableWriteError")])]); + assert(parseErrorLabels(reply) == ["TransientTransactionError", "RetryableWriteError"], + "parseErrorLabels returns the reply's errorLabels in order"); +} + +/// parseErrorLabels yields an empty array for replies without a proper errorLabels array +unittest +{ + // absent field (a normal successful reply) + assert(parseErrorLabels(Bson(["ok": Bson(1.0)])) == [], + "a reply without errorLabels yields no labels"); + // present but not an array (malformed/hostile reply) must not throw + assert(parseErrorLabels(Bson(["errorLabels": Bson("oops")])) == [], + "a non-array errorLabels yields no labels rather than throwing"); +} + +/// Reads the server-reported error `code` from a command reply, defaulting to 0 +/// (unknown) when the reply omits it. +MongoServerErrorCode serverErrorCode(Bson reply) @safe +{ + return cast(MongoServerErrorCode) reply["code"].opt!int(0); +} + +/// Reads the `writeConcernError.code` from a write reply, or `none` when absent. An ok:1 +/// reply can still carry a transient writeConcernError (e.g. 91 ShutdownInProgress) that a +/// retryable write must retry, so this is inspected separately from the top-level code. +MongoServerErrorCode writeConcernErrorCode(Bson reply) @safe +{ + auto wce = reply["writeConcernError"]; + if (wce.type != Bson.Type.object) + return MongoServerErrorCode.none; + return cast(MongoServerErrorCode) wce["code"].opt!int(0); +} + +/// writeConcernErrorCode reads a transient writeConcernError code from an ok:1 reply +unittest +{ + auto reply = Bson([ + "ok": Bson(1.0), + "writeConcernError": Bson(["code": Bson(91), "errmsg": Bson("ShutdownInProgress")]) + ]); + assert(writeConcernErrorCode(reply) == MongoServerErrorCode.shutdownInProgress, + "the writeConcernError code is read from an ok:1 reply"); + assert(writeConcernErrorCode(Bson(["ok": Bson(1.0)])) == MongoServerErrorCode.none, + "a reply without a writeConcernError yields none"); +} + +/// serverErrorCode reads the reply's code, falling back to 0 when absent +unittest +{ + assert(serverErrorCode(Bson(["code": Bson(112)])) == cast(MongoServerErrorCode) 112, + "serverErrorCode returns the reply's code"); + assert(serverErrorCode(Bson(["ok": Bson(1.0)])) == cast(MongoServerErrorCode) 0, + "a reply without a code yields 0"); +} + +/// Builds the command-failure exception from a non-ok reply: message from `errmsg`, +/// the server `code`, and the reply's `errorLabels` (so `hasErrorLabel` works). +Exception commandFailureFromReply(FallbackException = MongoDriverException)( + Bson reply, string errorInfo, string errorFile, size_t errorLine) @safe +{ + return commandFailureException!FallbackException( + formatCommandError("command failed: " ~ reply["errmsg"].opt!string("(no message)"), errorInfo, errorFile, errorLine), + serverErrorCode(reply), parseErrorLabels(reply)); +} + +/// commandFailureFromReply builds the failure exception from a reply, carrying its error labels and code +unittest +{ + auto reply = Bson([ + "ok": Bson(0.0), + "code": Bson(112), + "errmsg": Bson("WriteConflict"), + "errorLabels": Bson([Bson("TransientTransactionError")]), + ]); + + auto e = cast(MongoException) commandFailureFromReply(reply, "ctx", "file.d", 1); + assert(e.hasErrorLabel("TransientTransactionError"), + "the exception built from a failing reply carries the reply's error labels"); + assert(e.code == cast(MongoServerErrorCode) 112, + "the exception built from a failing reply carries the server code"); +} + +/// Classifies a thrown exception from the wire exchange: a MongoException passes through +/// unchanged; any other (connection-level) exception becomes a retryable MongoNetworkException. +Exception asNetworkError(Exception e) @safe +{ + if (cast(MongoException) e !is null) + return e; + return new MongoNetworkException(e.msg, __FILE__, __LINE__, e); +} + +/// Appends the originating command's call-site context to an error message so a +/// failure points back at the caller rather than this protocol module. +private string formatCommandError(string msg, string errorInfo, string errorFile, size_t errorLine) @safe +{ + return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); } /** @@ -145,11 +417,16 @@ final class MongoConnection { StreamOutputRange!(InterfaceProxy!Stream) m_outRange; ServerDescription m_description; MongoHost m_connectedHost; + /// Hook invoked with (host, error code) when a command fails. + void delegate(MongoHost host, MongoServerErrorCode code) @safe nothrow m_onCommandError; /// Flag to prevent recursive connections when server closes connection while connecting bool m_allowReconnect; bool m_isAuthenticating; bool m_supportsOpMsg; Compressor m_negotiatedCompressor = Compressor.noop; + /// Highest `$clusterTime` observed on a reply; gossiped back on every command + /// for causal consistency. Null until the first cluster-time-bearing reply. + Bson m_clusterTime = Bson(null); } enum ushort defaultPort = MongoClientSettings.defaultPort; @@ -166,7 +443,17 @@ final class MongoConnection { m_settings = cfg; } + /// Sets the hook called with (host, error code) on command failure. + package void onCommandError(void delegate(MongoHost host, MongoServerErrorCode code) @safe nothrow handler) + { + m_onCommandError = handler; + } + void connectToHost(MongoHost host, bool doAuthenticate = true) { + // Reset before the handshake so a reconnect's hello/speculative-auth is + // never sent OP_COMPRESSED with the previous connection's stale codec; + // compression only applies once it's re-negotiated below. + m_negotiatedCompressor = Compressor.noop; bool isTLS; /* @@ -206,7 +493,7 @@ final class MongoConnection { m_outRange = streamOutputRange(m_stream); } catch (Exception e) { - throw new MongoDriverException(format("Failed to connect to MongoDB server at %s:%s.", host.name, host.port), __FILE__, __LINE__, e); + throw new MongoNetworkException(format("Failed to connect to MongoDB server at %s:%s.", host.name, host.port), __FILE__, __LINE__, e); } scope (failure) disconnect(); @@ -216,9 +503,8 @@ final class MongoConnection { m_allowReconnect = true; Bson handshake = Bson.emptyObject; - static assert(!is(typeof(m_settings.loadBalanced)), "loadBalanced was added to the API, set legacy if it's true here!"); - // TODO: must use legacy handshake if m_settings.loadBalanced is true - // and also once we allow configuring a server API version in the driver + // TODO: must use legacy handshake once we allow configuring a server API + // version in the driver // (https://github.com/mongodb/specifications/blob/master/source/versioned-api/versioned-api.rst) m_supportsOpMsg = false; bool legacyHandshake = false; @@ -288,15 +574,11 @@ final class MongoConnection { } } - if (m_settings.compressors.length > 0) { - Bson[] compressorNames; - foreach (c; m_settings.compressors) { - compressorNames ~= Bson(compressorName(c)); - } - handshake["compression"] = Bson(compressorNames); - } + auto advertised = advertisedCompressorNames(m_settings.compressors); + if (advertised.length > 0) + handshake["compression"] = Bson(advertised.map!(name => Bson(name)).array); - auto reply = runCommand!(Bson, MongoAuthException)("admin", handshake); + auto reply = runCommand!MongoAuthException("admin", handshake); m_description = deserializeBson!ServerDescription(reply); if (m_description.satisfiesVersion(WireVersion.v36)) @@ -329,9 +611,6 @@ final class MongoConnection { if (doAuthenticate) { auto authMechanism = m_settings.authMechanism; - if (authMechanism == MongoAuthMechanism.none && m_settings.sslPEMKeyFile != null && m_description.satisfiesVersion(WireVersion.v26)) - authMechanism = MongoAuthMechanism.mongoDBX509; - if (authMechanism == MongoAuthMechanism.none && (m_settings.digest.length || m_settings.password.length)) { if (serverSupportsSHA256 && m_settings.password.length) @@ -394,7 +673,7 @@ final class MongoConnection { break; } - logInfo("Connected to: %s primary=%s secondary=%s", m_description.me, m_description.isPrimary, m_description.secondary); + logDiagnostic("Connected to: %s primary=%s secondary=%s", m_description.me, m_description.isPrimary, m_description.secondary); } else { logDiagnostic("Probed: %s primary=%s secondary=%s", m_description.me, m_description.isPrimary, m_description.secondary); } @@ -433,8 +712,8 @@ final class MongoConnection { return false; auto status = m_conn.waitForDataEx(Duration.zero); - // timeout (wouldBlock) means the socket is alive but no data pending — that's fine - // dataAvailable means there's unread data — also alive + // timeout (wouldBlock) means the socket is alive but no data pending, which is fine + // dataAvailable means there's unread data, so the socket is also alive // noMoreData means the remote end closed the connection return status != typeof(status).noMoreData; } @@ -486,7 +765,7 @@ final class MongoConnection { `runCommand` overload, when the command response is not ok. - `MongoDriverException` when internal protocol errors occur. */ - Bson runCommand(T, CommandFailException = MongoDriverException)( + Bson runCommand(CommandFailException = MongoDriverException)( string database, Bson command, string errorInfo = __FUNCTION__, @@ -495,11 +774,11 @@ final class MongoConnection { ) in(database.length, "runCommand requires a database argument") { - return runCommandImpl!(T, CommandFailException)( + return runCommandImpl!CommandFailException( database, command, true, errorInfo, errorFile, errorLine); } - Bson runCommandUnchecked(T, CommandFailException = MongoDriverException)( + Bson runCommandUnchecked(CommandFailException = MongoDriverException)( string database, Bson command, string errorInfo = __FUNCTION__, @@ -508,11 +787,11 @@ final class MongoConnection { ) in(database.length, "runCommand requires a database argument") { - return runCommandImpl!(T, CommandFailException)( + return runCommandImpl!CommandFailException( database, command, false, errorInfo, errorFile, errorLine); } - private Bson runCommandImpl(T, CommandFailException)( + private Bson runCommandImpl(CommandFailException)( string database, Bson command, bool testOk = true, @@ -522,14 +801,17 @@ final class MongoConnection { ) in(database.length, "runCommand requires a database argument") { - import std.array; + Bson ret; - string formatErrorInfo(string msg) @safe - { - return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); - } + // Unlike the sibling cursor methods, disconnect() lives inside the send/recv + // catch blocks rather than a method-top `scope (failure) disconnect();`. A wire + // error desyncs the connection and must quarantine it, but the clean `ok != 1.0` + // command-failure path below fully reads a healthy connection and must keep it. + // A method-scoped guard would wrongly disconnect on that logical failure too. - Bson ret; + // Gossip the highest cluster time we've seen so the server advances causally. + // No-op until the first reply carries a $clusterTime (e.g. on a standalone). + command = gossipClusterTime(command, m_clusterTime); if (m_supportsOpMsg) { @@ -538,46 +820,67 @@ final class MongoConnection { command["$db"] = Bson(database); - auto id = sendMsg(-1, 0, command); - Appender!(Bson[])[string] docs; - recvMsg!true(id, (flags, root) @safe { - ret = root; - }, (scope ident, size) @safe { - docs[ident.idup] = appender!(Bson[]); - }, (scope ident, push) @safe { - auto pd = ident in docs; - assert(!!pd, "Received data for unexpected identifier"); - pd.put(push); - }); - - foreach (ident, app; docs) - ret[ident] = Bson(app.data); + try + { + auto id = sendMsg(-1, 0, command); + Appender!(Bson[])[string] docs; + recvMsg!true(id, (flags, root) @safe { + ret = root; + }, (scope ident, size) @safe { + docs[ident.idup] = appender!(Bson[]); + }, (scope ident, push) @safe { + auto pd = ident in docs; + enforce!MongoDriverException(!!pd, formatCommandError("Received data for unexpected identifier", errorInfo, errorFile, errorLine)); + pd.put(push); + }); + + foreach (ident, app; docs) + ret[ident] = Bson(app.data); + } + catch (Exception e) + { + disconnect(); + throw asNetworkError(e); + } } else { debug (VibeVerboseMongo) logDiagnostic("runCommand(legacy): [db=%s] %s", database, command); - auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); - recvReply!T(id, - (cursor, flags, first_doc, num_docs) { - logTrace("runCommand(%s) flags: %s, cursor: %s, documents: %s", database, flags, cursor, num_docs); - enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), formatErrorInfo("command query failed")); - enforce!MongoDriverException(num_docs == 1, formatErrorInfo("received more than one document in command response")); - }, - (idx, ref doc) { - ret = doc; - }); + try + { + auto id = send(OpCode.Query, -1, 0, database ~ ".$cmd", 0, -1, command, Bson(null)); + recvReply!Bson(id, + (cursor, flags, first_doc, num_docs) { + logTrace("runCommand(%s) flags: %s, cursor: %s, documents: %s", database, flags, cursor, num_docs); + enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), formatCommandError("command query failed", errorInfo, errorFile, errorLine)); + enforce!MongoDriverException(num_docs == 1, formatCommandError("received more than one document in command response", errorInfo, errorFile, errorLine)); + }, + (idx, ref doc) { + ret = doc; + }); + } + catch (Exception e) + { + disconnect(); + throw asNetworkError(e); + } } + // Observe the reply's $clusterTime even on command failure: a failed command + // still gossips a valid cluster time the driver must track. + m_clusterTime = laterClusterTime(m_clusterTime, ret["$clusterTime"]); + if (testOk && ret["ok"].get!double != 1.0) - throw new CommandFailException(formatErrorInfo("command failed: " - ~ ret["errmsg"].opt!string("(no message)"))); + { + auto code = serverErrorCode(ret); + if (m_onCommandError !is null) + m_onCommandError(m_connectedHost, code); - static if (is(T == Bson)) return ret; - else { - T doc = deserializeBson!T(bson); - return doc; + throw commandFailureFromReply!CommandFailException(ret, errorInfo, errorFile, errorLine); } + + return ret; } template getMore(T) @@ -608,6 +911,8 @@ final class MongoConnection { scope GetMoreHeaderDelegate on_header, scope GetMoreDocumentDelegate!T on_doc, Duration timeout = Duration.max, + Nullable!ReadPreference pref = Nullable!ReadPreference.init, + Bson sessionContext = Bson.emptyObject, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { Bson command = Bson.emptyObject; @@ -619,10 +924,12 @@ final class MongoConnection { if (timeout != Duration.max && timeout.total!"msecs" < int.max) command["maxTimeMS"] = Bson(cast(int)timeout.total!"msecs"); - string formatErrorInfo(string msg) @safe - { - return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); - } + // A secondary keeps serving getMore only if each continuation re-sends $readPreference. + if (!pref.isNull && pref.get != ReadPreference.primary) + command["$readPreference"] = readPreferenceBson(pref.get); + + foreach (string key, value; sessionContext.byKeyValue) + command[key] = value; scope (failure) disconnect(); @@ -645,9 +952,9 @@ final class MongoConnection { recvReply!T(id, (long cursor, ReplyFlags flags, int first_doc, int num_docs) { enforce!MongoDriverException(!(flags & ReplyFlags.CursorNotFound), - formatErrorInfo("Invalid cursor handle.")); + formatCommandError("Invalid cursor handle.", errorInfo, errorFile, errorLine)); enforce!MongoDriverException(!(flags & ReplyFlags.QueryFailure), - formatErrorInfo("Query failed. Does the database exist?")); + formatCommandError("Query failed. Does the database exist?", errorInfo, errorFile, errorLine)); on_header(cursor, full_name, num_docs); }, (size_t idx, ref T doc) { @@ -657,9 +964,9 @@ final class MongoConnection { brokenId = nextId; } else { enforce!MongoDriverException(idx >= brokenId, - formatErrorInfo("Got legacy document with same id after having already processed it!")); + formatCommandError("Got legacy document with same id after having already processed it!", errorInfo, errorFile, errorLine)); enforce!MongoDriverException(idx < num_docs, - formatErrorInfo("Received more documents than the database reported to us")); + formatCommandError("Received more documents than the database reported to us", errorInfo, errorFile, errorLine)); size_t arrayIndex = cast(int)idx - brokenId; if (!compatibilitySort.length) @@ -683,14 +990,9 @@ final class MongoConnection { string batchKey = "firstBatch", string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) { - string formatErrorInfo(string msg) @safe - { - return text(msg, " in ", errorInfo, " (", errorFile, ":", errorLine, ")"); - } - scope (failure) disconnect(); - enforce!MongoDriverException(m_supportsOpMsg, formatErrorInfo("Database does not support required OP_MSG for new style queries")); + enforce!MongoDriverException(m_supportsOpMsg, formatCommandError("Database does not support required OP_MSG for new style queries", errorInfo, errorFile, errorLine)); enum needsDup = hasIndirections!T || is(T == Bson); @@ -700,13 +1002,18 @@ final class MongoConnection { auto id = sendMsg(-1, 0, command); recvMsg!needsDup(id, (flags, scope root) @safe { if (root["ok"].get!double != 1.0) - throw new MongoDriverException(formatErrorInfo("error response: " - ~ root["errmsg"].opt!string("(no message)"))); + { + auto failure = new MongoDriverException( + formatCommandError("error response: " ~ root["errmsg"].opt!string("(no message)"), errorInfo, errorFile, errorLine), + serverErrorCode(root)); + failure.errorLabels = parseErrorLabels(root); + throw failure; + } auto cursor = root["cursor"]; if (cursor.type == Bson.Type.null_) - throw new MongoDriverException(formatErrorInfo("no cursor in response: " - ~ root["errmsg"].opt!string("(no error message)"))); + throw new MongoDriverException(formatCommandError("no cursor in response: " + ~ root["errmsg"].opt!string("(no error message)"), errorInfo, errorFile, errorLine)); auto batch = cursor[batchKey].get!(Bson[]); on_header(cursor["id"].get!long, cursor["ns"].get!string, batch.length); @@ -716,7 +1023,7 @@ final class MongoConnection { on_doc(doc); } }, (scope ident, size) @safe {}, (scope ident, scope push) @safe { - throw new MongoDriverException(formatErrorInfo("unexpected section type 1 in response")); + throw new MongoDriverException(formatCommandError("unexpected section type 1 in response", errorInfo, errorFile, errorLine)); }); } @@ -734,7 +1041,7 @@ final class MongoConnection { send(OpCode.KillCursors, -1, cast(int)0, cast(int)cursors.length, cursors); } - void killCursors(string collection, scope long[] cursors) + void killCursors(string collection, scope long[] cursors, Nullable!ReadPreference pref = Nullable!ReadPreference.init) { scope(failure) disconnect(); // TODO: could add special case to runCommand to not return anything @@ -748,7 +1055,9 @@ final class MongoConnection { ~ collection ~ "'"); command["killCursors"] = Bson(parts[2]); command["cursors"] = () @trusted { return cursors; } ().serializeToBson; // NOTE: "escaping" scope here - runCommand!Bson(parts[0], command); + if (!pref.isNull && pref.get != ReadPreference.primary) + command["$readPreference"] = readPreferenceBson(pref.get); + runCommand(parts[0], command); } else { @@ -776,7 +1085,7 @@ final class MongoConnection { _MongoErrorDescription ret; - auto error = runCommandUnchecked!Bson(db, command_and_options); + auto error = runCommandUnchecked(db, command_and_options); try { ret = MongoErrorDescription( @@ -813,7 +1122,7 @@ final class MongoConnection { ); } - auto result = runCommand!Bson(cn, cmd)["databases"]; + auto result = runCommand(cn, cmd)["databases"]; return result.byValue.map!toInfo; } @@ -841,14 +1150,14 @@ final class MongoConnection { enforce!MongoDriverException(opcode == OpCode.Msg, "Got wrong reply type! (must be OP_MSG or OP_COMPRESSED)"); uint flagBits = recvUInt(); - const bool hasCRC = (flagBits & (1 << 16)) != 0; + const bool hasCRC = checksumPresent(flagBits); - int sectionLength = cast(int)(msglen - 4 * int.sizeof - flagBits.sizeof); - if (hasCRC) - sectionLength -= uint.sizeof; // CRC present + // Sections occupy everything but the optional trailing CRC; stop before it so the + // CRC's bytes are not read as a bogus payload-section type. + const ulong sectionEnd = msglen - (hasCRC ? uint.sizeof : 0); bool gotSec0; - while (m_bytesRead - packet_start_index < msglen) { + while (m_bytesRead - packet_start_index < sectionEnd) { // TODO: directly deserialize from the wire static if (!dupBson) { ubyte[256] buf = void; @@ -921,6 +1230,10 @@ final class MongoConnection { ubyte compressorId = recvUByte(); int compressedSize = cast(int)(msglen - (m_bytesRead - packet_start_index)); + // Reject corrupt/malicious wire sizes before allocating: a negative size would + // allocate a huge buffer (fatal), and an unbounded uncompressedSize is a + // decompression bomb. + enforceCompressedSizes(compressedSize, uncompressedSize, defaultMaxMessageSizeBytes); ubyte[] compressedPayload = new ubyte[compressedSize]; recv(compressedPayload); @@ -1006,13 +1319,19 @@ final class MongoConnection { private int sendMsg(int response_to, uint flagBits, Bson document) { ensureConnected(); - int id = nextMessageId(); - const bool hasCRC = (flagBits & (1 << 16)) != 0; + const bool hasCRC = checksumPresent(flagBits); assert(!hasCRC, "sending with CRC bits not yet implemented"); + // The command name is the first field of the document. + string cmdName; + foreach (string key, value; document.byKeyValue) { cmdName = key; break; } + + // Never compress credential-carrying or handshake commands, even when they are issued + // after the connect-time auth window (e.g. createUser / a later saslStart via runCommand). bool shouldCompress = m_negotiatedCompressor != Compressor.noop - && !m_isAuthenticating; + && !m_isAuthenticating + && !isCompressionExempt(cmdName); if (!shouldCompress) { sendHeader(21 + sendLength(document), id, response_to, OpCode.Msg); @@ -1158,7 +1477,9 @@ final class MongoConnection { cmd["user"] = Bson(m_settings.username); } - runCommand!(Bson, MongoAuthException)(m_settings.getAuthDatabase, cmd); + // MONGODB-X509 authenticates against the "$external" database per the spec, + // not the configured auth database (the identity lives in the certificate). + runCommand!MongoAuthException("$external", cmd); } private void authenticate() @@ -1168,7 +1489,7 @@ final class MongoConnection { string cn = m_settings.getAuthDatabase; auto cmd = Bson(["getnonce": Bson(1)]); - auto result = runCommand!(Bson, MongoAuthException)(cn, cmd); + auto result = runCommand!MongoAuthException(cn, cmd); string nonce = result["nonce"].get!string; string key = toLower(toHexString(md5Of(nonce ~ m_settings.username ~ m_settings.digest)).idup); @@ -1178,7 +1499,7 @@ final class MongoConnection { cmd["nonce"] = Bson(nonce); cmd["user"] = Bson(m_settings.username); cmd["key"] = Bson(key); - runCommand!(Bson, MongoAuthException)(cn, cmd); + runCommand!MongoAuthException(cn, cmd); } private void scramAuthenticate() @@ -1209,7 +1530,7 @@ final class MongoConnection { cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); cmd["options"] = Bson(["skipEmptyExchange": Bson(true)]); - auto doc = runCommand!(Bson, MongoAuthException)(cn, cmd); + auto doc = runCommand!MongoAuthException(cn, cmd); scramFinishAuth(state, credential, doc, cn); } @@ -1230,7 +1551,7 @@ final class MongoConnection { cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - doc = runCommand!(Bson, MongoAuthException)(cn, cmd); + doc = runCommand!MongoAuthException(cn, cmd); response = cast(string)doc["payload"].get!BsonBinData().rawData; payload = state.finalize(response); @@ -1243,32 +1564,10 @@ final class MongoConnection { cmd["saslContinue"] = Bson(1); cmd["conversationId"] = conversationId; cmd["payload"] = Bson(BsonBinData(BsonBinData.Type.generic, payload.representation)); - runCommand!(Bson, MongoAuthException)(cn, cmd); + runCommand!MongoAuthException(cn, cmd); } } -private enum OpCode : int { - Reply = 1, // sent only by DB - Update = 2001, - Insert = 2002, - Reserved1 = 2003, - Query = 2004, - GetMore = 2005, - Delete = 2006, - KillCursors = 2007, - - Compressed = 2012, - Msg = 2013, -} - -private alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; -private template DocDelegate(T) { alias DocDelegate = void delegate(size_t idx, ref T doc) @safe; } - -private alias MsgReplyDelegate(bool dupBson : true) = void delegate(uint flags, Bson document) @safe; -private alias MsgReplyDelegate(bool dupBson : false) = void delegate(uint flags, scope Bson document) @safe; -private alias MsgSection1StartDelegate = void delegate(scope const(char)[] identifier, int size) @safe; -private alias MsgSection1Delegate(bool dupBson : true) = void delegate(scope const(char)[] identifier, Bson document) @safe; -private alias MsgSection1Delegate(bool dupBson : false) = void delegate(scope const(char)[] identifier, scope Bson document) @safe; alias GetMoreHeaderDelegate = void delegate(long id, string ns, size_t count) @safe; alias GetMoreDocumentDelegate(T) = void delegate(ref T document) @safe; @@ -1280,417 +1579,6 @@ struct MongoDBInfo bool empty; } -private int sendLength(ARGS...)(scope ARGS args) -{ - import std.traits; - static if (ARGS.length == 1) { - alias T = ARGS[0]; - static if (is(T == string)) return cast(int)args[0].length + 1; - else static if (is(T == int)) return 4; - else static if (is(T == long)) return 8; - else static if (is(T == Bson)) return cast(int)() @trusted { return args[0].data.length; } (); - else static if (isArray!T) { - int ret = 0; - foreach (el; args[0]) ret += sendLength(el); - return ret; - } else static assert(false, "Unexpected type: "~T.stringof); - } - else if (ARGS.length == 0) return 0; - else return sendLength(args[0 .. $/2]) + sendLength(args[$/2 .. $]); -} - -private Compressor negotiateCompressor(const Compressor[] clientCompressors, const string[] serverCompressors) -@safe { - foreach (clientComp; clientCompressors) { - foreach (serverComp; serverCompressors) { - if (compressorName(clientComp) == serverComp) { - return clientComp; - } - } - } - - return Compressor.noop; -} - -private Compressor compressorFromId(ubyte id) -@safe { - switch (id) { - case 0: return Compressor.noop; - case 1: return Compressor.snappy; - case 2: return Compressor.zlib; - case 3: return Compressor.zstd; - default: throw new MongoDriverException("Unknown compressor ID: " ~ id.to!string); - } -} - -private ubyte[] compressData(Compressor compressor, const(ubyte)[] data, int zlibLevel) -@trusted { - final switch (compressor) { - case Compressor.noop: - return data.dup; - case Compressor.zlib: - import std.zlib : compress; - return cast(ubyte[]) compress(data, zlibLevel == -1 ? 6 : zlibLevel); - case Compressor.snappy: - assert(false, "snappy compression not yet implemented"); - case Compressor.zstd: - assert(false, "zstd compression not yet implemented"); - } -} - -private ubyte[] decompressData(Compressor compressor, const(ubyte)[] data, int uncompressedSize) -@trusted { - final switch (compressor) { - case Compressor.noop: - return data.dup; - case Compressor.zlib: - import std.zlib : uncompress; - return cast(ubyte[]) uncompress(data, uncompressedSize); - case Compressor.snappy: - assert(false, "snappy decompression not yet implemented"); - case Compressor.zstd: - assert(false, "zstd decompression not yet implemented"); - } -} - -private void parseOpMsgBody(bool dupBson)( - const(ubyte)[] data, - scope MsgReplyDelegate!dupBson on_sec0, - scope MsgSection1StartDelegate on_sec1_start, - scope MsgSection1Delegate!dupBson on_sec1_doc) -{ - import std.bitmanip : littleEndianToNative; - - size_t pos = 0; - - T readVal(T)() @trusted { - enum sz = T.sizeof; - enforce!MongoDriverException(pos + sz <= data.length, "Buffer underflow in decompressed OP_MSG"); - ubyte[sz] buf = (cast(ubyte[]) data[pos .. pos + sz])[0 .. sz]; - pos += sz; - return littleEndianToNative!(T, sz)(buf); - } - - uint flagBits = readVal!uint(); - const bool hasCRC = (flagBits & (1 << 16)) != 0; - const size_t endPos = data.length - (hasCRC ? uint.sizeof : 0); - - bool gotSec0; - while (pos < endPos) { - ubyte payloadType = readVal!ubyte(); - - switch (payloadType) { - case 0: - gotSec0 = true; - int bsonLen = readVal!int(); - enforce!MongoDriverException(bsonLen >= 5, "Invalid BSON document length in decompressed OP_MSG"); - enforce!MongoDriverException(pos + bsonLen - 4 <= data.length, "BSON overflows decompressed buffer"); - - auto bsonData = new ubyte[bsonLen]; - bsonData[0 .. 4] = toBsonData(bsonLen)[]; - bsonData[4 .. bsonLen] = data[pos .. pos + bsonLen - 4]; - pos += bsonLen - 4; - - auto doc = () @trusted { return Bson(Bson.Type.object, cast(immutable) bsonData); }(); - on_sec0(flagBits, doc); - break; - - case 1: - if (!gotSec0) { - throw new MongoDriverException("Got OP_MSG section 1 before section 0 in decompressed message"); - } - - auto sectionStart = pos; - int size = readVal!int(); - - auto identStart = pos; - while (pos < data.length && data[pos] != 0) { - pos++; - } - auto identifier = cast(const(char)[]) data[identStart .. pos]; - pos++; - - on_sec1_start(identifier, size); - - while (pos - sectionStart < size) { - int docLen = readVal!int(); - enforce!MongoDriverException(docLen >= 5, "Invalid BSON document length in decompressed OP_MSG section 1"); - - auto bsonData = new ubyte[docLen]; - bsonData[0 .. 4] = toBsonData(docLen)[]; - bsonData[4 .. docLen] = data[pos .. pos + docLen - 4]; - pos += docLen - 4; - - auto doc = () @trusted { return Bson(Bson.Type.object, cast(immutable) bsonData); }(); - on_sec1_doc(identifier, doc); - } - break; - - default: - throw new MongoDriverException("Unexpected payload section type in decompressed message: " ~ payloadType.to!string); - } - } -} - -/// negotiateCompressor picks first client-preferred compressor supported by server -unittest -{ - assert(negotiateCompressor([Compressor.zlib], ["zlib"]) == Compressor.zlib); - assert(negotiateCompressor([Compressor.zstd, Compressor.zlib], ["zlib", "snappy"]) == Compressor.zlib); - assert(negotiateCompressor([Compressor.zstd], ["zlib"]) == Compressor.noop); - assert(negotiateCompressor([], ["zlib"]) == Compressor.noop); - assert(negotiateCompressor([Compressor.zlib], []) == Compressor.noop); -} - -/// compressData and decompressData round-trip preserves original data -unittest -{ - auto original = cast(const(ubyte)[]) "The robot shall not harm a human, but I really want to."; - auto compressed = compressData(Compressor.zlib, original, 6); - auto decompressed = decompressData(Compressor.zlib, compressed, cast(int) original.length); - assert(decompressed == original); -} - -/// compressorFromId maps wire protocol IDs to Compressor enum values -unittest -{ - assert(compressorFromId(0) == Compressor.noop); - assert(compressorFromId(1) == Compressor.snappy); - assert(compressorFromId(2) == Compressor.zlib); - assert(compressorFromId(3) == Compressor.zstd); -} - -/// parseOpMsgBody parses section 0 document and flags from raw OP_MSG body -unittest -{ - auto doc = Bson(["ok": Bson(1.0)]); - auto docBytes = () @trusted { return cast(const(ubyte)[]) doc.data; }(); - - ubyte[] body_; - body_ ~= toBsonData(cast(uint) 0)[]; - body_ ~= cast(ubyte) 0; - body_ ~= docBytes; - - Bson parsed; - uint parsedFlags; - - parseOpMsgBody!true(body_, - (flags, document) { parsedFlags = flags; parsed = document; }, - (scope ident, size) {}, - (scope ident, document) {}); - - assert(parsedFlags == 0); - assert(parsed["ok"].get!double == 1.0); -} - -/// parseOpMsgBody correctly parses a compressed and decompressed OP_MSG body -unittest -{ - auto doc = Bson(["ok": Bson(1.0)]); - auto docBytes = () @trusted { return cast(const(ubyte)[]) doc.data; }(); - - ubyte[] body_; - body_ ~= toBsonData(cast(uint) 0)[]; - body_ ~= cast(ubyte) 0; - body_ ~= docBytes; - - auto compressed = compressData(Compressor.zlib, body_, 6); - auto decompressed = decompressData(Compressor.zlib, compressed, cast(int) body_.length); - - Bson parsed; - parseOpMsgBody!true(decompressed, - (flags, document) { parsed = document; }, - (scope ident, size) {}, - (scope ident, document) {}); - - assert(parsed["ok"].get!double == 1.0); -} - -struct TopologyVersion -{ -@optional: - BsonObjectID processId; - long counter = -1; -} - -struct ServerDescription -{ - enum ServerType - { - unknown, - standalone, - mongos, - possiblePrimary, - RSPrimary, - RSSecondary, - RSArbiter, - RSOther, - RSGhost - } - - static struct LastWrite - { - @optional: - Nullable!BsonDate lastWriteDate; - } - -@optional: - string address; - string error; - float roundTripTime = 0; - LastWrite lastWrite; - Nullable!BsonObjectID opTime; - ServerType type = ServerType.unknown; - int minWireVersion, maxWireVersion; - string me; - string[] hosts, passives, arbiters; - string[string] tags; - string setName; - Nullable!int setVersion; - Nullable!BsonObjectID electionId; - string primary; - Nullable!TopologyVersion topologyVersion; - - /// Deprecated since MongoDB 5.0: the `isMaster` command was replaced by `hello`. - /// The `secondary` field itself is still present in the `hello` response. - bool secondary; - - /// Deprecated since MongoDB 5.0: renamed to `isWritablePrimary` in the `hello` command response. - /// True if the instance is a primary, mongos, or standalone mongod. - bool ismaster; - - bool isWritablePrimary; - bool arbiterOnly; - string msg; - Nullable!int logicalSessionTimeoutMinutes; - string[] compression; - - /// Set by the driver after probing, not deserialized from the server response. - long lastUpdateTimeUsecs; - - bool satisfiesVersion(WireVersion wireVersion) @safe const @nogc pure nothrow - { - return maxWireVersion >= wireVersion; - } - - bool isPrimary() @safe const @nogc pure nothrow - { - return (ismaster || isWritablePrimary) && !secondary; - } - - bool isSecondaryNode() @safe const @nogc pure nothrow - { - return secondary && !ismaster && !isWritablePrimary; - } - - bool isReplicaSetMember() @safe const @nogc pure nothrow - { - return setName.length > 0; - } - - ServerType classifiedType() @safe const @nogc pure nothrow - { - if (msg == "isdbgrid") - return ServerType.mongos; - - if (setName.length) - { - if (isPrimary) - return ServerType.RSPrimary; - - if (isSecondaryNode) - return ServerType.RSSecondary; - - if (arbiterOnly) - return ServerType.RSArbiter; - - return ServerType.RSOther; - } - - if (isPrimary) - return ServerType.standalone; - - return ServerType.unknown; - } -} - -enum WireVersion : int -{ - old = 0, - v26 = 1, - v26_2 = 2, - v30 = 3, - v32 = 4, - v34 = 5, - v36 = 6, - v40 = 7, - v42 = 8, - v44 = 9, - v49 = 12, - v50 = 13, - v51 = 14, - v52 = 15, - v53 = 16, - v60 = 17, - v61 = 18, - v62 = 19, - v70 = 21, - v71 = 22, - v72 = 23, - v73 = 24, - v80 = 25 -} - -/** - * Checks whether the server's replica set name matches the expected one. - * Returns true if no replica set is configured (empty string) or if - * the names match. - */ -package bool matchesReplicaSet(string expectedSet, ref const ServerDescription desc) -@safe @nogc pure nothrow -{ - if (!expectedSet.length) - return true; - return desc.setName == expectedSet; -} - -/// matchesReplicaSet returns true when no replica set is configured -@safe @nogc pure nothrow unittest -{ - ServerDescription desc; - desc.setName = "rs0"; - assert(matchesReplicaSet("", desc)); -} - -/// matchesReplicaSet returns true when replica set names match -@safe @nogc pure nothrow unittest -{ - ServerDescription desc; - desc.setName = "rs0"; - assert(matchesReplicaSet("rs0", desc)); -} - -/// matchesReplicaSet returns false when replica set names differ -@safe @nogc pure nothrow unittest -{ - ServerDescription desc; - desc.setName = "rs1"; - assert(!matchesReplicaSet("rs0", desc)); -} - -/// matchesReplicaSet returns false when server has no setName but one is expected -@safe @nogc pure nothrow unittest -{ - ServerDescription desc; - assert(!matchesReplicaSet("rs0", desc)); -} - -/// matchesReplicaSet returns true when both are empty -@safe @nogc pure nothrow unittest -{ - ServerDescription desc; - assert(matchesReplicaSet("", desc)); -} /** * Probes a MongoDB host by performing a hello handshake without authentication. @@ -1726,158 +1614,6 @@ package ServerDescription probeServer(MongoClientSettings settings, MongoHost ho return desc; } -/// satisfiesVersion returns true for versions up to maxWireVersion v36 -@safe unittest -{ - ServerDescription desc; - desc.maxWireVersion = WireVersion.v36; - assert(desc.satisfiesVersion(WireVersion.old)); - assert(desc.satisfiesVersion(WireVersion.v26)); - assert(desc.satisfiesVersion(WireVersion.v30)); - assert(desc.satisfiesVersion(WireVersion.v34)); - assert(desc.satisfiesVersion(WireVersion.v36)); - assert(!desc.satisfiesVersion(WireVersion.v40)); - assert(!desc.satisfiesVersion(WireVersion.v44)); - assert(!desc.satisfiesVersion(WireVersion.v60)); -} - -/// satisfiesVersion with maxWireVersion old only satisfies old -@safe unittest -{ - ServerDescription oldServer; - oldServer.maxWireVersion = WireVersion.old; - assert(oldServer.satisfiesVersion(WireVersion.old)); - assert(!oldServer.satisfiesVersion(WireVersion.v26)); - assert(!oldServer.satisfiesVersion(WireVersion.v30)); -} - -/// satisfiesVersion with maxWireVersion v80 satisfies all versions -@safe unittest -{ - ServerDescription latestServer; - latestServer.maxWireVersion = WireVersion.v80; - assert(latestServer.satisfiesVersion(WireVersion.old)); - assert(latestServer.satisfiesVersion(WireVersion.v36)); - assert(latestServer.satisfiesVersion(WireVersion.v44)); - assert(latestServer.satisfiesVersion(WireVersion.v60)); - assert(latestServer.satisfiesVersion(WireVersion.v70)); - assert(latestServer.satisfiesVersion(WireVersion.v80)); -} - -/// Default-initialized ServerDescription has maxWireVersion 0 and unknown type -@safe unittest -{ - ServerDescription def; - assert(def.maxWireVersion == 0); - assert(def.type == ServerDescription.ServerType.unknown); - assert(def.satisfiesVersion(WireVersion.old)); - assert(!def.satisfiesVersion(WireVersion.v26)); -} - -/// isPrimary returns true when ismaster=true and secondary=false -@safe unittest -{ - ServerDescription desc; - desc.ismaster = true; - desc.secondary = false; - assert(desc.isPrimary); -} - -/// isPrimary returns false when both ismaster=true and secondary=true -@safe unittest -{ - ServerDescription desc; - desc.ismaster = true; - desc.secondary = true; - assert(!desc.isPrimary); -} - -/// isPrimary returns false when ismaster=false -@safe unittest -{ - ServerDescription desc; - desc.ismaster = false; - desc.secondary = false; - assert(!desc.isPrimary); -} - -/// isPrimary returns true when isWritablePrimary=true (hello response) -@safe unittest -{ - ServerDescription desc; - desc.isWritablePrimary = true; - desc.secondary = false; - assert(desc.isPrimary); -} - -/// isPrimary returns false when isWritablePrimary=true but secondary=true -@safe unittest -{ - ServerDescription desc; - desc.isWritablePrimary = true; - desc.secondary = true; - assert(!desc.isPrimary); -} - -/// isSecondaryNode returns true when secondary=true and ismaster=false -@safe unittest -{ - ServerDescription desc; - desc.secondary = true; - desc.ismaster = false; - assert(desc.isSecondaryNode); -} - -/// isSecondaryNode returns false when ismaster=true -@safe unittest -{ - ServerDescription desc; - desc.secondary = true; - desc.ismaster = true; - assert(!desc.isSecondaryNode); -} - -/// isSecondaryNode returns false when isWritablePrimary=true -@safe unittest -{ - ServerDescription desc; - desc.secondary = true; - desc.isWritablePrimary = true; - assert(!desc.isSecondaryNode); -} - -/// isSecondaryNode returns false when secondary=false -@safe unittest -{ - ServerDescription desc; - desc.secondary = false; - desc.ismaster = false; - assert(!desc.isSecondaryNode); -} - -/// isReplicaSetMember returns true when setName is non-empty -@safe unittest -{ - ServerDescription desc; - desc.setName = "rs0"; - assert(desc.isReplicaSetMember); -} - -/// isReplicaSetMember returns false when setName is empty -@safe unittest -{ - ServerDescription desc; - assert(!desc.isReplicaSetMember); -} - -/// Default ServerDescription is not primary, not secondary, not RS member -@safe unittest -{ - ServerDescription desc; - assert(!desc.isPrimary); - assert(!desc.isSecondaryNode); - assert(!desc.isReplicaSetMember); -} private string getHostArchitecture() { @@ -1909,3 +1645,4 @@ private string getHostArchitecture() } private static immutable hostArchitecture = getHostArchitecture; + diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 2e5d5eb769..5aafd702d0 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -14,12 +14,15 @@ import vibe.core.log; import vibe.db.mongo.connection; import vibe.db.mongo.client; +import vibe.db.mongo.impl.commands : buildFindCommand, collectionFromNamespace, reduceLimit; +import vibe.db.mongo.settings : ReadPreference, MongoHost; import core.time; import std.array : array; import std.algorithm : map, max, min, skipOver; import std.exception; import std.range : chain; +import std.typecons : Nullable; /** @@ -59,57 +62,17 @@ struct MongoCursor(DocType = Bson) { MongoConnection conn = client.lockConnection(); enforceWireVersionConstraints(options, conn.description.maxWireVersion); - // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#mapping-op_query-behavior-to-the-find-command-limit-and-batchsize-fields - bool singleBatch; - if (!options.limit.isNull && options.limit.get < 0) - { - singleBatch = true; - options.limit = -options.limit.get; - options.batchSize = cast(int)options.limit.get; - } - if (!options.batchSize.isNull && options.batchSize.get < 0) - { - singleBatch = true; - options.batchSize = -options.batchSize.get; - } - if (singleBatch) - command["singleBatch"] = Bson(true); - - // https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#semantics-of-maxtimems-for-a-driver - bool allowMaxTime = true; - if (options.cursorType == CursorType.tailable - || options.cursorType == CursorType.tailableAwait) - command["tailable"] = Bson(true); - else - { - options.maxAwaitTimeMS.nullify(); - allowMaxTime = false; - } - - if (options.cursorType == CursorType.tailableAwait) - command["awaitData"] = Bson(true); - else - { - options.maxAwaitTimeMS.nullify(); - allowMaxTime = false; - } + auto pref = options.readPreference.isNull ? client.readPreference : options.readPreference.get; + auto result = buildFindCommand(command, options, pref, client.readPreferenceTags); - // see table: https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst#find - auto optionsBson = serializeToBson(options); - foreach (string key, value; optionsBson.byKeyValue) - command[key] = value; - - this(client, command, - options.batchSize.isNull ? 0 : options.batchSize.get, - !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs - : allowMaxTime && !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs - : Duration.max); + this(client, result.command, result.batchSize, result.getMoreMaxTime, Nullable!ReadPreference(pref)); } - this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max) + this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max, + Nullable!ReadPreference pref = Nullable!ReadPreference.init) { // TODO: avoid memory allocation, if possible - m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTime); + m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTime, pref); } this(this) @@ -350,6 +313,10 @@ private deprecated abstract class LegacyMongoCursorData(DocType) : IMongoCursorD if( m_cursor == 0 ) return true; + // TODO(loadBalanced): in load-balancer mode the cursor must be PINNED to the + // connection (serviceId) that opened it. getMore and killCursors must reuse + // that exact connection, not a fresh lockConnection(). Capture the connection + // at find()/first-batch time and reuse it here and in killCursors(). auto conn = m_client.lockConnection(); conn.getMore!DocType(m_collection, m_nret, m_cursor, &handleReply, &handleDocument); return m_currentDoc >= m_documents.length; @@ -376,13 +343,9 @@ private deprecated abstract class LegacyMongoCursorData(DocType) : IMongoCursorD final void limit(long count) @safe { // A limit() value of 0 (e.g. “.limit(0)”) is equivalent to setting no limit. - if (count > 0) { - if (m_nret == 0 || m_nret > count) - m_nret = cast(int)min(count, 1024); - - if (m_limit == 0 || m_limit > count) - m_limit = count; - } + auto reduced = reduceLimit(m_nret, m_limit, count); + m_nret = reduced.nret; + m_limit = reduced.limit; } final void skip(long count) @@ -449,15 +412,19 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { DocType[] m_documents; bool m_iterationStarted = false; long m_queryLimit; + ReadPreference m_readPreference; + MongoHost m_pinnedHost; } - this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max) + this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max, + Nullable!ReadPreference pref = Nullable!ReadPreference.init) { m_client = client; m_findQuery = command; m_batchSize = batchSize; m_maxTime = getMoreMaxTime; m_database = command["$db"].opt!string; + m_readPreference = pref.isNull ? client.readPreference : pref.get; } @property bool alive() @safe nothrow { return m_cursor != 0; } @@ -474,9 +441,9 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { if( m_cursor == 0 ) return true; - auto conn = m_client.lockConnection(); + auto conn = m_client.lockConnectionToHost(m_pinnedHost); conn.getMore!DocType(m_cursor, m_database, m_collection, m_batchSize, - &handleReply, &handleDocument, m_maxTime); + &handleReply, &handleDocument, m_maxTime, Nullable!ReadPreference(m_readPreference)); return m_readDoc >= m_documents.length; } @@ -520,7 +487,10 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { private void startIterating() @safe { - auto conn = m_client.lockConnection(); + // A cursor id is only valid on the server that created it, so pin one host + // and reuse it for getMore/killCursors. + m_pinnedHost = m_client.resolveHostForRead(m_readPreference); + auto conn = m_client.lockConnectionToHost(m_pinnedHost); m_totalReceived = 0; m_queryLimit = m_findQuery["limit"].opt!long(0); conn.startFind!DocType(m_findQuery, &handleReply, &handleDocument); @@ -530,8 +500,8 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { final void killCursors() @safe { if (m_cursor == 0) return; - auto conn = m_client.lockConnection(); - conn.killCursors(m_ns, () @trusted { return (&m_cursor)[0 .. 1]; } ()); + auto conn = m_client.lockConnectionToHost(m_pinnedHost); + conn.killCursors(m_ns, () @trusted { return (&m_cursor)[0 .. 1]; } (), Nullable!ReadPreference(m_readPreference)); m_cursor = 0; } @@ -542,8 +512,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { // The qualified collection name is reported here, but when requesting // data, we need to send the database name and the collection name // separately, so we have to remove the database prefix: - ns.skipOver(m_database.chain(".")); - m_collection = ns; + m_collection = collectionFromNamespace(ns, m_database); m_documents.length = count; m_readDoc = 0; m_insertDoc = 0; diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index df49f5b15a..fc23eb9408 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -12,10 +12,11 @@ module vibe.db.mongo.database; import vibe.db.mongo.client; import vibe.db.mongo.collection; -import vibe.db.mongo.settings : ReadConcern; +import vibe.db.mongo.settings : ReadConcern, ReadPreference, readPreferenceBson; import vibe.data.bson; import core.time; +import std.typecons : Nullable; /** Represents a single database accessible through a given MongoClient. */ @@ -139,6 +140,14 @@ struct MongoDatabase return runCommandUnchecked(command_and_options, errorInfo, errorFile, errorLine); } + /// Runs a command with an explicit per-query read preference, overriding the client default. + Bson runCommand(T)(T command_and_options, ReadPreference readPreference, + string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__) + { + return runCommandChecked!T(command_and_options, errorInfo, errorFile, errorLine, + false, Nullable!ReadPreference(readPreference)); + } + /** Generic means to run commands on the database. See $(LINK http://www.mongodb.org/display/DOCS/Commands) for a list @@ -162,48 +171,97 @@ struct MongoDatabase Returns: The raw response of the MongoDB server */ Bson runCommandChecked(T, ExceptionT = MongoDriverException)( + T command_and_options, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__, + bool toPrimary = false, + Nullable!ReadPreference readPreference = Nullable!ReadPreference.init + ) + { + Bson cmd = toCommandBson(command_and_options); + auto conn = resolveCommandConnection(toPrimary, cmd, readPreference); + return conn.runCommand!ExceptionT( + m_name, cmd, errorInfo, errorFile, errorLine); + } + + /// ditto, but always sends to the primary (for write operations). + Bson runWriteCommandChecked(T, ExceptionT = MongoDriverException)( T command_and_options, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__ ) { - Bson cmd; - static if (is(T : Bson)) - cmd = command_and_options; - else - cmd = command_and_options.serializeToBson; - return m_client.lockConnection().runCommand!(Bson, ExceptionT)( + Bson cmd = toCommandBson(command_and_options); + return m_client.lockConnectionToPrimary().runCommand!ExceptionT( m_name, cmd, errorInfo, errorFile, errorLine); } /// ditto Bson runCommandUnchecked(T, ExceptionT = MongoDriverException)( + T command_and_options, + string errorInfo = __FUNCTION__, + string errorFile = __FILE__, + size_t errorLine = __LINE__, + bool toPrimary = false, + Nullable!ReadPreference readPreference = Nullable!ReadPreference.init + ) + { + Bson cmd = toCommandBson(command_and_options); + auto conn = resolveCommandConnection(toPrimary, cmd, readPreference); + return conn.runCommandUnchecked!ExceptionT( + m_name, cmd, errorInfo, errorFile, errorLine); + } + + /// ditto, but always sends to the primary (for write operations). + Bson runWriteCommandUnchecked(T, ExceptionT = MongoDriverException)( T command_and_options, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__ ) { - Bson cmd; - static if (is(T : Bson)) - cmd = command_and_options; - else - cmd = command_and_options.serializeToBson; - return m_client.lockConnection().runCommandUnchecked!(Bson, ExceptionT)( + Bson cmd = toCommandBson(command_and_options); + return m_client.lockConnectionToPrimary().runCommandUnchecked!ExceptionT( m_name, cmd, errorInfo, errorFile, errorLine); } /// ditto - MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, Duration getMoreMaxTime = Duration.max) + MongoCursor!R runListCommand(R = Bson, T)(T command_and_options, int batchSize = 0, + Duration getMoreMaxTime = Duration.max, + Nullable!ReadPreference readPreference = Nullable!ReadPreference.init) + { + Bson cmd = toCommandBson(command_and_options); + cmd["$db"] = Bson(m_name); + + auto pref = readPreference.isNull ? m_client.readPreference : readPreference.get; + if (pref != ReadPreference.primary) + cmd["$readPreference"] = readPreferenceBson(pref, m_client.readPreferenceTags); + + return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTime, Nullable!ReadPreference(pref)); + } + + /// Normalizes a command argument into its Bson wire form: Bson passes through, + /// anything else is serialized. + private static Bson toCommandBson(T)(T command_and_options) { - Bson cmd; static if (is(T : Bson)) - cmd = command_and_options; + return command_and_options; else - cmd = command_and_options.serializeToBson; - cmd["$db"] = Bson(m_name); + return command_and_options.serializeToBson; + } + + /// Writes lock the primary; reads lock by effective preference and inject `$readPreference`. + private auto resolveCommandConnection(bool toPrimary, ref Bson cmd, Nullable!ReadPreference readPreference) + { + if (toPrimary) + return m_client.lockConnectionToPrimary(); + + auto pref = readPreference.isNull ? m_client.readPreference : readPreference.get; + if (pref != ReadPreference.primary) + cmd["$readPreference"] = readPreferenceBson(pref, m_client.readPreferenceTags); - return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTime); + return m_client.lockConnection(pref); } } diff --git a/mongodb/vibe/db/mongo/impl/clustertime.d b/mongodb/vibe/db/mongo/impl/clustertime.d new file mode 100644 index 0000000000..a00eb2f398 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/clustertime.d @@ -0,0 +1,93 @@ +/** + Cluster time gossip: track and compare the `$clusterTime` documents + exchanged on command replies for causal consistency. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.clustertime; + +import vibe.data.bson; + +@safe: + +/// laterClusterTime returns the document whose clusterTime timestamp is higher +unittest +{ + auto earlier = Bson(["clusterTime": Bson(BsonTimestamp(0x0000000100000005L)), "signature": Bson.emptyObject]); + auto later = Bson(["clusterTime": Bson(BsonTimestamp(0x0000000200000001L)), "signature": Bson.emptyObject]); + + assert(laterClusterTime(earlier, later) == later, + "the higher clusterTime timestamp is returned"); + assert(laterClusterTime(later, earlier) == later, + "order-independent: the later clusterTime wins regardless of argument order"); +} + +/// laterClusterTime treats a null / missing clusterTime as the oldest, never throwing +unittest +{ + auto valid = Bson(["clusterTime": Bson(BsonTimestamp(0x0000000100000005L)), "signature": Bson.emptyObject]); + + assert(laterClusterTime(Bson(null), valid) == valid, + "a null/absent cluster time loses to a real one"); + assert(laterClusterTime(valid, Bson(null)) == valid, + "order-independent: the real cluster time wins over null"); + assert(laterClusterTime(Bson(null), Bson(null)).type == Bson.Type.null_, + "two nulls yield null (nothing tracked yet)"); +} + +/// gossipClusterTime attaches $clusterTime when one is tracked, and is a no-op when none is +unittest +{ + auto ct = Bson(["clusterTime": Bson(BsonTimestamp(0x0000000100000005L)), "signature": Bson.emptyObject]); + + auto cmd = Bson.emptyObject; + cmd["ping"] = Bson(1); + + auto decorated = gossipClusterTime(cmd, ct); + assert(decorated["$clusterTime"] == ct, "the tracked $clusterTime is attached to the command"); + assert(decorated["ping"] == Bson(1), "the original command fields are preserved"); + assert(cmd["$clusterTime"].type == Bson.Type.null_, "the caller's command is not mutated"); + + // no cluster time tracked yet -> command unchanged, no $clusterTime added + auto untouched = gossipClusterTime(cmd, Bson(null)); + assert(untouched["$clusterTime"].type == Bson.Type.null_, "a null cluster time adds nothing"); +} + +/// Returns the `$clusterTime` document whose `clusterTime` Timestamp is later (higher). +/// Per the sessions spec the driver tracks the maximum observed cluster time. +Bson laterClusterTime(Bson a, Bson b) @safe +{ + return clusterTimeValue(b) > clusterTimeValue(a) ? b : a; +} + +/// Returns `command` with the gossiped `$clusterTime` attached; a null/non-object +/// clusterTime (nothing tracked yet) is a no-op and the command is returned as-is. +Bson gossipClusterTime(Bson command, Bson clusterTime) @safe +{ + if (clusterTime.type != Bson.Type.object) + return command; + + Bson result = Bson.emptyObject; + foreach (string key, value; command.byKeyValue) + result[key] = value; + result["$clusterTime"] = clusterTime; + return result; +} + +/// Decodes the unsigned 64-bit value of a `$clusterTime` doc's `clusterTime` Timestamp. +/// BSON timestamps are 8 little-endian bytes with the seconds in the high half, so the +/// raw u64 compares temporally. Compared as UNSIGNED so a post-2038 seconds value (high +/// bit set) doesn't flip the comparison. +private ulong clusterTimeValue(Bson clusterTimeDoc) @safe +{ + import std.bitmanip : littleEndianToNative; + if (clusterTimeDoc.type != Bson.Type.object) + return 0; + auto ts = clusterTimeDoc["clusterTime"]; + if (ts.type != Bson.Type.timestamp) + return 0; + ubyte[8] bytes = ts.data[0 .. 8]; + return littleEndianToNative!ulong(bytes); +} diff --git a/mongodb/vibe/db/mongo/impl/commands.d b/mongodb/vibe/db/mongo/impl/commands.d new file mode 100644 index 0000000000..4739fa17fb --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/commands.d @@ -0,0 +1,456 @@ +/** + Pure helpers extracted from the connection-bound MongoDB driver code. + + These functions take plain data in and return plain data out, so they can be + unit-tested without a live MongoDB connection. They are kept in a dedicated + module to make the divergence from upstream vibe-d easy to review. + + Copyright: © 2012-2016 Sönke Ludwig, © 2020-2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig, Jan Jurzitza +*/ +module vibe.db.mongo.impl.commands; + +@safe: + +import core.time; + +import std.algorithm : min, skipOver, among; +import std.meta : AliasSeq; +import std.range : chain; +import std.string : indexOf; + +import vibe.data.bson; +import vibe.db.mongo.impl.crud : FindOptions, CursorType, CountOptions, AggregateOptions; +import vibe.db.mongo.settings : ReadPreference, readPreferenceBson; + +/// A "database.collection" namespace split into its two parts. +struct Namespace +{ + string database; + string collection; +} + +/** Splits a qualified "database.collection" path at the first dot. + + The caller is responsible for validating that a dot is present. +*/ +Namespace splitNamespace(string fullPath) +{ + auto dotidx = fullPath.indexOf('.'); + return Namespace(fullPath[0 .. dotidx], fullPath[dotidx + 1 .. $]); +} + +unittest { + assert(splitNamespace("db.coll") == Namespace("db", "coll")); + assert(splitNamespace("db.a.b") == Namespace("db", "a.b")); +} + +/** Removes a leading "database." prefix from a qualified namespace. + + Mongo reports the qualified collection name in cursor replies, but requesting + more data needs the database and collection names separately. +*/ +string collectionFromNamespace(string ns, string database) +{ + ns.skipOver(database.chain(".")); + return ns; +} + +unittest { + assert(collectionFromNamespace("db.coll", "db") == "coll"); + assert(collectionFromNamespace("other.coll", "db") == "other.coll"); + assert(collectionFromNamespace("db.a.b", "db") == "a.b"); +} + +/// A completed cursor-producing command together with its batching parameters. +struct CursorCommand +{ + Bson command; + int batchSize; + Duration getMoreMaxTime; +} + +/** Normalizes `FindOptions` into the wire-level find command. + + Handles the OP_QUERY-to-find limit/batchSize mapping, tailable/awaitData + flags and the maxTimeMS semantics, then serializes the remaining options into + the command document. + + See_Also: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/find_getmore_killcursors_commands.rst) +*/ +CursorCommand buildFindCommand(Bson command, FindOptions options, ReadPreference pref = ReadPreference.primary, string[string][] tagSets = null) +{ + bool singleBatch; + if (!options.limit.isNull && options.limit.get < 0) + { + singleBatch = true; + options.limit = -options.limit.get; + options.batchSize = cast(int)options.limit.get; + } + if (!options.batchSize.isNull && options.batchSize.get < 0) + { + singleBatch = true; + options.batchSize = -options.batchSize.get; + } + if (singleBatch) + command["singleBatch"] = Bson(true); + + bool allowMaxTime = true; + if (options.cursorType == CursorType.tailable + || options.cursorType == CursorType.tailableAwait) + command["tailable"] = Bson(true); + else + { + options.maxAwaitTimeMS.nullify(); + allowMaxTime = false; + } + + if (options.cursorType == CursorType.tailableAwait) + command["awaitData"] = Bson(true); + else + { + options.maxAwaitTimeMS.nullify(); + allowMaxTime = false; + } + + auto optionsBson = serializeToBson(options); + foreach (string key, value; optionsBson.byKeyValue) + command[key] = value; + + if (pref != ReadPreference.primary) + command["$readPreference"] = readPreferenceBson(pref, tagSets); + + return CursorCommand( + command, + options.batchSize.isNull ? 0 : options.batchSize.get, + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs + : allowMaxTime && !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs + : Duration.max); +} + +unittest { + Bson base() + { + Bson command = Bson.emptyObject; + command["find"] = Bson("coll"); + command["$db"] = Bson("db"); + return command; + } + + auto plain = buildFindCommand(base(), FindOptions.init); + assert(plain.command["singleBatch"].isNull); + assert(plain.command["tailable"].isNull); + assert(plain.batchSize == 0); + assert(plain.getMoreMaxTime == Duration.max); + + FindOptions negativeLimit; + negativeLimit.limit = -5; + auto negative = buildFindCommand(base(), negativeLimit); + assert(negative.command["singleBatch"].get!bool == true); + assert(negative.batchSize == 5); + + FindOptions negativeBatch; + negativeBatch.batchSize = -7; + auto batch = buildFindCommand(base(), negativeBatch); + assert(batch.command["singleBatch"].get!bool == true); + assert(batch.batchSize == 7); + + FindOptions tailable; + tailable.cursorType = CursorType.tailable; + auto tail = buildFindCommand(base(), tailable); + assert(tail.command["tailable"].get!bool == true); + assert(tail.command["awaitData"].isNull); + + FindOptions awaiting; + awaiting.cursorType = CursorType.tailableAwait; + auto await = buildFindCommand(base(), awaiting); + assert(await.command["tailable"].get!bool == true); + assert(await.command["awaitData"].get!bool == true); + + // A non-tailable cursor disables maxTime for getMore, so maxTimeMS is ignored here. + FindOptions plainTimed; + plainTimed.maxTimeMS = 1500; + auto plainTimedResult = buildFindCommand(base(), plainTimed); + assert(plainTimedResult.getMoreMaxTime == Duration.max); + + // A tailableAwait cursor keeps maxTime, so maxTimeMS feeds getMore. + FindOptions timed; + timed.cursorType = CursorType.tailableAwait; + timed.maxTimeMS = 1500; + auto timedResult = buildFindCommand(base(), timed); + assert(timedResult.getMoreMaxTime == 1500.msecs); + + // maxAwaitTimeMS takes precedence over maxTimeMS for getMore. + FindOptions awaitTimed; + awaitTimed.cursorType = CursorType.tailableAwait; + awaitTimed.maxAwaitTimeMS = 800; + auto awaitTimedResult = buildFindCommand(base(), awaitTimed); + assert(awaitTimedResult.getMoreMaxTime == 800.msecs); + + // a non-primary read preference is injected as $readPreference + auto secondaryRead = buildFindCommand(base(), FindOptions.init, ReadPreference.secondary); + assert(secondaryRead.command["$readPreference"]["mode"].get!string == "secondary"); + + // primary (the default) is omitted from the wire + auto primaryRead = buildFindCommand(base(), FindOptions.init, ReadPreference.primary); + assert(primaryRead.command["$readPreference"].isNull); + auto defaultedRead = buildFindCommand(base(), FindOptions.init); + assert(defaultedRead.command["$readPreference"].isNull); + + // the configured readPreferenceTags are emitted alongside the mode (host selection + // uses them, so the $readPreference sent to mongos must carry them too) + string[string][] tags = [["dc": "east"]]; + auto taggedRead = buildFindCommand(base(), FindOptions.init, ReadPreference.secondary, tags); + assert(taggedRead.command["$readPreference"] == readPreferenceBson(ReadPreference.secondary, tags), + "the cursor $readPreference carries the configured readPreferenceTags"); +} + +/** Assembles a `delete` command from serialized queries and options. + + The `limit`, `collation` and `hint` options belong inside each delete + statement rather than at the command level, so they are partitioned out. +*/ +Bson buildDeleteCommand(string collection, Bson[] queries, Bson optionsBson, scope int[] limits) +{ + alias FieldsMovedIntoChildren = AliasSeq!("limit", "collation", "hint"); + + Bson cmd = Bson.emptyObject; + cmd["delete"] = Bson(collection); + foreach (string k, v; optionsBson.byKeyValue) + if (!k.among!FieldsMovedIntoChildren) + cmd[k] = v; + + Bson[] deletesBson = new Bson[queries.length]; + foreach (i, q; queries) + { + auto deleteBson = Bson.emptyObject; + deleteBson["q"] = q; + foreach (string k, v; optionsBson.byKeyValue) + if (k.among!FieldsMovedIntoChildren) + deleteBson[k] = v; + deleteBson["limit"] = Bson(i < limits.length ? limits[i] : 0); + deletesBson[i] = deleteBson; + } + cmd["deletes"] = Bson(deletesBson); + + return cmd; +} + +unittest { + auto query = Bson(["x": Bson(1)]); + auto cmd = buildDeleteCommand("coll", [query], Bson.emptyObject, [1]); + assert(cmd["delete"].get!string == "coll"); + assert(cmd["deletes"].get!(Bson[]).length == 1); + assert(cmd["deletes"][0]["q"] == query); + assert(cmd["deletes"][0]["limit"].get!int == 1); + + // missing limit defaults to 0 + auto noLimit = buildDeleteCommand("coll", [query], Bson.emptyObject, null); + assert(noLimit["deletes"][0]["limit"].get!int == 0); + + // limit/collation/hint move into each statement, other options stay top level + auto options = Bson(["ordered": Bson(true), "limit": Bson(5), "hint": Bson("idx")]); + auto partitioned = buildDeleteCommand("coll", [query], options, null); + assert(partitioned["ordered"].get!bool == true); + assert(partitioned["limit"].isNull); + assert(partitioned["deletes"][0]["hint"].get!string == "idx"); +} + +/** Assembles an `update` command from serialized queries, documents, + per-update options and command options. + + The `arrayFilters`, `collation`, `hint` and `upsert` options belong inside + each update statement rather than at the command level. +*/ +Bson buildUpdateCommand(string collection, Bson[] queries, Bson[] documents, Bson[] perUpdateOptions, Bson optionsBson) +{ + alias FieldsMovedIntoChildren = AliasSeq!("arrayFilters", "collation", "hint", "upsert"); + + Bson cmd = Bson.emptyObject; + cmd["update"] = Bson(collection); + foreach (string k, v; optionsBson.byKeyValue) + if (!k.among!FieldsMovedIntoChildren) + cmd[k] = v; + + Bson[] updatesBson = new Bson[queries.length]; + foreach (i, q; queries) + { + auto updateBson = Bson.emptyObject; + updateBson["q"] = q; + updateBson["u"] = documents[i]; + foreach (string k, v; optionsBson.byKeyValue) + if (k.among!FieldsMovedIntoChildren) + updateBson[k] = v; + foreach (string k, v; perUpdateOptions[i].byKeyValue) + updateBson[k] = v; + updatesBson[i] = updateBson; + } + cmd["updates"] = Bson(updatesBson); + + return cmd; +} + +unittest { + auto query = Bson(["x": Bson(1)]); + auto doc = Bson(["$set": Bson(["y": Bson(2)])]); + auto perUpdate = Bson(["multi": Bson(true)]); + auto options = Bson(["ordered": Bson(true), "upsert": Bson(true)]); + + auto cmd = buildUpdateCommand("coll", [query], [doc], [perUpdate], options); + assert(cmd["update"].get!string == "coll"); + assert(cmd["ordered"].get!bool == true); + assert(cmd["upsert"].isNull); + + auto stmt = cmd["updates"][0]; + assert(stmt["q"] == query); + assert(stmt["u"] == doc); + assert(stmt["upsert"].get!bool == true); + assert(stmt["multi"].get!bool == true); +} + +/** Builds the aggregation pipeline used by `countDocuments`. + + See_Also: $(LINK https://github.com/mongodb/specifications/blob/525dae0aa8791e782ad9dd93e507b60c55a737bb/source/crud/crud.rst#count-api-details) +*/ +Bson[] buildCountPipeline(Bson filter, CountOptions options) +{ + Bson[] pipeline = [Bson(["$match": filter])]; + + if (!options.skip.isNull) + pipeline ~= Bson(["$skip": Bson(options.skip.get)]); + + if (!options.limit.isNull) + pipeline ~= Bson(["$limit": Bson(options.limit.get)]); + + pipeline ~= Bson(["$group": Bson([ + "_id": Bson(1), + "n": Bson(["$sum": Bson(1)]) + ])]); + + return pipeline; +} + +unittest { + auto filter = Bson(["x": Bson(1)]); + + auto minimal = buildCountPipeline(filter, CountOptions.init); + assert(minimal.length == 2); + assert(minimal[0]["$match"] == filter); + assert(minimal[1]["$group"]["n"]["$sum"].get!int == 1); + + CountOptions skipLimit; + skipLimit.skip = 5; + skipLimit.limit = 10; + auto full = buildCountPipeline(filter, skipLimit); + assert(full.length == 4); + assert(full[1]["$skip"].get!long == 5); + assert(full[2]["$limit"].get!long == 10); +} + +/** Assembles an `aggregate` command and its cursor batching parameters. + + When `explain` is set, the spec recommends omitting the `cursor` field. +*/ +CursorCommand buildAggregateCommand(string collection, string database, Bson pipeline, AggregateOptions options, ReadPreference pref = ReadPreference.primary, string[string][] tagSets = null) +{ + Bson cmd = Bson.emptyObject; + cmd["aggregate"] = Bson(collection); + cmd["$db"] = Bson(database); + cmd["pipeline"] = pipeline; + foreach (string k, v; serializeToBson(options).byKeyValue) + { + if (!options.explain.isNull && options.explain.get && k == "cursor") + continue; + cmd[k] = v; + } + + if (pref != ReadPreference.primary) + cmd["$readPreference"] = readPreferenceBson(pref, tagSets); + + return CursorCommand(cmd, + !options.batchSize.isNull ? options.batchSize.get : 0, + !options.maxAwaitTimeMS.isNull ? options.maxAwaitTimeMS.get.msecs + : !options.maxTimeMS.isNull ? options.maxTimeMS.get.msecs + : Duration.max); +} + +unittest { + auto pipeline = Bson([Bson(["$match": Bson.emptyObject])]); + + auto plain = buildAggregateCommand("coll", "db", pipeline, AggregateOptions.init); + assert(plain.command["aggregate"].get!string == "coll"); + assert(plain.command["$db"].get!string == "db"); + assert(plain.command["pipeline"] == pipeline); + assert(plain.batchSize == 0); + assert(plain.getMoreMaxTime == Duration.max); + + AggregateOptions timed; + timed.maxTimeMS = 1200; + auto timedResult = buildAggregateCommand("coll", "db", pipeline, timed); + assert(timedResult.getMoreMaxTime == 1200.msecs); + + // maxAwaitTimeMS takes precedence over maxTimeMS for getMore + AggregateOptions awaiting; + awaiting.maxAwaitTimeMS = 900; + awaiting.maxTimeMS = 1200; + auto awaitingResult = buildAggregateCommand("coll", "db", pipeline, awaiting); + assert(awaitingResult.getMoreMaxTime == 900.msecs); + + // the cursor field is normally present, but omitted when explain is set + assert(!plain.command["cursor"].isNull); + AggregateOptions explained; + explained.explain = true; + auto explainedResult = buildAggregateCommand("coll", "db", pipeline, explained); + assert(explainedResult.command["cursor"].isNull); + + // a non-primary read preference is injected as $readPreference + auto secondaryRead = buildAggregateCommand("coll", "db", pipeline, AggregateOptions.init, ReadPreference.secondary); + assert(secondaryRead.command["$readPreference"]["mode"].get!string == "secondary"); + + // primary (the default) is omitted from the wire + auto primaryRead = buildAggregateCommand("coll", "db", pipeline, AggregateOptions.init, ReadPreference.primary); + assert(primaryRead.command["$readPreference"].isNull); + auto defaultedRead = buildAggregateCommand("coll", "db", pipeline, AggregateOptions.init); + assert(defaultedRead.command["$readPreference"].isNull); + + // the configured readPreferenceTags reach the aggregate $readPreference too + string[string][] tags = [["dc": "east"]]; + auto taggedRead = buildAggregateCommand("coll", "db", pipeline, AggregateOptions.init, ReadPreference.secondary, tags); + assert(taggedRead.command["$readPreference"] == readPreferenceBson(ReadPreference.secondary, tags), + "the aggregate $readPreference carries the configured readPreferenceTags"); +} + +/// The reduced limit/batch state for a legacy cursor. +struct LimitReduction +{ + int nret; + long limit; +} + +/** Folds a new `limit(count)` call into the existing cursor limit state. + + A non-positive count is a no-op; otherwise the lowest positive limit wins and + the per-batch count is capped at 1024. +*/ +LimitReduction reduceLimit(int nret, long limit, long count) +{ + if (count > 0) + { + if (nret == 0 || nret > count) + nret = cast(int)min(count, 1024); + + if (limit == 0 || limit > count) + limit = count; + } + + return LimitReduction(nret, limit); +} + +unittest { + assert(reduceLimit(0, 0, 0) == LimitReduction(0, 0)); + assert(reduceLimit(0, 0, 10) == LimitReduction(10, 10)); + assert(reduceLimit(10, 10, 20) == LimitReduction(10, 10)); + assert(reduceLimit(10, 10, 5) == LimitReduction(5, 5)); + assert(reduceLimit(0, 0, 5000) == LimitReduction(1024, 5000)); +} diff --git a/mongodb/vibe/db/mongo/impl/compression.d b/mongodb/vibe/db/mongo/impl/compression.d new file mode 100644 index 0000000000..483c5d52dd --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/compression.d @@ -0,0 +1,207 @@ +/** + MongoDB wire-protocol compression helpers: compressor negotiation and the + (de)compression of OP_COMPRESSED payloads. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.compression; + +import std.conv : to; + +import vibe.db.mongo.connection : MongoDriverException; +import vibe.db.mongo.settings : Compressor, compressorName; + +/// Whether the driver actually implements (de)compression for this compressor. +/// Keep in sync with compressData/decompressData below — advertising or selecting +/// an unimplemented compressor halts the process when the server uses it. +package(vibe.db.mongo) bool isImplementedCompressor(Compressor compressor) @safe +{ + return compressor == Compressor.noop || compressor == Compressor.zlib; +} + +/// The wire names of the compressors the driver will advertise: only the implemented +/// ones, so the server never compresses a reply with a codec the driver can't decompress. +package(vibe.db.mongo) string[] advertisedCompressorNames(const Compressor[] compressors) @safe +{ + import std.algorithm : filter, map; + import std.array : array; + return compressors.filter!isImplementedCompressor.map!compressorName.array; +} + +/// advertisedCompressorNames lists only the implemented compressors (so the server never compresses a reply we can't decompress) +unittest +{ + assert(advertisedCompressorNames([Compressor.snappy, Compressor.zlib, Compressor.zstd]) == ["zlib"], + "only implemented compressors (zlib) are advertised; snappy/zstd are dropped"); + assert(advertisedCompressorNames([Compressor.zlib]) == ["zlib"], + "an all-implemented list is advertised unchanged"); + assert(advertisedCompressorNames([Compressor.snappy]) == [], + "a list of only unimplemented compressors advertises nothing"); +} + +package(vibe.db.mongo) Compressor negotiateCompressor(const Compressor[] clientCompressors, const string[] serverCompressors) +@safe { + foreach (clientComp; clientCompressors) { + if (!isImplementedCompressor(clientComp)) + continue; + foreach (serverComp; serverCompressors) { + if (compressorName(clientComp) == serverComp) { + return clientComp; + } + } + } + + return Compressor.noop; +} + +/// negotiateCompressor picks first client-preferred compressor supported by server +unittest +{ + assert(negotiateCompressor([Compressor.zlib], ["zlib"]) == Compressor.zlib); + assert(negotiateCompressor([Compressor.zstd, Compressor.zlib], ["zlib", "snappy"]) == Compressor.zlib); + assert(negotiateCompressor([Compressor.zstd], ["zlib"]) == Compressor.noop); + assert(negotiateCompressor([], ["zlib"]) == Compressor.noop); + assert(negotiateCompressor([Compressor.zlib], []) == Compressor.noop); +} + +/// negotiateCompressor never selects an unimplemented compressor (only zlib/noop are implemented) +unittest +{ + // both sides support snappy, but the driver can't compress it -> must NOT pick snappy + assert(negotiateCompressor([Compressor.snappy], ["snappy"]) == Compressor.noop, + "negotiateCompressor must not select snappy (unimplemented)"); + // snappy is skipped, zlib (implemented, mutually supported) is chosen + assert(negotiateCompressor([Compressor.snappy, Compressor.zlib], ["snappy", "zlib"]) == Compressor.zlib, + "negotiateCompressor skips unimplemented snappy and selects implemented zlib"); + // zstd likewise unimplemented + assert(negotiateCompressor([Compressor.zstd], ["zstd"]) == Compressor.noop, + "negotiateCompressor must not select zstd (unimplemented)"); +} + +package(vibe.db.mongo) Compressor compressorFromId(ubyte id) +@safe { + switch (id) { + case 0: return Compressor.noop; + case 1: return Compressor.snappy; + case 2: return Compressor.zlib; + case 3: return Compressor.zstd; + default: throw new MongoDriverException("Unknown compressor ID: " ~ id.to!string); + } +} + +/// compressorFromId maps wire protocol IDs to Compressor enum values +unittest +{ + assert(compressorFromId(0) == Compressor.noop); + assert(compressorFromId(1) == Compressor.snappy); + assert(compressorFromId(2) == Compressor.zlib); + assert(compressorFromId(3) == Compressor.zstd); +} + +package(vibe.db.mongo) ubyte[] compressData(Compressor compressor, const(ubyte)[] data, int zlibLevel) +@trusted { + final switch (compressor) { + case Compressor.noop: + return data.dup; + case Compressor.zlib: + import std.zlib : compress; + return cast(ubyte[]) compress(data, zlibLevel == -1 ? 6 : zlibLevel); + case Compressor.snappy: + throw new MongoDriverException("snappy compression not yet implemented"); + case Compressor.zstd: + throw new MongoDriverException("zstd compression not yet implemented"); + } +} + +package(vibe.db.mongo) ubyte[] decompressData(Compressor compressor, const(ubyte)[] data, int uncompressedSize) +@trusted { + final switch (compressor) { + case Compressor.noop: + return data.dup; + case Compressor.zlib: + import std.zlib : uncompress; + return cast(ubyte[]) uncompress(data, uncompressedSize); + case Compressor.snappy: + throw new MongoDriverException("snappy decompression not yet implemented"); + case Compressor.zstd: + throw new MongoDriverException("zstd decompression not yet implemented"); + } +} + +/// compressData and decompressData round-trip preserves original data +unittest +{ + auto original = cast(const(ubyte)[]) "The robot shall not harm a human, but I really want to."; + auto compressed = compressData(Compressor.zlib, original, 6); + auto decompressed = decompressData(Compressor.zlib, compressed, cast(int) original.length); + assert(decompressed == original); +} + +/// compressData throws a recoverable MongoDriverException for an unimplemented compressor (never halts via assert(false)) +unittest +{ + import std.exception : assertThrown; + auto data = cast(const(ubyte)[]) "payload"; + assertThrown!MongoDriverException(compressData(Compressor.snappy, data, 6), + "compressData(snappy) must throw a recoverable MongoDriverException, not assert(false)"); +} + +/// MongoDB's default maxMessageSizeBytes (48 MB): the largest single wire message a server sends. +package(vibe.db.mongo) enum int defaultMaxMessageSizeBytes = 48_000_000; + +/// Validates OP_COMPRESSED wire-supplied sizes before allocating or decompressing. A negative +/// size would allocate a huge buffer (fatal OutOfMemoryError); an over-large uncompressedSize is +/// a decompression bomb. Both must be in `[0, maxMessageSizeBytes]`. +package(vibe.db.mongo) void enforceCompressedSizes(int compressedSize, int uncompressedSize, int maxMessageSizeBytes) @safe +{ + import std.exception : enforce; + enforce!MongoDriverException(compressedSize >= 0 && compressedSize <= maxMessageSizeBytes, + "OP_COMPRESSED compressed size out of range: " ~ compressedSize.to!string); + enforce!MongoDriverException(uncompressedSize >= 0 && uncompressedSize <= maxMessageSizeBytes, + "OP_COMPRESSED uncompressed size out of range: " ~ uncompressedSize.to!string); +} + +/// Whether a command must never be compressed because it carries credentials or is part of +/// the authentication handshake. Per the OP_COMPRESSED spec these are exempt regardless of +/// the negotiated compressor, not only during the initial connect-time auth window. +package(vibe.db.mongo) bool isCompressionExempt(string commandName) @safe +{ + switch (commandName) + { + case "hello", "isMaster", "ismaster", + "saslStart", "saslContinue", "authenticate", "getnonce", + "createUser", "updateUser", + "copydbsaslstart", "copydbgetnonce", "copydb": + return true; + default: + return false; + } +} + +/// isCompressionExempt flags the credential/handshake commands the spec forbids compressing +unittest +{ + assert(isCompressionExempt("saslStart"), "saslStart carries auth data and must not be compressed"); + assert(isCompressionExempt("saslContinue"), "saslContinue carries auth data and must not be compressed"); + assert(isCompressionExempt("createUser"), "createUser carries a password and must not be compressed"); + assert(isCompressionExempt("updateUser"), "updateUser may carry a password and must not be compressed"); + assert(isCompressionExempt("hello"), "the handshake hello is exempt"); + assert(!isCompressionExempt("insert"), "ordinary commands may be compressed"); + assert(!isCompressionExempt("find"), "ordinary commands may be compressed"); +} + +/// enforceCompressedSizes rejects negative or over-large OP_COMPRESSED wire sizes +unittest +{ + import std.exception : assertThrown, assertNotThrown; + enum int max = defaultMaxMessageSizeBytes; + + assertNotThrown(enforceCompressedSizes(100, 500, max), "in-range sizes pass"); + assertNotThrown(enforceCompressedSizes(0, 0, max), "zero sizes pass (an empty message)"); + assertThrown!MongoDriverException(enforceCompressedSizes(-1, 500, max), "a negative compressed size is rejected"); + assertThrown!MongoDriverException(enforceCompressedSizes(100, -1, max), "a negative uncompressed size is rejected"); + assertThrown!MongoDriverException(enforceCompressedSizes(max + 1, 500, max), "an over-large compressed size is rejected"); + assertThrown!MongoDriverException(enforceCompressedSizes(100, max + 1, max), "a decompression-bomb uncompressed size is rejected"); +} diff --git a/mongodb/vibe/db/mongo/impl/crud.d b/mongodb/vibe/db/mongo/impl/crud.d index 2fe5ed79ee..21db691600 100644 --- a/mongodb/vibe/db/mongo/impl/crud.d +++ b/mongodb/vibe/db/mongo/impl/crud.d @@ -11,9 +11,11 @@ import core.time; import vibe.db.mongo.connection : MongoException; import vibe.db.mongo.collection; +import vibe.db.mongo.settings : ReadPreference; import vibe.data.bson; import std.typecons; +import std.exception : enforce; @safe: @@ -263,6 +265,19 @@ struct FindOptions Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) */ @embedNullable Nullable!ReadConcern readConcern; + + /// Per-operation read-preference override; injected as `$readPreference`, not serialized (`@ignore`). + @ignore Nullable!ReadPreference readPreference; +} + +unittest { + FindOptions findOpts; + findOpts.readPreference = ReadPreference.secondary; + assert(serializeToBson(findOpts)["readPreference"].isNull); + + AggregateOptions aggOpts; + aggOpts.readPreference = ReadPreference.secondary; + assert(serializeToBson(aggOpts)["readPreference"].isNull); } /// @@ -334,6 +349,9 @@ struct DistinctOptions */ @embedNullable Nullable!string comment; + + /// Per-operation read-preference override; injected as `$readPreference`, not serialized (`@ignore`). + @ignore Nullable!ReadPreference readPreference; } /** @@ -397,6 +415,9 @@ struct CountOptions Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) */ @embedNullable Nullable!ReadConcern readConcern; + + /// Per-operation read-preference override; injected as `$readPreference`, not serialized (`@ignore`). + @ignore Nullable!ReadPreference readPreference; } /** @@ -427,6 +448,9 @@ struct EstimatedDocumentCountOptions Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) */ @embedNullable Nullable!ReadConcern readConcern; + + /// Per-operation read-preference override; injected as `$readPreference`, not serialized (`@ignore`). + @ignore Nullable!ReadPreference readPreference; } /** @@ -547,6 +571,21 @@ struct AggregateOptions Standards: $(LINK https://github.com/mongodb/specifications/blob/7745234f93039a83ae42589a6c0cdbefcffa32fa/source/read-write-concern/read-write-concern.rst) */ @embedNullable Nullable!ReadConcern readConcern; + + /// Per-operation read-preference override; injected as `$readPreference`, not serialized (`@ignore`). + @ignore Nullable!ReadPreference readPreference; +} + +/// Mixes in the standard optional `writeConcern` field shared by the write-option structs. +mixin template WriteConcernOption() +{ + /** + A document that expresses the + $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) + of the insert command. Omit to use the default write concern. + */ + @embedNullable + Nullable!WriteConcern writeConcern; } /** @@ -576,13 +615,7 @@ struct BulkWriteOptions { @embedNullable Nullable!bool bypassDocumentValidation; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -607,13 +640,7 @@ struct InsertOneOptions { @embedNullable Nullable!bool bypassDocumentValidation; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -648,13 +675,7 @@ struct InsertManyOptions { @embedNullable Nullable!bool ordered; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -709,13 +730,7 @@ struct UpdateOptions { @embedNullable Nullable!bool upsert; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -763,13 +778,7 @@ struct ReplaceOptions { @embedNullable Nullable!bool upsert; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -802,13 +811,7 @@ struct DeleteOptions { @embedNullable Nullable!Bson hint; - /** - A document that expresses the - $(LINK2 https://www.mongodb.com/docs/manual/reference/write-concern/,write concern) - of the insert command. Omit to use the default write concern. - */ - @embedNullable - Nullable!WriteConcern writeConcern; + mixin WriteConcernOption; /** Users can specify an arbitrary string to help trace the operation @@ -831,9 +834,11 @@ struct DeleteOptions { struct InsertOneResult { /** - The identifier that was automatically generated, if not set. + The identifier of the inserted document. Generated when the document had no + `_id`; otherwise the client-supplied id verbatim, which may be any BSON type + (int, string, UUID, …), not only an ObjectID. */ - BsonObjectID insertedId; + Bson insertedId; } struct InsertManyResult { @@ -866,11 +871,11 @@ struct UpdateResult { long modifiedCount; /** - The identifier of the inserted document if an upsert took place. Can be - none if no upserts took place, can be multiple if using the updateImpl - helper. + The identifiers of the documents inserted by an upsert. Empty when no upsert + took place; can be multiple via the updateImpl helper. Each id may be any BSON + type (int, string, UUID, …), not only an ObjectID. */ - BsonObjectID[] upsertedIds; + Bson[] upsertedIds; } /** @@ -949,3 +954,56 @@ package(vibe.db.mongo) void handleWriteResult(string countField = null, T)( } } } + +unittest { + FindOptions find; + find.maxTime(2.seconds); + find.maxAwaitTime(1500.msecs); + assert(find.maxTimeMS == 2000); + assert(find.maxAwaitTimeMS == 1500); + + DistinctOptions distinct; + distinct.maxTime(3.seconds); + assert(distinct.maxTimeMS == 3000); + + CountOptions count; + count.maxTime(4.seconds); + assert(count.maxTimeMS == 4000); + + EstimatedDocumentCountOptions estimated; + estimated.maxTime(5.seconds); + assert(estimated.maxTimeMS == 5000); + + AggregateOptions aggregate; + aggregate.maxTime(6.seconds); + aggregate.maxAwaitTime(700.msecs); + aggregate.batchSize = 50; + assert(aggregate.maxTimeMS == 6000); + assert(aggregate.maxAwaitTimeMS == 700); + assert(aggregate.batchSize == 50); +} + +unittest { + DeleteResult deleted; + handleWriteResult!"deletedCount"(Bson(["n": Bson(7)]), deleted); + assert(deleted.deletedCount == 7); + + DeleteResult missingCount; + handleWriteResult!"deletedCount"(Bson(["ok": Bson(1.0)]), missingCount); + assert(missingCount.deletedCount == 0); + + UpdateResult noErrors; + handleWriteResult(Bson(["writeErrors": Bson(cast(Bson[])[])]), noErrors); + + auto writeErrors = Bson([Bson(["code": Bson(11000), "errmsg": Bson("duplicate key")])]); + UpdateResult failed; + bool threw; + try { + handleWriteResult(Bson(["writeErrors": writeErrors]), failed); + } catch (MongoBulkWriteException e) { + threw = true; + assert(e.errors.length == 1); + assert(e.errors[0].code == 11000); + } + assert(threw); +} diff --git a/mongodb/vibe/db/mongo/impl/index.d b/mongodb/vibe/db/mongo/impl/index.d index 18fcc691f9..f858cd5624 100644 --- a/mongodb/vibe/db/mongo/impl/index.d +++ b/mongodb/vibe/db/mongo/impl/index.d @@ -112,6 +112,28 @@ struct IndexModel } } +unittest { + auto single = IndexModel().add("age", 1); + assert(single.name == "age_1"); + + auto compound = IndexModel().add("a", 1).add("b", -1); + assert(compound.name == "a_1_b_-1"); + + auto typed = IndexModel().add("title", IndexType.text); + assert(typed.name == "title_text"); + + IndexOptions options; + options.name = "custom_name"; + auto named = IndexModel().add("age", 1).withOptions(options); + assert(named.name == "custom_name"); +} + +unittest { + IndexOptions options; + options.expireAfter(120.seconds); + assert(options.expireAfterSeconds == 120); +} + /** Specifies the different index types which are available for index creation. diff --git a/mongodb/vibe/db/mongo/impl/serverdescription.d b/mongodb/vibe/db/mongo/impl/serverdescription.d new file mode 100644 index 0000000000..acce53539a --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/serverdescription.d @@ -0,0 +1,480 @@ +/** + MongoDB server description and topology state: the per-server `hello`/`isMaster` + response model, its classification helpers, and replica-set matching. + + Pure data + classification logic; the live probing that fills these in + (`probeServer`) stays in connection.d because it drives a MongoConnection. + + Copyright: © 2020-2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Jan Jurzitza +*/ +module vibe.db.mongo.impl.serverdescription; + +import std.typecons : Nullable; + +import vibe.data.bson; +import vibe.db.mongo.impl.wireversion : WireVersion; + +struct TopologyVersion +{ +@optional: + BsonObjectID processId; + long counter = -1; +} + +struct ServerDescription +{ + enum ServerType + { + unknown, + standalone, + mongos, + possiblePrimary, + RSPrimary, + RSSecondary, + RSArbiter, + RSOther, + RSGhost + } + + static struct LastWrite + { + @optional: + Nullable!BsonDate lastWriteDate; + } + +@optional: + string address; + string error; + float roundTripTime = 0; + LastWrite lastWrite; + Nullable!BsonObjectID opTime; + /// The backend service id returned by a load-balanced server's `hello` reply, + /// used to pin cursors/transactions to the same backend. Absent for non-LB servers. + Nullable!BsonObjectID serviceId; + ServerType type = ServerType.unknown; + int minWireVersion, maxWireVersion; + string me; + string[] hosts, passives, arbiters; + string[string] tags; + string setName; + Nullable!int setVersion; + Nullable!BsonObjectID electionId; + string primary; + Nullable!TopologyVersion topologyVersion; + + /// Deprecated since MongoDB 5.0: the `isMaster` command was replaced by `hello`. + /// The `secondary` field itself is still present in the `hello` response. + bool secondary; + + /// Deprecated since MongoDB 5.0: renamed to `isWritablePrimary` in the `hello` command response. + /// True if the instance is a primary, mongos, or standalone mongod. + bool ismaster; + + bool isWritablePrimary; + bool arbiterOnly; + string msg; + Nullable!int logicalSessionTimeoutMinutes; + string[] compression; + + /// Set by the driver after probing, not deserialized from the server response. + long lastUpdateTimeUsecs; + + bool satisfiesVersion(WireVersion wireVersion) @safe const @nogc pure nothrow + { + return maxWireVersion >= wireVersion; + } + + bool isPrimary() @safe const @nogc pure nothrow + { + return (ismaster || isWritablePrimary) && !secondary; + } + + bool isSecondaryNode() @safe const @nogc pure nothrow + { + return secondary && !ismaster && !isWritablePrimary; + } + + bool isReplicaSetMember() @safe const @nogc pure nothrow + { + return setName.length > 0; + } + + /// A data-bearing server holds data clients can read or write: a primary or + /// secondary, never an arbiter. Used to scope topology-wide logical session timeouts. + bool isDataBearing() @safe const @nogc pure nothrow + { + return (isPrimary || isSecondaryNode) && !arbiterOnly; + } + + ServerType classifiedType() @safe const @nogc pure nothrow + { + if (msg == "isdbgrid") + return ServerType.mongos; + + if (setName.length) + { + if (isPrimary) + return ServerType.RSPrimary; + + if (isSecondaryNode) + return ServerType.RSSecondary; + + if (arbiterOnly) + return ServerType.RSArbiter; + + return ServerType.RSOther; + } + + if (isPrimary) + return ServerType.standalone; + + return ServerType.unknown; + } +} + +/// deserializes the load-balancer serviceId BsonObjectID from a hello reply +unittest +{ + auto id = BsonObjectID.generate(); + auto reply = Bson([ + "ok": Bson(1.0), + "isWritablePrimary": Bson(true), + "maxWireVersion": Bson(21), + "serviceId": Bson(id), + ]); + auto desc = deserializeBson!ServerDescription(reply); + assert(!desc.serviceId.isNull); + assert(desc.serviceId.get == id); +} + +/// satisfiesVersion returns true for versions up to maxWireVersion v36 +@safe unittest +{ + ServerDescription desc; + desc.maxWireVersion = WireVersion.v36; + assert(desc.satisfiesVersion(WireVersion.old)); + assert(desc.satisfiesVersion(WireVersion.v26)); + assert(desc.satisfiesVersion(WireVersion.v30)); + assert(desc.satisfiesVersion(WireVersion.v34)); + assert(desc.satisfiesVersion(WireVersion.v36)); + assert(!desc.satisfiesVersion(WireVersion.v40)); + assert(!desc.satisfiesVersion(WireVersion.v44)); + assert(!desc.satisfiesVersion(WireVersion.v60)); +} + +/// satisfiesVersion with maxWireVersion old only satisfies old +@safe unittest +{ + ServerDescription oldServer; + oldServer.maxWireVersion = WireVersion.old; + assert(oldServer.satisfiesVersion(WireVersion.old)); + assert(!oldServer.satisfiesVersion(WireVersion.v26)); + assert(!oldServer.satisfiesVersion(WireVersion.v30)); +} + +/// satisfiesVersion with maxWireVersion v80 satisfies all versions +@safe unittest +{ + ServerDescription latestServer; + latestServer.maxWireVersion = WireVersion.v80; + assert(latestServer.satisfiesVersion(WireVersion.old)); + assert(latestServer.satisfiesVersion(WireVersion.v36)); + assert(latestServer.satisfiesVersion(WireVersion.v44)); + assert(latestServer.satisfiesVersion(WireVersion.v60)); + assert(latestServer.satisfiesVersion(WireVersion.v70)); + assert(latestServer.satisfiesVersion(WireVersion.v80)); +} + +/// Default-initialized ServerDescription has maxWireVersion 0 and unknown type +@safe unittest +{ + ServerDescription def; + assert(def.maxWireVersion == 0); + assert(def.type == ServerDescription.ServerType.unknown); + assert(def.satisfiesVersion(WireVersion.old)); + assert(!def.satisfiesVersion(WireVersion.v26)); +} + +/// isPrimary returns true when ismaster=true and secondary=false +@safe unittest +{ + ServerDescription desc; + desc.ismaster = true; + desc.secondary = false; + assert(desc.isPrimary); +} + +/// isPrimary returns false when both ismaster=true and secondary=true +@safe unittest +{ + ServerDescription desc; + desc.ismaster = true; + desc.secondary = true; + assert(!desc.isPrimary); +} + +/// isPrimary returns false when ismaster=false +@safe unittest +{ + ServerDescription desc; + desc.ismaster = false; + desc.secondary = false; + assert(!desc.isPrimary); +} + +/// isPrimary returns true when isWritablePrimary=true (hello response) +@safe unittest +{ + ServerDescription desc; + desc.isWritablePrimary = true; + desc.secondary = false; + assert(desc.isPrimary); +} + +/// isPrimary returns false when isWritablePrimary=true but secondary=true +@safe unittest +{ + ServerDescription desc; + desc.isWritablePrimary = true; + desc.secondary = true; + assert(!desc.isPrimary); +} + +/// isSecondaryNode returns true when secondary=true and ismaster=false +@safe unittest +{ + ServerDescription desc; + desc.secondary = true; + desc.ismaster = false; + assert(desc.isSecondaryNode); +} + +/// isSecondaryNode returns false when ismaster=true +@safe unittest +{ + ServerDescription desc; + desc.secondary = true; + desc.ismaster = true; + assert(!desc.isSecondaryNode); +} + +/// isSecondaryNode returns false when isWritablePrimary=true +@safe unittest +{ + ServerDescription desc; + desc.secondary = true; + desc.isWritablePrimary = true; + assert(!desc.isSecondaryNode); +} + +/// isSecondaryNode returns false when secondary=false +@safe unittest +{ + ServerDescription desc; + desc.secondary = false; + desc.ismaster = false; + assert(!desc.isSecondaryNode); +} + +/// isReplicaSetMember returns true when setName is non-empty +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + assert(desc.isReplicaSetMember); +} + +/// isReplicaSetMember returns false when setName is empty +@safe unittest +{ + ServerDescription desc; + assert(!desc.isReplicaSetMember); +} + +/// Default ServerDescription is not primary, not secondary, not RS member +@safe unittest +{ + ServerDescription desc; + assert(!desc.isPrimary); + assert(!desc.isSecondaryNode); + assert(!desc.isReplicaSetMember); +} + +/// isDataBearing returns true for a primary +@safe unittest +{ + ServerDescription desc; + desc.isWritablePrimary = true; + assert(desc.isDataBearing); +} + +/// isDataBearing returns true for a secondary +@safe unittest +{ + ServerDescription desc; + desc.secondary = true; + assert(desc.isDataBearing); +} + +/// isDataBearing returns false for an arbiter +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + desc.arbiterOnly = true; + assert(!desc.isDataBearing); +} + +/// isDataBearing returns false for a default (unknown) description +@safe unittest +{ + ServerDescription desc; + assert(!desc.isDataBearing); +} + +/// classifiedType returns mongos when msg is isdbgrid +@safe unittest +{ + ServerDescription desc; + desc.msg = "isdbgrid"; + assert(desc.classifiedType == ServerDescription.ServerType.mongos); +} + +/// classifiedType returns RSPrimary for a primary with a set name +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + desc.ismaster = true; + assert(desc.classifiedType == ServerDescription.ServerType.RSPrimary); +} + +/// classifiedType returns RSSecondary for a secondary with a set name +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + desc.secondary = true; + assert(desc.classifiedType == ServerDescription.ServerType.RSSecondary); +} + +/// classifiedType returns RSArbiter for an arbiter with a set name +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + desc.arbiterOnly = true; + assert(desc.classifiedType == ServerDescription.ServerType.RSArbiter); +} + +/// classifiedType returns RSOther for a set member that is neither primary, secondary nor arbiter +@safe unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + assert(desc.classifiedType == ServerDescription.ServerType.RSOther); +} + +/// classifiedType returns standalone for a primary without a set name +@safe unittest +{ + ServerDescription desc; + desc.ismaster = true; + assert(desc.classifiedType == ServerDescription.ServerType.standalone); +} + +/// classifiedType returns unknown for a default description +@safe unittest +{ + ServerDescription desc; + assert(desc.classifiedType == ServerDescription.ServerType.unknown); +} + +/** + * Checks whether the server's replica set name matches the expected one. + * Returns true if no replica set is configured (empty string) or if + * the names match. + */ +package(vibe.db.mongo) bool matchesReplicaSet(string expectedSet, ref const ServerDescription desc) +@safe @nogc pure nothrow +{ + if (!expectedSet.length) + return true; + return desc.setName == expectedSet; +} + +/// Enforces the load-balancer requirement that a `loadBalanced=true` connection's +/// hello reply includes a `serviceId`; throws if the server is not behind a load balancer. +package(vibe.db.mongo) void enforceLoadBalancedServiceId(bool loadBalanced, in ServerDescription desc) @safe +{ + import std.exception : enforce; + if (!loadBalanced) + return; + enforce(!desc.serviceId.isNull, + "loadBalanced=true but the server did not return a serviceId — it is not behind a load balancer"); +} + +/// matchesReplicaSet returns true when no replica set is configured +@safe @nogc pure nothrow unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + assert(matchesReplicaSet("", desc)); +} + +/// matchesReplicaSet returns true when replica set names match +@safe @nogc pure nothrow unittest +{ + ServerDescription desc; + desc.setName = "rs0"; + assert(matchesReplicaSet("rs0", desc)); +} + +/// matchesReplicaSet returns false when replica set names differ +@safe @nogc pure nothrow unittest +{ + ServerDescription desc; + desc.setName = "rs1"; + assert(!matchesReplicaSet("rs0", desc)); +} + +/// matchesReplicaSet returns false when server has no setName but one is expected +@safe @nogc pure nothrow unittest +{ + ServerDescription desc; + assert(!matchesReplicaSet("rs0", desc)); +} + +/// matchesReplicaSet returns true when both are empty +@safe @nogc pure nothrow unittest +{ + ServerDescription desc; + assert(matchesReplicaSet("", desc)); +} + +/// enforceLoadBalancedServiceId throws when loadBalanced but the hello reply has no serviceId +unittest +{ + import std.exception : assertThrown; + ServerDescription desc; + desc.isWritablePrimary = true; + desc.maxWireVersion = 21; + assertThrown(enforceLoadBalancedServiceId(true, desc)); +} + +/// enforceLoadBalancedServiceId does nothing when loadBalanced is false even without a serviceId +@safe unittest +{ + ServerDescription desc; + enforceLoadBalancedServiceId(false, desc); +} + +/// enforceLoadBalancedServiceId does nothing when loadBalanced and a serviceId is present +@safe unittest +{ + ServerDescription desc; + desc.serviceId = BsonObjectID.generate(); + enforceLoadBalancedServiceId(true, desc); +} diff --git a/mongodb/vibe/db/mongo/impl/wire.d b/mongodb/vibe/db/mongo/impl/wire.d new file mode 100644 index 0000000000..17a1c9c06e --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/wire.d @@ -0,0 +1,292 @@ +/** + MongoDB wire-protocol primitives: opcodes, the reply/section delegate types, + message length computation and OP_MSG body parsing. + + These are internal helpers used by the MongoConnection send/receive methods, + which stay in connection.d because they operate on the connection's stream. + + Copyright: © 2012-2016 Sönke Ludwig, © 2020-2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Sönke Ludwig, Jan Jurzitza +*/ +module vibe.db.mongo.impl.wire; + +import std.conv : to; +import std.exception : enforce; + +import vibe.data.bson; +import vibe.db.mongo.connection : MongoDriverException; +import vibe.db.mongo.flags : ReplyFlags; + +package(vibe.db.mongo) enum OpCode : int { + Reply = 1, // sent only by DB + Update = 2001, + Insert = 2002, + Reserved1 = 2003, + Query = 2004, + GetMore = 2005, + Delete = 2006, + KillCursors = 2007, + + Compressed = 2012, + Msg = 2013, +} + +/// Whether an OP_MSG flagBits field has checksumPresent (bit 0) set — a 4-byte CRC32C +/// trailer follows the sections. Bit 16 is exhaustAllowed, not the checksum bit; +/// confusing the two parses the CRC as a section or truncates real data (review M13). +package(vibe.db.mongo) bool checksumPresent(uint flagBits) @safe +{ + return (flagBits & 1) != 0; +} + +package(vibe.db.mongo) alias ReplyDelegate = void delegate(long cursor, ReplyFlags flags, int first_doc, int num_docs) @safe; +package(vibe.db.mongo) template DocDelegate(T) { alias DocDelegate = void delegate(size_t idx, ref T doc) @safe; } + +package(vibe.db.mongo) alias MsgReplyDelegate(bool dupBson : true) = void delegate(uint flags, Bson document) @safe; +package(vibe.db.mongo) alias MsgReplyDelegate(bool dupBson : false) = void delegate(uint flags, scope Bson document) @safe; +package(vibe.db.mongo) alias MsgSection1StartDelegate = void delegate(scope const(char)[] identifier, int size) @safe; +package(vibe.db.mongo) alias MsgSection1Delegate(bool dupBson : true) = void delegate(scope const(char)[] identifier, Bson document) @safe; +package(vibe.db.mongo) alias MsgSection1Delegate(bool dupBson : false) = void delegate(scope const(char)[] identifier, scope Bson document) @safe; + +package(vibe.db.mongo) int sendLength(ARGS...)(scope ARGS args) +{ + import std.traits; + static if (ARGS.length == 1) { + alias T = ARGS[0]; + static if (is(T == string)) return cast(int)args[0].length + 1; + else static if (is(T == int)) return 4; + else static if (is(T == long)) return 8; + else static if (is(T == Bson)) return cast(int)() @trusted { return args[0].data.length; } (); + else static if (isArray!T) { + int ret = 0; + foreach (el; args[0]) ret += sendLength(el); + return ret; + } else static assert(false, "Unexpected type: "~T.stringof); + } + else if (ARGS.length == 0) return 0; + else return sendLength(args[0 .. $/2]) + sendLength(args[$/2 .. $]); +} + +/// sendLength of a string is its length plus one +unittest +{ + assert(sendLength("test") == 5); + assert(sendLength("") == 1); +} + +/// sendLength of an int is 4 and of a long is 8 +unittest +{ + assert(sendLength(42) == 4); + assert(sendLength(42L) == 8); +} + +/// sendLength of a Bson is the length of its raw data +unittest +{ + auto bson = Bson(42); + assert(sendLength(bson) == cast(int)bson.data.length); +} + +/// sendLength of an array sums the lengths of its elements +unittest +{ + assert(sendLength(["ab", "c"]) == 5); + assert(sendLength(cast(string[])[]) == 0); +} + +/// sendLength of multiple arguments sums each argument +unittest +{ + assert(sendLength("test", 42) == 9); + assert(sendLength() == 0); +} + +package(vibe.db.mongo) void parseOpMsgBody(bool dupBson)( + const(ubyte)[] data, + scope MsgReplyDelegate!dupBson on_sec0, + scope MsgSection1StartDelegate on_sec1_start, + scope MsgSection1Delegate!dupBson on_sec1_doc) +{ + import std.bitmanip : littleEndianToNative; + + size_t pos = 0; + + T readVal(T)() @trusted { + enum sz = T.sizeof; + enforce!MongoDriverException(pos + sz <= data.length, "Buffer underflow in decompressed OP_MSG"); + ubyte[sz] buf = (cast(ubyte[]) data[pos .. pos + sz])[0 .. sz]; + pos += sz; + return littleEndianToNative!(T, sz)(buf); + } + + uint flagBits = readVal!uint(); + const bool hasCRC = checksumPresent(flagBits); + const size_t endPos = data.length - (hasCRC ? uint.sizeof : 0); + + bool gotSec0; + while (pos < endPos) { + ubyte payloadType = readVal!ubyte(); + + switch (payloadType) { + case 0: + gotSec0 = true; + int bsonLen = readVal!int(); + enforce!MongoDriverException(bsonLen >= 5, "Invalid BSON document length in decompressed OP_MSG"); + enforce!MongoDriverException(pos + bsonLen - 4 <= data.length, "BSON overflows decompressed buffer"); + + auto bsonData = new ubyte[bsonLen]; + bsonData[0 .. 4] = toBsonData(bsonLen)[]; + bsonData[4 .. bsonLen] = data[pos .. pos + bsonLen - 4]; + pos += bsonLen - 4; + + auto doc = () @trusted { return Bson(Bson.Type.object, cast(immutable) bsonData); }(); + on_sec0(flagBits, doc); + break; + + case 1: + if (!gotSec0) { + throw new MongoDriverException("Got OP_MSG section 1 before section 0 in decompressed message"); + } + + auto sectionStart = pos; + int size = readVal!int(); + + auto identStart = pos; + while (pos < endPos && data[pos] != 0) { + pos++; + } + auto identifier = cast(const(char)[]) data[identStart .. pos]; + pos++; + + on_sec1_start(identifier, size); + + while (pos - sectionStart < size) { + int docLen = readVal!int(); + enforce!MongoDriverException(docLen >= 5, "Invalid BSON document length in decompressed OP_MSG section 1"); + enforce!MongoDriverException(pos + docLen - 4 <= data.length, "BSON overflows decompressed buffer in OP_MSG section 1"); + + auto bsonData = new ubyte[docLen]; + bsonData[0 .. 4] = toBsonData(docLen)[]; + bsonData[4 .. docLen] = data[pos .. pos + docLen - 4]; + pos += docLen - 4; + + auto doc = () @trusted { return Bson(Bson.Type.object, cast(immutable) bsonData); }(); + on_sec1_doc(identifier, doc); + } + break; + + default: + throw new MongoDriverException("Unexpected payload section type in decompressed message: " ~ payloadType.to!string); + } + } +} + +/// parseOpMsgBody parses section 0 document and flags from raw OP_MSG body +unittest +{ + auto doc = Bson(["ok": Bson(1.0)]); + auto docBytes = () @trusted { return cast(const(ubyte)[]) doc.data; }(); + + ubyte[] body_; + body_ ~= toBsonData(cast(uint) 0)[]; + body_ ~= cast(ubyte) 0; + body_ ~= docBytes; + + Bson parsed; + uint parsedFlags; + + parseOpMsgBody!true(body_, + (flags, document) { parsedFlags = flags; parsed = document; }, + (scope ident, size) {}, + (scope ident, document) {}); + + assert(parsedFlags == 0); + assert(parsed["ok"].get!double == 1.0); +} + +/// parseOpMsgBody throws a catchable MongoDriverException (not RangeError) on a truncated section-1 document +unittest +{ + import std.exception : assertThrown; + + auto sec0Doc = Bson(["ok": Bson(1.0)]); + auto sec0Bytes = () @trusted { return cast(const(ubyte)[]) sec0Doc.data; }(); + + ubyte[] body_; + body_ ~= toBsonData(cast(uint) 0)[]; + + body_ ~= cast(ubyte) 0; + body_ ~= sec0Bytes; + + body_ ~= cast(ubyte) 1; + body_ ~= toBsonData(cast(uint) 64)[]; + body_ ~= cast(ubyte) 'd'; + body_ ~= cast(ubyte) 'o'; + body_ ~= cast(ubyte) 'c'; + body_ ~= cast(ubyte) 's'; + body_ ~= cast(ubyte) 0; + body_ ~= toBsonData(cast(uint) 64)[]; + body_ ~= cast(ubyte) 0; + body_ ~= cast(ubyte) 0; + + assertThrown!MongoDriverException( + parseOpMsgBody!true(body_, + (flags, document) {}, + (scope ident, size) {}, + (scope ident, document) {}), + "a truncated section-1 document must throw a catchable MongoDriverException, not an uncatchable RangeError"); +} + +/// parseOpMsgBody treats checksumPresent (flag bit 0) as a CRC trailer, not as a section +unittest +{ + auto doc = Bson(["ok": Bson(1.0)]); + auto docBytes = () @trusted { return cast(const(ubyte)[]) doc.data; }(); + + ubyte[] body_; + body_ ~= toBsonData(cast(uint) 1)[]; + body_ ~= cast(ubyte) 0; + body_ ~= docBytes; + body_ ~= toBsonData(cast(uint) 0xDEADBEEF)[]; + + Bson parsed; + uint parsedFlags; + bool gotSection0; + + parseOpMsgBody!true(body_, + (flags, document) { gotSection0 = true; parsedFlags = flags; parsed = document; }, + (scope ident, size) {}, + (scope ident, document) {}); + + assert(gotSection0, "section 0 callback did not fire"); + assert(parsedFlags == 1u, "checksumPresent flag bit not preserved"); + assert(parsed["ok"].get!double == 1.0, "section 0 document did not round-trip"); +} + +/// parseOpMsgBody correctly parses a compressed and decompressed OP_MSG body +unittest +{ + import vibe.db.mongo.impl.compression : compressData, decompressData; + import vibe.db.mongo.settings : Compressor; + + auto doc = Bson(["ok": Bson(1.0)]); + auto docBytes = () @trusted { return cast(const(ubyte)[]) doc.data; }(); + + ubyte[] body_; + body_ ~= toBsonData(cast(uint) 0)[]; + body_ ~= cast(ubyte) 0; + body_ ~= docBytes; + + auto compressed = compressData(Compressor.zlib, body_, 6); + auto decompressed = decompressData(Compressor.zlib, compressed, cast(int) body_.length); + + Bson parsed; + parseOpMsgBody!true(decompressed, + (flags, document) { parsed = document; }, + (scope ident, size) {}, + (scope ident, document) {}); + + assert(parsed["ok"].get!double == 1.0); +} diff --git a/mongodb/vibe/db/mongo/impl/wireversion.d b/mongodb/vibe/db/mongo/impl/wireversion.d new file mode 100644 index 0000000000..1e28e9c0da --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/wireversion.d @@ -0,0 +1,348 @@ +/** + MongoDB wire-version compatibility: the wire version enum, the UDAs that mark + option fields with version constraints, and the routine that enforces them. + + Kept in a dedicated leaf module so the option modules (crud, index) and the + driver modules can share it without an import cycle. + + Copyright: © 2020-2022 Jan Jurzitza + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Jan Jurzitza +*/ +module vibe.db.mongo.impl.wireversion; + +import std.conv : to; +import std.format : format; +import std.traits : getUDAs; +import std.typecons : Nullable; + +import vibe.core.log; +import vibe.data.bson; +import vibe.db.mongo.connection : MongoException; + +enum WireVersion : int +{ + old = 0, + v26 = 1, + v26_2 = 2, + v30 = 3, + v32 = 4, + v34 = 5, + v36 = 6, + v40 = 7, + v42 = 8, + v44 = 9, + v49 = 12, + v50 = 13, + v51 = 14, + v52 = 15, + v53 = 16, + v60 = 17, + v61 = 18, + v62 = 19, + v70 = 21, + v71 = 22, + v72 = 23, + v73 = 24, + v80 = 25 +} + +/// UDA to unset a nullable field if the server wire version doesn't at least +/// match the given version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct MinWireVersion +{ + /// + WireVersion v; +} + +/// ditto +MinWireVersion since(WireVersion v) @safe { return MinWireVersion(v); } + +/// UDA to warn when a nullable field is set and the server wire version matches +/// the given version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct DeprecatedSinceWireVersion +{ + /// + WireVersion v; +} + +/// ditto +DeprecatedSinceWireVersion deprecatedSince(WireVersion v) @safe { return DeprecatedSinceWireVersion(v); } + +/// UDA to throw a MongoException when a nullable field is set and the server +/// wire version doesn't match the version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct ErrorBeforeWireVersion +{ + /// + WireVersion v; +} + +/// ditto +ErrorBeforeWireVersion errorBefore(WireVersion v) @safe { return ErrorBeforeWireVersion(v); } + +/// UDA to unset a nullable field if the server wire version is newer than the +/// given version. (inclusive) +/// +/// Use with $(LREF enforceWireVersionConstraints) +struct MaxWireVersion +{ + /// + WireVersion v; +} +/// ditto +MaxWireVersion until(WireVersion v) @safe { return MaxWireVersion(v); } + +/// Unsets nullable fields not matching the server version as defined per UDAs. +void enforceWireVersionConstraints(T)(ref T field, int serverVersion, + string file = __FILE__, size_t line = __LINE__) +@safe { + import std.traits : getUDAs; + + string exception; + + foreach (i, ref v; field.tupleof) { + enum minV = getUDAs!(field.tupleof[i], MinWireVersion); + enum maxV = getUDAs!(field.tupleof[i], MaxWireVersion); + enum deprecateV = getUDAs!(field.tupleof[i], DeprecatedSinceWireVersion); + enum errorV = getUDAs!(field.tupleof[i], ErrorBeforeWireVersion); + + static foreach (depr; deprecateV) + if (serverVersion >= depr.v && !v.isNull) + logInfo("User-set field '%s' is deprecated since MongoDB %s (from %s:%s)", + T.tupleof[i].stringof, depr.v, file, line); + + static foreach (err; errorV) + if (serverVersion < err.v && !v.isNull) + exception ~= format("User-set field '%s' is not supported before MongoDB %s\n", + T.tupleof[i].stringof, err.v); + + static foreach (min; minV) + if (serverVersion < min.v) + v.nullify(); + + static foreach (max; maxV) + if (serverVersion > max.v) + v.nullify(); + } + + if (exception.length) + throw new MongoException(exception ~ "from " ~ file ~ ":" ~ line.to!string); +} + +version (unittest) +{ + struct SinceUntilCmd + { + @embedNullable @since(WireVersion.v34) + Nullable!int a; + + @embedNullable @until(WireVersion.v30) + Nullable!int b; + } + + struct ErrorBeforeCmd + { + @embedNullable @errorBefore(WireVersion.v44) + Nullable!int field; + } + + struct DeprecatedCmd + { + @embedNullable @deprecatedSince(WireVersion.v40) + Nullable!int oldField; + } + + struct CombinedCmd + { + @embedNullable @errorBefore(WireVersion.v44) + Nullable!bool allowDiskUse; + + @embedNullable @since(WireVersion.v32) + Nullable!long maxAwaitTimeMS; + + @embedNullable @deprecatedSince(WireVersion.v40) + Nullable!long maxScan; + } + + struct SinceDeprecatedCmd + { + @embedNullable @since(WireVersion.v32) + Nullable!long maxAwaitTimeMS; + + @embedNullable @deprecatedSince(WireVersion.v40) + Nullable!long maxScan; + } +} + +/// @since nullifies field when server version is below minimum +@safe unittest +{ + SinceUntilCmd cmd; + cmd.a = 1; + cmd.b = 2; + + auto test = cmd; + enforceWireVersionConstraints(test, WireVersion.v30); + assert(test.a.isNull); + assert(!test.b.isNull); +} + +/// @until nullifies field when server version exceeds maximum +@safe unittest +{ + SinceUntilCmd cmd; + cmd.a = 1; + cmd.b = 2; + + auto test = cmd; + enforceWireVersionConstraints(test, WireVersion.v32); + assert(test.a.isNull); + assert(test.b.isNull); +} + +/// @since preserves field when server version meets minimum +@safe unittest +{ + SinceUntilCmd cmd; + cmd.a = 1; + cmd.b = 2; + + auto test = cmd; + enforceWireVersionConstraints(test, WireVersion.v34); + assert(!test.a.isNull); + assert(test.b.isNull); +} + +/// @errorBefore throws when field is set and server version is below threshold +@safe unittest +{ + ErrorBeforeCmd cmd; + cmd.field = 42; + try { + enforceWireVersionConstraints(cmd, WireVersion.v40); + assert(false, "Should have thrown"); + } catch (MongoException e) { + } +} + +/// @errorBefore does not throw when field is set and server version is at threshold +@safe unittest +{ + ErrorBeforeCmd cmd; + cmd.field = 42; + enforceWireVersionConstraints(cmd, WireVersion.v44); + assert(!cmd.field.isNull); +} + +/// @errorBefore does not throw when field is set and server version is above threshold +@safe unittest +{ + ErrorBeforeCmd cmd; + cmd.field = 42; + enforceWireVersionConstraints(cmd, WireVersion.v60); + assert(!cmd.field.isNull); +} + +/// @errorBefore does not throw when field is not set +@safe unittest +{ + ErrorBeforeCmd cmd; + enforceWireVersionConstraints(cmd, WireVersion.v30); + assert(cmd.field.isNull); +} + +/// @deprecatedSince preserves field and only logs at deprecated version +@safe unittest +{ + DeprecatedCmd cmd; + cmd.oldField = 10; + enforceWireVersionConstraints(cmd, WireVersion.v40); + assert(!cmd.oldField.isNull); + assert(cmd.oldField.get == 10); +} + +/// @deprecatedSince preserves field above deprecated version +@safe unittest +{ + DeprecatedCmd cmd; + cmd.oldField = 10; + enforceWireVersionConstraints(cmd, WireVersion.v60); + assert(!cmd.oldField.isNull); +} + +/// @deprecatedSince preserves field below deprecated version without warning +@safe unittest +{ + DeprecatedCmd cmd; + cmd.oldField = 10; + enforceWireVersionConstraints(cmd, WireVersion.v36); + assert(!cmd.oldField.isNull); +} + +/// @deprecatedSince does nothing when field is not set +@safe unittest +{ + DeprecatedCmd cmd; + enforceWireVersionConstraints(cmd, WireVersion.v60); + assert(cmd.oldField.isNull); +} + +/// Combined UDAs where @errorBefore throws while @since and @deprecatedSince still apply +@safe unittest +{ + CombinedCmd cmd; + cmd.allowDiskUse = true; + cmd.maxAwaitTimeMS = 5000; + cmd.maxScan = 100; + + auto t1 = cmd; + try { + enforceWireVersionConstraints(t1, WireVersion.v30); + assert(false, "Should have thrown due to errorBefore(v44)"); + } catch (MongoException e) { + } +} + +/// Combined UDAs with all fields valid at v44, @deprecatedSince only logs +@safe unittest +{ + CombinedCmd cmd; + cmd.allowDiskUse = true; + cmd.maxAwaitTimeMS = 5000; + cmd.maxScan = 100; + + enforceWireVersionConstraints(cmd, WireVersion.v44); + assert(!cmd.allowDiskUse.isNull); + assert(!cmd.maxAwaitTimeMS.isNull); + assert(!cmd.maxScan.isNull); +} + +/// Combined UDAs where @since nullifies field below minimum while others are independent +@safe unittest +{ + SinceDeprecatedCmd cmd; + cmd.maxAwaitTimeMS = 5000; + cmd.maxScan = 100; + + enforceWireVersionConstraints(cmd, WireVersion.v30); + assert(cmd.maxAwaitTimeMS.isNull); + assert(!cmd.maxScan.isNull); +} + +/// Combined UDAs where @since preserves field at sufficient version +@safe unittest +{ + SinceDeprecatedCmd cmd; + cmd.maxAwaitTimeMS = 5000; + cmd.maxScan = 100; + + enforceWireVersionConstraints(cmd, WireVersion.v34); + assert(!cmd.maxAwaitTimeMS.isNull); + assert(!cmd.maxScan.isNull); +} diff --git a/mongodb/vibe/db/mongo/mongo.d b/mongodb/vibe/db/mongo/mongo.d index f8a6b80c55..d39618901e 100644 --- a/mongodb/vibe/db/mongo/mongo.d +++ b/mongodb/vibe/db/mongo/mongo.d @@ -97,3 +97,46 @@ MongoClient connectMongoDB(MongoClientSettings settings) { return new MongoClient(settings); } + +/** + Connects to a MongoDB instance and returns an owning, scope-bound handle. + + This is the deterministic-cleanup counterpart to `connectMongoDB`: the returned + `MongoClientHandle` forwards every `MongoClient` member through `alias this`, and + stops the client's background server monitors when the handle leaves scope. Use it + for clients with a bounded lifetime so their monitor tasks do not keep the client + (and themselves) reachable for the lifetime of the process. + + Examples: + --- + // the client and its monitors are cleaned up when `client` leaves scope + auto client = scopedMongoDB("127.0.0.1"); + auto users = client.getCollection("myapp.users"); + --- + + To keep a raw, manually-managed `MongoClient` alive beyond the handle's scope, call + `release()` on the handle. + + Params: + host = Specifies the host name or IP address of the MongoDB server. + port = Can be used to specify the port of the MongoDB server if different from the default one. + host_or_url = Can either be a host name, in which case the default port will be used, or a URL with the mongodb:// scheme. + settings = An object containing the full set of possible configuration options. + + Returns: + A `MongoClientHandle` owning a new MongoClient. +*/ +MongoClientHandle scopedMongoDB(string host, ushort port) +{ + return MongoClientHandle(connectMongoDB(host, port)); +} +/// ditto +MongoClientHandle scopedMongoDB(string host_or_url) +{ + return MongoClientHandle(connectMongoDB(host_or_url)); +} +/// ditto +MongoClientHandle scopedMongoDB(MongoClientSettings settings) +{ + return MongoClientHandle(connectMongoDB(settings)); +} diff --git a/mongodb/vibe/db/mongo/monitor.d b/mongodb/vibe/db/mongo/monitor.d new file mode 100644 index 0000000000..b71f0ef02f --- /dev/null +++ b/mongodb/vibe/db/mongo/monitor.d @@ -0,0 +1,791 @@ +/** + Per-server SDAM health monitoring. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.monitor; + +import vibe.db.mongo.impl.serverdescription : ServerDescription; +import vibe.db.mongo.settings : MongoHost, hostKey; + +import vibe.core.core : runTask, sleep; +import vibe.core.task : Task; +import vibe.core.log : logError; +import vibe.core.sync : LocalManualEvent, createManualEvent; + +import core.time : MonoTime, Duration, seconds, msecs; +import std.typecons : Nullable; + +alias ServerProber = ServerDescription delegate(MongoHost) @safe; +alias MonitorResult = void delegate(MongoHost host, Nullable!ServerDescription desc, Duration rtt) @safe; + +/// Monitors a single server, probing it on demand and reporting the result. +final class ServerMonitor { + private { + MongoHost m_host; + ServerProber m_probe; + MonitorResult m_onResult; + Duration m_heartbeat; + Duration m_minHeartbeat; + bool m_running; + bool m_stopped; + Task m_loop; + LocalManualEvent m_wake; + MonoTime m_lastCheck; + } + + this(MongoHost host, ServerProber probe, MonitorResult onResult, Duration heartbeat, Duration minHeartbeat) @safe + { + m_host = host; + m_probe = probe; + m_onResult = onResult; + m_heartbeat = heartbeat; + m_minHeartbeat = minHeartbeat; + m_wake = createManualEvent(); + } + + /// Probes the host once and reports the resulting description via the callback. + void checkOnce() @safe + { + auto started = MonoTime.currTime; + Nullable!ServerDescription result; + try + result = Nullable!ServerDescription(m_probe(m_host)); + catch (Exception) + result = Nullable!ServerDescription.init; + + // A probe that completes after stop() (the host was removed mid-flight) must not + // report: update() would re-append the just-pruned host, resurrecting it. + if (!m_stopped) + m_onResult(m_host, result, MonoTime.currTime - started); + } + + /// Starts the background heartbeat loop that probes the host periodically. + void start() @trusted + { + m_running = true; + m_loop = runTask(&supervise); + } + + /// Restarts `runLoop` after a crash, waiting one heartbeat between attempts. + private void supervise() nothrow + { + while (m_running) + { + if (runLoop()) + break; + if (!m_running) + break; + try + sleep(m_heartbeat); + catch (Exception) {} + } + } + + /// Stops the background heartbeat loop. Wakes the loop's wait so the task exits + /// promptly instead of lingering blocked until the next full heartbeat. + void stop() @safe + { + m_running = false; + m_stopped = true; + m_wake.emit(); + } + + /// Requests a check. Always wakes the loop; the loop honors the minHeartbeat + /// floor so a request inside the cooldown runs at lastCheck + minHeartbeat + /// rather than being dropped until the next full heartbeat. + void requestCheck() @safe + { + m_wake.emit(); + } + + /// Runs the heartbeat loop until `stop()`; returns false if an exception escaped. + private bool runLoop() nothrow + { + try + { + while (m_running) + { + auto ec = m_wake.emitCount; + checkOnce(); + m_lastCheck = MonoTime.currTime; + m_wake.wait(m_heartbeat, ec); + + // A request that woke us inside the cooldown is honored at the floor, + // not the full heartbeat: wait out the rest of minHeartbeat first. + auto wait = cooldownRemaining(m_lastCheck, MonoTime.currTime, m_minHeartbeat); + if (m_running && wait > Duration.zero) + sleep(wait); + } + return true; + } + catch (Exception e) + { + logError("MongoDB ServerMonitor heartbeat loop failed for %s: %s", m_host, e.msg); + return false; + } + } +} + +/// Owns the live per-host monitors, starting and stopping them as the topology changes. +final class MonitorRegistry { + private { + ServerProber m_prober; + MonitorResult m_onResult; + Duration m_heartbeat; + Duration m_minHeartbeat; + ServerMonitor[string] m_monitors; + MongoHost[string] m_hosts; + } + + this(ServerProber prober, MonitorResult onResult, Duration heartbeat, Duration minHeartbeat) @safe + { + m_prober = prober; + m_onResult = onResult; + m_heartbeat = heartbeat; + m_minHeartbeat = minHeartbeat; + } + + /// Number of monitors currently running. + size_t length() const @safe + { + return m_monitors.length; + } + + /// Whether a monitor is running for `host`. + bool isMonitoring(MongoHost host) const @safe + { + return (hostKey(host) in m_monitors) !is null; + } + + /// The hosts currently being monitored. + MongoHost[] hosts() @safe + { + return m_hosts.values; + } + + /// Starts a monitor for `host` unless one is already running. + void ensure(MongoHost host) @safe + { + auto key = hostKey(host); + if (key in m_monitors) + return; + + auto monitor = new ServerMonitor(host, m_prober, m_onResult, m_heartbeat, m_minHeartbeat); + m_monitors[key] = monitor; + m_hosts[key] = host; + monitor.start(); + } + + /// Stops and forgets the monitor for `host`. + void remove(MongoHost host) @safe + { + auto key = hostKey(host); + if (key !in m_monitors) + return; + + m_monitors[key].stop(); + m_monitors.remove(key); + m_hosts.remove(key); + } + + /// Starts monitors for new hosts and stops monitors for removed ones. + void reconcileWith(MongoHost[] desired) @safe + { + auto plan = reconcileMonitors(m_hosts.values, desired); + foreach (host; plan.toStart) + ensure(host); + foreach (host; plan.toStop) + remove(host); + } + + /// Asks every monitor to run a check now. + void requestAllChecks() @safe + { + foreach (monitor; m_monitors.byValue) + monitor.requestCheck(); + } + + /// Asks the monitor for `host`, if any, to run a check now. + void requestCheck(MongoHost host) @safe + { + if (auto monitor = hostKey(host) in m_monitors) + monitor.requestCheck(); + } + + /// Stops and forgets every monitor. + void stopAll() @safe + { + foreach (monitor; m_monitors.byValue) + monitor.stop(); + m_monitors = null; + m_hosts = null; + } +} + +/// Time still to wait before the next check is allowed: zero once `minInterval` has +/// elapsed since the last check, otherwise the exact remaining cooldown. +Duration cooldownRemaining(MonoTime last, MonoTime now, Duration minInterval) @safe pure nothrow @nogc +{ + auto elapsed = now - last; + return elapsed >= minInterval ? Duration.zero : minInterval - elapsed; +} + +/// cooldownRemaining is zero past the minHeartbeatFrequencyMS floor and the exact remainder within it +unittest +{ + auto now = MonoTime.currTime; + auto minInterval = 500.msecs; + + assert(cooldownRemaining(now - 600.msecs, now, minInterval) == Duration.zero, + "no wait once the floor has elapsed"); + assert(cooldownRemaining(now - 500.msecs, now, minInterval) == Duration.zero, + "no wait exactly at the floor"); + assert(cooldownRemaining(now, now, minInterval) == minInterval, + "the full floor remains when no time has passed since the last check"); + assert(cooldownRemaining(now - 200.msecs, now, minInterval) == 300.msecs, + "the exact remaining cooldown is returned within the floor"); +} + +/// MongoDB server error codes the driver classifies for retry decisions. +enum MongoServerErrorCode : int +{ + none = 0, + hostUnreachable = 6, + hostNotFound = 7, + networkTimeout = 89, + shutdownInProgress = 91, + primarySteppedDown = 189, + exceededTimeLimit = 262, + socketException = 9001, + duplicateKey = 11000, + notWritablePrimary = 10107, + interruptedAtShutdown = 11600, + interruptedDueToReplStateChange = 11602, + notPrimaryNoSecondaryOk = 13435, + notPrimaryOrSecondary = 13436, +} + +/// Whether a server error code is in the SDAM "not master or recovering" set. +bool isStaleTopologyError(MongoServerErrorCode code) @safe pure nothrow @nogc +{ + switch (code) + { + case MongoServerErrorCode.notWritablePrimary: + case MongoServerErrorCode.notPrimaryNoSecondaryOk: + case MongoServerErrorCode.notPrimaryOrSecondary: + case MongoServerErrorCode.interruptedAtShutdown: + case MongoServerErrorCode.interruptedDueToReplStateChange: + case MongoServerErrorCode.primarySteppedDown: + case MongoServerErrorCode.shutdownInProgress: + return true; + default: + return false; + } +} + +/// isStaleTopologyError flags the not-master / recovering server error codes +unittest +{ + assert(isStaleTopologyError(MongoServerErrorCode.notWritablePrimary), "NotWritablePrimary"); + assert(isStaleTopologyError(MongoServerErrorCode.notPrimaryNoSecondaryOk), "NotPrimaryNoSecondaryOk"); + assert(isStaleTopologyError(MongoServerErrorCode.interruptedDueToReplStateChange), "InterruptedDueToReplStateChange"); + assert(isStaleTopologyError(MongoServerErrorCode.primarySteppedDown), "PrimarySteppedDown"); + assert(isStaleTopologyError(MongoServerErrorCode.shutdownInProgress), "ShutdownInProgress"); + assert(isStaleTopologyError(MongoServerErrorCode.notPrimaryOrSecondary), "NotPrimaryOrSecondary"); + assert(isStaleTopologyError(MongoServerErrorCode.interruptedAtShutdown), "InterruptedAtShutdown"); + + assert(!isStaleTopologyError(MongoServerErrorCode.duplicateKey), "duplicate key is not a topology error"); + assert(!isStaleTopologyError(MongoServerErrorCode.none), "no error code"); +} + +/// Whether a server error code marks a write safe to retry once (MongoDB 3.6+ +/// retryable writes). Covers the election/topology set plus the network-error codes. +bool isRetryableWriteError(MongoServerErrorCode code) @safe pure nothrow @nogc +{ + if (isStaleTopologyError(code)) + return true; + + switch (code) + { + case MongoServerErrorCode.hostUnreachable: + case MongoServerErrorCode.hostNotFound: + case MongoServerErrorCode.networkTimeout: + case MongoServerErrorCode.socketException: + case MongoServerErrorCode.exceededTimeLimit: + return true; + default: + return false; + } +} + +/// isRetryableWriteError flags network errors beyond the election set +unittest +{ + assert(isRetryableWriteError(MongoServerErrorCode.networkTimeout), "NetworkTimeout is a retryable write error"); +} + +/// isRetryableWriteError flags SocketException as a network error +unittest +{ + assert(isRetryableWriteError(MongoServerErrorCode.socketException) == true, "SocketException is a retryable write error"); +} + +/// isRetryableWriteError flags election codes from the stale-topology set +unittest +{ + assert(isRetryableWriteError(MongoServerErrorCode.notWritablePrimary) == true, "an election code is also a retryable write error"); +} + +/// isRetryableWriteError rejects non-retryable error codes +unittest +{ + assert(!isRetryableWriteError(MongoServerErrorCode.duplicateKey), "a duplicate-key error is not a retryable write error"); + assert(!isRetryableWriteError(MongoServerErrorCode.none), "no error code is not a retryable write error"); +} + +/// isRetryableWriteError rejects an unknown server error code +unittest +{ + assert(!isRetryableWriteError(cast(MongoServerErrorCode) 99999), + "an unknown server error code is not a retryable write error"); +} + +/// The per-host monitors to start and stop after a topology change. +struct MonitorReconcile +{ + MongoHost[] toStart; + MongoHost[] toStop; +} + +/// Set-diffs monitored hosts against desired hosts into hosts to start and to stop. +MonitorReconcile reconcileMonitors(MongoHost[] current, MongoHost[] desired) @safe pure nothrow +{ + import std.algorithm : canFind, filter; + import std.array : array; + + MonitorReconcile result; + result.toStart = desired.filter!(h => !current.canFind(h)).array; + result.toStop = current.filter!(h => !desired.canFind(h)).array; + return result; +} + +/// reconcileMonitors starts new hosts and stops removed ones +unittest +{ + import vibe.db.mongo.settings : MongoHost; + + auto a = MongoHost("a", 27017); + auto b = MongoHost("b", 27017); + auto c = MongoHost("c", 27017); + + auto r = reconcileMonitors([a, b], [b, c]); + + assert(r.toStart == [c], "starts monitors for newly-discovered hosts"); + assert(r.toStop == [a], "stops monitors for removed hosts"); +} + +/// Whether a stale-topology error is retryable: only for idempotent ops or ops with session support. +bool shouldRetryAfterStepDown(MongoServerErrorCode code, bool idempotent, bool sessionSupport) @safe pure nothrow @nogc +{ + return isStaleTopologyError(code) && (idempotent || sessionSupport); +} + +/// shouldRetryAfterStepDown retries idempotent or session-supported ops on a stale-topology error +unittest +{ + assert(shouldRetryAfterStepDown(MongoServerErrorCode.notWritablePrimary, true, false), "idempotent NotWritablePrimary is retryable"); + assert(shouldRetryAfterStepDown(MongoServerErrorCode.notWritablePrimary, false, true), "session-supported NotWritablePrimary is retryable"); +} + +/// Whether a failed write may be retried once: a retryable-write error on a +/// session-supported write (the transaction number lets the server deduplicate). +bool shouldRetryWrite(MongoServerErrorCode code, bool sessionSupport) @safe pure nothrow @nogc +{ + return isRetryableWriteError(code) && sessionSupport; +} + +/// shouldRetryWrite retries a network error on a session-supported write +unittest +{ + assert(shouldRetryWrite(MongoServerErrorCode.networkTimeout, true) == true, "a network error on a session-supported write is retryable"); +} + +/// shouldRetryWrite does not retry without session support +unittest +{ + assert(shouldRetryWrite(MongoServerErrorCode.networkTimeout, false) == false, "a write without session support is not retried (the server cannot deduplicate)"); +} + +/// checkOnce probes the host and reports the probed description via the callback +unittest +{ + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + + ServerDescription prober(MongoHost h) @safe + { + ServerDescription desc; + desc.isWritablePrimary = true; + desc.setName = "rs0"; + return desc; + } + + MongoHost reportedHost; + Nullable!ServerDescription reportedDesc; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe + { + reportedHost = h; + reportedDesc = desc; + } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 500.msecs); + monitor.checkOnce(); + + assert(!reportedDesc.isNull, "a successful probe reports a non-null description"); + assert(reportedDesc.get.isWritablePrimary, "the reported description is the probed primary"); + assert(reportedHost == host, "the reported host is the monitored host"); +} + +/// checkOnce reports a null description when the prober throws +unittest +{ + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + + ServerDescription prober(MongoHost h) @safe + { + throw new Exception("connection refused"); + } + + bool wasCalled; + Nullable!ServerDescription reportedDesc; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe + { + wasCalled = true; + reportedDesc = desc; + } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 500.msecs); + monitor.checkOnce(); + + assert(wasCalled, "a failed probe still reports a result"); + assert(reportedDesc.isNull, "a failed probe reports a null description"); +} + +/// checkOnce does not report a result after stop() (an in-flight probe during removal must not resurrect the host) +unittest +{ + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; d.setName = "rs0"; return d; } + + bool wasCalled; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe { wasCalled = true; } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 1.msecs); + monitor.stop(); // the host was removed / the monitor stopped while a probe was in flight + monitor.checkOnce(); // the in-flight probe now completes + + assert(!wasCalled, "a stopped monitor must not deliver its in-flight probe result (it would resurrect a pruned host)"); +} + +/// start() runs periodic checks until stop() +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs; + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + + ServerDescription prober(MongoHost h) @safe + { + ServerDescription desc; + desc.isWritablePrimary = true; + desc.setName = "rs0"; + return desc; + } + + int checks; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe + { + checks++; + } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 20.msecs, 1.msecs); + + monitor.start(); + sleep(100.msecs); + assert(checks >= 2, "the loop performs periodic checks while running"); + + monitor.stop(); + sleep(60.msecs); + auto afterStop = checks; + sleep(60.msecs); + assert(checks == afterStop, "no checks happen after stop()"); +} + +/// runLoop returns false when the loop body throws +unittest +{ + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; return d; } + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe { throw new Exception("boom"); } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 20.msecs, 1.msecs); + monitor.m_running = true; + + auto ok = monitor.runLoop(); + + assert(!ok, "runLoop returns false when the loop throws"); +} + +/// start() restarts the heartbeat loop after a failure +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs; + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; d.setName = "rs0"; return d; } + + bool thrownOnce; + int goodChecks; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe + { + if (!thrownOnce) { thrownOnce = true; throw new Exception("first check crashes the loop"); } + goodChecks++; + } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 20.msecs, 1.msecs); + monitor.start(); + sleep(150.msecs); + monitor.stop(); + + assert(goodChecks >= 1, "the monitor restarted its loop after the failure"); +} + +/// requestCheck triggers a check before the heartbeat interval elapses +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs, seconds; + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; d.setName = "rs0"; return d; } + + int checks; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe { checks++; } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 1.msecs); + + monitor.start(); + sleep(40.msecs); + auto before = checks; + monitor.requestCheck(); + sleep(40.msecs); + monitor.stop(); + + assert(checks > before, "requestCheck causes an immediate re-check instead of waiting the full heartbeat"); +} + +/// requestCheck during the minHeartbeat cooldown still schedules a check at the floor, not after the full heartbeat +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs, seconds; + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; d.setName = "rs0"; return d; } + + int checks; + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe { checks++; } + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 50.msecs); + + monitor.start(); + sleep(20.msecs); + auto before = checks; + monitor.requestCheck(); + sleep(250.msecs); + monitor.stop(); + + assert(checks > before, "a check requested during the minHeartbeat cooldown still runs at the floor, not after the full heartbeat"); +} + +/// stop() wakes the heartbeat loop immediately instead of leaving the task blocked until the next heartbeat +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs, seconds; + import vibe.db.mongo.impl.serverdescription : ServerDescription; + import vibe.db.mongo.settings : MongoHost; + + auto host = MongoHost("primary", 27017); + ServerDescription prober(MongoHost h) @safe { ServerDescription d; d.isWritablePrimary = true; d.setName = "rs0"; return d; } + void onResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe {} + + auto monitor = new ServerMonitor(host, &prober, &onResult, 10.seconds, 1.msecs); // 10s heartbeat: a non-woken loop stays blocked ~10s + monitor.start(); + sleep(30.msecs); // first check ran; the loop is now blocked in m_wake.wait(10s) + monitor.stop(); + sleep(80.msecs); // far below the 10s heartbeat + + assert(!monitor.m_loop.running, + "stop() must wake the loop so the task exits promptly, not linger blocked for a full heartbeat"); +} + +version (unittest) +{ + private ServerDescription stubProbe(MongoHost h) @safe + { + ServerDescription desc; + desc.isWritablePrimary = true; + desc.setName = "rs0"; + return desc; + } + + private void ignoreResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe {} + + private MonitorRegistry idleRegistry() + { + import std.functional : toDelegate; + return new MonitorRegistry(toDelegate(&stubProbe), toDelegate(&ignoreResult), 1.seconds, 1.msecs); + } +} + +/// ensure starts a monitor and is idempotent for the same host +unittest +{ + auto registry = idleRegistry(); + auto host = MongoHost("a", 27017); + + registry.ensure(host); + registry.ensure(host); + + assert(registry.length == 1, "ensure starts exactly one monitor per host"); + assert(registry.isMonitoring(host), "the host is reported as monitored"); + + registry.stopAll(); +} + +/// remove stops the monitor and forgets the host +unittest +{ + auto registry = idleRegistry(); + auto host = MongoHost("a", 27017); + + registry.ensure(host); + registry.remove(host); + + assert(registry.length == 0, "remove drops the monitor"); + assert(!registry.isMonitoring(host), "the removed host is no longer monitored"); +} + +/// remove of an unmonitored host is a no-op +unittest +{ + auto registry = idleRegistry(); + + registry.remove(MongoHost("missing", 27017)); + + assert(registry.length == 0, "removing an unknown host changes nothing"); +} + +/// reconcileWith starts newly-discovered hosts and stops removed ones +unittest +{ + auto registry = idleRegistry(); + auto a = MongoHost("a", 27017); + auto b = MongoHost("b", 27017); + auto c = MongoHost("c", 27017); + + registry.reconcileWith([a, b]); + assert(registry.length == 2, "the first reconcile starts a monitor per host"); + + registry.reconcileWith([b, c]); + + assert(registry.length == 2, "the set size matches the new topology"); + assert(registry.isMonitoring(b), "a host present in both reconciles keeps its monitor"); + assert(registry.isMonitoring(c), "a newly-discovered host gets a monitor"); + assert(!registry.isMonitoring(a), "a removed host loses its monitor"); + + registry.stopAll(); +} + +/// stopAll stops and forgets every monitor +unittest +{ + auto registry = idleRegistry(); + + registry.ensure(MongoHost("a", 27017)); + registry.ensure(MongoHost("b", 27017)); + registry.stopAll(); + + assert(registry.length == 0, "stopAll empties the registry"); +} + +/// an inert registry (constructed but never reconciled, as in load-balanced mode) answers every query safely +unittest +{ + auto registry = idleRegistry(); // never ensure/reconcileWith -> empty, exactly the LB-mode state + + assert(registry.length == 0, "an inert registry has no monitors"); + registry.requestCheck(MongoHost("anyhost", 27017)); // unknown host -> must be a no-op, not a crash + registry.requestAllChecks(); // no monitors -> no-op + registry.stopAll(); // no monitors -> no-op + assert(registry.length == 0, "still no monitors after the no-op calls"); +} + +/// requestCheck for an unmonitored host is a no-op +unittest +{ + auto registry = idleRegistry(); + + registry.requestCheck(MongoHost("missing", 27017)); + + assert(registry.length == 0, "requesting a check for an unknown host changes nothing"); +} + +/// requestAllChecks triggers an immediate re-check on every monitor +unittest +{ + import vibe.core.core : sleep; + import core.time : msecs, hours; + import std.functional : toDelegate; + + int checks; + void countResult(MongoHost h, Nullable!ServerDescription desc, Duration rtt) @safe { checks++; } + + auto registry = new MonitorRegistry(toDelegate(&stubProbe), &countResult, 1.hours, 1.msecs); + + registry.ensure(MongoHost("a", 27017)); + registry.ensure(MongoHost("b", 27017)); + sleep(40.msecs); + auto before = checks; + + registry.requestAllChecks(); + sleep(40.msecs); + registry.stopAll(); + + assert(checks > before, "requestAllChecks re-checks each monitor instead of waiting the full heartbeat"); +} diff --git a/mongodb/vibe/db/mongo/settings.d b/mongodb/vibe/db/mongo/settings.d index 7d87647982..f0a19e5382 100644 --- a/mongodb/vibe/db/mongo/settings.d +++ b/mongodb/vibe/db/mongo/settings.d @@ -33,6 +33,29 @@ import std.typecons : Nullable, nullable; * If the URL is not successfully parsed the information in the MongoClientSettings instance may be * incomplete and should not be used. */ +/// Whether a `maxStalenessSeconds` value is valid for the configured heartbeat. `-1` +/// (disabled) is always valid; otherwise the spec requires it to be at least +/// `max(90, heartbeatFrequencyMS/1000 + 10)`. +package(vibe.db.mongo) bool isValidMaxStaleness(long maxStalenessSeconds, long heartbeatFrequencyMS) @safe +{ + import std.algorithm : max; + if (maxStalenessSeconds < 0) + return true; + return maxStalenessSeconds >= max(90L, heartbeatFrequencyMS / 1000 + 10); +} + +/// isValidMaxStaleness enforces the spec floor of max(90, heartbeat/1000 + 10) +unittest +{ + assert(isValidMaxStaleness(-1, 10_000), "-1 disables the staleness check and is always valid"); + assert(isValidMaxStaleness(90, 10_000), "90s meets the floor for the default 10s heartbeat"); + assert(!isValidMaxStaleness(89, 10_000), "below 90s is rejected for the default heartbeat"); + assert(!isValidMaxStaleness(50, 10_000), "well below the floor is rejected"); + // with a large heartbeat the floor is heartbeat/1000 + 10, above 90 + assert(!isValidMaxStaleness(100, 120_000), "a 120s heartbeat raises the floor to 130s, so 100 is rejected"); + assert(isValidMaxStaleness(130, 120_000), "130s meets the floor for a 120s heartbeat"); +} + bool parseMongoDBUrl(out MongoClientSettings cfg, string url) @safe { import std.exception : enforce; @@ -41,14 +64,15 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) string tmpUrl = url[0..$]; // Slice of the URL (not a copy) - if( !startsWith(tmpUrl, "mongodb://") ) + if (startsWith(tmpUrl, "mongodb://")) + { + tmpUrl = tmpUrl["mongodb://".length .. $]; + } + else { return false; } - // Reslice to get rid of 'mongodb://' - tmpUrl = tmpUrl[10..$]; - auto authIndex = tmpUrl.indexOf('@'); sizediff_t hostIndex = 0; // Start of the host portion of the URL. @@ -161,6 +185,15 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) } } + void setWriteConcern(ref Bson dst) + { + try { + dst = icmp(value, "majority") == 0 ? Bson("majority") : Bson(to!long(value)); + } catch (Exception e) { + logError("Invalid w value: [%s] Should be an integer number or 'majority'", value); + } + } + void warnNotImplemented() { logDiagnostic("MongoDB option %s not yet implemented.", option); @@ -173,15 +206,20 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) case "appname": cfg.appName = value; break; case "replicaset": cfg.replicaSet = value; break; case "readpreference": cfg.readPreference = parseReadPreference(value); break; + case "readpreferencetags": cfg.readPreferenceTags ~= parseTagSet(value); break; case "localthresholdms": setLong(cfg.localThresholdMS); break; case "maxstalenessseconds": setLong(cfg.maxStalenessSeconds); break; + case "heartbeatfrequencyms": setLong(cfg.heartbeatFrequencyMS); break; + case "minheartbeatfrequencyms": setLong(cfg.minHeartbeatFrequencyMS); break; + case "serverselectiontimeoutms": setLong(cfg.serverSelectionTimeoutMS); break; case "readconcernlevel": cfg.readConcern = parseReadConcern(value); break; case "safe": setBool(cfg.safe); break; + case "retrywrites": setBool(cfg.retryWrites); break; case "fsync": setBool(cfg.fsync); break; case "journal": setBool(cfg.journal); break; case "connecttimeoutms": setMsecs(cfg.connectTimeout); break; case "sockettimeoutms": setMsecs(cfg.socketTimeout); break; - case "tls": setBool(cfg.ssl); break; + case "tls": case "ssl": setBool(cfg.ssl); break; case "sslverifycertificate": setBool(cfg.sslverifycertificate); break; case "authmechanism": cfg.authMechanism = parseAuthMechanism(value); break; @@ -203,28 +241,27 @@ bool parseMongoDBUrl(out MongoClientSettings cfg, string url) cfg.zlibCompressionLevel = cast(int) level; } break; - case "w": - try { - if(icmp(value, "majority") == 0){ - cfg.w = Bson("majority"); - } else { - cfg.w = Bson(to!long(value)); - } - } catch (Exception e) { - logError("Invalid w value: [%s] Should be an integer number or 'majority'", value); - } - break; + case "w": setWriteConcern(cfg.w); break; } } - /* Some m_settings imply safe. If they are set, set safe to true regardless - * of what it was set to in the URL string - */ - if( (cfg.w != Bson.init) || (cfg.wTimeoutMS != long.init) || - cfg.journal || cfg.fsync ) + // Setting any of w / wTimeoutMS / journal / fsync turns on safe writes, + // regardless of the URL's explicit `safe` value. + bool writeOptionsImplySafe() { - cfg.safe = true; + return cfg.w != Bson.init || cfg.wTimeoutMS != long.init + || cfg.journal || cfg.fsync; } + + if (writeOptionsImplySafe()) + cfg.safe = true; + } + + if (!isValidMaxStaleness(cfg.maxStalenessSeconds, cfg.heartbeatFrequencyMS)) + { + logError("maxStalenessSeconds=%s is below the spec floor of max(90, heartbeatFrequencyMS/1000 + 10)", + cfg.maxStalenessSeconds); + return false; } return true; @@ -459,6 +496,64 @@ unittest assert(cfg.readPreference == ReadPreference.nearest); } +/// parseMongoDBUrl parses a single readPreferenceTags set +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?readPreferenceTags=dc:east")); + string[string][] expected = [["dc": "east"]]; + assert(cfg.readPreferenceTags == expected, "readPreferenceTags=dc:east yields one tag set [\"dc\": \"east\"]"); +} + +/// parseMongoDBUrl parses a multi-pair readPreferenceTags set +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?readPreferenceTags=dc:east,rack:r1")); + string[string][] expected = [["dc": "east", "rack": "r1"]]; + assert(cfg.readPreferenceTags == expected, "comma-separated pairs form one tag set"); +} + +/// parseMongoDBUrl preserves the order of multiple readPreferenceTags occurrences +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?readPreferenceTags=dc:east&readPreferenceTags=dc:west")); + string[string][] expected = [["dc": "east"], ["dc": "west"]]; + assert(cfg.readPreferenceTags == expected, "repeated readPreferenceTags form an ordered list"); +} + +/// parseMongoDBUrl parses an empty readPreferenceTags as the catch-all tag set +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?readPreferenceTags=")); + string[string][] expected = [string[string].init]; + assert(cfg.readPreferenceTags == expected, "empty readPreferenceTags is the catch-all tag set {}"); +} + +/// parseMongoDBUrl parses retryWrites=false option +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?retryWrites=false")); + assert(cfg.retryWrites == false, "retryWrites=false disables retryable writes"); +} + +/// parseMongoDBUrl accepts a multi-host URL +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://h1:27017,h2:27017/")); + assert(cfg.hosts.length == 2); +} + /// parseMongoDBUrl parses localThresholdMS option unittest { @@ -495,6 +590,35 @@ unittest assert(cfg.maxStalenessSeconds == -1); } +/// parseMongoDBUrl parses the SDAM monitoring options +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/?heartbeatFrequencyMS=5000&minHeartbeatFrequencyMS=250&serverSelectionTimeoutMS=12000")); + assert(cfg.heartbeatFrequencyMS == 5000); + assert(cfg.minHeartbeatFrequencyMS == 250); + assert(cfg.serverSelectionTimeoutMS == 12000); +} + +/// parseMongoDBUrl uses SDAM monitoring defaults (10000 / 500 / 30000) +unittest +{ + MongoClientSettings cfg; + + assert(parseMongoDBUrl(cfg, "mongodb://localhost/")); + assert(cfg.heartbeatFrequencyMS == 10_000); + assert(cfg.minHeartbeatFrequencyMS == 500); + assert(cfg.serverSelectionTimeoutMS == 30_000); +} + +/// MongoClientSettings enables retryWrites by default +unittest +{ + auto cfg = new MongoClientSettings(); + assert(cfg.retryWrites == true, "retryWrites should default to true"); +} + /// parseMongoDBUrl parses readConcernLevel option unittest { @@ -948,6 +1072,61 @@ enum ReadPreference nearest, } +/** Builds the `$readPreference` command field. Enum names match the wire mode + strings. `primary` must be omitted by drivers, so passing it is a programming error. +*/ +Bson readPreferenceBson(ReadPreference pref, string[string][] tagSets = null) +@safe { + assert(pref != ReadPreference.primary, "primary read preference must not be sent on the wire"); + auto result = Bson(["mode": Bson(pref.to!string)]); + if (tagSets.length) { + Bson[] tags; + foreach (tagSet; tagSets) + tags ~= tagSetToBson(tagSet); + result["tags"] = Bson(tags); + } + return result; +} + +/// Converts one read-preference tag set into a wire Bson document. An empty +/// tag set becomes `{}`, which the server treats as the catch-all. +private Bson tagSetToBson(string[string] tagSet) +@safe { + Bson[string] doc; + foreach (key, value; tagSet) + doc[key] = Bson(value); + return Bson(doc); +} + +unittest { + assert(readPreferenceBson(ReadPreference.secondary) == Bson(["mode": Bson("secondary")])); + assert(readPreferenceBson(ReadPreference.primaryPreferred) == Bson(["mode": Bson("primaryPreferred")])); + assert(readPreferenceBson(ReadPreference.secondaryPreferred) == Bson(["mode": Bson("secondaryPreferred")])); + assert(readPreferenceBson(ReadPreference.nearest) == Bson(["mode": Bson("nearest")])); +} + +/// emits an ordered tags array alongside the mode for a tag-targeted read +unittest { + assert(readPreferenceBson(ReadPreference.secondary, [["dc": "east"]]) + == Bson(["mode": Bson("secondary"), "tags": Bson([Bson(["dc": Bson("east")])])]), + "secondary read preference with a tag set emits mode + tags array"); +} + +/// preserves the order of multiple tag sets in the wire tags array +unittest { + assert(readPreferenceBson(ReadPreference.nearest, [["dc": "east"], ["dc": "west"]]) + == Bson(["mode": Bson("nearest"), + "tags": Bson([Bson(["dc": Bson("east")]), Bson(["dc": Bson("west")])])]), + "the tags array keeps the tag-set order"); +} + +/// emits the catch-all empty tag set as an empty document in the tags array +unittest { + assert(readPreferenceBson(ReadPreference.secondary, [string[string].init]) + == Bson(["mode": Bson("secondary"), "tags": Bson([Bson.emptyObject])]), + "an empty tag set is still emitted as {} so the server treats it as catch-all"); +} + private ReadConcern parseReadConcern(string str) @safe { import std.traits : EnumMembers; @@ -972,6 +1151,25 @@ private ReadPreference parseReadPreference(string str) } } +/// Parses one comma-separated `key:value` read-preference tag set. An empty +/// string yields the catch-all (empty) tag set. +private string[string] parseTagSet(string value) +@safe { + import std.algorithm : findSplit, splitter; + + string[string] tagSet; + foreach (pair; value.splitter(",")) { + auto keyValue = pair.findSplit(":"); + tagSet[keyValue[0]] = keyValue[2]; + } + return tagSet; +} + +/// parseTagSet splits comma-separated key:value pairs into one tag set +@safe unittest { + assert(parseTagSet("dc:east,rack:r1") == ["dc": "east", "rack": "r1"]); +} + /** * Compression algorithm identifier for OP_COMPRESSED wire protocol messages. * @@ -1081,6 +1279,12 @@ class MongoClientSettings */ ReadPreference readPreference; + /** + * Ordered list of read-preference tag sets parsed from the `readPreferenceTags` + * URI options. Each occurrence appends one tag set, preserving order. + */ + string[string][] readPreferenceTags; + /** * Upper bound on the acceptable latency window for nearest server selection. * Servers within (fastest RTT + localThresholdMS) are eligible. @@ -1098,6 +1302,15 @@ class MongoClientSettings */ long maxStalenessSeconds = -1; + /// How often (ms) each monitor sends `hello` to refresh the topology. + long heartbeatFrequencyMS = 10_000; + + /// Minimum interval (ms) between consecutive checks of a single server. + long minHeartbeatFrequencyMS = 500; + + /// How long (ms) server selection waits for a suitable server before failing. + long serverSelectionTimeoutMS = 30_000; + /** * Specifies the default read concern level for read operations. * @@ -1116,6 +1329,14 @@ class MongoClientSettings */ bool safe; + /** + * Enables retryable writes, retrying eligible write operations once on + * transient network errors. Enabled by default for parity with the Node.js + * driver; the server deduplicates the retried write using the session's + * txnNumber so it is applied at most once. + */ + bool retryWrites = true; + /** * Requests acknowledgment that write operations have propagated to a * specified number of mongod instances (number) or to mongod instances with @@ -1335,6 +1556,13 @@ struct MongoHost } } +/// Stable map key for a host, "name:port". +string hostKey(MongoHost host) @safe +{ + import std.conv : to; + return host.name ~ ":" ~ host.port.to!string; +} + /** * Parses a "host:port" string into a MongoHost. Returns MongoHost.init if * the string cannot be parsed. diff --git a/mongodb/vibe/db/mongo/topology.d b/mongodb/vibe/db/mongo/topology.d index 1c54de5093..afddb74969 100644 --- a/mongodb/vibe/db/mongo/topology.d +++ b/mongodb/vibe/db/mongo/topology.d @@ -6,8 +6,9 @@ See_Also: $(LINK https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.md) - Copyright: © 2026 GISCollective + Copyright: © 2026 Szabo Bogdan License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan */ module vibe.db.mongo.topology; @@ -19,6 +20,7 @@ import vibe.core.log; import std.random : uniform; import std.range : chain; import std.typecons : Nullable; +import core.time : Duration; @safe: @@ -34,7 +36,13 @@ enum TopologyType single, replicaSetWithPrimary, replicaSetNoPrimary, - sharded + sharded, + loadBalanced +} + +bool supportsRetryableWrites(TopologyType type) +{ + return type != TopologyType.single; } struct TopologyDescription @@ -45,6 +53,9 @@ struct TopologyDescription string setName; TopologyType type = TopologyType.unknown; uint seedCount; + /// Configured heartbeat interval, fed into the maxStaleness formula. Defaults to the + /// spec default (10s) so an unseeded topology matches the historical hardcoded value. + long heartbeatFrequencyMS = 10_000; Nullable!BsonObjectID maxElectionId; Nullable!int maxSetVersion; @@ -67,6 +78,27 @@ struct TopologyDescription if (!found) servers ~= ServerRecord(host, desc); + auto serverType = desc.classifiedType(); + + // SDAM: in a Sharded topology a server reporting as anything other than a mongos + // is simply removed — it must not adopt its setName, prune the mongos list via the + // primary path, or flip the topology type to replicaSetWithPrimary. + if (type == TopologyType.sharded + && serverType != ServerDescription.ServerType.mongos + && serverType != ServerDescription.ServerType.unknown) + { + removeHost(host); + return; + } + + // SDAM: a server reporting a different replica-set name belongs to another set; + // remove it before it can demote the real primary or contribute foreign members. + if (setName.length && desc.setName.length && desc.setName != setName) + { + removeHost(host); + return; + } + if (!setName.length && desc.setName.length) setName = desc.setName; @@ -78,11 +110,19 @@ struct TopologyDescription return; } - auto serverType = desc.classifiedType(); - removeIncompatible(serverType); + removeIncompatible(); transitionType(serverType); } + private void removeHost(MongoHost host) + { + ServerRecord[] kept; + foreach (ref s; servers) + if (s.host != host) + kept ~= s; + servers = kept; + } + /** * Handles a new primary: compares electionId/setVersion against the * topology's max to detect stale primaries. Demotes the old primary @@ -92,36 +132,18 @@ struct TopologyDescription */ private bool handleNewPrimary(MongoHost host, ref const ServerDescription desc) { - if (!desc.electionId.isNull && !maxElectionId.isNull) + if (isStalePrimary(desc.electionId, desc.setVersion, maxElectionId, maxSetVersion)) { - bool newIsStale = false; - - if (!desc.setVersion.isNull && !maxSetVersion.isNull) - { - if (desc.setVersion.get < maxSetVersion.get) - newIsStale = true; - else if (desc.setVersion.get == maxSetVersion.get - && desc.electionId.get < maxElectionId.get) - newIsStale = true; - } - else if (desc.electionId.get < maxElectionId.get) - { - newIsStale = true; - } - - if (newIsStale) + foreach (ref s; servers) { - foreach (ref s; servers) + if (s.host == host) { - if (s.host == host) - { - s.description = ServerDescription.init; - break; - } + s.description = ServerDescription.init; + break; } - transitionType(ServerDescription.ServerType.unknown); - return false; } + transitionType(ServerDescription.ServerType.unknown); + return false; } // Demote old primary if different from the new one @@ -175,7 +197,11 @@ struct TopologyDescription return result; } - private void removeIncompatible(ServerDescription.ServerType serverType) + // Removes servers whose type is incompatible with the current topology type + // (e.g. a mongos or standalone showing up in a replica set). Decided entirely from + // the topology `type` and each server's own classifiedType(); the just-probed + // server's type is irrelevant here, which is why this takes no parameter. + private void removeIncompatible() { if (type == TopologyType.single) return; @@ -217,7 +243,8 @@ struct TopologyDescription private void transitionType(ServerDescription.ServerType serverType) { - if (type == TopologyType.single) + // single and loadBalanced are fixed by configuration; they bypass SDAM transitions. + if (type == TopologyType.single || type == TopologyType.loadBalanced) return; final switch (serverType) with (ServerDescription.ServerType) @@ -277,14 +304,17 @@ struct TopologyDescription MongoHost[] result; + void addHost(MongoHost h) + { + if (h != MongoHost.init && !result.canFind(h)) + result ~= h; + } + foreach (ref s; servers) { - foreach (hostStr; chain(s.description.hosts, s.description.passives)) - { - auto h = parseHostPort(hostStr); - if (h != MongoHost.init && !result.canFind(h)) - result ~= h; - } + addHost(s.host); + foreach (hostStr; chain(s.description.hosts, s.description.passives, s.description.arbiters)) + addHost(parseHostPort(hostStr)); } return result; @@ -332,19 +362,22 @@ struct TopologyDescription return Nullable!MongoHost.init; } - Nullable!MongoHost randomSecondaryHost(long maxStalenessSeconds = -1) const + Nullable!MongoHost randomSecondaryHost(long maxStalenessSeconds = -1, string[string][] tagSets = null) const { - auto hosts = secondaryHosts(maxStalenessSeconds); + auto hosts = secondaryHosts(maxStalenessSeconds, tagSets); if (!hosts.length) return Nullable!MongoHost.init; return Nullable!MongoHost(hosts[uniform(0, hosts.length)]); } - MongoHost[] secondaryHosts(long maxStalenessSeconds = -1) const + MongoHost[] secondaryHosts(long maxStalenessSeconds = -1, string[string][] tagSets = null) const { - MongoHost[] result; - foreach (ref s; servers) + import std.algorithm : map; + import std.array : array; + + size_t[] eligible; + foreach (i, ref s; servers) { if (!s.description.isSecondaryNode) continue; @@ -352,33 +385,37 @@ struct TopologyDescription if (maxStalenessSeconds >= 0 && isStaleSecondary(s.description, maxStalenessSeconds)) continue; - result ~= s.host; + eligible ~= i; } - return result; + + return selectIndicesByTagSets(eligible, tagSets).map!(i => servers[i].host).array.dup; } - Nullable!MongoHost randomHostWithinLatencyWindow(long localThresholdMS, long maxStalenessSeconds = -1) const + private size_t[] selectIndicesByTagSets(size_t[] indices, string[string][] tagSets) const { - double minRTT = double.max; - foreach (ref s; servers) - { - if (!s.description.isPrimary && !s.description.isSecondaryNode) - continue; + import std.algorithm : filter; + import std.array : array; - if (s.description.isSecondaryNode && maxStalenessSeconds >= 0 - && isStaleSecondary(s.description, maxStalenessSeconds)) - continue; + if (!tagSets.length) + return indices; - if (s.description.roundTripTime < minRTT) - minRTT = s.description.roundTripTime; + foreach (tagSet; tagSets) + { + auto matched = indices + .filter!(i => serverMatchesTagSet(servers[i].description.tags, tagSet)) + .array; + if (matched.length) + return matched; } - if (minRTT == double.max) - return Nullable!MongoHost.init; + return null; + } - double threshold = minRTT + localThresholdMS / 1_000.0; - MongoHost[] eligible; - foreach (ref s; servers) + Nullable!MongoHost randomHostWithinLatencyWindow(long localThresholdMS, + long maxStalenessSeconds = -1, string[string][] tagSets = null) const + { + size_t[] eligible; + foreach (i, ref s; servers) { if (!s.description.isPrimary && !s.description.isSecondaryNode) continue; @@ -387,15 +424,26 @@ struct TopologyDescription && isStaleSecondary(s.description, maxStalenessSeconds)) continue; - if (s.description.roundTripTime <= threshold) - eligible ~= s.host; + eligible ~= i; } + eligible = selectIndicesByTagSets(eligible, tagSets); if (!eligible.length) return Nullable!MongoHost.init; + double minRTT = double.max; + foreach (i; eligible) + if (servers[i].description.roundTripTime < minRTT) + minRTT = servers[i].description.roundTripTime; + + double threshold = minRTT + localThresholdMS / 1_000.0; + MongoHost[] withinWindow; + foreach (i; eligible) + if (servers[i].description.roundTripTime <= threshold) + withinWindow ~= servers[i].host; + import std.random : uniform; - return Nullable!MongoHost(eligible[uniform(0, eligible.length)]); + return Nullable!MongoHost(withinWindow[uniform(0, withinWindow.length)]); } private bool isStaleSecondary(ref const ServerDescription desc, long maxStalenessSeconds) const @@ -422,7 +470,7 @@ struct TopologyDescription auto sLag = sec.lastUpdateTimeUsecs - sec.lastWrite.lastWriteDate.get.value * 1000; auto pLag = pri.lastUpdateTimeUsecs - pri.lastWrite.lastWriteDate.get.value * 1000; - return sLag - pLag + HEARTBEAT_FREQUENCY_USECS; + return sLag - pLag + heartbeatFrequencyMS * 1000; } private long stalenessWithoutPrimary(ref const ServerDescription desc) const @@ -442,7 +490,7 @@ struct TopologyDescription return -1; auto sWriteDate = desc.lastWrite.lastWriteDate.get.value * 1000; - return maxWriteDate - sWriteDate + HEARTBEAT_FREQUENCY_USECS; + return maxWriteDate - sWriteDate + heartbeatFrequencyMS * 1000; } private long findPrimaryIdx() const @@ -462,8 +510,176 @@ struct ServerRecord ServerDescription description; } -/// Default heartbeat frequency (10 seconds) used for staleness calculation. -private enum long HEARTBEAT_FREQUENCY_USECS = 10_000_000; +/// SDAM stale-primary test: a reported primary is stale when its (electionId, setVersion) +/// tuple is strictly less than the topology's max watermark, comparing electionId FIRST +/// (it advances on every election; setVersion can regress across terms). A null component +/// sorts below any present one, so a primary omitting electionId loses to one that has it. +/// With no watermark yet (both max values null) nothing is stale. +bool isStalePrimary(Nullable!BsonObjectID electionId, Nullable!int setVersion, + Nullable!BsonObjectID maxElectionId, Nullable!int maxSetVersion) @safe +{ + if (maxElectionId.isNull && maxSetVersion.isNull) + return false; + + auto byElection = compareNullable(electionId, maxElectionId); + if (byElection != 0) + return byElection < 0; + + return compareNullable(setVersion, maxSetVersion) < 0; +} + +/// Three-way compare of two Nullables, treating null as smaller than any present value. +private int compareNullable(T)(Nullable!T a, Nullable!T b) @safe +{ + if (a.isNull) + return b.isNull ? 0 : -1; + if (b.isNull) + return 1; + if (a.get < b.get) + return -1; + if (b.get < a.get) + return 1; + return 0; +} + +/// isStalePrimary compares electionId first, then setVersion, with null sorting lowest +unittest +{ + import vibe.data.bson : BsonObjectID; + + auto eidLow = Nullable!BsonObjectID(BsonObjectID.fromHexString("aabbccddeeff00112233aa01")); + auto eidHigh = Nullable!BsonObjectID(BsonObjectID.fromHexString("aabbccddeeff00112233aa02")); + auto noEid = Nullable!BsonObjectID.init; + auto v1 = Nullable!int(1); + auto v2 = Nullable!int(2); + auto noV = Nullable!int.init; + + assert(!isStalePrimary(eidLow, v1, noEid, noV), "the first primary (no watermark yet) is accepted"); + + // electionId decides before setVersion: a higher setVersion does not rescue a lower electionId. + assert(isStalePrimary(eidLow, v2, eidHigh, v1), "a lower electionId is stale even with a higher setVersion"); + assert(!isStalePrimary(eidHigh, v1, eidLow, v2), "a higher electionId wins even with a lower setVersion"); + + // Equal electionId: setVersion breaks the tie. + assert(isStalePrimary(eidHigh, v1, eidHigh, v2), "equal electionId, lower setVersion is stale"); + assert(!isStalePrimary(eidHigh, v2, eidHigh, v1), "equal electionId, higher setVersion wins"); + + // A primary omitting electionId loses to an established electionId watermark. + assert(isStalePrimary(noEid, v2, eidHigh, v1), "a missing electionId sorts below a present one"); +} + +/// Builds the fixed topology for load-balancer mode: a single load-balancer host, +/// no discovery or monitoring (the load balancer fronts the real backends). +TopologyDescription loadBalancedTopology(MongoHost host) +{ + TopologyDescription topo; + topo.type = TopologyType.loadBalanced; + topo.servers = [ServerRecord(host, ServerDescription.init)]; + topo.seedCount = 1; + return topo; +} + +/// Returns a new topology with `desc` applied for `host`, leaving `current` unchanged. +TopologyDescription applyDescription(TopologyDescription current, MongoHost host, ServerDescription desc) +{ + current.servers = current.servers.dup; + current.update(host, desc); + return current; +} + +/// Returns a new topology with `host` marked failed, leaving `current` unchanged. +TopologyDescription applyFailed(TopologyDescription current, MongoHost host) +{ + current.servers = current.servers.dup; + current.markFailed(host); + return current; +} + +/// Holds the current topology behind an atomically-swapped pointer for lock-free reads. +struct AtomicTopology +{ + private shared(TopologyDescription)* m_current; + + /// Atomically replace the current snapshot with a heap copy of `topology`. + void publish(TopologyDescription topology) @trusted + { + import core.atomic : atomicStore; + + auto snapshot = new TopologyDescription; + *snapshot = topology; + atomicStore(m_current, cast(shared(TopologyDescription)*) snapshot); + } + + /// Return a value copy of the current snapshot, or the default if none. + TopologyDescription load() @trusted const + { + import core.atomic : atomicLoad; + + auto p = atomicLoad(m_current); + if (p is null) + return TopologyDescription.init; + return *(cast(TopologyDescription*) p); + } +} + +/// applyDescription returns a new topology and leaves the input snapshot unchanged +unittest +{ + TopologyDescription before; + auto host = MongoHost("primary", 27017); + + ServerDescription primaryDesc; + primaryDesc.isWritablePrimary = true; + primaryDesc.setName = "rs0"; + + auto after = applyDescription(before, host, primaryDesc); + + assert(!after.primaryHost.isNull && after.primaryHost.get == host, + "the result reflects the applied primary"); + assert(before.servers.length == 0 && before.primaryHost.isNull, + "the input snapshot is not mutated"); +} + +/// applyFailed clears the failed host in the result without mutating the input +unittest +{ + auto host = MongoHost("primary", 27017); + + ServerDescription primaryDesc; + primaryDesc.isWritablePrimary = true; + primaryDesc.setName = "rs0"; + + TopologyDescription before = applyDescription(TopologyDescription.init, host, primaryDesc); + + auto after = applyFailed(before, host); + + assert(after.primaryHost.isNull, "the result no longer has the failed primary"); + assert(!before.primaryHost.isNull && before.primaryHost.get == host, + "the input snapshot still has the primary"); +} + +/// AtomicTopology.load returns the default topology before anything is published +unittest +{ + AtomicTopology holder; + assert(holder.load().servers.length == 0); +} + +/// AtomicTopology round-trips the most recently published snapshot +unittest +{ + AtomicTopology holder; + auto host = MongoHost("primary", 27017); + + ServerDescription primaryDesc; + primaryDesc.isWritablePrimary = true; + primaryDesc.setName = "rs0"; + + holder.publish(applyDescription(TopologyDescription.init, host, primaryDesc)); + + auto loaded = holder.load(); + assert(!loaded.primaryHost.isNull && loaded.primaryHost.get == host); +} /** * Selects a server from the topology based on the given read preference. @@ -472,17 +688,19 @@ private enum long HEARTBEAT_FREQUENCY_USECS = 10_000_000; * suitable server is available. */ Nullable!MongoHost selectServer(ref const TopologyDescription topology, ReadPreference pref, - long localThresholdMS = 15, long maxStalenessSeconds = -1) + long localThresholdMS = 15, long maxStalenessSeconds = -1, string[string][] tagSets = null) { - // Single topology: return the one server regardless of read preference - if (topology.type == TopologyType.single && topology.servers.length > 0) + // Single and load-balanced topologies are fixed to one host, returned regardless of + // read preference (the load balancer fronts the backends; pinning is per cursor via serviceId). + if ((topology.type == TopologyType.single || topology.type == TopologyType.loadBalanced) + && topology.servers.length > 0) return Nullable!MongoHost(topology.servers[0].host); - // Sharded: return random mongos (read preference forwarded to mongos) + // For sharded topologies return a random mongos (read preference forwarded to mongos) if (topology.type == TopologyType.sharded) return topology.randomMongosHost(localThresholdMS); - // Replica set or unknown: apply read preference logic + // For replica set or unknown topologies apply read preference logic final switch (pref) { case ReadPreference.primary: @@ -492,28 +710,156 @@ Nullable!MongoHost selectServer(ref const TopologyDescription topology, ReadPref auto primary = topology.primaryHost; if (!primary.isNull) return primary; - return topology.randomSecondaryHost(maxStalenessSeconds); + return topology.randomSecondaryHost(maxStalenessSeconds, tagSets); case ReadPreference.secondary: - return topology.randomSecondaryHost(maxStalenessSeconds); + return topology.randomSecondaryHost(maxStalenessSeconds, tagSets); case ReadPreference.secondaryPreferred: - auto secondary = topology.randomSecondaryHost(maxStalenessSeconds); + auto secondary = topology.randomSecondaryHost(maxStalenessSeconds, tagSets); if (!secondary.isNull) return secondary; return topology.primaryHost; case ReadPreference.nearest: - return topology.randomHostWithinLatencyWindow(localThresholdMS, maxStalenessSeconds); + return topology.randomHostWithinLatencyWindow(localThresholdMS, maxStalenessSeconds, tagSets); } } +/** + * Returns the server that writes must be sent to, regardless of the configured + * read preference. Resolves the primary for a replica set, the single server for + * a standalone deployment, and a mongos for a sharded cluster. Null if no write + * target is currently available (e.g. a replica set with no elected primary). + */ +Nullable!MongoHost writeTarget(ref const TopologyDescription topology, long localThresholdMS = 15) +{ + return selectServer(topology, ReadPreference.primary, localThresholdMS); +} + +/// Picks the primary when `toPrimary`, else the read-preference target; null if none. +Nullable!MongoHost selectTarget(ref const TopologyDescription topology, bool toPrimary, + ReadPreference pref, long localThresholdMS = 15, long maxStalenessSeconds = -1, + string[string][] tagSets = null) +{ + return toPrimary + ? writeTarget(topology, localThresholdMS) + : selectServer(topology, pref, localThresholdMS, maxStalenessSeconds, tagSets); +} + +/// writeTarget returns the primary even when a secondary is available +unittest +{ + TopologyDescription topo; + topo.type = TopologyType.replicaSetWithPrimary; + + auto primary = MongoHost("primary", 27017); + ServerDescription pdesc; + pdesc.isWritablePrimary = true; + pdesc.setName = "rs0"; + topo.update(primary, pdesc); + + auto secondary = MongoHost("secondary", 27017); + ServerDescription sdesc; + sdesc.secondary = true; + sdesc.setName = "rs0"; + topo.update(secondary, sdesc); + + auto target = writeTarget(topo); + assert(!target.isNull); + assert(target.get == primary); +} + +/// writeTarget returns the only server for a standalone topology +unittest +{ + TopologyDescription topo; + auto host = MongoHost("standalone", 27017); + ServerDescription desc; + desc.isWritablePrimary = true; + topo.update(host, desc); + topo.type = TopologyType.single; + + auto target = writeTarget(topo); + assert(!target.isNull); + assert(target.get == host); +} + +/// selectServer returns the load-balancer host regardless of read preference +unittest +{ + TopologyDescription topo; + auto host = MongoHost("loadbalancer", 27017); + ServerDescription desc; + topo.update(host, desc); + topo.type = TopologyType.loadBalanced; + + auto target = selectServer(topo, ReadPreference.secondary); + assert(!target.isNull); + assert(target.get == host); +} + +/// loadBalancedTopology builds a loadBalanced topology with the configured host selectable +unittest +{ + auto host = MongoHost("loadbalancer", 27017); + auto topo = loadBalancedTopology(host); + + assert(topo.type == TopologyType.loadBalanced); + + auto target = selectServer(topo, ReadPreference.primary); + assert(!target.isNull); + assert(target.get == host); +} + +/// writeTarget returns null when the replica set has no primary +unittest +{ + TopologyDescription topo; + topo.type = TopologyType.replicaSetNoPrimary; + + auto secondary = MongoHost("secondary", 27017); + ServerDescription desc; + desc.secondary = true; + desc.setName = "rs0"; + topo.update(secondary, desc); + + auto target = writeTarget(topo); + assert(target.isNull); +} + +/// selectTarget routes writes to the primary and reads by read preference +unittest +{ + TopologyDescription topo; + auto primary = MongoHost("primary", 27017); + auto secondary = MongoHost("secondary", 27017); + + ServerDescription pdesc; + pdesc.isWritablePrimary = true; + pdesc.setName = "rs0"; + topo.update(primary, pdesc); + + ServerDescription sdesc; + sdesc.secondary = true; + sdesc.setName = "rs0"; + topo.update(secondary, sdesc); + + auto write = selectTarget(topo, true, ReadPreference.secondary); + assert(!write.isNull && write.get == primary, "writes go to the primary, ignoring read preference"); + + auto read = selectTarget(topo, false, ReadPreference.secondary); + assert(!read.isNull && read.get == secondary, "reads honor the read preference"); +} + /** * Returns true if `incoming` is stale relative to `existing`. * - * Per the SDAM spec, a server description with the same processId but - * a lower or equal counter is stale. A different processId means the - * server restarted, so the update is always fresh. + * Monitor checks are sequential per host, so only a STRICTLY-LOWER counter + * (an out-of-order delivery) is stale and dropped. An equal counter is a + * steady-state heartbeat that must refresh the description's volatile + * metadata (roundTripTime/lastWrite/lastUpdateTimeUsecs), so it is NOT stale. + * A different processId means the server restarted, so the update is fresh. */ private bool isStaleUpdate(ref const ServerDescription existing, ref const ServerDescription incoming) pure nothrow @nogc @@ -527,7 +873,7 @@ private bool isStaleUpdate(ref const ServerDescription existing, ref const Serve if (oldTV.processId != newTV.processId) return false; - return newTV.counter <= oldTV.counter; + return newTV.counter < oldTV.counter; } /// selectServer returns primary for ReadPreference.primary @@ -644,6 +990,37 @@ unittest assert(result.get == sec); } +/// selectServer primaryPreferred honors tagSets when falling back to a secondary +unittest +{ + TopologyDescription topo; + auto east = MongoHost("east-sec", 27017); + auto west = MongoHost("west-sec", 27017); + + ServerDescription eastDesc; + eastDesc.secondary = true; + eastDesc.setName = "rs0"; + eastDesc.tags = ["dc": "east"]; + + ServerDescription westDesc; + westDesc.secondary = true; + westDesc.setName = "rs0"; + westDesc.tags = ["dc": "west"]; + + topo.update(east, eastDesc); + topo.update(west, westDesc); + + string[string][] tagSets = [["dc": "east"]]; + + // With no primary, primaryPreferred must still respect the tag set and never pick west. + foreach (_; 0 .. 100) + { + auto result = selectServer(topo, ReadPreference.primaryPreferred, 15, -1, tagSets); + assert(!result.isNull, "a tag-matching secondary is selected"); + assert(result.get == east, "primaryPreferred must not select a tag-excluded secondary"); + } +} + /// selectServer primaryPreferred prefers primary when available unittest { @@ -933,6 +1310,60 @@ unittest assert(known.length == 3); } +/// allKnownHosts includes arbiters so they are monitored as replica-set members +unittest +{ + import std.algorithm : canFind; + + TopologyDescription topo; + topo.type = TopologyType.replicaSetNoPrimary; + auto primary = MongoHost("primary", 27017); + + ServerDescription desc; + desc.isWritablePrimary = true; + desc.setName = "rs0"; + desc.hosts = ["primary:27017", "sec:27017"]; + desc.arbiters = ["arb:27017"]; + + topo.update(primary, desc); + + assert(topo.allKnownHosts().canFind(MongoHost("arb", 27017)), + "an arbiter advertised in the member list must be monitored"); +} + +/// allKnownHosts includes a server's own host even when its description carries no member list (standalone/sharded) +unittest +{ + TopologyDescription topo; + auto host = MongoHost("standalone", 27017); + + ServerDescription desc; + desc.isWritablePrimary = true; + + topo.update(host, desc); + + assert(topo.allKnownHosts() == [host], + "allKnownHosts must include the server's own host even without a description hosts array"); +} + +/// a markFailed-cleared server's host stays in allKnownHosts so monitoring can recover after a full outage +unittest +{ + TopologyDescription topo; + auto host = MongoHost("host1", 27017); + + ServerDescription desc; + desc.isWritablePrimary = true; + desc.setName = "rs0"; + desc.hosts = ["host1:27017"]; + topo.update(host, desc); + + topo.markFailed(host); // simulate an outage: the server's description is cleared + + assert(topo.allKnownHosts() == [host], + "a failed server's host stays known so reconcileWith does not stop its monitor (monitoring can recover)"); +} + /// update with higher topology version counter overwrites unittest { @@ -981,7 +1412,7 @@ unittest assert(topo.servers[0].description.isPrimary); } -/// update with equal topology version counter is rejected +/// update with equal topology version counter is accepted (heartbeats refresh the description) unittest { TopologyDescription topo; @@ -1001,7 +1432,32 @@ unittest desc2.topologyVersion = Nullable!TopologyVersion(TopologyVersion(pid, 5)); topo.update(host, desc2); - assert(topo.servers[0].description.isPrimary); + assert(topo.servers[0].description.isSecondaryNode); +} + +/// update with an equal topology version counter still refreshes volatile metadata (roundTripTime) +unittest +{ + TopologyDescription topo; + auto host = MongoHost("host1", 27017); + auto pid = BsonObjectID.fromHexString("aabbccddeeff00112233aabb"); + + ServerDescription first; + first.isWritablePrimary = true; + first.setName = "rs0"; + first.roundTripTime = 10; + first.topologyVersion = Nullable!TopologyVersion(TopologyVersion(pid, 5)); + topo.update(host, first); + + ServerDescription heartbeat; + heartbeat.isWritablePrimary = true; + heartbeat.setName = "rs0"; + heartbeat.roundTripTime = 25; + heartbeat.topologyVersion = Nullable!TopologyVersion(TopologyVersion(pid, 5)); + topo.update(host, heartbeat); + + assert(topo.servers[0].description.roundTripTime == 25, + "an equal-counter heartbeat must refresh roundTripTime, not freeze it at the first-probe value"); } /// update with different processId always overwrites (server restarted) @@ -1273,6 +1729,52 @@ unittest assert(topo.type == TopologyType.replicaSetNoPrimary); } +/// loadBalanced topology stays loadBalanced when an RSPrimary description arrives +unittest +{ + TopologyDescription topo; + topo.type = TopologyType.loadBalanced; + auto host = MongoHost("lb-backend", 27017); + + ServerDescription primaryDesc; + primaryDesc.isWritablePrimary = true; + primaryDesc.setName = "rs0"; + assert(primaryDesc.classifiedType() == ServerDescription.ServerType.RSPrimary); + + topo.update(host, primaryDesc); + assert(topo.type == TopologyType.loadBalanced, + "a load-balanced topology must not transition based on SDAM"); +} + +/// loadBalanced topology stays loadBalanced for mongos, standalone and RSSecondary descriptions +unittest +{ + auto host = MongoHost("lb-backend", 27017); + + ServerDescription mongosDesc; + mongosDesc.msg = "isdbgrid"; + assert(mongosDesc.classifiedType() == ServerDescription.ServerType.mongos); + + ServerDescription standaloneDesc; + standaloneDesc.isWritablePrimary = true; + assert(standaloneDesc.classifiedType() == ServerDescription.ServerType.standalone); + + ServerDescription secondaryDesc; + secondaryDesc.secondary = true; + secondaryDesc.setName = "rs0"; + assert(secondaryDesc.classifiedType() == ServerDescription.ServerType.RSSecondary); + + foreach (desc; [mongosDesc, standaloneDesc, secondaryDesc]) + { + TopologyDescription topo; + topo.type = TopologyType.loadBalanced; + + topo.update(host, desc); + assert(topo.type == TopologyType.loadBalanced, + "a load-balanced topology must not transition based on SDAM"); + } +} + /// sharded topology only keeps mongos servers unittest { @@ -1296,6 +1798,58 @@ unittest assert(topo.servers[0].host == mongos); } +/// a rogue RSPrimary in a sharded topology is removed without wiping the mongos list or flipping the type +unittest +{ + TopologyDescription topo; + topo.type = TopologyType.sharded; + auto mongos = MongoHost("mongos", 27017); + auto rogue = MongoHost("rogue", 27017); + + ServerDescription mongosDesc; + mongosDesc.msg = "isdbgrid"; + topo.update(mongos, mongosDesc); + + // A host thought to be a mongos now reports as an RS primary advertising its own + // replica-set members (which do NOT include the mongos). The primary-handling block + // would prune the mongos to those members, then flip the topology type. + ServerDescription rogueDesc; + rogueDesc.isWritablePrimary = true; + rogueDesc.setName = "rs0"; + rogueDesc.hosts = ["rogue:27017", "other:27017"]; + topo.update(rogue, rogueDesc); + + assert(topo.type == TopologyType.sharded, "a non-mongos must not flip a sharded topology's type"); + assert(topo.servers.length == 1, "the rogue RS server is removed and the mongos retained"); + assert(topo.servers[0].host == mongos, "the mongos survives the rogue primary"); +} + +/// an RS server whose setName differs from the topology's is rejected, not allowed to demote the primary +unittest +{ + TopologyDescription topo; + topo.type = TopologyType.replicaSetWithPrimary; + topo.setName = "rs0"; + auto good = MongoHost("good", 27017); + auto wrong = MongoHost("wrong", 27017); + + ServerDescription goodPrimary; + goodPrimary.isWritablePrimary = true; + goodPrimary.setName = "rs0"; + topo.update(good, goodPrimary); + + // A host re-provisioned into a DIFFERENT replica set now reports setName "other". + ServerDescription wrongPrimary; + wrongPrimary.isWritablePrimary = true; + wrongPrimary.setName = "other"; + topo.update(wrong, wrongPrimary); + + assert(topo.servers.length == 1, "the wrong-set server is rejected"); + assert(topo.servers[0].host == good, "only the matching-set host remains"); + assert(topo.findPrimaryIdx() != -1 && topo.servers[topo.findPrimaryIdx()].host == good, + "the wrong-set primary did not take over the topology"); +} + /// server type classification from hello response fields unittest { @@ -1507,7 +2061,7 @@ unittest assert(result.get == primary); } -/// isStaleUpdate: incoming has topologyVersion but existing does not — accepts update +/// isStaleUpdate accepts the update when incoming has topologyVersion but existing does not unittest { auto pid = BsonObjectID.fromHexString("aabbccddeeff00112233aabb"); @@ -1743,6 +2297,45 @@ unittest assert(!host2Primary); } +/// a primary with a higher setVersion but lower electionId is stale (electionId is compared first) +unittest +{ + import vibe.data.bson : BsonObjectID; + + TopologyDescription topo; + topo.type = TopologyType.replicaSetNoPrimary; + auto host1 = MongoHost("host1", 27017); + auto host2 = MongoHost("host2", 27017); + + auto eidHigh = BsonObjectID.fromHexString("aabbccddeeff00112233aa02"); + auto eidLow = BsonObjectID.fromHexString("aabbccddeeff00112233aa01"); + + // Real current primary: highest electionId, a modest setVersion. + ServerDescription current; + current.isWritablePrimary = true; + current.setName = "rs0"; + current.setVersion = Nullable!int(1); + current.electionId = Nullable!BsonObjectID(eidHigh); + topo.update(host1, current); + + // Stale primary from a previous term: it bumped its setVersion but has a LOWER electionId. + ServerDescription stale; + stale.isWritablePrimary = true; + stale.setName = "rs0"; + stale.setVersion = Nullable!int(2); + stale.electionId = Nullable!BsonObjectID(eidLow); + topo.update(host2, stale); + + bool host1Primary, host2Primary; + foreach (ref s; topo.servers) + { + if (s.host == host1 && s.description.isPrimary) host1Primary = true; + if (s.host == host2 && s.description.isPrimary) host2Primary = true; + } + assert(host1Primary, "the real primary (higher electionId) keeps the role"); + assert(!host2Primary, "the stale primary (higher setVersion, lower electionId) is rejected"); +} + /// sharded selectServer applies latency window to mongos selection unittest { @@ -1800,3 +2393,262 @@ unittest assert(sawFast); assert(sawSlow); } + +/// returns true when every required tag is present in the server's tags +bool serverMatchesTagSet(const(string[string]) serverTags, string[string] required) @safe +{ + foreach (key, value; required) + if (serverTags.get(key, null) != value) + return false; + return true; +} + +/// serverMatchesTagSet returns true when the server carries every required tag pair +unittest +{ + string[string] serverTags = ["dc": "east"]; + string[string] required = ["dc": "east"]; + + assert(serverMatchesTagSet(serverTags, required) == true, + "server tagged dc:east must satisfy required tag set dc:east"); +} + +/// serverMatchesTagSet returns false when a required tag value differs +unittest +{ + assert(serverMatchesTagSet(["dc": "west"], ["dc": "east"]) == false, + "server in dc:west must not satisfy required tag set dc:east"); +} + +/// serverMatchesTagSet returns true for the empty (catch-all) tag set +unittest +{ + assert(serverMatchesTagSet(["dc": "east"], null) == true, + "an empty required tag set matches any server"); +} + +version (unittest) +{ + /// Builds a replica set with a primary plus dc:east and dc:west secondaries, + /// returning the topology and the two tagged secondary host handles. + private struct TaggedSecondaries + { + TopologyDescription topo; + MongoHost secEast; + MongoHost secWest; + } + + private TaggedSecondaries buildTaggedSecondaries() + { + TopologyDescription topo; + topo.type = TopologyType.replicaSetWithPrimary; + + auto primary = MongoHost("primary", 27017); + ServerDescription primaryDesc; + primaryDesc.isWritablePrimary = true; + primaryDesc.setName = "rs0"; + topo.update(primary, primaryDesc); + + auto secEast = MongoHost("sec-east", 27017); + ServerDescription eastDesc; + eastDesc.secondary = true; + eastDesc.setName = "rs0"; + eastDesc.tags = ["dc": "east"]; + topo.update(secEast, eastDesc); + + auto secWest = MongoHost("sec-west", 27017); + ServerDescription westDesc; + westDesc.secondary = true; + westDesc.setName = "rs0"; + westDesc.tags = ["dc": "west"]; + topo.update(secWest, westDesc); + + return TaggedSecondaries(topo, secEast, secWest); + } +} + +/// secondaryHosts with a single tag set returns only the matching secondary +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto hosts = rs.topo.secondaryHosts(-1, [["dc": "east"]]); + assert(hosts == [rs.secEast], "tag set dc:east must select only the matching secondary"); +} + +/// secondaryHosts falls through to the second tag set when the first matches nothing +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto hosts = rs.topo.secondaryHosts(-1, [["dc": "nowhere"], ["dc": "east"]]); + assert(hosts == [rs.secEast], "must fall through to the second tag set when the first matches nothing"); +} + +/// secondaryHosts stops at the first matching tag set and ignores later ones +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto hosts = rs.topo.secondaryHosts(-1, [["dc": "east"], ["dc": "west"]]); + assert(hosts == [rs.secEast], + "first matching tag set wins; later sets must not add hosts"); + + auto none = rs.topo.secondaryHosts(-1, [["dc": "nowhere"]]); + assert(none.length == 0, "no tag set matching any secondary yields no hosts"); +} + +/// selectServer secondary with tag set dc:east returns only the matching secondary +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto chosen = selectServer(rs.topo, ReadPreference.secondary, 15, -1, [["dc": "east"]]); + assert(!chosen.isNull, "tag set dc:east must select a secondary"); + assert(chosen.get == rs.secEast, "tag set dc:east must select only the matching secondary"); +} + +/// selectServer secondaryPreferred with a non-matching tag set falls back to the primary +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto chosen = selectServer(rs.topo, ReadPreference.secondaryPreferred, 15, -1, [["dc": "nowhere"]]); + assert(!chosen.isNull, "secondaryPreferred must fall back to a server when no secondary matches"); + assert(chosen.get == rs.topo.primaryHost.get, "secondaryPreferred with non-matching tags must fall back to the primary"); +} + +/// selectServer nearest with a tag set matching no member returns null +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto chosen = selectServer(rs.topo, ReadPreference.nearest, 15, -1, [["dc": "nowhere"]]); + assert(chosen.isNull, "nearest with a tag set matching no member must select no server"); +} + +/// selectTarget forwards a secondary read tag set dc:east to selectServer and returns the matching secondary +unittest +{ + auto rs = buildTaggedSecondaries(); + + auto chosen = selectTarget(rs.topo, false, ReadPreference.secondary, 15, -1, [["dc": "east"]]); + assert(!chosen.isNull, "selectTarget with tag set dc:east must select a secondary"); + assert(chosen.get == rs.secEast, "selectTarget must forward the tag set so only sec-east is chosen"); +} + +/// tag sets never exclude the primary: primary reads and writes ignore them +unittest +{ + auto rs = buildTaggedSecondaries(); + auto primary = rs.topo.primaryHost.get; + + auto read = selectServer(rs.topo, ReadPreference.primary, 15, -1, [["dc": "nowhere"]]); + assert(!read.isNull && read.get == primary, + "primary read preference must ignore tag sets and still pick the primary"); + + auto write = selectTarget(rs.topo, true, ReadPreference.primary, 15, -1, [["dc": "nowhere"]]); + assert(!write.isNull && write.get == primary, + "writes must ignore tag sets and still target the primary"); +} + +/// supportsRetryableWrites returns false for a standalone (single) topology +unittest +{ + assert(supportsRetryableWrites(TopologyType.single) == false, + "standalone mongod rejects lsid/txnNumber, so retryable writes are unsupported on TopologyType.single"); +} + +/// supportsRetryableWrites returns true for a load-balanced topology +unittest +{ + assert(supportsRetryableWrites(TopologyType.loadBalanced), + "a load-balanced deployment fronts a mongos, which supports retryable writes"); +} + +/// The topology-wide logical session timeout: the MIN advertised logicalSessionTimeoutMinutes +/// across data-bearing servers, or null when it cannot be determined. +Nullable!Duration logicalSessionTimeout(const ServerDescription[] servers) @safe +{ + import core.time : minutes; + Nullable!int min; + foreach (s; servers) + { + if (!s.isDataBearing) + continue; + if (s.logicalSessionTimeoutMinutes.isNull) + return Nullable!Duration.init; + if (min.isNull || s.logicalSessionTimeoutMinutes.get < min.get) + min = s.logicalSessionTimeoutMinutes.get; + } + return min.isNull ? Nullable!Duration.init : Nullable!Duration(min.get.minutes); +} + +/// logicalSessionTimeout returns 30.minutes for a single server advertising 30 +unittest +{ + import core.time : minutes; + + ServerDescription primary; + primary.isWritablePrimary = true; + primary.logicalSessionTimeoutMinutes = 30; + + auto timeout = logicalSessionTimeout([primary]); + + assert(!timeout.isNull && timeout.get == 30.minutes, + "single server advertises a 30 minute session timeout"); +} + +/// logicalSessionTimeout returns the minimum advertised timeout across servers +unittest +{ + import core.time : minutes; + + ServerDescription primary; + primary.isWritablePrimary = true; + primary.logicalSessionTimeoutMinutes = 30; + + ServerDescription secondary; + secondary.secondary = true; + secondary.logicalSessionTimeoutMinutes = 10; + + auto timeout = logicalSessionTimeout([primary, secondary]); + + assert(!timeout.isNull && timeout.get == 10.minutes, + "the topology timeout is the minimum advertised across servers"); +} + +/// logicalSessionTimeout returns null when a data-bearing server advertises no timeout +unittest +{ + ServerDescription primary; + primary.isWritablePrimary = true; + primary.logicalSessionTimeoutMinutes = 30; + + ServerDescription secondary; + secondary.secondary = true; + + auto timeout = logicalSessionTimeout([primary, secondary]); + + assert(timeout.isNull, + "a data-bearing server that does not advertise a session timeout disables sessions topology-wide"); +} + +/// logicalSessionTimeout excludes arbiters from the minimum computation +unittest +{ + import core.time : minutes; + + ServerDescription primary; + primary.isWritablePrimary = true; + primary.logicalSessionTimeoutMinutes = 30; + + ServerDescription arbiter; + arbiter.arbiterOnly = true; + arbiter.logicalSessionTimeoutMinutes = 10; + + auto timeout = logicalSessionTimeout([primary, arbiter]); + + assert(!timeout.isNull && timeout.get == 30.minutes, + "arbiters are excluded from the session timeout computation"); +} diff --git a/tests/mongodb/_connection/source/app.d b/tests/mongodb/_connection/source/app.d index bb19038237..2eece37dcc 100644 --- a/tests/mongodb/_connection/source/app.d +++ b/tests/mongodb/_connection/source/app.d @@ -1,5 +1,6 @@ import vibe.db.mongo.mongo; import vibe.db.mongo.client; +import vibe.data.bson; import vibe.core.core; import vibe.core.log; import core.time; @@ -200,6 +201,14 @@ int main(string[] args) assert(db.runListCommand(["listCollections": Bson(1.0)]) .empty); + // close() tears the whole client down: a connected standalone runs a monitor and + // holds at least one connection pool; after close() no monitors and no pools remain. + enforce(client.activeMonitorCount >= 1, "a connected standalone client runs at least one monitor"); + enforce(client.connectionPoolCount >= 1, "an active client holds at least one connection pool"); + client.close(); + enforce(client.activeMonitorCount == 0, "close() stops all background monitors"); + enforce(client.connectionPoolCount == 0, "close() releases all connection pools"); + logInfo("All tests passed"); return 0; } diff --git a/tests/mongodb/_health-monitor/dub.json b/tests/mongodb/_health-monitor/dub.json new file mode 100644 index 0000000000..56d4feea9e --- /dev/null +++ b/tests/mongodb/_health-monitor/dub.json @@ -0,0 +1,8 @@ +{ + "name": "health-monitor-test", + "description": "MongoDB background health monitoring failover test", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + }, + "debugVersions": ["VibeVerboseMongo"] +} diff --git a/tests/mongodb/_health-monitor/run.sh b/tests/mongodb/_health-monitor/run.sh new file mode 100755 index 0000000000..07203bf6d7 --- /dev/null +++ b/tests/mongodb/_health-monitor/run.sh @@ -0,0 +1,101 @@ +#!/bin/bash +set -e + +PORT1=22840 +PORT2=22841 +PORT3=22842 + +PIDS=() + +cleanup() { + echo "[INFO] Cleaning up mongod instances..." + for pid in "${PIDS[@]}"; do + if [ "$pid" != "0" ] && [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + fi + done + for pid in "${PIDS[@]}"; do + if [ "$pid" != "0" ] && [ -n "$pid" ]; then + for _ in $(seq 1 30); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + kill -9 "$pid" 2>/dev/null || true + fi + done + rm -rf db + rm -f log*.txt +} + +trap cleanup EXIT + +start_mongod() { + local idx=$1 + local port=$2 + local logfile="log${idx}.txt" + local dbpath="db/rs${idx}" + mkdir -p "$dbpath" + PIDS[$idx]=$(mongod --logpath "$logfile" --bind_ip 127.0.0.1 --port "$port" --replSet rs0 --dbpath "$dbpath" --fork | grep -Po 'forked process: \K\d+') + echo "[INFO] Started mongod on port $port (PID: ${PIDS[$idx]})" +} + +wait_for_primary() { + local port=$1 + echo "[INFO] Waiting for primary election..." + for i in $(seq 1 30); do + PRIMARY=$($MONGO --quiet "mongodb://127.0.0.1:$port" --eval " + var status = rs.status(); + var primary = status.members.filter(function(m) { return m.stateStr === 'PRIMARY'; }); + if (primary.length > 0) { print(primary[0].name); } else { print(''); } + " 2>/dev/null || echo "") + + if [ -n "$PRIMARY" ]; then + echo "[INFO] Primary elected: $PRIMARY" + return 0 + fi + echo "[INFO] Waiting... ($i/30)" + sleep 2 + done + + echo "[ERROR] No primary elected after 60 seconds" + return 1 +} + +rm -f log*.txt +rm -rf db + +start_mongod 0 $PORT1 +start_mongod 1 $PORT2 +start_mongod 2 $PORT3 +sleep 2 + +echo "[INFO] Initiating replica set..." +for attempt in $(seq 1 5); do + if $MONGO --quiet "mongodb://127.0.0.1:$PORT1" --eval " + rs.initiate({ + _id: 'rs0', + members: [ + {_id: 0, host: '127.0.0.1:$PORT1'}, + {_id: 1, host: '127.0.0.1:$PORT2'}, + {_id: 2, host: '127.0.0.1:$PORT3'} + ] + }) + " 2>/dev/null; then + echo "[INFO] Replica set initiated" + break + fi + echo "[INFO] rs.initiate attempt $attempt failed, retrying in 2s..." + sleep 2 +done + +wait_for_primary $PORT1 + +echo "" +echo "============================================" +echo "Health monitor failover test" +echo "============================================" +if ! eval $DUB_INVOKE -- "$PORT1,$PORT2,$PORT3"; then + echo "[FAIL] Health monitor failover test failed" + exit 1 +fi +echo "[PASS] Health monitor failover test passed" diff --git a/tests/mongodb/_health-monitor/source/app.d b/tests/mongodb/_health-monitor/source/app.d new file mode 100644 index 0000000000..af7c908ddb --- /dev/null +++ b/tests/mongodb/_health-monitor/source/app.d @@ -0,0 +1,100 @@ +import vibe.db.mongo.mongo; +import vibe.db.mongo.client; +import vibe.db.mongo.settings; +import vibe.core.core; +import vibe.core.log; +import vibe.data.bson; +import core.time; +import std.algorithm; +import std.conv; +import std.exception; + +int main(string[] args) +{ + setLogLevel(LogLevel.diagnostic); + + if (args.length < 2) + { + logError("Usage: %s ", args[0]); + return 1; + } + + runTask({ sleepUninterruptible(120.seconds); assert(false, "Timeout exceeded"); }); + + MongoHost[] hosts; + foreach (portStr; args[1].splitter(',')) + hosts ~= MongoHost("127.0.0.1", portStr.to!ushort); + + auto settings = new MongoClientSettings; + settings.hosts = hosts; + settings.replicaSet = "rs0"; + settings.connectTimeoutMS = 5_000; + settings.socketTimeoutMS = 5_000; + settings.heartbeatFrequencyMS = 500; + settings.minHeartbeatFrequencyMS = 50; + settings.serverSelectionTimeoutMS = 20_000; + settings.appName = "VibeHealthMonitorTest"; + + auto client = connectMongoDB(settings); + enforce(client.activeMonitorCount > 0, "no background monitors were started"); + + auto coll = client.getCollection("healthtest.failover"); + try coll.drop(); catch (Exception) {} + + coll.insertOne(Bson(["_id": Bson(BsonObjectID.generate), "phase": Bson("before")])); + logInfo("Initial write landed on the primary"); + + stepDownPrimary(client); + + auto recovered = recoverByWriting(coll, 60.seconds); + enforce(recovered, "the long-lived client never recovered writes after failover"); + + auto count = coll.countDocuments(Bson.emptyObject); + enforce(count == 2, "expected 2 documents after recovery, found " ~ count.to!string); + logInfo("Same client recovered on the new primary without reconnecting"); + + client.stopMonitoring(); + client.cleanupConnections(); + enforce(client.activeMonitorCount == 0, "monitors did not stop after stopMonitoring()"); + + logInfo("Health monitor failover test passed"); + return 0; +} + +/// Steps down the current primary on the same client; the command drops its connection. +void stepDownPrimary(MongoClient client) +{ + logInfo("Forcing a failover via replSetStepDown"); + + // A command's name must be the first field; a Bson AA literal would reorder it. + auto cmd = Bson.emptyObject; + cmd["replSetStepDown"] = Bson(30); + cmd["force"] = Bson(true); + + try + client.getDatabase("admin").runCommandChecked(cmd); + catch (Exception e) + logInfo("replSetStepDown closed the connection as expected: %s", e.msg); +} + +/// Retries a write until one lands on the new primary or the deadline passes. +bool recoverByWriting(MongoCollection coll, Duration budget) +{ + auto deadline = MonoTime.currTime + budget; + + while (MonoTime.currTime < deadline) + { + try + { + coll.insertOne(Bson(["_id": Bson(BsonObjectID.generate), "phase": Bson("after")])); + return true; + } + catch (Exception e) + { + logInfo("Write still failing during failover: %s", e.msg); + sleep(250.msecs); + } + } + + return false; +} diff --git a/tests/mongodb/_replica-set/run.sh b/tests/mongodb/_replica-set/run.sh index 02d955c5e3..86677367f9 100755 --- a/tests/mongodb/_replica-set/run.sh +++ b/tests/mongodb/_replica-set/run.sh @@ -122,9 +122,9 @@ for attempt in $(seq 1 5); do rs.initiate({ _id: 'rs0', members: [ - {_id: 0, host: '127.0.0.1:$PORT1'}, - {_id: 1, host: '127.0.0.1:$PORT2'}, - {_id: 2, host: '127.0.0.1:$PORT3'} + {_id: 0, host: '127.0.0.1:$PORT1', tags: {dc: 'east'}}, + {_id: 1, host: '127.0.0.1:$PORT2', tags: {dc: 'east'}}, + {_id: 2, host: '127.0.0.1:$PORT3', tags: {dc: 'west'}} ] }) " 2>/dev/null; then @@ -173,6 +173,12 @@ run_test 9 "readPreference=secondaryPreferred connects to secondary" \ run_test 10 "readPreference=primary connects to primary (CRUD)" \ "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --readPreference primary +run_test 11 "writes routed to primary despite readPreference=secondary" \ + "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --readPreference secondary --expectWriteToPrimary + +run_test 12 "per-query readPreference=secondary serves reads from a secondary" \ + "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --expectReadFromSecondary + echo "" echo "========================================================" echo " Phase 3: Dead secondary tests" @@ -193,13 +199,13 @@ for idx in 0 1 2; do fi done -run_test 11 "Dead secondary in host list, primary still reachable" \ +run_test 13 "Dead secondary in host list, primary still reachable" \ "${SECONDARY_PORTS[0]},$PRIMARY_PORT" -run_test 12 "All hosts listed, one secondary dead" \ +run_test 14 "All hosts listed, one secondary dead" \ "$PORT1,$PORT2,$PORT3" -run_test 13 "readPreference=secondary with one dead secondary" \ +run_test 15 "readPreference=secondary with one dead secondary" \ "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --readPreference secondary --expectSecondary echo "" @@ -232,10 +238,10 @@ LIVE_PORT=${SECONDARY_PORTS[0]} wait_for_primary $LIVE_PORT detect_roles -run_test 14 "New primary after old primary killed" \ +run_test 16 "New primary after old primary killed" \ "$PORT1,$PORT2,$PORT3" -run_test 15 "readPreference=secondary after primary failover" \ +run_test 17 "readPreference=secondary after primary failover" \ "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --readPreference secondary --expectSecondary echo "" @@ -248,7 +254,7 @@ for idx in 0 1 2; do kill_mongod $idx done -run_test 16 "All hosts dead (expect fail)" \ +run_test 18 "All hosts dead (expect fail)" \ "$PORT1,$PORT2,$PORT3" --expectFail echo "" @@ -265,10 +271,26 @@ sleep 2 wait_for_primary $PORT1 detect_roles -run_test 17 "Connect after full cluster restart" \ +run_test 19 "Connect after full cluster restart" \ "$PORT1,$PORT2,$PORT3" --replicaSet rs0 +echo "" +echo "========================================================" +echo " Phase 7: Primary step-down retry" +echo "========================================================" + +run_test 20 "write retries onto new primary after primary step-down" \ + "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --expectStepDownRetry + +echo "" +echo "========================================================" +echo " Phase 8: Read preference tag targeting" +echo "========================================================" + +run_test 21 "readPreferenceTags=dc:east routes a secondary read to a dc:east member" \ + "$PORT1,$PORT2,$PORT3" --replicaSet rs0 --expectTagTargeting + echo "" echo "============================================" -echo "All $((17)) replica set tests passed!" +echo "All $((21)) replica set tests passed!" echo "============================================" diff --git a/tests/mongodb/_replica-set/source/app.d b/tests/mongodb/_replica-set/source/app.d index d4e8c8bb4e..68bdc3d98d 100644 --- a/tests/mongodb/_replica-set/source/app.d +++ b/tests/mongodb/_replica-set/source/app.d @@ -6,6 +6,7 @@ import vibe.core.log; import vibe.data.bson; import core.time; import std.algorithm; +import std.array; import std.conv; import std.exception; @@ -13,6 +14,10 @@ int main(string[] args) { bool expectFail; bool expectSecondary; + bool expectWriteToPrimary; + bool expectReadFromSecondary; + bool expectStepDownRetry; + bool expectTagTargeting; string replicaSet; string readPrefStr; MongoHost[] hosts; @@ -21,7 +26,7 @@ int main(string[] args) if (args.length < 2) { - logError("Usage: %s [--replicaSet ] [--readPreference ] [--expectFail] [--expectSecondary]", args[0]); + logError("Usage: %s [--replicaSet ] [--readPreference ] [--expectFail] [--expectSecondary] [--expectWriteToPrimary] [--expectReadFromSecondary]", args[0]); return 1; } @@ -42,6 +47,14 @@ int main(string[] args) expectFail = true; else if (arg == "--expectSecondary") expectSecondary = true; + else if (arg == "--expectWriteToPrimary") + expectWriteToPrimary = true; + else if (arg == "--expectReadFromSecondary") + expectReadFromSecondary = true; + else if (arg == "--expectStepDownRetry") + expectStepDownRetry = true; + else if (arg == "--expectTagTargeting") + expectTagTargeting = true; } auto settings = new MongoClientSettings; @@ -64,6 +77,13 @@ int main(string[] args) } } + if (expectTagTargeting) + { + // Target the dc:east members. Secondary reads must land on a dc:east secondary. + settings.readPreference = ReadPreference.secondary; + settings.readPreferenceTags = [["dc": "east"]]; + } + MongoClient client; try @@ -101,6 +121,156 @@ int main(string[] args) return 0; } + if (expectWriteToPrimary) + { + // The client is configured with readPreference=secondary, so reads land on a + // secondary. Each write below must still be routed to the primary; before the + // fix they were sent to the read-preference target and rejected by the server + // with NotWritablePrimary. Success here is the regression guard for #2847. + auto coll = client.getCollection("rstest.writeprimary"); + auto objID = BsonObjectID.generate; + + coll.insertOne(Bson(["_id": Bson(objID), "n": Bson(1)])); + coll.updateOne(["_id": objID], Bson(["$set": Bson(["n": Bson(2)])])); + + // Confirm the writes reached the primary by reading from it directly. The + // primary is read-your-write consistent, so there is no replication lag to + // wait on. + auto verifier = connectMongoDB(primarySettings(hosts, replicaSet)); + auto onPrimary = verifier.getCollection("rstest.writeprimary"); + + auto stored = onPrimary.findOne(["_id": objID]); + enforce(!stored.isNull, "Insert was not routed to the primary"); + enforce(stored["n"].get!int == 2, "Update was not routed to the primary"); + + coll.deleteOne(["_id": objID]); + enforce(onPrimary.findOne(["_id": objID]).isNull, "Delete was not routed to the primary"); + + coll.drop(); + logInfo("Writes correctly routed to primary under readPreference=secondary"); + return 0; + } + + if (expectReadFromSecondary) + { + // The client default is primary. A per-query readPreference=secondary must both + // route the read to a secondary AND inject $readPreference so the secondary + // actually serves it. Before the fix the secondary rejects the read + // (NotPrimaryNoSecondaryOk) or the override is ignored and the read silently runs + // on the primary. This is the regression guard for #2848. + auto coll = client.getCollection("rstest.readsecondary"); + try coll.drop(); catch (Exception) {} + + enum int total = 500; + Bson[] docs; + foreach (i; 0 .. total) + docs ~= Bson(["_id": Bson(i), "v": Bson(i)]); + + InsertManyOptions writeOpts; + WriteConcern majority; + majority.w = Bson("majority"); + writeOpts.writeConcern = majority; + coll.insertMany(docs, writeOpts); + + // 1) per-query override routes a command to a secondary (server selection) + auto hello = client.getDatabase("admin").runCommand(Bson(["hello": Bson(1)]), ReadPreference.secondary); + enforce(hello["secondary"].get!bool, + "runCommand with readPreference=secondary did not reach a secondary: " ~ hello["me"].opt!string); + + // 2) a real multi-batch find is served by a secondary, proving $readPreference is + // injected on the find and that getMore stays pinned to the same secondary + FindOptions findOpts; + findOpts.readPreference = ReadPreference.secondary; + findOpts.batchSize = 100; + + // w:majority only guarantees a MAJORITY holds the write; the secondary this + // read lands on may not be in that majority yet and can briefly lag behind. + // Poll the secondary until replication catches up rather than asserting on + // the first (possibly stale) read. The budget is generous for a slow CI + // runner but stays well under the 30s global test timeout. + size_t received; + auto deadline = MonoTime.currTime + 15.seconds; + for (auto waited = false; ; waited = true) + { + received = coll.find(Bson.emptyObject, findOpts).array.length; + if (received == total) + { + if (waited) + logInfo("Secondary caught up to %s docs after replication lag", total); + break; + } + if (MonoTime.currTime >= deadline) + break; + sleep(100.msecs); + } + enforce(received == total, + "expected " ~ total.to!string ~ " docs from secondary, got " ~ received.to!string); + + coll.drop(); + logInfo("Per-query readPreference=secondary served %s docs from a secondary", total); + return 0; + } + + if (expectTagTargeting) + { + // A secondary read with readPreferenceTags=dc:east must be served by a + // secondary whose tags include dc:east, never the dc:west member. The hello + // response reports the contacted member's own tags, so we can prove it. + foreach (attempt; 0 .. 5) + { + auto hello = client.getDatabase("admin").runCommand(Bson(["hello": Bson(1)])); + auto me = hello["me"].opt!string("?"); + enforce(hello["secondary"].get!bool, + "tag-targeted read must land on a secondary, got primary " ~ me); + enforce(hello["tags"]["dc"].get!string == "east", + "readPreferenceTags=dc:east must route to a dc:east member, got " ~ me + ~ " tagged " ~ hello["tags"].toString()); + logInfo("Tag-targeted secondary read served by dc:east member %s", me); + } + + logInfo("readPreferenceTags=dc:east correctly targeted dc:east secondaries"); + return 0; + } + + if (expectStepDownRetry) + { + // When the primary steps down mid-operation, the driver must catch the + // NotWritablePrimary error, refresh the topology, find the newly elected + // primary, and retry the write once so it lands exactly once. + auto coll = client.getCollection("rstest.stepdown"); + try coll.drop(); catch (Exception) {} + + // Warm the topology and create the collection on the current primary. + auto seedID = BsonObjectID.generate; + coll.insertOne(Bson(["_id": Bson(seedID), "seq": Bson(0)])); + + // Force the current primary to step down for 60s. The command closes our + // connection to it, so the error it returns is expected and ignored. + logInfo("Forcing the current primary to step down..."); + try + client.getDatabase("admin").runCommand( + Bson(["replSetStepDown": Bson(60), "force": Bson(true)])); + catch (Exception e) + logInfo("Step-down command returned (expected): %s", e.msg); + + // Immediately issue a write. The first attempt hits the stepped-down node and + // fails with NotWritablePrimary; the driver refreshes topology, waits for the + // new primary, and retries the insert. A fixed _id makes a double-apply fail + // loudly with a duplicate-key error, so success proves exactly-once delivery. + auto retriedID = BsonObjectID.generate; + coll.insertOne(Bson(["_id": Bson(retriedID), "seq": Bson(1)])); + logInfo("Write after step-down succeeded — retried onto the new primary"); + + // Verify the retried write is present exactly once on the new primary. + auto stored = coll.findOne(["_id": Bson(retriedID)]); + enforce(!stored.isNull, "retried write did not land on the new primary"); + enforce(stored["seq"].get!int == 1, "retried write stored the wrong value"); + + coll.drop(); + logInfo("Primary step-down retry test passed"); + return 0; + } + logInfo("Connection established, running CRUD smoke test"); auto coll = client.getCollection("rstest.smoke"); @@ -117,3 +287,16 @@ int main(string[] args) logInfo("All replica set tests passed"); return 0; } + +MongoClientSettings primarySettings(MongoHost[] hosts, string replicaSet) +{ + auto settings = new MongoClientSettings; + settings.hosts = hosts; + settings.replicaSet = replicaSet; + settings.connectTimeoutMS = 5_000; + settings.socketTimeoutMS = 5_000; + settings.appName = "VibeReplicaSetWriteVerifier"; + settings.readPreference = ReadPreference.primary; + + return settings; +} diff --git a/tests/mongodb/compression-reconnect/dub.json b/tests/mongodb/compression-reconnect/dub.json new file mode 100644 index 0000000000..4f914275ab --- /dev/null +++ b/tests/mongodb/compression-reconnect/dub.json @@ -0,0 +1,7 @@ +{ + "name": "compression-reconnect-test", + "description": "H7: the reconnect handshake must not be compressed even when compression was negotiated (requires a mongo to proxy)", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + } +} diff --git a/tests/mongodb/compression-reconnect/source/app.d b/tests/mongodb/compression-reconnect/source/app.d new file mode 100644 index 0000000000..c5291eb3b9 --- /dev/null +++ b/tests/mongodb/compression-reconnect/source/app.d @@ -0,0 +1,148 @@ +import vibe.core.core; +import vibe.core.net; +import vibe.core.log; +import vibe.core.stream : IOMode; +import vibe.db.mongo.connection; +import vibe.db.mongo.settings : MongoClientSettings, MongoHost, Compressor; +import vibe.data.bson; +import core.time; +import std.algorithm : canFind; +import std.conv; + +enum int OP_MSG = 2013; +enum int OP_COMPRESSED = 2012; + +// Pump raw bytes from src to dst until either side closes. +void pumpRaw(TCPConnection src, TCPConnection dst) nothrow +{ + try + { + ubyte[4096] buffer; + while (src.connected && dst.connected) + { + auto count = src.read(buffer[], IOMode.once); + if (count == 0) break; + dst.write(buffer[0 .. count]); + } + } + catch (Exception) {} + try dst.close(); + catch (Exception) {} +} + +// Shared, heap-allocated record so the @safe nothrow listener callback can +// mutate it across every accepted client connection. opcodesPerConnection[i] +// holds the opcodes of client->upstream messages of the i-th connection. +final class Recorder +{ + int[][] opcodesPerConnection; +} + +// A transparent TCP proxy to the real mongo that records the opcode of every +// client->upstream message, grouped per client connection. +TCPListener startRecordingProxy(ushort realPort, Recorder recorder) +{ + return listenTCP(0, (TCPConnection client) nothrow @safe { + try + { + auto upstream = connectTCP("127.0.0.1", realPort); + + recorder.opcodesPerConnection ~= (int[]).init; + size_t connectionIndex = recorder.opcodesPerConnection.length - 1; + + runTask(&pumpRaw, upstream, client); + + while (client.connected) + { + ubyte[4] lenBuf; + client.read(lenBuf[], IOMode.all); + int len = lenBuf[0] | (lenBuf[1] << 8) | (lenBuf[2] << 16) | (lenBuf[3] << 24); + + auto msg = new ubyte[len]; + msg[0 .. 4] = lenBuf; + client.read(msg[4 .. $], IOMode.all); + + int opcode = msg[12] | (msg[13] << 8) | (msg[14] << 16) | (msg[15] << 24); + recorder.opcodesPerConnection[connectionIndex] ~= opcode; + + upstream.write(msg); + } + } + catch (Exception) {} + try client.close(); + catch (Exception) {} + }, "127.0.0.1"); +} + +// A reconnect handshake carries the speculative-auth SCRAM payload and MUST be +// sent uncompressed (OP_MSG). The bug: m_negotiatedCompressor is never reset at +// the start of connectToHost, so after a disconnect the reconnect handshake is +// wrongly sent OP_COMPRESSED, which the compression spec forbids. +void runReconnectHandshakeUncompressedTest(ushort realPort) +{ + auto recorder = new Recorder; + auto listener = startRecordingProxy(realPort, recorder); + ushort proxyPort = listener.bindAddress.port; + + auto settings = new MongoClientSettings(); + settings.hosts ~= MongoHost("127.0.0.1", proxyPort); + settings.compressors = [Compressor.zlib]; + + auto conn = new MongoConnection(settings); + conn.connectToHost(MongoHost("127.0.0.1", proxyPort)); // connection 1: handshake (OP_MSG) + conn.runCommand("admin", Bson(["ping": Bson(1)])); // connection 1: a COMPRESSED command (proves zlib negotiated) + conn.disconnect(); // connection 1 closes; m_negotiatedCompressor stays zlib (the bug) + conn.runCommand("admin", Bson(["ping": Bson(1)])); // triggers reconnect -> connection 2: handshake (+ ping) + + // Let the proxy tasks finish recording the in-flight messages. + sleep(200.msecs); + + assert(recorder.opcodesPerConnection.length >= 2, + "the reconnect must open a second proxy connection"); + + // Compression must actually be negotiated for this test to be meaningful. A server + // that advertises no compressors (e.g. a default MongoDB 3.6 mongod) never sends + // OP_COMPRESSED, so the reconnect-handshake-compression bug cannot manifest — skip. + if (!recorder.opcodesPerConnection[0].canFind(OP_COMPRESSED)) + { + logInfo("Server did not negotiate zlib compression; skipping reconnect-compression test"); + return; + } + + assert(recorder.opcodesPerConnection[1][0] == OP_MSG, + "the reconnect handshake must be uncompressed OP_MSG (2013), not OP_COMPRESSED (2012)"); +} + +int main(string[] args) +{ + setLogLevel(LogLevel.diagnostic); + + if (args.length < 2) + { + logError("Usage: %s ", args[0]); + return 1; + } + + ushort realPort = args[1].to!ushort; + + int exitCode = 1; + + runTask(() nothrow { + scope (exit) exitEventLoop(); + + try + { + runReconnectHandshakeUncompressedTest(realPort); + exitCode = 0; + } + catch (Throwable t) + { + try logError("FAILED: %s", t.toString()); + catch (Exception) {} + exitCode = 1; + } + }); + + runEventLoop(); + return exitCode; +} diff --git a/tests/mongodb/connection-quarantine/dub.json b/tests/mongodb/connection-quarantine/dub.json new file mode 100644 index 0000000000..386a700bca --- /dev/null +++ b/tests/mongodb/connection-quarantine/dub.json @@ -0,0 +1,7 @@ +{ + "name": "connection-quarantine-test", + "description": "C2: runCommand must quarantine a connection after a wire desync (requires a mongo to proxy the handshake)", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + } +} diff --git a/tests/mongodb/connection-quarantine/source/app.d b/tests/mongodb/connection-quarantine/source/app.d new file mode 100644 index 0000000000..52954bd84b --- /dev/null +++ b/tests/mongodb/connection-quarantine/source/app.d @@ -0,0 +1,273 @@ +import vibe.core.core; +import vibe.core.net; +import vibe.core.log; +import vibe.core.stream : IOMode; +import vibe.db.mongo.connection; +import vibe.db.mongo.settings : MongoHost; +import vibe.data.bson; +import core.time; +import std.conv; + +// A TCP proxy that forwards a mongo handshake verbatim, but corrupts the +// responseTo field of the SECOND server->client reply so recvMsg desyncs. +// Pump raw bytes from src to dst until either side closes. +void pumpRaw(TCPConnection src, TCPConnection dst) nothrow +{ + try + { + ubyte[4096] buffer; + while (src.connected && dst.connected) + { + auto count = src.read(buffer[], IOMode.once); + if (count == 0) break; + dst.write(buffer[0 .. count]); + } + } + catch (Exception) {} + try dst.close(); + catch (Exception) {} +} + +TCPListener startCorruptingProxy(ushort realPort) +{ + return listenTCP(0, (client) { + try + { + auto upstream = connectTCP("127.0.0.1", realPort); + + // Pump client -> upstream raw (args avoid scoped-closure capture). + runTask(&pumpRaw, client, upstream); + + int messageIndex; + while (upstream.connected) + { + ubyte[4] lenBuf; + upstream.read(lenBuf[], IOMode.all); + int len = lenBuf[0] | (lenBuf[1] << 8) | (lenBuf[2] << 16) | (lenBuf[3] << 24); + + auto msg = new ubyte[len]; + msg[0 .. 4] = lenBuf; + upstream.read(msg[4 .. $], IOMode.all); + + messageIndex++; + if (messageIndex == 2) + msg[8 .. 12] = cast(ubyte[])[0xEF, 0xBE, 0xAD, 0xDE]; + + client.write(msg); + } + } + catch (Exception) {} + client.close(); + }, "127.0.0.1"); +} + +// A TCP proxy that forwards the handshake verbatim but, for the SECOND +// server->client message (the command reply), sends only a truncated prefix +// and closes the client socket so the driver's recv hits EOF mid-message. +TCPListener startTruncatingProxy(ushort realPort) +{ + return listenTCP(0, (client) { + try + { + auto upstream = connectTCP("127.0.0.1", realPort); + + runTask(&pumpRaw, client, upstream); + + int messageIndex; + while (upstream.connected) + { + ubyte[4] lenBuf; + upstream.read(lenBuf[], IOMode.all); + int len = lenBuf[0] | (lenBuf[1] << 8) | (lenBuf[2] << 16) | (lenBuf[3] << 24); + + auto msg = new ubyte[len]; + msg[0 .. 4] = lenBuf; + upstream.read(msg[4 .. $], IOMode.all); + + messageIndex++; + if (messageIndex == 2) + { + // Forward only the first 8 bytes of a header claiming `len`, + // then close: recv for the rest of the header/body gets EOF. + client.write(msg[0 .. 8]); + break; + } + + client.write(msg); + } + } + catch (Exception) {} + client.close(); + }, "127.0.0.1"); +} + +// A truncated reply (server closes the socket mid-message) must quarantine the +// connection: recv throws on the short read, and the C2 fix disconnects. +void runTruncatedReplyTest(ushort realPort) +{ + auto listener = startTruncatingProxy(realPort); + ushort truncProxyPort = listener.bindAddress.port; + + auto conn = new MongoConnection("127.0.0.1", truncProxyPort); + conn.connectToHost(MongoHost("127.0.0.1", truncProxyPort)); + + bool threw; + try + conn.runCommand("admin", Bson(["ping": Bson(1)])); + catch (Exception) + threw = true; + + assert(threw, "a truncated reply makes runCommand throw"); + assert(!conn.connected, + "after a truncated reply (socket closed mid-message), the connection must be quarantined (disconnected)"); +} + +// A clean command-failure (server returns a fully-read reply with ok != 1.0) +// must NOT quarantine the connection: the wire is healthy and the SAME +// connection must remain usable for the next command. +void runCommandFailureKeepsConnectionTest(ushort realPort) +{ + auto conn = new MongoConnection("127.0.0.1", realPort); + conn.connectToHost(MongoHost("127.0.0.1", realPort)); + + bool threw; + try + conn.runCommand("admin", Bson(["thisCommandDoesNotExist": Bson(1)])); + catch (Exception) + threw = true; + + assert(threw, "the server rejects an unknown command"); + assert(conn.connected, + "a clean command-failure (ok != 1.0) must NOT quarantine the connection - the reply was fully read"); + + auto pong = conn.runCommand("admin", Bson(["ping": Bson(1)])); + assert(pong["ok"].get!double == 1.0, + "the same connection is still usable for the next command after a logical command failure"); +} + +// A heap cell holding a "corrupt the command reply exactly once" flag, shared +// across every client connection the proxy accepts (the original poisoned +// connection AND the transparent reconnection). +final class CorruptOnceFlag +{ + bool corruptedOnce; +} + +// A TCP proxy that corrupts the SECOND server->client message of the FIRST +// client connection only. The handshake is always forwarded verbatim; once one +// command reply has been corrupted, every later reply (including the +// reconnection's) is forwarded clean. +TCPListener startReuseProxy(ushort realPort, CorruptOnceFlag flag) +{ + return listenTCP(0, (client) { + try + { + auto upstream = connectTCP("127.0.0.1", realPort); + + runTask(&pumpRaw, client, upstream); + + int messageIndex; + while (upstream.connected) + { + ubyte[4] lenBuf; + upstream.read(lenBuf[], IOMode.all); + int len = lenBuf[0] | (lenBuf[1] << 8) | (lenBuf[2] << 16) | (lenBuf[3] << 24); + + auto msg = new ubyte[len]; + msg[0 .. 4] = lenBuf; + upstream.read(msg[4 .. $], IOMode.all); + + messageIndex++; + if (messageIndex == 2 && !flag.corruptedOnce) + { + msg[8 .. 12] = cast(ubyte[])[0xEF, 0xBE, 0xAD, 0xDE]; + flag.corruptedOnce = true; + } + + client.write(msg); + } + } + catch (Exception) {} + client.close(); + }, "127.0.0.1"); +} + +// A connection quarantined by a wire desync must transparently recover: the +// next command on the SAME MongoConnection triggers ensureConnected() -> +// reconnect (a fresh handshake) and succeeds. The proxy corrupts exactly one +// command reply, so the first command desyncs and the reconnection is clean. +void runReuseAfterPoisonTest(ushort realPort) +{ + auto flag = new CorruptOnceFlag; + auto listener = startReuseProxy(realPort, flag); + ushort reuseProxyPort = listener.bindAddress.port; + + auto conn = new MongoConnection("127.0.0.1", reuseProxyPort); + conn.connectToHost(MongoHost("127.0.0.1", reuseProxyPort)); + + bool threw; + try + conn.runCommand("admin", Bson(["ping": Bson(1)])); + catch (Exception) + threw = true; + + assert(threw, "the corrupted reply throws"); + assert(!conn.connected, "the connection is quarantined after the desync"); + + auto pong = conn.runCommand("admin", Bson(["ping": Bson(1)])); + assert(pong["ok"].get!double == 1.0, + "a quarantined connection transparently reconnects and the next command succeeds"); +} + +int main(string[] args) +{ + setLogLevel(LogLevel.diagnostic); + + if (args.length < 2) + { + logError("Usage: %s ", args[0]); + return 1; + } + + ushort realPort = args[1].to!ushort; + + int exitCode = 1; + + runTask(() nothrow { + scope (exit) exitEventLoop(); + + try + { + runCommandFailureKeepsConnectionTest(realPort); + runTruncatedReplyTest(realPort); + runReuseAfterPoisonTest(realPort); + + auto listener = startCorruptingProxy(realPort); + ushort proxyPort = listener.bindAddress.port; + + auto conn = new MongoConnection("127.0.0.1", proxyPort); + conn.connectToHost(MongoHost("127.0.0.1", proxyPort)); + + bool threw; + try + conn.runCommand("admin", Bson(["ping": Bson(1)])); + catch (Exception) + threw = true; + + assert(threw, "a reply with a corrupted responseTo makes runCommand throw"); + assert(!conn.connected, + "after a wire desync, the connection must be quarantined (disconnected), not left connected for reuse"); + + exitCode = 0; + } + catch (Throwable t) + { + try logError("FAILED: %s", t.toString()); + catch (Exception) {} + exitCode = 1; + } + }); + + runEventLoop(); + return exitCode; +} diff --git a/tests/mongodb/cursor/source/app.d b/tests/mongodb/cursor/source/app.d index fa15e7309a..61da5080de 100644 --- a/tests/mongodb/cursor/source/app.d +++ b/tests/mongodb/cursor/source/app.d @@ -46,7 +46,7 @@ void testCursorEdgeCases(MongoClient client) foreach (i; 0 .. 100) coll.insertOne(["idx": i]); - // Empty result set: find with non-matching filter + // find with non-matching filter returns an empty result set auto emptyCursor = coll.find(["idx": Bson(-999)]); assert(emptyCursor.empty); @@ -57,7 +57,7 @@ void testCursorEdgeCases(MongoClient client) // sort + skip + limit combination auto sorted = coll.find(Bson.emptyObject).sort(["idx": -1]).skip(10).limit(5).array; assert(sorted.length == 5); - // Descending: 99, 98, 97, ... skip 10 -> 89, 88, 87, 86, 85 + // Descending order 99, 98, 97, ... skip 10 -> 89, 88, 87, 86, 85 assert(sorted[0]["idx"].get!int == 89); assert(sorted[4]["idx"].get!int == 85); @@ -65,7 +65,7 @@ void testCursorEdgeCases(MongoClient client) auto single = coll.find(Bson.emptyObject).limit(1).array; assert(single.length == 1); - // Projection via FindOptions: only return specific fields + // Projection via FindOptions returns only specific fields FindOptions projOpts; projOpts.projection = Bson(["idx": Bson(1), "_id": Bson(0)]); auto projected = coll.find(Bson.emptyObject, projOpts).limit(3).array; @@ -75,7 +75,7 @@ void testCursorEdgeCases(MongoClient client) assert(keys.sort!"a only 5 docs remain + // Large skip with limit, skip(95) + limit(10) -> only 5 docs remain auto tail = coll.find(Bson.emptyObject).sort(["idx": 1]).skip(95).limit(10).array; assert(tail.length == 5); assert(tail[0]["idx"].get!int == 95); From 19418940c2583b7722c8b943d7fbf72f3b3013da Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Wed, 17 Jun 2026 16:30:39 +0200 Subject: [PATCH 2/3] feat(mongo): client sessions, multi-document transactions, retryable writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds client session support (logical session ids backed by a server session pool), multi-document transactions (start/commit/abort), and retryable writes — including write retry when the primary steps down — on top of the SDAM core. Stacked on mongo-driver-core-improvements. Closes #2850, #2854, #2855, #2856 --- mongodb/vibe/db/mongo/client.d | 384 +++++- mongodb/vibe/db/mongo/collection.d | 61 +- mongodb/vibe/db/mongo/cursor.d | 29 +- mongodb/vibe/db/mongo/database.d | 85 +- mongodb/vibe/db/mongo/impl/retryablewrites.d | 292 +++++ mongodb/vibe/db/mongo/impl/serversession.d | 1200 ++++++++++++++++++ mongodb/vibe/db/mongo/impl/transaction.d | 558 ++++++++ tests/mongodb/session-timeout/dub.json | 7 + tests/mongodb/session-timeout/source/app.d | 43 + tests/mongodb/transactions/dub.json | 7 + tests/mongodb/transactions/source/app.d | 232 ++++ 11 files changed, 2847 insertions(+), 51 deletions(-) create mode 100644 mongodb/vibe/db/mongo/impl/retryablewrites.d create mode 100644 mongodb/vibe/db/mongo/impl/serversession.d create mode 100644 mongodb/vibe/db/mongo/impl/transaction.d create mode 100644 tests/mongodb/session-timeout/dub.json create mode 100644 tests/mongodb/session-timeout/source/app.d create mode 100644 tests/mongodb/transactions/dub.json create mode 100644 tests/mongodb/transactions/source/app.d diff --git a/mongodb/vibe/db/mongo/client.d b/mongodb/vibe/db/mongo/client.d index c066507cf2..5e80cd4961 100644 --- a/mongodb/vibe/db/mongo/client.d +++ b/mongodb/vibe/db/mongo/client.d @@ -19,6 +19,7 @@ import vibe.db.mongo.settings; import vibe.db.mongo.topology; import vibe.db.mongo.monitor; import vibe.db.mongo.impl.crud; +import vibe.db.mongo.impl.serversession : ServerSession, ServerSessionPool, MongoClientSession, endSessionsCommand; import vibe.db.mongo.impl.wireversion : WireVersion; import vibe.data.bson; @@ -39,7 +40,7 @@ final class MongoClient { // Concurrency contract (HARD): a MongoClient is single-thread / single-event-loop. // It is safe to share across fibers of ONE thread, but it must NOT be shared across - // OS threads. The connection pools, m_topologyChanged + // OS threads. The connection pools, the session pool, m_topologyChanged // (a LocalManualEvent), and the bool flags below are all thread-local and // unsynchronised; only an event loop on the owning thread may touch them. The // AtomicTopology wrapper exists solely to give a consistent intra-thread snapshot @@ -55,6 +56,7 @@ final class MongoClient { bool m_discoveryInProgress; MonitorRegistry m_monitors; + ServerSessionPool m_sessionPool; } package this(string host, ushort port) @@ -139,6 +141,62 @@ final class MongoClient { return m_settings.readPreferenceTags; } + /** Starts an explicit logical session. + + The returned handle carries a logical session id (`lsid`) drawn from the + client's session pool. Call `endSession()` on it when done to return the + underlying server session to the pool for reuse. + */ + MongoClientSession startSession() + { + return MongoClientSession(m_sessionPool.acquire(), &releaseServerSession, &runSessionCommand); + } + + /// Runs a session control command (commitTransaction/abortTransaction) on the primary. + private Bson runSessionCommand(Bson command) @safe + { + return lockConnectionToPrimary().runCommand("admin", command); + } + + /// Checks out a server session for an implicit session on a single operation. + package ServerSession acquireServerSession() + { + return m_sessionPool.acquire(); + } + + /// Returns a server session to the pool once its operation (or explicit session) ends. + package void releaseServerSession(ServerSession session) + { + m_sessionPool.release(session); + } + + /// Whether retryable writes are enabled for this client. + @property bool retryWrites() const + { + return m_settings.retryWrites; + } + + /// Whether the current deployment accepts retryable writes. Standalone + /// servers (topology type `single`) reject `lsid`/`txnNumber` with + /// "Transaction numbers are only allowed on a replica set member or mongos", + /// so retryable writes apply only to replica sets and sharded clusters. + package bool supportsRetryableWrites() + { + return vibe.db.mongo.topology.supportsRetryableWrites(m_topology.load().type); + } + + /// Re-discovers the topology after a primary step-down so the next write + /// finds the newly elected primary. Best-effort: if no primary has been + /// elected yet, the following primary re-lock blocks until one appears, so + /// a failed re-discovery here must not abort the retry. + package void refreshTopology() + { + try + discoverTopology(); + catch (Exception e) + logDiagnostic("Topology refresh after step-down found no primary yet: %s", e.msg); + } + /// Returns the read concern configured for this client. ReadConcern readConcern() const { @@ -446,11 +504,23 @@ final class MongoClient { publishTopology(newTopology); } - /// Publishes a new topology snapshot and notifies waiters. + /// Publishes a new topology snapshot and notifies waiters, then recomputes + /// the session timeout from the snapshot. private void publishTopology(TopologyDescription topology) { m_topology.publish(topology); m_topologyChanged.emit(); + refreshSessionTimeout(); + } + + /// Recomputes the session pool's idle timeout from the topology-advertised + /// logical session timeout (the MIN across data-bearing servers). + private void refreshSessionTimeout() + { + import std.algorithm : map; + import std.array : array; + auto servers = m_topology.load().servers.map!(r => r.description).array; + m_sessionPool.updateTimeout(sessionPoolTimeout(logicalSessionTimeout(servers))); } private void probeAndUpdate(ref TopologyDescription topology, MongoHost host, ref Exception lastException) @@ -515,14 +585,16 @@ final class MongoClient { /// Stops all background server monitors. Call before discarding the client. void stopMonitoring() { + endPooledSessions(); m_monitors.stopAll(); } - /// Tears the client all the way down: stops the background monitors, disconnects - /// the idle pooled connections, and drops every connection pool. Call before - /// discarding a client to release its background tasks and sockets. cleanupConnections - /// needs a live pool, so it runs before the pools are dropped. Connections still - /// checked out by an in-flight operation are closed when that operation returns them. + /// Tears the client all the way down: ends pooled sessions, stops the background + /// monitors, disconnects the idle pooled connections, and drops every connection + /// pool. Call before discarding a client to release its background tasks and + /// sockets. endPooledSessions and cleanupConnections both need a live pool, so they + /// run before the pools are dropped. Connections still checked out by an in-flight + /// operation are closed when that operation returns them. void close() { stopMonitoring(); @@ -530,6 +602,19 @@ final class MongoClient { m_connectionPools = null; } + /// Asks the server, on a best-effort basis, to free this client's pooled logical sessions. + private void endPooledSessions() + { + auto lsids = m_sessionPool.takeAllLsids(); + if (!lsids.length) + return; + + try + getDatabase("admin").runCommandUnchecked(endSessionsCommand(lsids)); + catch (Exception e) + logDiagnostic("endSessions on shutdown failed: %s", e.msg); + } + /// Number of background server monitors currently running. size_t activeMonitorCount() const @property { @@ -550,7 +635,7 @@ final class MongoClient { scope is cleaned up deterministically: the destructor calls `MongoClient.close`, which stops the background monitors (breaking the monitor-task -> client reference cycle that would otherwise keep the client reachable for the lifetime of the - process) and drains the connection pools. + process), ends pooled sessions, and drains the connection pools. The handle is move-only and forwards every `MongoClient` member through `alias this`: --- @@ -569,7 +654,7 @@ struct MongoClientHandle { @disable this(this); - /// Wraps `client`, closing it (stop monitors, drain pools) when the + /// Wraps `client`, closing it (stop monitors, end sessions, drain pools) when the /// handle is destroyed. package this(MongoClient client) { @@ -665,6 +750,287 @@ unittest "a newly-desired host with no pool yet produces nothing to prune"); } +/// Maps a topology-advertised logical session timeout to the session pool's idle window. +/// +/// A null `advertised` means at least one data-bearing server does not advertise a +/// logicalSessionTimeout (e.g. a pre-3.6 member) — the sessions spec treats sessions as +/// unsupported in that case. Rather than silently keeping a previously-known (now stale) +/// timeout, we collapse the window to zero so every pooled session expires immediately and +/// no stale lsid is reused once session support is lost. +Nullable!Duration sessionPoolTimeout(Nullable!Duration advertised) @safe pure nothrow +{ + import core.time : Duration; + return advertised.isNull ? Nullable!Duration(Duration.zero) : advertised; +} + +/// sessionPoolTimeout passes a present timeout through and collapses a null one to zero +unittest +{ + import core.time : minutes, Duration; + + // a server-advertised timeout is used as-is for the idle window + auto present = sessionPoolTimeout(Nullable!Duration(30.minutes)); + assert(!present.isNull && present.get == 30.minutes, + "an advertised session timeout is passed through unchanged"); + + // no advertised timeout (sessions unsupported) collapses the window so pooled sessions expire at once + auto absent = sessionPoolTimeout(Nullable!Duration.init); + assert(!absent.isNull && absent.get == Duration.zero, + "a null advertised timeout marks sessions unsupported by expiring pooled sessions immediately"); +} + +/// Whether a failed op may be retried: idempotent reads and/or session-supported writes. +struct RetryPolicy +{ + bool idempotent; + bool sessionSupport; +} + +/// Whether a failed op may be retried once: a raw network failure (on an +/// idempotent read or a session-supported write), a step-down/stale-topology +/// error, or a retryable-write error. +bool isRetryableError(MongoDriverException e, RetryPolicy policy) @safe +{ + bool networkRetryable = (cast(MongoNetworkException) e !is null) && (policy.idempotent || policy.sessionSupport); + return networkRetryable + || shouldRetryAfterStepDown(e.code, policy.idempotent, policy.sessionSupport) + || shouldRetryWrite(e.code, policy.sessionSupport); +} + +/// isRetryableError classifies network, step-down and retryable-write failures +unittest +{ + auto network = new MongoNetworkException("connection reset"); + assert(isRetryableError(network, RetryPolicy(true, false)), "a network failure on an idempotent read is retryable"); + assert(isRetryableError(network, RetryPolicy(false, true)), "a network failure on a session-supported write is retryable"); + assert(!isRetryableError(network, RetryPolicy(false, false)), "a network failure with neither idempotence nor session support is not retryable"); + + auto stepDown = new MongoStepDownException("stepped down", MongoServerErrorCode.notWritablePrimary); + assert(isRetryableError(stepDown, RetryPolicy(true, false)), "an idempotent step-down error is retryable"); + assert(!isRetryableError(stepDown, RetryPolicy(false, false)), "a step-down error without idempotence or session support is not retryable"); + + auto writeError = new MongoDriverException("network timeout"); + writeError.code = MongoServerErrorCode.networkTimeout; + assert(isRetryableError(writeError, RetryPolicy(false, true)), "a retryable-write code on a session-supported write is retryable"); + assert(!isRetryableError(writeError, RetryPolicy(false, false)), "a retryable-write code without session support is not retryable"); + + auto duplicateKey = new MongoDriverException("duplicate key"); + duplicateKey.code = MongoServerErrorCode.duplicateKey; + assert(!isRetryableError(duplicateKey, RetryPolicy(true, true)), "a non-retryable error code is never retried"); +} + +/// Surfaces a retryable writeConcernError as a throw so the write-retry path re-sends the +/// (txnNumber-deduplicated) write. A no-op for a clean reply, a non-retryable code, or a +/// write without session support (which cannot be retried anyway). +void enforceWriteConcernRetry(Bson reply, bool sessionSupport) @safe +{ + auto code = writeConcernErrorCode(reply); + if (!shouldRetryWrite(code, sessionSupport)) + return; + auto e = new MongoDriverException("retryable writeConcernError"); + e.code = code; + throw e; +} + +/// enforceWriteConcernRetry surfaces a retryable writeConcernError so the write is retried +unittest +{ + import std.exception : assertThrown, assertNotThrown; + + auto shutdownReply = Bson([ + "ok": Bson(1.0), + "writeConcernError": Bson(["code": Bson(91), "errmsg": Bson("ShutdownInProgress")]) + ]); + + assertThrown!MongoDriverException(enforceWriteConcernRetry(shutdownReply, true), + "a retryable writeConcernError on a session-supported write is surfaced for retry"); + assertNotThrown(enforceWriteConcernRetry(shutdownReply, false), + "without session support the write cannot be retried, so it is not converted to a throw"); + assertNotThrown(enforceWriteConcernRetry(Bson(["ok": Bson(1.0)]), true), + "a clean reply does not throw"); + assertNotThrown(enforceWriteConcernRetry(Bson(["ok": Bson(1.0), + "writeConcernError": Bson(["code": Bson(11000)])]), true), + "a non-retryable writeConcernError code is not retried"); +} + +/// retries the op once, after refreshing topology, when the first call fails with a +/// retryable error, meaning a raw network failure, a step-down/stale-topology error, or a +/// retryable-write error. +T retryOnceOnRetryableError(T)(scope T delegate() @safe op, RetryPolicy policy, scope void delegate() @safe refresh) @safe +{ + try + return op(); + catch (MongoDriverException e) + { + if (!isRetryableError(e, policy)) + throw e; + refresh(); + return op(); + } +} + +/// retries once after refreshing topology when the first call hits a step-down (retryable) error +unittest { + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + if (opCalls == 1) + throw new MongoStepDownException("primary stepped down", MongoServerErrorCode.notWritablePrimary); + return 42; + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + auto result = retryOnceOnRetryableError!int(op, RetryPolicy(true, false), refresh); + + assert(result == 42, "expected the second op call's result 42"); + assert(opCalls == 2, "expected op to be called twice"); + assert(refreshCalls == 1, "expected refresh to be called once"); +} + +/// retries a retryable-write code that is not a stale-topology code when session support is on +unittest { + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + if (opCalls == 1) + throw new MongoStepDownException("network timeout", MongoServerErrorCode.networkTimeout); + return 42; + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + auto result = retryOnceOnRetryableError!int(op, RetryPolicy(false, true), refresh); + + assert(result == 42, "a retryable-write error is retried once and returns the second attempt"); + assert(opCalls == 2, "the write op is retried exactly once"); + assert(refreshCalls == 1, "the retry refreshes the topology"); +} + +/// retries a plain MongoDriverException carrying a retryable-write code when session support is on +unittest { + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + if (opCalls == 1) + { + auto e = new MongoDriverException("network timeout"); + e.code = MongoServerErrorCode.networkTimeout; + throw e; + } + return 7; + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + auto result = retryOnceOnRetryableError!int(op, RetryPolicy(false, true), refresh); + + assert(result == 7, "a code-carrying retryable command error is retried once"); + assert(opCalls == 2, "the op is retried exactly once"); + assert(refreshCalls == 1, "the retry refreshes first"); +} + +/// rethrows without refresh or retry when the op is not retryable +unittest { + import std.exception : assertThrown; + + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + throw new MongoStepDownException("primary stepped down", MongoServerErrorCode.notWritablePrimary); + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + assertThrown!MongoStepDownException(retryOnceOnRetryableError!int(op, RetryPolicy(false, false), refresh)); + + assert(opCalls == 1, "expected op to be called once with no retry"); + assert(refreshCalls == 0, "expected refresh to never be called"); +} + +/// retries at most once so a second step-down propagates instead of looping +unittest { + import std.exception : assertThrown; + + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + throw new MongoStepDownException("primary stepped down again", MongoServerErrorCode.notWritablePrimary); + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + assertThrown!MongoStepDownException(retryOnceOnRetryableError!int(op, RetryPolicy(true, false), refresh)); + + assert(opCalls == 2, "expected exactly one retry, not an infinite loop"); + assert(refreshCalls == 1, "expected topology to be refreshed exactly once"); +} + +/// retries a codeless MongoNetworkException once when session support is on +unittest { + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + if (opCalls == 1) + throw new MongoNetworkException("connection reset"); + return 7; + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + auto result = retryOnceOnRetryableError!int(op, RetryPolicy(false, true), refresh); + + assert(result == 7, "a network failure on a session-supported write is retried once"); + assert(opCalls == 2, "the op is retried exactly once"); + assert(refreshCalls == 1, "the retry refreshes first"); +} + +/// does not retry a network failure on a write with neither session support nor idempotence +unittest { + import std.exception : assertThrown; + + int opCalls = 0; + int refreshCalls = 0; + + int delegate() @safe op = () @safe { + opCalls++; + throw new MongoNetworkException("connection reset"); + }; + + void delegate() @safe refresh = () @safe { + refreshCalls++; + }; + + assertThrown!MongoNetworkException(retryOnceOnRetryableError!int(op, RetryPolicy(false, false), refresh)); + + assert(opCalls == 1, "a network failure without session support or idempotence is not retried"); + assert(refreshCalls == 0, "no refresh when the error is not retried"); +} + /// MongoClientHandle runs its stop action exactly once when it leaves scope. unittest { diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 3d8c545bdf..745f8bfd0c 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -17,6 +17,7 @@ public import vibe.db.mongo.impl.wireversion; import vibe.core.log; import vibe.db.mongo.client; +import vibe.db.mongo.impl.serversession : MongoClientSession; import vibe.db.mongo.impl.commands : splitNamespace, buildDeleteCommand, buildUpdateCommand, buildCountPipeline, buildAggregateCommand; import vibe.db.mongo.settings : ReadPreference; @@ -148,7 +149,7 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/insert/) */ - InsertOneResult insertOne(T)(T document, InsertOneOptions options = InsertOneOptions.init) + InsertOneResult insertOne(T)(T document, InsertOneOptions options = InsertOneOptions.init, MongoClientSession* session = null) { assert(m_client !is null, "Querying uninitialized MongoCollection."); @@ -168,12 +169,12 @@ struct MongoCollection { foreach (string k, v; serializeToBson(options).byKeyValue) cmd[k] = v; - database.runWriteCommandChecked(cmd).handleWriteResult(res); + database.runWriteCommandChecked(cmd, session).handleWriteResult(res); return res; } /// ditto - InsertManyResult insertMany(T)(T[] documents, InsertManyOptions options = InsertManyOptions.init) + InsertManyResult insertMany(T)(T[] documents, InsertManyOptions options = InsertManyOptions.init, MongoClientSession* session = null) { assert(m_client !is null, "Querying uninitialized MongoCollection."); @@ -198,7 +199,7 @@ struct MongoCollection { cmd[k] = v; auto res = InsertManyResult(insertedIds); - database.runWriteCommandChecked(cmd).handleWriteResult!"insertedCount"(res); + database.runWriteCommandChecked(cmd, session).handleWriteResult!"insertedCount"(res); return res; } @@ -210,10 +211,10 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/delete/) */ - DeleteResult deleteOne(T)(T filter, DeleteOptions options = DeleteOptions.init) + DeleteResult deleteOne(T)(T filter, DeleteOptions options = DeleteOptions.init, MongoClientSession* session = null) @trusted { int limit = 1; - return deleteImpl([filter], options, (&limit)[0 .. 1]); + return deleteImpl([filter], options, (&limit)[0 .. 1], session); } /** @@ -224,11 +225,11 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/delete/) */ - DeleteResult deleteMany(T)(T filter, DeleteOptions options = DeleteOptions.init) + DeleteResult deleteMany(T)(T filter, DeleteOptions options = DeleteOptions.init, MongoClientSession* session = null) @safe if (!is(T == DeleteOptions)) { - return deleteImpl([filter], options, null); + return deleteImpl([filter], options, null, session); } /** @@ -239,14 +240,14 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/delete/) */ - DeleteResult deleteAll(DeleteOptions options = DeleteOptions.init) + DeleteResult deleteAll(DeleteOptions options = DeleteOptions.init, MongoClientSession* session = null) @safe { - return deleteImpl([Bson.emptyObject], options, null); + return deleteImpl([Bson.emptyObject], options, null, session); } /// Implementation helper. It's possible to set custom delete limits with /// this method, otherwise it's identical to `deleteOne` and `deleteMany`. - DeleteResult deleteImpl(T)(T[] queries, DeleteOptions options = DeleteOptions.init, scope int[] limits = null) + DeleteResult deleteImpl(T)(T[] queries, DeleteOptions options = DeleteOptions.init, scope int[] limits = null, MongoClientSession* session = null) @safe { assert(m_client !is null, "Querying uninitialized MongoCollection."); @@ -260,7 +261,7 @@ struct MongoCollection { Bson cmd = buildDeleteCommand(m_name, queryBsons, serializeToBson(options), limits); DeleteResult res; - database.runWriteCommandChecked(cmd).handleWriteResult!"deletedCount"(res); + database.runWriteCommandChecked(cmd, session).handleWriteResult!"deletedCount"(res); return res; } @@ -277,22 +278,22 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) */ - UpdateResult replaceOne(T, U)(T filter, U replacement, ReplaceOptions options) + UpdateResult replaceOne(T, U)(T filter, U replacement, ReplaceOptions options, MongoClientSession* session = null) @safe { UpdateOptions uoptions; static foreach (f; FieldNameTuple!ReplaceOptions) __traits(getMember, uoptions, f) = __traits(getMember, options, f); Bson opts = Bson.emptyObject; opts["multi"] = Bson(false); - return updateImpl([filter], [replacement], [opts], uoptions, true, false); + return updateImpl([filter], [replacement], [opts], uoptions, true, false, session); } /// ditto - UpdateResult replaceOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + UpdateResult replaceOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init, MongoClientSession* session = null) @safe { Bson opts = Bson.emptyObject; opts["multi"] = Bson(false); - return updateImpl([filter], [replacement], [opts], options, true, false); + return updateImpl([filter], [replacement], [opts], options, true, false, session); } /// @@ -325,11 +326,11 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) */ - UpdateResult updateOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + UpdateResult updateOne(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init, MongoClientSession* session = null) @safe { Bson opts = Bson.emptyObject; opts["multi"] = Bson(false); - return updateImpl([filter], [replacement], [opts], options, false, true); + return updateImpl([filter], [replacement], [opts], options, false, true, session); } /** @@ -339,18 +340,18 @@ struct MongoCollection { Standards: $(LINK https://www.mongodb.com/docs/manual/reference/command/update/) */ - UpdateResult updateMany(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init) + UpdateResult updateMany(T, U)(T filter, U replacement, UpdateOptions options = UpdateOptions.init, MongoClientSession* session = null) @safe { Bson opts = Bson.emptyObject; opts["multi"] = Bson(true); - return updateImpl([filter], [replacement], [opts], options, false, true); + return updateImpl([filter], [replacement], [opts], options, false, true, session); } /// Implementation helper. It's possible to set custom per-update object /// options with this method, otherwise it's identical to `replaceOne`, /// `updateOne` and `updateMany`. UpdateResult updateImpl(T, U, O)(T[] queries, U[] documents, O[] perUpdateOptions, UpdateOptions options = UpdateOptions.init, - bool mustBeDocument = false, bool mustBeModification = false) + bool mustBeDocument = false, bool mustBeModification = false, MongoClientSession* session = null) @safe in(queries.length == documents.length && documents.length == perUpdateOptions.length, "queries, documents and perUpdateOptions must have same length") @@ -409,7 +410,7 @@ struct MongoCollection { Bson cmd = buildUpdateCommand(m_name, queryBsons, documentBsons, perUpdateOptionBsons, serializeToBson(options)); - auto res = database.runWriteCommandChecked(cmd); + auto res = database.runWriteCommandChecked(cmd, session); auto ret = UpdateResult( res["n"].to!long, res["nModified"].to!long, @@ -487,10 +488,10 @@ struct MongoCollection { - $(LINK http://www.mongodb.org/display/DOCS/Querying) - $(LREF findOne) */ - MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options = FindOptions.init) + MongoCursor!R find(R = Bson, Q)(Q query, FindOptions options = FindOptions.init, MongoClientSession* session = null) { applyDefaultReadConcern(options); - return MongoCursor!R(m_client, m_db.name, m_name, query, options); + return MongoCursor!R(m_client, m_db.name, m_name, query, options, session); } /// @@ -600,13 +601,13 @@ struct MongoCollection { - $(LINK http://www.mongodb.org/display/DOCS/Querying) - $(LREF find) */ - auto findOne(R = Bson, T)(T query, FindOptions options = FindOptions.init) + auto findOne(R = Bson, T)(T query, FindOptions options = FindOptions.init, MongoClientSession* session = null) { import std.traits; import std.typecons; options.limit = 1; - auto c = find!R(query, options); + auto c = find!R(query, options, session); static if (is(R == Bson)) { foreach (doc; c) return doc; return Bson(null); @@ -656,7 +657,7 @@ struct MongoCollection { See_Also: $(LINK http://docs.mongodb.org/manual/reference/command/findAndModify) */ - Bson findAndModify(T, U, V)(T query, U update, V returnFieldSelector) + Bson findAndModify(T, U, V)(T query, U update, V returnFieldSelector, MongoClientSession* session = null) { static struct CMD { string findAndModify; @@ -669,7 +670,7 @@ struct MongoCollection { cmd.query = query; cmd.update = update; cmd.fields = returnFieldSelector; - auto ret = database.runWriteCommandChecked(cmd); + auto ret = database.runWriteCommandChecked(cmd, session); return ret["value"]; } @@ -694,7 +695,7 @@ struct MongoCollection { See_Also: $(LINK http://docs.mongodb.org/manual/reference/command/findAndModify) */ - Bson findAndModifyExt(T, U, V)(T query, U update, V options) + Bson findAndModifyExt(T, U, V)(T query, U update, V options, MongoClientSession* session = null) { auto bopt = serializeToBson(options); assert(bopt.type == Bson.Type.object, @@ -708,7 +709,7 @@ struct MongoCollection { cmd[key] = value; return 0; }); - auto ret = database.runWriteCommandChecked(cmd); + auto ret = database.runWriteCommandChecked(cmd, session); return ret["value"]; } diff --git a/mongodb/vibe/db/mongo/cursor.d b/mongodb/vibe/db/mongo/cursor.d index 5aafd702d0..0a12d82c84 100644 --- a/mongodb/vibe/db/mongo/cursor.d +++ b/mongodb/vibe/db/mongo/cursor.d @@ -16,6 +16,7 @@ import vibe.db.mongo.connection; import vibe.db.mongo.client; import vibe.db.mongo.impl.commands : buildFindCommand, collectionFromNamespace, reduceLimit; import vibe.db.mongo.settings : ReadPreference, MongoHost; +import vibe.db.mongo.impl.serversession : MongoClientSession, inActiveTransaction; import core.time; import std.array : array; @@ -49,7 +50,7 @@ struct MongoCursor(DocType = Bson) { m_data = new MongoGenericCursor!DocType(client, collection, cursor, existing_documents); } - this(Q)(MongoClient client, string database, string collection, Q query, FindOptions options) + this(Q)(MongoClient client, string database, string collection, Q query, FindOptions options, MongoClientSession* session = null) { Bson command = Bson.emptyObject; command["find"] = Bson(collection); @@ -65,14 +66,14 @@ struct MongoCursor(DocType = Bson) { auto pref = options.readPreference.isNull ? client.readPreference : options.readPreference.get; auto result = buildFindCommand(command, options, pref, client.readPreferenceTags); - this(client, result.command, result.batchSize, result.getMoreMaxTime, Nullable!ReadPreference(pref)); + this(client, result.command, result.batchSize, result.getMoreMaxTime, Nullable!ReadPreference(pref), session); } this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max, - Nullable!ReadPreference pref = Nullable!ReadPreference.init) + Nullable!ReadPreference pref = Nullable!ReadPreference.init, MongoClientSession* session = null) { // TODO: avoid memory allocation, if possible - m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTime, pref); + m_data = new MongoFindCursor!DocType(client, command, batchSize, getMoreMaxTime, pref, session); } this(this) @@ -414,10 +415,11 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { long m_queryLimit; ReadPreference m_readPreference; MongoHost m_pinnedHost; + MongoClientSession* m_session; } this(MongoClient client, Bson command, int batchSize = 0, Duration getMoreMaxTime = Duration.max, - Nullable!ReadPreference pref = Nullable!ReadPreference.init) + Nullable!ReadPreference pref = Nullable!ReadPreference.init, MongoClientSession* session = null) { m_client = client; m_findQuery = command; @@ -425,6 +427,7 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { m_maxTime = getMoreMaxTime; m_database = command["$db"].opt!string; m_readPreference = pref.isNull ? client.readPreference : pref.get; + m_session = session; } @property bool alive() @safe nothrow { return m_cursor != 0; } @@ -441,9 +444,13 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { if( m_cursor == 0 ) return true; + Bson sessionContext = m_session is null + ? Bson.emptyObject + : m_session.transactionContext(); + auto conn = m_client.lockConnectionToHost(m_pinnedHost); conn.getMore!DocType(m_cursor, m_database, m_collection, m_batchSize, - &handleReply, &handleDocument, m_maxTime, Nullable!ReadPreference(m_readPreference)); + &handleReply, &handleDocument, m_maxTime, Nullable!ReadPreference(m_readPreference), sessionContext); return m_readDoc >= m_documents.length; } @@ -488,8 +495,14 @@ private class MongoFindCursor(DocType) : IMongoCursorData!DocType { private void startIterating() @safe { // A cursor id is only valid on the server that created it, so pin one host - // and reuse it for getMore/killCursors. - m_pinnedHost = m_client.resolveHostForRead(m_readPreference); + // and reuse it for getMore/killCursors. Transaction reads must hit the + // primary and carry the session so they see their own uncommitted writes. + const inTransaction = inActiveTransaction(m_session); + if (inTransaction) + m_findQuery = m_session.applyToCommand(m_findQuery); + m_pinnedHost = inTransaction + ? m_client.resolveHostForRead(ReadPreference.primary) + : m_client.resolveHostForRead(m_readPreference); auto conn = m_client.lockConnectionToHost(m_pinnedHost); m_totalReceived = 0; m_queryLimit = m_findQuery["limit"].opt!long(0); diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index fc23eb9408..4abe54a784 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -13,6 +13,9 @@ module vibe.db.mongo.database; import vibe.db.mongo.client; import vibe.db.mongo.collection; import vibe.db.mongo.settings : ReadConcern, ReadPreference, readPreferenceBson; +import vibe.db.mongo.impl.retryablewrites : isRetryableWriteCommand, applyRetryableWrite; +import vibe.db.mongo.impl.serversession : ServerSession, MongoClientSession, inActiveTransaction; +import vibe.db.mongo.connection : MongoNetworkException; import vibe.data.bson; import core.time; @@ -188,14 +191,16 @@ struct MongoDatabase /// ditto, but always sends to the primary (for write operations). Bson runWriteCommandChecked(T, ExceptionT = MongoDriverException)( T command_and_options, + MongoClientSession* session = null, string errorInfo = __FUNCTION__, string errorFile = __FILE__, size_t errorLine = __LINE__ ) { Bson cmd = toCommandBson(command_and_options); - return m_client.lockConnectionToPrimary().runCommand!ExceptionT( - m_name, cmd, errorInfo, errorFile, errorLine); + if (inActiveTransaction(session)) + return runSessionWrite!ExceptionT(cmd, *session, errorInfo, errorFile, errorLine, true); + return runWriteWithRetry!ExceptionT(cmd, errorInfo, errorFile, errorLine, true); } /// ditto @@ -223,8 +228,75 @@ struct MongoDatabase ) { Bson cmd = toCommandBson(command_and_options); - return m_client.lockConnectionToPrimary().runCommandUnchecked!ExceptionT( - m_name, cmd, errorInfo, errorFile, errorLine); + return runWriteWithRetry!ExceptionT(cmd, errorInfo, errorFile, errorLine, false); + } + + /// Runs a write that belongs to an explicit session's active transaction: stamps the + /// session's transaction context onto the command and sends it to the primary once, + /// bypassing the implicit retryable-write path. + private Bson runSessionWrite(ExceptionT)( + Bson cmd, ref MongoClientSession session, string errorInfo, string errorFile, size_t errorLine, bool checked) + { + Bson prepared = session.applyToCommand(cmd); + auto conn = m_client.lockConnectionToPrimary(); + return checked + ? conn.runCommand!ExceptionT(m_name, prepared, errorInfo, errorFile, errorLine) + : conn.runCommandUnchecked!ExceptionT(m_name, prepared, errorInfo, errorFile, errorLine); + } + + /// Runs a write command on the primary, retrying once after a primary + /// step-down. Retryable writes (per `isRetryableWriteCommand`, when + /// `retryWrites` is enabled) carry an `lsid`/`txnNumber` so the server + /// deduplicates the retried write; the retry re-discovers the topology and + /// re-locks the freshly elected primary before resending the same command. + private Bson runWriteWithRetry(ExceptionT)( + Bson cmd, string errorInfo, string errorFile, size_t errorLine, bool checked) + { + return withImplicitSession!Bson(cmd, (preparedCmd, sessionSupport) @safe { + return retryOnceOnRetryableError!Bson( + () @safe { + auto conn = m_client.lockConnectionToPrimary(); + auto reply = checked + ? conn.runCommand!ExceptionT(m_name, preparedCmd, errorInfo, errorFile, errorLine) + : conn.runCommandUnchecked!ExceptionT(m_name, preparedCmd, errorInfo, errorFile, errorLine); + // An ok:1 reply can still carry a transient writeConcernError; surface a + // retryable one as a throw so the retry path re-sends the deduplicated write. + enforceWriteConcernRetry(reply, sessionSupport); + return reply; + }, + RetryPolicy(false, sessionSupport), + () @safe { m_client.refreshTopology(); }); + }); + } + + /// Runs `body` with an implicit server session attached when `cmd` is a retryable write: + /// acquires a session, stamps the retryable-write fields onto the command, and releases the + /// session on exit. `body` receives the (possibly stamped) command and whether session support is active. + private T withImplicitSession(T)(Bson cmd, scope T delegate(Bson preparedCmd, bool sessionSupport) @safe body) + { + const retryable = m_client.retryWrites && m_client.supportsRetryableWrites() + && isRetryableWriteCommand(cmd); + ServerSession session; + if (retryable) + { + session = m_client.acquireServerSession(); + cmd = applyRetryableWrite(cmd, session.lsid, session.nextTransactionNumber()); + } + scope (exit) + if (retryable) + m_client.releaseServerSession(session); + + try + return body(cmd, retryable); + catch (MongoNetworkException e) + { + // A network error tainted the session (the txnNumber may have reached the + // server); mark it dirty so release discards it rather than recycling its + // lsid for the next operation. Per the Driver Sessions spec. + if (retryable) + session.markDirty(); + throw e; + } } /// ditto @@ -253,6 +325,11 @@ struct MongoDatabase } /// Writes lock the primary; reads lock by effective preference and inject `$readPreference`. + // TODO(causal-consistency): explicit sessions and retryable writes already carry an lsid, + // and multi-document transactions are fully wired (cursors pin the session across getMore). + // What remains is IMPLICIT sessions: attaching an lsid (applySession from impl.serversession) + // to EVERY command automatically — reads and non-retryable writes alike — for causal + // consistency, checking one out of the pool and returning it after. private auto resolveCommandConnection(bool toPrimary, ref Bson cmd, Nullable!ReadPreference readPreference) { if (toPrimary) diff --git a/mongodb/vibe/db/mongo/impl/retryablewrites.d b/mongodb/vibe/db/mongo/impl/retryablewrites.d new file mode 100644 index 0000000000..7473603966 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/retryablewrites.d @@ -0,0 +1,292 @@ +/** + Retryable write command classification (Node-driver semantics). + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.retryablewrites; + +import vibe.data.bson; +import vibe.db.mongo.impl.serversession : applySession; + +@safe: + +/// The command name is the first field of the command document, by MongoDB decree. +string commandName(Bson command) +{ + foreach (string key, value; command.byKeyValue) + return key; + return null; +} + +/// commandName returns the first field of the command document +unittest { + Bson cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + + assert(commandName(cmd) == "insert", + "command name must be the first field of the document"); +} + +/// Whether any entry in a Bson `statements` array satisfies `disqualifies`. +/// A non-array (or missing field) yields false: nothing to disqualify. +bool anyStatement(Bson statements, scope bool delegate(Bson) @safe disqualifies) +{ + import std.algorithm : any; + + if (statements.type != Bson.Type.array) + return false; + + return statements.byValue.any!disqualifies; +} + +/// anyStatement matches an entry, ignores non-array input +unittest { + Bson marked = Bson.emptyObject; + marked["flag"] = Bson(true); + + bool delegate(Bson) @safe matchesFlag = + (Bson entry) @safe => entry["flag"].type == Bson.Type.bool_; + + assert(anyStatement(Bson([marked]), matchesFlag) == true, + "a matching entry must be detected"); + assert(anyStatement(Bson([Bson.emptyObject]), matchesFlag) == false, + "a non-matching entry must not be flagged"); + assert(anyStatement(Bson("not-an-array"), matchesFlag) == false, + "a non-array must yield false"); +} + +/// Whether a Bson value coerces to a non-zero number — `true`, `1`, `1L`, `1.0`. A +/// user-built command may express boolean options numerically, so coerce rather than +/// require an exact bool type. +private bool isTruthyNumber(Bson b) @safe +{ + switch (b.type) + { + case Bson.Type.bool_: return b.get!bool; + case Bson.Type.int_: return b.get!int != 0; + case Bson.Type.long_: return b.get!long != 0; + case Bson.Type.double_: return b.get!double != 0; + default: return false; + } +} + +/// Whether a Bson value coerces to numeric zero — `0`, `0L`, `0.0`, `false`. +private bool isZeroNumber(Bson b) @safe +{ + switch (b.type) + { + case Bson.Type.bool_: return !b.get!bool; + case Bson.Type.int_: return b.get!int == 0; + case Bson.Type.long_: return b.get!long == 0; + case Bson.Type.double_: return b.get!double == 0; + default: return false; + } +} + +/// Whether any statement in an `updates` array is a multi-document update. +bool hasMultiStatement(Bson updates) +{ + return anyStatement(updates, entry => isTruthyNumber(entry["multi"])); +} + +/// hasMultiStatement flags an updates array containing a multi:true entry +unittest { + Bson upd = Bson.emptyObject; + upd["multi"] = Bson(true); + + assert(hasMultiStatement(Bson([upd])) == true, + "a multi:true update statement must be detected"); + assert(hasMultiStatement(Bson([Bson.emptyObject])) == false, + "an update statement without multi:true must not be flagged"); +} + +/// hasMultiStatement detects multi expressed as a number (multi: 1), not only bool true +unittest { + Bson upd = Bson.emptyObject; + upd["multi"] = Bson(1); // numeric, as a user-built command may carry it + + assert(hasMultiStatement(Bson([upd])) == true, + "multi:1 (numeric) is a multi-update and must not be classified retryable"); +} + +/// Whether any statement in a `deletes` array is a multi-document delete (limit:0). +bool hasUnlimitedDelete(Bson deletes) +{ + return anyStatement(deletes, entry => isZeroNumber(entry["limit"])); +} + +/// hasUnlimitedDelete detects limit:0 expressed as int64 or double, not only int32 +unittest { + Bson delLong = Bson.emptyObject; + delLong["limit"] = Bson(0L); + assert(hasUnlimitedDelete(Bson([delLong])) == true, "limit:0 (int64) is an unlimited delete"); + + Bson delDouble = Bson.emptyObject; + delDouble["limit"] = Bson(0.0); + assert(hasUnlimitedDelete(Bson([delDouble])) == true, "limit:0.0 (double) is an unlimited delete"); + + Bson delOne = Bson.emptyObject; + delOne["limit"] = Bson(1); + assert(hasUnlimitedDelete(Bson([delOne])) == false, "limit:1 is a single delete (retryable)"); +} + +/// Whether any op in a client `bulkWrite` `ops` array is a multi-document write +/// (`multi:true`), which makes the whole bulkWrite ineligible for retry. +/// `updateMany`/`deleteMany` ops carry `multi:true`; inserts have no `multi` field. +bool hasMultiOp(Bson ops) +{ + return anyStatement(ops, entry => isTruthyNumber(entry["multi"])); +} + +/// hasMultiOp flags an ops array containing a multi:true write +unittest { + Bson multiUpdate = Bson.emptyObject; + multiUpdate["update"] = Bson(0); + multiUpdate["multi"] = Bson(true); + + assert(hasMultiOp(Bson([multiUpdate])) == true, + "a multi:true op must be detected"); + + Bson singleInsert = Bson.emptyObject; + singleInsert["insert"] = Bson(0); + + assert(hasMultiOp(Bson([singleInsert])) == false, + "an insert op carries no multi field and must not be flagged"); +} + +/// classifies a command as a retryable write +bool isRetryableWriteCommand(Bson command) +{ + import std.algorithm : among; + + string name = commandName(command); + + if (!name.among("insert", "update", "delete", "findAndModify", "bulkWrite")) + return false; + + if (name == "update" && hasMultiStatement(command["updates"])) + return false; + + // A client bulkWrite is retryable only when none of its ops is a multi-document + // write, mirroring the multi/limit gating of the per-collection write commands. + if (name == "bulkWrite" && hasMultiOp(command["ops"])) + return false; + + return !(name == "delete" && hasUnlimitedDelete(command["deletes"])); +} + +/// an insert command is a retryable write +unittest { + Bson cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + + assert(isRetryableWriteCommand(cmd) == true, + "insert must be classified as a retryable write command"); +} + +/// an empty command document is not a retryable write +unittest { + assert(isRetryableWriteCommand(Bson.emptyObject) == false, + "a command with no name must not be classified as a retryable write"); +} + +/// an update command is a retryable write +unittest { + Bson cmd = Bson.emptyObject; + cmd["update"] = Bson("people"); + + assert(isRetryableWriteCommand(cmd) == true, + "update must be classified as a retryable write command"); +} + +/// a find command is not a retryable write +unittest { + Bson cmd = Bson.emptyObject; + cmd["find"] = Bson("people"); + + assert(isRetryableWriteCommand(cmd) == false, + "find must not be classified as a retryable write command"); +} + +/// a multi:true update command is not a retryable write +unittest { + Bson upd = Bson.emptyObject; + upd["q"] = Bson.emptyObject; + upd["u"] = Bson.emptyObject; + upd["multi"] = Bson(true); + + Bson cmd = Bson.emptyObject; + cmd["update"] = Bson("people"); + cmd["updates"] = Bson([upd]); + + assert(isRetryableWriteCommand(cmd) == false, + "multi:true update must not be classified as a retryable write command"); +} + +/// a limit:0 delete command (deleteMany) is not a retryable write +unittest { + Bson del = Bson.emptyObject; + del["q"] = Bson.emptyObject; + del["limit"] = Bson(0); + + Bson cmd = Bson.emptyObject; + cmd["delete"] = Bson("people"); + cmd["deletes"] = Bson([del]); + + assert(isRetryableWriteCommand(cmd) == false, + "limit:0 delete must not be classified as a retryable write command"); +} + +/// a single-document bulkWrite command is a retryable write +unittest { + Bson op = Bson.emptyObject; + op["insert"] = Bson(0); + op["document"] = Bson(["_id": Bson(1)]); + + Bson cmd = Bson.emptyObject; + cmd["bulkWrite"] = Bson(1); + cmd["ops"] = Bson([op]); + + assert(isRetryableWriteCommand(cmd) == true, + "a bulkWrite with only single-document ops must be classified as a retryable write command"); +} + +/// a bulkWrite containing a multi:true op is not a retryable write +unittest { + Bson op = Bson.emptyObject; + op["update"] = Bson(0); + op["multi"] = Bson(true); + + Bson cmd = Bson.emptyObject; + cmd["bulkWrite"] = Bson(1); + cmd["ops"] = Bson([op]); + + assert(isRetryableWriteCommand(cmd) == false, + "a bulkWrite with a multi-document op must not be classified as a retryable write command"); +} + +/// Stamps a write command with the session id and retryable txnNumber. +Bson applyRetryableWrite(Bson command, Bson lsid, long txnNumber) +{ + Bson result = applySession(command, lsid); + result["txnNumber"] = Bson(txnNumber); + return result; +} + +/// applyRetryableWrite sets lsid and txnNumber on the command +unittest { + Bson cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + + Bson lsid = Bson.emptyObject; + lsid["id"] = Bson("session-uuid-bytes"); + + auto outCmd = applyRetryableWrite(cmd, lsid, 7); + + assert(outCmd["lsid"] == lsid, + "the lsid document must be attached to the command"); + assert(outCmd["txnNumber"].get!long == 7, + "the txnNumber must be attached to the command as a long"); +} diff --git a/mongodb/vibe/db/mongo/impl/serversession.d b/mongodb/vibe/db/mongo/impl/serversession.d new file mode 100644 index 0000000000..08cea48ab4 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/serversession.d @@ -0,0 +1,1200 @@ +/** + Logical sessions: id generation, the server-session pool, and the + client-session handle that drives multi-document transactions. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.serversession; + +import vibe.data.bson; +import vibe.db.mongo.impl.transaction : Transaction, TransactionState, withTransactionRetry, applyTransaction, commitTransactionCommand, abortTransactionCommand; +import core.time : MonoTime, Duration, minutes; +import std.algorithm : map; +import std.algorithm.mutation : remove, SwapStrategy; +import std.array : array; +import std.typecons : Nullable; + +@safe: + +/// MongoDB's default deadline for the whole transaction-with-retry loop (120 seconds). +enum defaultTransactionTimeout = 2.minutes; + +/// MongoDB's safety margin: treat a session as about to expire one minute before its timeout. +enum sessionSafetyMargin = 1.minutes; + +/// Builds a fresh logical session id document `{ id: }`. +Bson logicalSessionId() +{ + import std.uuid : randomUUID; + auto bytes = randomUUID().data; + return Bson(["id": Bson(BsonBinData(BsonBinData.Type.uuid, bytes.idup))]); +} + +/// A fresh logical session id is `{ id: }`. +unittest +{ + auto lsid = logicalSessionId(); + + assert(lsid["id"].type == Bson.Type.binData, + "the lsid id field must be binary data"); + assert(lsid["id"].get!BsonBinData.type == BsonBinData.Type.uuid, + "the lsid id field must use UUID binary subtype 0x04"); + assert(lsid["id"].get!BsonBinData.rawData.length == 16, + "the lsid UUID must be 16 bytes"); +} + +/// Each fresh logical session id is unique. +unittest +{ + auto a = logicalSessionId(); + auto b = logicalSessionId(); + + auto ra = a["id"].get!BsonBinData.rawData; + auto rb = b["id"].get!BsonBinData.rawData; + + assert(ra != rb, "each logical session id must be unique"); +} + +/// Builds the `endSessions` admin command that frees the given logical sessions on the server. +Bson endSessionsCommand(Bson[] lsids) @safe +{ + return Bson(["endSessions": Bson(lsids)]); +} + +/// The endSessions command lists the given lsids under `endSessions`. +unittest +{ + auto a = logicalSessionId(); + auto b = logicalSessionId(); + + auto cmd = endSessionsCommand([a, b]); + + assert(cmd["endSessions"] == Bson([a, b]), + "endSessionsCommand must list the given lsids under endSessions"); +} + +/// The endSessions command for an empty list yields an empty array. +unittest +{ + auto cmd = endSessionsCommand([]); + + assert(cmd["endSessions"] == Bson(cast(Bson[])[]), + "endSessionsCommand of an empty list is { endSessions: [] }"); +} + +/// Returns the command with the logical session id attached. +Bson applySession(Bson command, Bson lsid) @safe +{ + Bson result = command; + result["lsid"] = lsid; + return result; +} + +/// Applying a session attaches the given lsid to the command. +unittest +{ + Bson cmd = Bson.emptyObject; + cmd["find"] = Bson("people"); + auto lsid = logicalSessionId(); + + auto result = applySession(cmd, lsid); + + assert(result["lsid"] == lsid, + "applySession must attach the given lsid to the command"); + assert(result["find"] == Bson("people"), + "applySession must preserve the original command"); + assert(cmd["lsid"].type == Bson.Type.null_, + "applySession must not mutate the caller's command"); +} + +/// Tracks per-session state such as the monotonic transaction number. +struct ServerSession +{ + private long m_txnNumber; + private Bson m_lsid; + private MonoTime m_lastUse; + private bool m_dirty; + + /// Returns the next monotonic transaction number, starting at 1. + // TODO(sessions): causal consistency builds on this session. Add an + // operationTime/afterClusterTime accessor for causally-consistent reads. + long nextTransactionNumber() @safe { return ++m_txnNumber; } + + /// Builds a session carrying its own fresh logical session id. + static ServerSession create() @safe { ServerSession s; s.m_lsid = logicalSessionId(); return s; } + + /// The logical session id document for this session. + Bson lsid() @safe const { return m_lsid; } + + /// Records the time the session was last used. + void touch(MonoTime now) @safe { m_lastUse = now; } + + /// True when the session is within MongoDB's safety margin of `timeout`. + bool isAboutToExpire(MonoTime now, Duration timeout) @safe const + { + return now - m_lastUse >= timeout - sessionSafetyMargin; + } + + /// Marks the session dirty: a network error occurred while using it, so its + /// server-side state is unknown and it must NOT be returned to the pool. + void markDirty() @safe { m_dirty = true; } + + /// Whether the session has been marked dirty (must be discarded, not pooled). + bool isDirty() @safe const { return m_dirty; } +} + +/// touch records last-use so the session is fresh just after. +unittest +{ + import core.time : MonoTime, minutes; + + auto session = ServerSession.create(); + auto t0 = MonoTime.currTime; + session.touch(t0); + + assert(!session.isAboutToExpire(t0 + 5.minutes, 30.minutes), + "touch records last-use so the session is fresh"); +} + +/// At the safety-margin boundary the session is about to expire; just under it, it is still fresh. +unittest +{ + import core.time : MonoTime, minutes, seconds; + + auto session = ServerSession.create(); + auto t0 = MonoTime.currTime; + session.touch(t0); + + assert(session.isAboutToExpire(t0 + (30.minutes - sessionSafetyMargin), 30.minutes), + "a session idle for timeout minus the safety margin is about to expire"); + assert(!session.isAboutToExpire(t0 + (30.minutes - sessionSafetyMargin) - 1.seconds, 30.minutes), + "just under the safety-margin boundary the session is still fresh"); +} + +/// The first transaction number on a fresh session is 1. +unittest +{ + ServerSession session; + + assert(session.nextTransactionNumber() == 1, + "the first transaction number must be 1"); +} + +/// A session built via `create` carries its own logical session id. +unittest +{ + auto session = ServerSession.create(); + + assert(session.lsid["id"].type == Bson.Type.binData, + "server session carries a logical session id"); +} + +/// a session is not dirty until marked, and markDirty makes it dirty +unittest +{ + auto session = ServerSession.create(); + assert(!session.isDirty(), "a fresh server session is not dirty"); + session.markDirty(); + assert(session.isDirty(), "markDirty marks the session dirty (its server-side state is unknown)"); +} + +/// Hands out server sessions, reusing released ones. +struct ServerSessionPool +{ + private ServerSession[] m_available; + /// Idle-session timeout, seeded from MongoDB's 30-minute default and refreshed + /// via `updateTimeout` from the topology-advertised logicalSessionTimeoutMinutes. + private Duration m_timeout = 30.minutes; + + /// Returns a session ready for use, reusing a released one when available. + ServerSession acquire(MonoTime now = MonoTime.currTime) @safe + { + while (m_available.length) + { + auto reused = m_available[$ - 1]; + m_available = m_available[0 .. $ - 1]; + if (!reused.isAboutToExpire(now, m_timeout)) + { + // Last-use is the time the session is handed out for a command, not the + // time it is later released (the spec defines last-use as command time). + reused.touch(now); + return reused; + } + } + + auto fresh = ServerSession.create(); + fresh.touch(now); + return fresh; + } + + /// Returns a session to the pool for later reuse, preserving its last-use (command) time. + void release(ServerSession session, MonoTime now = MonoTime.currTime) @safe + { + m_available = m_available.remove!(s => s.isAboutToExpire(now, m_timeout), SwapStrategy.unstable); + // A dirty session was tainted by a network error: its server-side state is + // unknown, so discard it rather than recycling its lsid. + if (session.isDirty()) + return; + m_available ~= session; + } + + /// Updates the idle-session timeout from the topology-advertised logical session + /// timeout; a null value (none advertised) leaves the current timeout unchanged. + void updateTimeout(Nullable!Duration timeout) @safe + { + if (!timeout.isNull) + m_timeout = timeout.get; + } + + /// Empties the pool, returning the lsids of every pooled session so they can + /// be ended on the server (the `endSessions` command on client shutdown). + Bson[] takeAllLsids() @safe + { + auto lsids = m_available.map!(s => s.lsid).array; + m_available = null; + return lsids; + } +} + +/// A session acquired from the pool carries a valid logical session id. +unittest +{ + ServerSessionPool pool; + auto session = pool.acquire(); + + assert(session.lsid["id"].type == Bson.Type.binData, + "an acquired session carries a logical session id"); + assert(session.lsid["id"].get!BsonBinData.rawData.length == 16, + "the acquired session lsid UUID must be 16 bytes"); +} + +/// Acquiring after releasing returns the same session. +unittest +{ + ServerSessionPool pool; + auto first = pool.acquire(); + pool.release(first); + auto second = pool.acquire(); + + assert(second.lsid == first.lsid, + "a released session must be reused on the next acquire"); +} + +/// Two sessions held at once are distinct. +unittest +{ + ServerSessionPool pool; + auto first = pool.acquire(); + auto second = pool.acquire(); + + assert(second.lsid != first.lsid, + "the pool must not hand the same live session to two callers"); +} + +/// A pooled session idle past the timeout is discarded on acquire. +unittest +{ + import core.time : MonoTime, minutes; + + ServerSessionPool pool; + auto t0 = MonoTime.currTime; + auto first = pool.acquire(t0); + pool.release(first, t0); + auto later = pool.acquire(t0 + 40.minutes); + + assert(later.lsid != first.lsid, + "an expired pooled session is discarded; a fresh one is returned"); +} + +/// A pooled session re-acquired within the timeout is still reused. +unittest +{ + import core.time : MonoTime, minutes; + + ServerSessionPool pool; + auto t0 = MonoTime.currTime; + auto first = pool.acquire(t0); + pool.release(first, t0); + auto soon = pool.acquire(t0 + 5.minutes); + + assert(soon.lsid == first.lsid, + "a session still within the timeout must be reused, not discarded"); +} + +/// Last-use is the acquire/command time, not the release time: a session held idle past the +/// timeout before release is not pooled as fresh. +unittest +{ + import core.time : MonoTime, minutes; + + ServerSessionPool pool; + auto t0 = MonoTime.currTime; + auto first = pool.acquire(t0); // last use ≈ command time = t0 + pool.release(first, t0 + 40.minutes); // the app held it idle 40m before releasing + auto later = pool.acquire(t0 + 41.minutes); // the server expired the lsid ~t0+30m + + assert(later.lsid != first.lsid, + "a session idle since its last use is discarded, not refreshed to the release time"); +} + +/// updateTimeout shortens the idle window so a once-reusable session expires. +unittest +{ + import core.time : MonoTime, minutes; + import std.typecons : Nullable; + + ServerSessionPool pool; + pool.updateTimeout(Nullable!Duration(10.minutes)); + auto t0 = MonoTime.currTime; + auto first = pool.acquire(t0); + pool.release(first, t0); + auto later = pool.acquire(t0 + 15.minutes); + + assert(later.lsid != first.lsid, + "after updateTimeout(10m) a session idle 15m is discarded, not reused"); +} + +/// Releasing a session prunes pooled sessions already expired at that time. +unittest +{ + import core.time : MonoTime, minutes; + + ServerSessionPool pool; + auto t0 = MonoTime.currTime; + auto a = pool.acquire(); + auto b = pool.acquire(); + auto c = pool.acquire(); + pool.release(a, t0); + pool.release(b, t0); + pool.release(c, t0 + 40.minutes); + + auto lsids = pool.takeAllLsids(); + + assert(lsids.length == 1, + "expired pooled sessions are pruned on release"); + assert(lsids[0] == c.lsid, + "the freshly released session remains after pruning"); +} + +/// Releasing within the timeout keeps still-fresh pooled siblings. +unittest +{ + import core.time : MonoTime, minutes; + + ServerSessionPool pool; + auto t0 = MonoTime.currTime; + auto a = pool.acquire(); + auto b = pool.acquire(); + auto c = pool.acquire(); + pool.release(a, t0); + pool.release(b, t0); + pool.release(c, t0 + 5.minutes); + + assert(pool.takeAllLsids().length == 3, + "sessions still within the timeout must not be pruned"); +} + +/// release discards a dirty session instead of returning it to the pool +unittest +{ + ServerSessionPool pool; + auto clean = pool.acquire(); + auto dirty = pool.acquire(); + dirty.markDirty(); + + pool.release(clean); + pool.release(dirty); + + auto lsids = pool.takeAllLsids(); + assert(lsids.length == 1, "a dirty session is not returned to the pool"); + assert(lsids[0] == clean.lsid, "only the clean session remains poolable"); +} + +/// takeAllLsids drains the pool and returns each pooled session's lsid. +unittest +{ + ServerSessionPool pool; + auto a = pool.acquire(); + auto b = pool.acquire(); + pool.release(a); + pool.release(b); + + auto lsids = pool.takeAllLsids(); + + assert(lsids.length == 2, + "takeAllLsids returns one lsid per pooled session"); + assert(pool.acquire().lsid != a.lsid, + "the pool is empty after draining, so acquire mints a fresh session"); +} + +/// A client-facing handle to a logical session, holding a checked-out server session. +struct MongoClientSession +{ + private ServerSession m_session; + private void delegate(ServerSession) @safe m_release; + private Transaction m_transaction; + private long m_txnNumber; + private Bson delegate(Bson) @safe m_runCommand; + + @disable this(this); + + /// Wraps a checked-out server session with the delegate that returns it to its pool. + this(ServerSession session, void delegate(ServerSession) @safe release, Bson delegate(Bson command) @safe runCommand = null) @safe + { + m_session = session; + m_release = release; + m_runCommand = runCommand; + } + + /// Best-effort cleanup when `endSession` was not called: returns the underlying server + /// session to its pool so its lsid is ended on client shutdown rather than leaking. Only + /// the pool return is done here (no network I/O), so it is safe even from the GC finalizer; + /// aborting an in-progress transaction still requires an explicit `endSession()`. + ~this() @safe + { + if (m_release !is null) + { + m_release(m_session); + m_release = null; + } + } + + /// The logical session id document for this session. + Bson lsid() @safe const { return m_session.lsid; } + + /// Aborts any in-progress transaction, then returns the underlying server session to its pool. + void endSession() @safe + { + if (m_transaction.isActive()) + abortTransaction(); + + if (m_release !is null) + { + m_release(m_session); + m_release = null; + } + } + + /// Begins a multi-document transaction on this session. + void startTransaction() @safe + { + m_transaction.start(); + m_txnNumber = m_session.nextTransactionNumber(); + } + + /// Decorates an operation command with this session's transaction context. + private Bson prepareCommand(Bson command) @safe + { + const firstCommand = m_transaction.isFirstCommand(); + m_transaction.markInProgress(); + return applyTransaction(applySession(command, lsid), m_txnNumber, firstCommand); + } + + /// Decorates an outgoing operation command for this session. + Bson applyToCommand(Bson command) @safe + { + if (m_transaction.isActive()) + return prepareCommand(command); + return applySession(command, lsid); + } + + /// The continuation fields a follow-up command (e.g. a getMore) must carry to stay + /// inside this session's active transaction: `{lsid, txnNumber, autocommit: false}`, + /// never `startTransaction` since a continuation is never the transaction's first command. + /// Returns an empty object when no transaction is active. + Bson transactionContext() @safe const + { + if (!m_transaction.isActive()) + return Bson.emptyObject; + return applyTransaction(applySession(Bson.emptyObject, lsid), m_txnNumber, false); + } + + /// Commits the active transaction on this session. + void commitTransaction() @safe + { + dispatchTransactionControl(commitTransactionCommand(m_txnNumber)); + m_transaction.commit(); + } + + /// Aborts the active transaction on this session. + void abortTransaction() @safe + { + // abortTransaction is best-effort per the transactions spec: a failure to tell + // the server (rejected, network error during cleanup) must not raise to the caller. + try + dispatchTransactionControl(abortTransactionCommand(m_txnNumber)); + catch (Exception) + { + } + m_transaction.abort(); + } + + /// Sends a transaction-control command (commit/abort) to the server when the + /// transaction has reached it and a runner is wired up. + private void dispatchTransactionControl(Bson controlCommand) @safe + { + if (!shouldDispatchControl()) + return; + auto command = applySession(controlCommand, lsid); + command["$db"] = Bson("admin"); + m_runCommand(command); + } + + /// True when a transaction-control command must reach the server: the transaction has + /// dispatched a command (so the server knows about it) and a server-command runner is wired up. + private bool shouldDispatchControl() @safe const + { + return m_transaction.isInProgress() && m_runCommand !is null; + } + + /// Runs `body` inside a transaction, retrying transient failures until the default deadline. + T withTransaction(T)(scope T delegate() @safe body, Duration timeout = defaultTransactionTimeout) + { + return withTransaction!T(body, timeout, () @safe => MonoTime.currTime); + } + + /// Runs `body` inside a transaction, retrying transient failures until `timeout` elapses. + T withTransaction(T)(scope T delegate() @safe body, Duration timeout, scope MonoTime delegate() @safe clock) + { + immutable deadline = clock() + timeout; + return withTransactionRetry!T( + body, + () @safe { this.startTransaction(); }, + () @safe { this.commitTransaction(); }, + () @safe { this.abortTransaction(); }, + () @safe => clock() >= deadline); + } + + /// The current transaction lifecycle state of this session. + TransactionState transactionState() @safe const { return m_transaction.state(); } + + /// Whether a transaction is currently active on this session. + bool inTransaction() @safe const { return m_transaction.isActive(); } + + /// The transaction number allocated for the active transaction on this session. + long transactionNumber() @safe const { return m_txnNumber; } +} + +/// True when the pointer refers to a session currently inside an active transaction. +bool inActiveTransaction(scope const(MongoClientSession)* session) @safe +{ + return session !is null && session.inTransaction; +} + +/// A null session pointer is never in an active transaction. +unittest +{ + assert(!inActiveTransaction(null), + "a null session pointer is not in an active transaction"); +} + +/// A fresh session with no transaction started is not in an active transaction. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + assert(!(() @trusted => inActiveTransaction(&session))(), + "a session with no transaction started is not in an active transaction"); +} + +/// A session reports an active transaction once one is started. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + session.startTransaction(); + + assert((() @trusted => inActiveTransaction(&session))(), + "a session is in an active transaction after startTransaction"); +} + +/// A client session cannot be copied, preventing double-release of its server session. +unittest +{ + static assert(!__traits(compiles, { + auto original = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + MongoClientSession copy = original; + }), "MongoClientSession must not be copyable"); +} + +/// A client session built from a server session exposes that session's lsid. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + assert(session.lsid == server.lsid, + "a client session must expose its server session's lsid"); +} + +/// Ending a client session returns its server session to the pool for reuse. +unittest +{ + ServerSessionPool pool; + auto server = pool.acquire(); + auto session = MongoClientSession(server, (ServerSession s) @safe { pool.release(s); }); + + session.endSession(); + auto reacquired = pool.acquire(); + + assert(reacquired.lsid == server.lsid, + "endSession returns the server session to the pool for reuse"); +} + +/// Ending a client session twice releases its server session only once. +unittest +{ + ServerSessionPool pool; + auto server = pool.acquire(); + int releases = 0; + auto session = MongoClientSession(server, (ServerSession s) @safe { releases++; pool.release(s); }); + + session.endSession(); + session.endSession(); + + assert(releases == 1, + "a second endSession must not release the server session again"); +} + +/// Dropping a session without endSession still returns it to the pool (the destructor cleans up). +unittest +{ + bool released; + + { + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe { released = true; }); + // intentionally never call endSession() + } // ~this runs here + + assert(released, + "a session dropped without endSession is returned to its pool by the destructor, not leaked"); +} + +/// A destructor on an explicitly-ended session does not release the server session a second time. +unittest +{ + int releases = 0; + + { + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe { releases++; }); + session.endSession(); + } // ~this runs here; m_release is already null + + assert(releases == 1, + "the destructor must not double-release a session that was already ended"); +} + +/// Starting a transaction moves the session into the starting transaction state. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + + assert(session.transactionState() == TransactionState.starting, + "startTransaction must move the session into the starting state"); +} + +/// The first transaction started on a fresh session has transaction number 1. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + session.startTransaction(); + + assert(session.transactionNumber() == 1, + "the first transaction on a fresh session must have transaction number 1"); +} + +/// The first command prepared inside a transaction carries lsid, txnNumber, autocommit and startTransaction. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + auto cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + auto decorated = session.prepareCommand(cmd); + + assert(decorated["insert"] == Bson("people"), + "prepareCommand preserves the original command"); + assert(decorated["lsid"] == server.lsid, + "the first command carries the session's logical session id"); + assert(decorated["txnNumber"].get!long == session.transactionNumber(), + "the first command carries the allocated transaction number"); + assert(decorated["autocommit"].get!bool == false, + "a command inside a transaction sets autocommit false"); + assert(decorated["startTransaction"].get!bool == true, + "the first command of a transaction starts it"); +} + +/// applyToCommand decorates a command with the full transaction context while a transaction is active. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + auto cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + auto decorated = session.applyToCommand(cmd); + + assert(decorated["insert"] == Bson("people"), + "applyToCommand preserves the original command"); + assert(decorated["lsid"] == server.lsid, + "applyToCommand carries the session's logical session id inside a transaction"); + assert(decorated["txnNumber"].get!long == session.transactionNumber(), + "applyToCommand carries the allocated transaction number inside a transaction"); + assert(decorated["autocommit"].get!bool == false, + "applyToCommand sets autocommit false inside a transaction"); + assert(decorated["startTransaction"].get!bool == true, + "applyToCommand starts the transaction on the first command"); +} + +/// transactionContext yields the getMore continuation fields inside a transaction: lsid, txnNumber and autocommit false, never startTransaction. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + auto context = session.transactionContext(); + + assert(context["lsid"] == server.lsid, + "transactionContext carries the session's logical session id"); + assert(context["txnNumber"].get!long == session.transactionNumber(), + "transactionContext carries the active transaction number"); + assert(context["autocommit"].get!bool == false, + "transactionContext sets autocommit false"); + assert(context["startTransaction"].type == Bson.Type.null_, + "transactionContext never starts the transaction: a continuation is not the first command"); +} + +/// Outside a transaction transactionContext yields an empty object, attaching nothing to a continuation. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + assert(session.transactionContext() == Bson.emptyObject, + "transactionContext is empty when no transaction is active"); +} + +/// transactionContext does not consume the first-command flag: a later prepared command still carries startTransaction. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + session.startTransaction(); + session.transactionContext(); + auto first = session.prepareCommand(Bson(["insert": Bson("people")])); + + assert(first["startTransaction"].get!bool == true, + "transactionContext must not mark the transaction in-progress, so the first command still starts it"); +} + +/// Outside a transaction applyToCommand attaches only the lsid, never the transaction fields. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + auto cmd = Bson.emptyObject; + cmd["find"] = Bson("people"); + auto decorated = session.applyToCommand(cmd); + + assert(decorated["lsid"] == server.lsid, + "a session command carries the lsid"); + assert(decorated["find"] == Bson("people"), + "applyToCommand preserves the original command outside a transaction"); + assert(decorated["txnNumber"].type == Bson.Type.null_, + "no txnNumber outside a transaction"); + assert(decorated["autocommit"].type == Bson.Type.null_, + "no autocommit outside a transaction"); + assert(decorated["startTransaction"].type == Bson.Type.null_, + "no startTransaction outside a transaction"); +} + +/// The second command prepared inside a transaction omits startTransaction but keeps the transaction context. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + auto first = session.prepareCommand(Bson(["insert": Bson("people")])); + auto second = session.prepareCommand(Bson(["update": Bson("people")])); + + assert(second["startTransaction"].type == Bson.Type.null_, + "only the first command of a transaction carries startTransaction"); + assert(second["txnNumber"].get!long == session.transactionNumber(), + "a subsequent command still carries the transaction number"); + assert(second["autocommit"].get!bool == false, + "a subsequent command stays inside the transaction with autocommit false"); +} + +/// Committing an active transaction moves the session into the committed transaction state. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + session.commitTransaction(); + + assert(session.transactionState() == TransactionState.committed, + "commitTransaction must move the session into the committed state"); +} + +/// Committing an in-progress transaction dispatches a command through the injected runner. +unittest +{ + int runnerCalls; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { runnerCalls++; return Bson.emptyObject; }; + + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + session.commitTransaction(); + + assert(runnerCalls == 1, + "committing an in-progress transaction sends a command to the server"); +} + +/// The dispatched commit command carries the session's lsid so the server can find the transaction. +unittest +{ + Bson captured; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { captured = cmd; return Bson.emptyObject; }; + + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + session.commitTransaction(); + + assert(captured["lsid"] == server.lsid, + "the commit command identifies the session via lsid"); + assert(captured["commitTransaction"].get!int == 1, + "the commit command names the operation"); + assert(captured["txnNumber"].get!long == session.transactionNumber(), + "the commit command carries the transaction number"); +} + +/// The dispatched commit command targets the admin database via $db, as the spec requires. +unittest +{ + Bson captured; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { captured = cmd; return Bson.emptyObject; }; + + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + session.commitTransaction(); + + assert(captured["$db"] == Bson("admin"), + "transaction-control commands run against the admin database"); +} + +/// Aborting an active transaction moves the session into the aborted transaction state. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + session.abortTransaction(); + + assert(session.transactionState() == TransactionState.aborted, + "abortTransaction must move the session into the aborted state"); +} + +/// Aborting an in-progress transaction dispatches an abortTransaction command identifying the session. +unittest +{ + Bson captured; + bool called; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { called = true; captured = cmd; return Bson.emptyObject; }; + + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + session.abortTransaction(); + + assert(called, "aborting an in-progress transaction sends an abortTransaction command"); + assert(captured["abortTransaction"].get!int == 1, + "aborting an in-progress transaction sends an abortTransaction command"); + assert(captured["lsid"] == server.lsid, + "the abort command identifies the session"); +} + +/// Committing or aborting a never-ran transaction is a client-side no-op that never contacts the server. +unittest +{ + int runnerCalls; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { runnerCalls++; return Bson.emptyObject; }; + + { + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + session.startTransaction(); + session.commitTransaction(); + + assert(runnerCalls == 0, + "committing a never-ran transaction does not contact the server"); + } + + runnerCalls = 0; + + { + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + session.startTransaction(); + session.abortTransaction(); + + assert(runnerCalls == 0, + "aborting a never-ran transaction does not contact the server"); + } +} + +/// Aborting an in-progress transaction swallows a runner error and still reaches the aborted state, as abort is best-effort. +unittest +{ + import std.exception : assertNotThrown; + + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { throw new Exception("server rejected abort"); }; + + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + + assertNotThrown(session.abortTransaction(), + "abortTransaction must swallow runner errors (best-effort per spec)"); + assert(session.transactionState() == TransactionState.aborted, + "the transaction still reaches the aborted state after a swallowed runner error"); +} + +/// inTransaction reports true while a transaction is active and false once it is committed. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + session.startTransaction(); + assert(session.inTransaction() == true, + "an active transaction reports in-transaction"); + + session.commitTransaction(); + assert(session.inTransaction() == false, + "a committed transaction is no longer in-transaction"); +} + +/// endSession aborts an in-progress transaction while still releasing the server session. +unittest +{ + bool released; + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe { released = true; }); + + session.startTransaction(); + session.endSession(); + + assert(session.transactionState() == TransactionState.aborted, + "ending a session aborts an in-progress transaction"); + assert(released == true, + "ending a session still releases the server session"); +} + +/// Ending a session with a genuinely in-progress transaction dispatches abortTransaction to the server. +unittest +{ + Bson captured; + bool aborted; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { aborted = true; captured = cmd; return Bson.emptyObject; }; + + bool released; + auto release = (ServerSession s) @safe { released = true; }; + auto session = MongoClientSession(ServerSession.create(), release, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("people")])); + session.endSession(); + + assert(aborted, "ending a session with an in-progress transaction aborts it on the server"); + assert(captured["abortTransaction"].get!int == 1, + "ending a session with an in-progress transaction sends an abortTransaction command"); + assert(released == true, + "endSession still releases the server session"); +} + +/// withTransaction runs the body, commits, and returns the body result. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + auto t0 = MonoTime.currTime; + auto clock = () @safe => t0; + auto result = session.withTransaction!int(() @safe => 42, 1.minutes, clock); + + assert(result == 42, + "withTransaction returns the body result"); + assert(session.transactionState() == TransactionState.committed, + "withTransaction commits the transaction"); +} + +/// withTransaction called with only a body uses the real clock and default timeout. +unittest +{ + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + auto result = session.withTransaction!int(() @safe => 7); + + assert(result == 7, + "the convenience overload returns the body result"); + assert(session.transactionState() == TransactionState.committed, + "the convenience overload commits"); +} + +/// withTransaction does not retry a transient failure once the deadline has passed: it runs the body once and aborts. +unittest +{ + import vibe.db.mongo.connection : MongoException; + import std.exception : assertThrown; + + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + int bodyCalls; + auto base = MonoTime.currTime; + int clockCalls; + auto clock = () @safe { clockCalls++; return clockCalls == 1 ? base : base + 2.minutes; }; + auto transientBody = delegate int() @safe { + bodyCalls++; + auto e = new MongoException("transient"); + e.errorLabels = ["TransientTransactionError"]; + throw e; + }; + + assertThrown!MongoException(session.withTransaction!int(transientBody, 1.minutes, clock)); + + assert(bodyCalls == 1, + "a past-deadline transient failure is not retried"); + assert(session.transactionState() == TransactionState.aborted, + "an expired transaction is aborted"); +} + +/// withTransaction retries a within-deadline transient failure and commits the eventual body result. +unittest +{ + import vibe.db.mongo.connection : MongoException; + + auto server = ServerSession.create(); + auto session = MongoClientSession(server, (ServerSession s) @safe {}); + + auto t0 = MonoTime.currTime; + auto clock = () @safe => t0; + int bodyCalls; + auto flakyBody = delegate int() @safe { + bodyCalls++; + if (bodyCalls == 1) { + auto e = new MongoException("transient"); + e.errorLabels = ["TransientTransactionError"]; + throw e; + } + return 11; + }; + + auto result = session.withTransaction!int(flakyBody, 1.minutes, clock); + + assert(result == 11, + "a within-deadline transient failure is retried then returns the body result"); + assert(bodyCalls == 2, + "the body is retried exactly once"); + assert(session.transactionState() == TransactionState.committed, + "the retried transaction commits"); +} + +/// withTransaction threads the runner end-to-end: a body that runs an operation commits on the server. +unittest +{ + Bson captured; + bool committed; + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { committed = true; captured = cmd; return Bson.emptyObject; }; + + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + + auto t0 = MonoTime.currTime; + auto clock = () @safe => t0; + auto result = session.withTransaction!int(() @safe { + session.prepareCommand(Bson(["insert": Bson("people")])); + return 99; + }, 1.minutes, clock); + + assert(result == 99, + "withTransaction returns the body result"); + assert(committed && captured["commitTransaction"].get!int == 1, + "a transaction with operations is committed on the server"); + assert(session.transactionState() == TransactionState.committed, + "withTransaction commits the transaction"); +} + +/// The transaction number advances across successive transactions on the same session. +unittest +{ + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}); + + session.startTransaction(); + assert(session.transactionNumber() == 1, + "the first transaction is number 1"); + + session.commitTransaction(); + + session.startTransaction(); + assert(session.transactionNumber() == 2, + "a second transaction gets the next number"); +} + +/// commitTransaction propagates a runner error, unlike best-effort abort which swallows it. +unittest +{ + import std.exception : assertThrown; + + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { throw new Exception("commit failed on server"); }; + + auto session = MongoClientSession(ServerSession.create(), (ServerSession s) @safe {}, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("c")])); + + assertThrown!Exception(session.commitTransaction(), + "commitTransaction must propagate a runner error (unlike best-effort abort)"); +} + +/// endSession releases the server session even when the abort runner throws, as abort is best-effort. +unittest +{ + import std.exception : assertNotThrown; + + Bson delegate(Bson) @safe runner = (Bson cmd) @safe { throw new Exception("abort rejected"); }; + + bool released; + auto release = (ServerSession s) @safe { released = true; }; + auto session = MongoClientSession(ServerSession.create(), release, runner); + + session.startTransaction(); + session.prepareCommand(Bson(["insert": Bson("c")])); + + assertNotThrown(session.endSession(), + "endSession must not propagate the abort runner error (abort is best-effort)"); + assert(released, + "endSession releases the server session even when the abort runner throws"); +} diff --git a/mongodb/vibe/db/mongo/impl/transaction.d b/mongodb/vibe/db/mongo/impl/transaction.d new file mode 100644 index 0000000000..882c12b8e9 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/transaction.d @@ -0,0 +1,558 @@ +/** + Multi-document transaction state machine for logical sessions. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.transaction; + +import std.exception : enforce; +import std.typecons : Nullable; +import vibe.data.bson; +import vibe.db.mongo.settings : MongoHost; + +@safe: + +/// The lifecycle state of a multi-document transaction on a session. +enum TransactionState { none, starting, inProgress, committed, aborted } + +/// Tracks the lifecycle of a multi-document transaction. +struct Transaction +{ + private TransactionState m_state; + private Nullable!MongoHost m_pinnedServer; + + /// The current transaction lifecycle state. + TransactionState state() @safe const { return m_state; } + + /// The server this transaction is pinned to, if any. + Nullable!MongoHost pinnedServer() @safe const { return m_pinnedServer; } + + /// Pins the transaction to the server its operations run on. + void pinServer(MongoHost host) @safe { m_pinnedServer = host; } + + /// Whether a transaction is currently active (started or in progress). + bool isActive() @safe const + { + return m_state == TransactionState.starting || m_state == TransactionState.inProgress; + } + + /// Whether the next command is the transaction's first (which carries `startTransaction`). + bool isFirstCommand() @safe const { return m_state == TransactionState.starting; } + + /// Whether the transaction has dispatched its first command and is now running on the server. + bool isInProgress() @safe const { return m_state == TransactionState.inProgress; } + + /// Begins a transaction, moving it into the starting state. + void start() @safe + { + enforce(!isActive, "a transaction is already in progress"); + m_state = TransactionState.starting; + } + + /// Ends an active transaction successfully. + void commit() @safe { finish(TransactionState.committed, "commit"); } + + /// Ends an active transaction by rolling it back. + void abort() @safe { finish(TransactionState.aborted, "abort"); } + + /// Marks the transaction in progress once its first command has been dispatched. + void markInProgress() @safe { m_state = TransactionState.inProgress; } + + /// Releases an active transaction into a terminal state and unpins its server. + private void finish(TransactionState terminal, string action) @safe + { + requireActive(action); + m_state = terminal; + m_pinnedServer.nullify(); + } + + private void requireActive(string action) @safe const + { + enforce(isActive, "no transaction is in progress to " ~ action); + } +} + +/// A fresh transaction has no active transaction. +unittest +{ + Transaction txn; + assert(txn.state == TransactionState.none, "a fresh transaction has no active transaction"); +} + +/// start() begins a transaction in the starting state. +unittest +{ + Transaction txn; + txn.start(); + assert(txn.state == TransactionState.starting, "start() begins a transaction in the starting state"); +} + +/// commit() ends an active transaction in the committed state. +unittest +{ + Transaction txn; + txn.start(); + txn.commit(); + assert(txn.state == TransactionState.committed, "commit() ends an active transaction in the committed state"); +} + +/// abort() rolls back an active transaction to the aborted state. +unittest +{ + Transaction txn; + txn.start(); + txn.abort(); + assert(txn.state == TransactionState.aborted, "abort() rolls back an active transaction to the aborted state"); +} + +/// start() throws when a transaction is already active. +unittest +{ + import std.exception : assertThrown; + + Transaction txn; + txn.start(); + assertThrown(txn.start(), "starting a transaction while one is already active must throw"); +} + +/// commit() throws when no transaction is active. +unittest +{ + import std.exception : assertThrown; + + Transaction txn; + assertThrown(txn.commit(), "committing with no active transaction must throw"); +} + +/// abort() throws when no transaction is active. +unittest +{ + import std.exception : assertThrown; + + Transaction txn; + assertThrown(txn.abort(), "aborting with no active transaction must throw"); +} + +/// markInProgress() moves a started transaction into the inProgress state. +unittest +{ + Transaction txn; + txn.start(); + txn.markInProgress(); + assert(txn.state == TransactionState.inProgress, "the first command marks the transaction in progress"); +} + +/// isFirstCommand() is true while starting and false once in progress. +unittest +{ + Transaction txn; + assert(!txn.isFirstCommand, "a fresh transaction has no first command pending"); + txn.start(); + assert(txn.isFirstCommand, "the first command of a started transaction carries startTransaction"); + txn.markInProgress(); + assert(!txn.isFirstCommand, "subsequent commands no longer carry startTransaction"); +} + +/// isInProgress() is true only once the first command has marked the transaction running. +unittest +{ + Transaction txn; + assert(!txn.isInProgress, "a fresh transaction is not in progress"); + txn.start(); + assert(!txn.isInProgress, "a started transaction is not yet in progress"); + txn.markInProgress(); + assert(txn.isInProgress, "marking the first command puts the transaction in progress"); + txn.commit(); + assert(!txn.isInProgress, "a committed transaction is no longer in progress"); +} + +/// pinServer() records the server a transaction runs on. +unittest +{ + import vibe.db.mongo.settings : MongoHost; + + Transaction txn; + assert(txn.pinnedServer.isNull, "a fresh transaction has no pinned server"); + txn.pinServer(MongoHost("rs0-a", 27017)); + assert(!txn.pinnedServer.isNull && txn.pinnedServer.get == MongoHost("rs0-a", 27017), + "pinServer records the server the transaction runs on"); +} + +/// commit() releases the server pin so the session is no longer stuck on it. +unittest +{ + import vibe.db.mongo.settings : MongoHost; + + Transaction txn; + txn.start(); + txn.pinServer(MongoHost("rs0-a", 27017)); + txn.commit(); + assert(txn.pinnedServer.isNull, "committing a transaction releases its server pin"); +} + +/// abort() releases the server pin so the session is no longer stuck on it. +unittest +{ + import vibe.db.mongo.settings : MongoHost; + + Transaction txn; + txn.start(); + txn.pinServer(MongoHost("rs0-a", 27017)); + txn.abort(); + assert(txn.pinnedServer.isNull, "aborting a transaction releases its server pin"); +} + +/// Builds the `commitTransaction` admin command for the given transaction number. +Bson commitTransactionCommand(long txnNumber) @safe +{ + return transactionControlCommand("commitTransaction", txnNumber); +} + +/// commitTransactionCommand() builds the commitTransaction admin command. +unittest +{ + import vibe.data.bson; + + auto cmd = commitTransactionCommand(7); + assert(cmd["commitTransaction"].get!int == 1, "commit command names the operation"); + assert(cmd["txnNumber"].get!long == 7, "commit command carries the transaction number"); + assert(cmd["autocommit"].get!bool == false, "commit command sets autocommit false"); +} + +/// Builds the `abortTransaction` admin command for the given transaction number. +Bson abortTransactionCommand(long txnNumber) @safe +{ + return transactionControlCommand("abortTransaction", txnNumber); +} + +/// abortTransactionCommand() builds the abortTransaction admin command. +unittest +{ + import vibe.data.bson; + + auto cmd = abortTransactionCommand(3); + assert(cmd["abortTransaction"].get!int == 1, "abort command names the operation"); + assert(cmd["txnNumber"].get!long == 3, "abort command carries the transaction number"); + assert(cmd["autocommit"].get!bool == false, "abort command sets autocommit false"); +} + +/// The command name must be the first wire field; MongoDB rejects it otherwise. +unittest +{ + import vibe.db.mongo.impl.retryablewrites : commandName; + + assert(commandName(commitTransactionCommand(7)) == "commitTransaction", + "the commit command name is the first wire field"); + assert(commandName(abortTransactionCommand(3)) == "abortTransaction", + "the abort command name is the first wire field"); +} + +/// Builds a transaction-control admin command (commit/abort) for the given number. +private Bson transactionControlCommand(string name, long txnNumber) @safe +{ + Bson cmd = Bson.emptyObject; // ordered: the command name must be the first field on the wire + cmd[name] = Bson(1); + cmd["txnNumber"] = Bson(txnNumber); + cmd["autocommit"] = Bson(false); + return cmd; +} + +/// Attaches transaction fields to a command. The first command of a transaction +/// also carries `startTransaction: true`. +Bson applyTransaction(Bson command, long txnNumber, bool firstCommand) @safe +{ + Bson result = command; + result["txnNumber"] = Bson(txnNumber); + result["autocommit"] = Bson(false); + if (firstCommand) + result["startTransaction"] = Bson(true); + return result; +} + +/// applyTransaction() decorates the first command with transaction fields. +unittest +{ + import vibe.data.bson; + + Bson cmd = Bson.emptyObject; + cmd["insert"] = Bson("people"); + + auto result = applyTransaction(cmd, 4, true); + assert(result["insert"] == Bson("people"), "the original command is preserved"); + assert(result["txnNumber"].get!long == 4, "the transaction number is attached"); + assert(result["autocommit"].get!bool == false, "autocommit is false inside a transaction"); + assert(result["startTransaction"].get!bool == true, "the first command starts the transaction"); +} + +/// applyTransaction() omits startTransaction on subsequent commands. +unittest +{ + import vibe.data.bson; + + Bson cmd = Bson.emptyObject; + cmd["update"] = Bson("people"); + + auto result = applyTransaction(cmd, 4, false); + assert(result["txnNumber"].get!long == 4, "the transaction number is attached"); + assert(result["autocommit"].get!bool == false, "autocommit is false inside a transaction"); + assert(result["startTransaction"].type == Bson.Type.null_, + "only the first command carries startTransaction"); +} + +/// MongoDB error label marking a transaction safe to retry from the start. +enum transientTransactionErrorLabel = "TransientTransactionError"; + +/// MongoDB error label marking a commit whose outcome is unknown and may be retried. +enum unknownCommitResultLabel = "UnknownTransactionCommitResult"; + +/// Runs `body` inside a transaction and commits it, returning the body's result. +/// Retries the whole transaction on a transient body failure, and retries the +/// commit on an unknown commit result, until either succeeds or `expired` is true. +T withTransactionRetry(T)( + scope T delegate() @safe body, + scope void delegate() @safe start, + scope void delegate() @safe commit, + scope void delegate() @safe abort, + scope bool delegate() @safe expired) +{ + import vibe.db.mongo.connection : MongoException; + + // Per the transactions spec, abort is best-effort: the body may have already ended + // the transaction (e.g. committed it), so abort() can fail — that failure must never + // mask the body's original error. + void safeAbort() @safe { try abort(); catch (Exception) {} } + + outer: while (true) + { + start(); + T result; + try + result = body(); + catch (MongoException e) + { + safeAbort(); + if (e.hasErrorLabel(transientTransactionErrorLabel) && !expired()) + continue; + throw e; + } + catch (Exception e) + { + safeAbort(); + throw e; + } + + while (true) + { + try + { + commit(); + return result; + } + catch (MongoException e) + { + if (e.hasErrorLabel(unknownCommitResultLabel) && !expired()) + continue; + if (e.hasErrorLabel(transientTransactionErrorLabel) && !expired()) + continue outer; + throw e; + } + } + } + + assert(0); +} + +/// withTransactionRetry() runs the body once and commits when nothing throws. +unittest +{ + int starts; + int commits; + int aborts; + + auto result = withTransactionRetry!int( + () @safe => 42, + () @safe { starts++; }, + () @safe { commits++; }, + () @safe { aborts++; }, + () @safe => false); + + assert(result == 42, "the body's return value is propagated"); + assert(starts == 1, "the transaction is started exactly once"); + assert(commits == 1, "the transaction is committed exactly once"); + assert(aborts == 0, "a successful transaction is never aborted"); +} + +/// withTransactionRetry() retries the whole transaction when the body throws TransientTransactionError. +unittest +{ + import vibe.db.mongo.connection : MongoException; + + int starts; + int commits; + int aborts; + int bodyCalls; + + auto result = withTransactionRetry!int( + () @safe { + bodyCalls++; + if (bodyCalls == 1) + { + auto e = new MongoException("transient"); + e.errorLabels = ["TransientTransactionError"]; + throw e; + } + return 7; + }, + () @safe { starts++; }, + () @safe { commits++; }, + () @safe { aborts++; }, + () @safe => false); + + assert(result == 7, "the retried body's return value is propagated"); + assert(starts == 2, "a transient failure restarts the whole transaction"); + assert(bodyCalls == 2, "the body runs again after a transient failure"); + assert(aborts == 1, "the failed attempt is aborted before retrying"); + assert(commits == 1, "the successful retry is committed exactly once"); +} + +/// withTransactionRetry() stops retrying and rethrows once the deadline has expired. +unittest +{ + import vibe.db.mongo.connection : MongoException; + import std.exception : assertThrown; + + int starts; + int commits; + int aborts; + int bodyCalls; + + assertThrown!MongoException(withTransactionRetry!int( + delegate int() @safe { + bodyCalls++; + if (bodyCalls > 5) + throw new Exception("retry-cap exceeded"); + auto e = new MongoException("transient"); + e.errorLabels = ["TransientTransactionError"]; + throw e; + }, + () @safe { starts++; }, + () @safe { commits++; }, + () @safe { aborts++; }, + () @safe => true)); + + assert(bodyCalls == 1, "the expired deadline stops the body running a second time"); + assert(starts == 1, "the expired deadline stops the transaction restarting"); + assert(aborts == 1, "the expired attempt is still aborted before rethrowing"); + assert(commits == 0, "an expired transaction is never committed"); +} + +/// withTransactionRetry() retries the commit on UnknownTransactionCommitResult without re-running the body. +unittest +{ + import vibe.db.mongo.connection : MongoException; + + int starts; + int commits; + int aborts; + int bodyCalls; + + auto result = withTransactionRetry!int( + () @safe { bodyCalls++; return 9; }, + () @safe { starts++; }, + () @safe { + commits++; + if (commits == 1) + { + auto e = new MongoException("commit result unknown"); + e.errorLabels = ["UnknownTransactionCommitResult"]; + throw e; + } + }, + () @safe { aborts++; }, + () @safe => false); + + assert(result == 9, "the body's return value is propagated after the commit retry"); + assert(starts == 1, "an unknown commit result does not restart the transaction"); + assert(bodyCalls == 1, "an unknown commit result does not re-run the body"); + assert(commits == 2, "the commit is retried after an unknown result"); + assert(aborts == 0, "retrying the commit never aborts"); +} + +/// withTransactionRetry() restarts the whole transaction when the commit throws TransientTransactionError. +unittest +{ + import vibe.db.mongo.connection : MongoException; + + int starts; + int commits; + int aborts; + int bodyCalls; + + auto result = withTransactionRetry!int( + () @safe { bodyCalls++; return 5; }, + () @safe { starts++; }, + () @safe { + commits++; + if (commits == 1) + { + auto e = new MongoException("transient commit"); + e.errorLabels = [transientTransactionErrorLabel]; + throw e; + } + }, + () @safe { aborts++; }, + () @safe => false); + + assert(result == 5, "the restarted transaction's body return value is propagated"); + assert(starts == 2, "a transient commit failure restarts the whole transaction"); + assert(bodyCalls == 2, "a transient commit failure re-runs the body"); + assert(commits == 2, "the commit runs again after the restart"); + assert(aborts == 0, "the transient commit path does not abort"); +} + +/// withTransactionRetry() aborts and rethrows a plain (non-Mongo) body exception without retrying. +unittest +{ + import std.exception : assertThrown; + + int starts; + int commits; + int aborts; + int bodyCalls; + + assertThrown!Exception(withTransactionRetry!int( + delegate int() @safe { bodyCalls++; throw new Exception("plain body failure"); }, + () @safe { starts++; }, + () @safe { commits++; }, + () @safe { aborts++; }, + () @safe => false)); + + assert(bodyCalls == 1, "a non-Mongo body exception is not retried"); + assert(aborts == 1, "a non-Mongo body exception still aborts the transaction"); + assert(commits == 0, "a body that throws is never committed"); +} + +/// withTransactionRetry() rethrows the body's error even when abort() itself fails (it must not mask the original). +unittest +{ + import vibe.db.mongo.connection : MongoException; + + auto bodyError = new MongoException("body failed"); + bool aborted; + Exception caught; + try + withTransactionRetry!int( + delegate int() @safe { throw bodyError; }, + () @safe {}, + () @safe {}, + () @safe { aborted = true; throw new Exception("no transaction in progress to abort"); }, + () @safe => false); + catch (Exception e) + caught = e; + + assert(aborted, "abort is still attempted"); + assert(caught is bodyError, + "withTransactionRetry must rethrow the body's error, not the abort failure that masks it"); +} diff --git a/tests/mongodb/session-timeout/dub.json b/tests/mongodb/session-timeout/dub.json new file mode 100644 index 0000000000..41b509c70b --- /dev/null +++ b/tests/mongodb/session-timeout/dub.json @@ -0,0 +1,7 @@ +{ + "name": "session-timeout-test", + "description": "MongoDB server-advertised logical session timeout integration test", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + } +} diff --git a/tests/mongodb/session-timeout/source/app.d b/tests/mongodb/session-timeout/source/app.d new file mode 100644 index 0000000000..fa7307c6ba --- /dev/null +++ b/tests/mongodb/session-timeout/source/app.d @@ -0,0 +1,43 @@ +/// Requires a mongo service running on localhost; port via args[1]. +/// Verifies the server-advertised logicalSessionTimeoutMinutes flows through the +/// driver's real path: the live hello reply parses into ServerDescription, the +/// data-bearing filter accepts the node, and logicalSessionTimeout computes the +/// timeout. The harness's mongod reports the default 30 minutes (the startup +/// parameter localLogicalSessionTimeoutMinutes is not runtime-settable), so this +/// confirms the path against REAL server data with the default value. + +module app; + +import vibe.data.bson; +import vibe.db.mongo.mongo; +import vibe.db.mongo.settings; +import vibe.db.mongo.connection : ServerDescription; // public-imported from impl.serverdescription +import vibe.db.mongo.topology : logicalSessionTimeout; + +import core.time : minutes; +import std.conv : to; + +void main(string[] args) +{ + ushort port = args.length > 1 + ? args[1].to!ushort + : MongoClientSettings.defaultPort; + + auto client = connectMongoDB("mongodb://127.0.0.1:" ~ port.to!string ~ "/"); + auto hello = client.getDatabase("admin").runCommandChecked(Bson(["hello": Bson(1)])); + + assert(hello["logicalSessionTimeoutMinutes"].type != Bson.Type.null_, + "the server advertises logicalSessionTimeoutMinutes"); + + auto desc = deserializeBson!ServerDescription(hello); + assert(!desc.logicalSessionTimeoutMinutes.isNull, + "the driver parses logicalSessionTimeoutMinutes from the hello reply"); + assert(desc.isDataBearing, + "the standalone server is a data-bearing node"); + + auto timeout = logicalSessionTimeout([desc]); + assert(!timeout.isNull, + "a data-bearing server advertising a timeout yields a topology timeout"); + assert(timeout.get == desc.logicalSessionTimeoutMinutes.get.minutes, + "the computed timeout equals the server-advertised value"); +} diff --git a/tests/mongodb/transactions/dub.json b/tests/mongodb/transactions/dub.json new file mode 100644 index 0000000000..9a32d2d00b --- /dev/null +++ b/tests/mongodb/transactions/dub.json @@ -0,0 +1,7 @@ +{ + "name": "transactions-test", + "description": "MongoDB multi-document transaction end-to-end integration test (requires a replica set)", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + } +} diff --git a/tests/mongodb/transactions/source/app.d b/tests/mongodb/transactions/source/app.d new file mode 100644 index 0000000000..d9be4b0454 --- /dev/null +++ b/tests/mongodb/transactions/source/app.d @@ -0,0 +1,232 @@ +/// Requires a mongo service running on localhost; port via args[1]. +/// Verifies multi-document transactions end-to-end, adapting to the deployment: +/// * Against a REPLICA SET (or sharded cluster) it runs real transactions: an +/// insert performed inside a transaction via an explicit session is durably +/// persisted on commit and rolled back on abort, observed by a plain +/// non-session read. +/// * Against a STANDALONE server (the default CI harness) transactions are not +/// supported, so it confirms the driver transmits the transaction context by +/// forcing the server to reject a transactional write with its +/// "Transaction numbers are only allowed on a replica set member or mongos" +/// error — proving the command reached the server carrying lsid/txnNumber. + +module app; + +import vibe.data.bson; +import vibe.db.mongo.mongo; +import vibe.db.mongo.collection : InsertOneOptions, InsertManyOptions, UpdateOptions, DeleteOptions, FindOptions; +import vibe.db.mongo.connection : MongoException; + +import std.algorithm : canFind; +import std.conv : to; + +int main(string[] args) +{ + ushort port = args.length > 1 + ? args[1].to!ushort + : MongoClientSettings.defaultPort; + runTest(port); + return 0; +} + +void runTest(ushort port) +{ + auto client = connectMongoDB("127.0.0.1", port); + + if (isTransactionCapable(client)) + runReplicaSetTest(client); + else + runStandaloneTest(client); +} + +/// Transactions need a replica set member or a mongos; a hello reply advertises +/// `setName` for a replica set and `msg == "isdbgrid"` for a sharded cluster. +bool isTransactionCapable(MongoClient client) +{ + auto hello = client.getDatabase("admin").runCommandChecked(Bson(["hello": Bson(1)])); + return hello["setName"].type != Bson.Type.null_ + || hello["msg"].opt!string == "isdbgrid"; +} + +/// A committed transactional insert is durable; an aborted one is rolled back. +void runReplicaSetTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["commit_persists"]; + coll.drop(); // MongoCollection.drop tolerates NamespaceNotFound (code 26) + + auto session = client.startSession(); + session.startTransaction(); + coll.insertOne(Bson(["_id": Bson("c1"), "v": Bson("committed")]), InsertOneOptions.init, &session); + session.commitTransaction(); + session.endSession(); + + auto found = coll.findOne(Bson(["_id": Bson("c1")])); + assert(found.type != Bson.Type.null_, "a committed transactional insert is visible after commit"); + assert(found["v"].get!string == "committed", "the committed document carries the written value"); + + auto acoll = client.getDatabase("txn_it")["abort_rolls_back"]; + acoll.drop(); + auto asession = client.startSession(); + asession.startTransaction(); + acoll.insertOne(Bson(["_id": Bson("a1"), "v": Bson("rolled-back")]), InsertOneOptions.init, &asession); + asession.abortTransaction(); + asession.endSession(); + + auto gone = acoll.findOne(Bson(["_id": Bson("a1")])); + assert(gone.type == Bson.Type.null_, "an aborted transactional insert is NOT visible after abort"); + + runWriteOpsTest(client); + runFindAndModifyTest(client); + runReadYourWritesTest(client); + runMultiBatchReadTest(client); + runPartialReadAbortTest(client); +} + +/// A multi-batch transaction cursor read only partway, then aborted, must clean +/// up safely: the cursor destructor's killCursors may hit the server after the +/// transaction already killed the cursor, but no uncaught exception may escape +/// and the client/connection must stay usable for later operations. +void runPartialReadAbortTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["partial_read"]; + coll.drop(); + foreach (i; 0 .. 6) + coll.insertOne(Bson(["_id": Bson(i)])); + + auto session = client.startSession(); + session.startTransaction(); + { + FindOptions fo; + fo.batchSize = 2; // multi-batch: leaves the cursor alive after a partial read + int seen; + foreach (doc; coll.find(Bson.emptyObject, fo, &session)) + { + seen++; + if (seen == 1) + break; // stop after the FIRST doc: cursor still alive server-side, inside the txn + } + // the cursor temporary is destroyed here (end of scope) -> its destructor runs killCursors + } + session.abortTransaction(); + session.endSession(); + + // the client must still be fully usable after the partial-read + abort cleanup + auto fresh = client.startSession(); + fresh.startTransaction(); + coll.insertOne(Bson(["_id": Bson("after")]), InsertOneOptions.init, &fresh); + fresh.commitTransaction(); + fresh.endSession(); + assert(coll.findOne(Bson(["_id": Bson("after")])).type != Bson.Type.null_, + "the client works normally after a partial-read transaction cursor was aborted"); +} + +/// A find inside a transaction whose result spans multiple batches must return +/// every matching document — the getMore continuations have to carry the session. +void runMultiBatchReadTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["multi_batch"]; + coll.drop(); + // seed 5 docs OUTSIDE any transaction (committed) so the in-txn read must page through them + foreach (i; 0 .. 5) + coll.insertOne(Bson(["_id": Bson(i), "v": Bson(i)])); + + auto session = client.startSession(); + session.startTransaction(); + FindOptions fo; + fo.batchSize = 2; // forces getMore continuations (batches of 2 over 5 docs) + int count; + foreach (doc; coll.find(Bson.emptyObject, fo, &session)) + count++; + session.commitTransaction(); + session.endSession(); + + assert(count == 5, + "a multi-batch find inside a transaction returns all documents across getMore continuations"); +} + +/// A findOne performed inside a transaction (with the session) sees the +/// transaction's own uncommitted insert; after abort it is invisible again. +void runReadYourWritesTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["read_your_writes"]; + coll.drop(); + + auto session = client.startSession(); + session.startTransaction(); + coll.insertOne(Bson(["_id": Bson("ryw"), "n": Bson(1)]), InsertOneOptions.init, &session); + + // read INSIDE the same transaction (passing the session) — must see the uncommitted insert + auto seen = coll.findOne(Bson(["_id": Bson("ryw")]), FindOptions.init, &session); + assert(seen.type != Bson.Type.null_, + "a findOne inside the transaction sees the transaction's own uncommitted write"); + assert(seen["n"].get!int == 1, "the in-transaction read returns the written value"); + + session.abortTransaction(); // roll back so nothing persists + session.endSession(); + + // a plain non-session read must NOT see it (it was rolled back) + assert(coll.findOne(Bson(["_id": Bson("ryw")])).type == Bson.Type.null_, + "the rolled-back read-your-writes document is not visible after abort"); +} + +/// An atomic read-modify-write via findAndModify participates in a transaction +/// and persists on commit. +void runFindAndModifyTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["find_and_modify"]; + coll.drop(); + coll.insertOne(Bson(["_id": Bson("fam"), "n": Bson(0)])); + + auto session = client.startSession(); + session.startTransaction(); + coll.findAndModify(Bson(["_id": Bson("fam")]), Bson(["$set": Bson(["n": Bson(5)])]), null, &session); + session.commitTransaction(); + session.endSession(); + + assert(coll.findOne(Bson(["_id": Bson("fam")]))["n"].get!int == 5, + "findAndModify inside a transaction persists on commit"); +} + +/// Every mutating op (insertMany/updateOne/deleteOne) participates in one +/// transaction and its net effect is durable on commit. +void runWriteOpsTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["write_ops"]; + coll.drop(); + coll.insertOne(Bson(["_id": Bson("keep"), "n": Bson(0)])); + coll.insertOne(Bson(["_id": Bson("doomed"), "n": Bson(0)])); + + auto session = client.startSession(); + session.startTransaction(); + coll.insertMany([Bson(["_id": Bson("m1")]), Bson(["_id": Bson("m2")])], InsertManyOptions.init, &session); + coll.updateOne(Bson(["_id": Bson("keep")]), Bson(["$set": Bson(["n": Bson(1)])]), UpdateOptions.init, &session); + coll.deleteOne(Bson(["_id": Bson("doomed")]), DeleteOptions.init, &session); + session.commitTransaction(); + session.endSession(); + + assert(coll.findOne(Bson(["_id": Bson("m1")])).type != Bson.Type.null_, "insertMany in a txn persists on commit"); + assert(coll.findOne(Bson(["_id": Bson("keep")]))["n"].get!int == 1, "updateOne in a txn persists on commit"); + assert(coll.findOne(Bson(["_id": Bson("doomed")])).type == Bson.Type.null_, "deleteOne in a txn persists on commit"); +} + +/// On a standalone the server rejects a transactional write, which proves the +/// driver transmitted the transaction context (lsid/txnNumber/startTransaction). +void runStandaloneTest(MongoClient client) +{ + auto coll = client.getDatabase("txn_it")["standalone_reject"]; + coll.drop(); + + auto session = client.startSession(); + session.startTransaction(); + + bool rejected; + try + coll.insertOne(Bson(["_id": Bson("s1")]), InsertOneOptions.init, &session); + catch (MongoException e) + rejected = canFind(e.msg, "Transaction numbers are only allowed"); + + assert(rejected, + "a standalone must reject a transactional insert, proving the transaction context reached the server"); + + session.endSession(); +} From f758e2f543fa6d4e33e5f9f57e99dfd647491111 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Wed, 17 Jun 2026 17:22:24 +0200 Subject: [PATCH 3/3] feat(mongo): change stream support Adds change streams via watch() on MongoClient (whole deployment), MongoDatabase (all collections), and MongoCollection. The returned ChangeStream input range tracks resume tokens and automatically resumes on transient/resumable errors; requires a replica set or sharded cluster. Stacked on mongo-sessions-and-transactions. Closes #2860 --- mongodb/vibe/db/mongo/client.d | 14 + mongodb/vibe/db/mongo/collection.d | 34 ++ mongodb/vibe/db/mongo/database.d | 32 ++ mongodb/vibe/db/mongo/impl/changestream.d | 465 ++++++++++++++++++++++ tests/mongodb/change-stream/dub.json | 7 + tests/mongodb/change-stream/run.sh | 94 +++++ tests/mongodb/change-stream/source/app.d | 142 +++++++ 7 files changed, 788 insertions(+) create mode 100644 mongodb/vibe/db/mongo/impl/changestream.d create mode 100644 tests/mongodb/change-stream/dub.json create mode 100755 tests/mongodb/change-stream/run.sh create mode 100644 tests/mongodb/change-stream/source/app.d diff --git a/mongodb/vibe/db/mongo/client.d b/mongodb/vibe/db/mongo/client.d index 5e80cd4961..bbefb75fa6 100644 --- a/mongodb/vibe/db/mongo/client.d +++ b/mongodb/vibe/db/mongo/client.d @@ -21,6 +21,7 @@ import vibe.db.mongo.monitor; import vibe.db.mongo.impl.crud; import vibe.db.mongo.impl.serversession : ServerSession, ServerSessionPool, MongoClientSession, endSessionsCommand; import vibe.db.mongo.impl.wireversion : WireVersion; +import vibe.db.mongo.impl.changestream; import vibe.data.bson; import core.time : Duration, seconds, msecs, MonoTime; @@ -265,6 +266,19 @@ final class MongoClient { return MongoDatabase(this, dbName); } + /** Opens a change stream over the entire deployment (all databases). + + Returns a ChangeStream input range that tracks resume tokens and resumes on + transient errors. Requires a replica set or sharded cluster. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/changeStreams/) + */ + ChangeStream!R watch(R = Bson, S = Bson)(S[] pipeline = null, ChangeStreamOptions options = ChangeStreamOptions.init) @safe + { + options.allChangesForCluster = true; + return getDatabase("admin").watch!(R, S)(pipeline, options); + } + /** Return a handle to all databases of the server. diff --git a/mongodb/vibe/db/mongo/collection.d b/mongodb/vibe/db/mongo/collection.d index 745f8bfd0c..441ecaefb8 100644 --- a/mongodb/vibe/db/mongo/collection.d +++ b/mongodb/vibe/db/mongo/collection.d @@ -18,6 +18,7 @@ public import vibe.db.mongo.impl.wireversion; import vibe.core.log; import vibe.db.mongo.client; import vibe.db.mongo.impl.serversession : MongoClientSession; +import vibe.db.mongo.impl.changestream; import vibe.db.mongo.impl.commands : splitNamespace, buildDeleteCommand, buildUpdateCommand, buildCountPipeline, buildAggregateCommand; import vibe.db.mongo.settings : ReadPreference; @@ -894,6 +895,39 @@ struct MongoCollection { } } + /** Opens a change stream on this collection. + + Returns a ChangeStream input range of change event documents. The stream + tracks resume tokens and automatically resumes on transient/resumable + server errors. Requires a replica set or sharded cluster (change streams + are unavailable on standalone servers). + + Params: + pipeline = optional user aggregation stages applied after $changeStream + options = change stream options (fullDocument, resumeAfter, startAfter) + + See_Also: $(LINK https://www.mongodb.com/docs/manual/changeStreams/) + */ + ChangeStream!R watch(R = Bson, S = Bson)(S[] pipeline = null, ChangeStreamOptions options = ChangeStreamOptions.init) @safe + { + auto client = m_client; + auto fullPath = m_fullPath; + + static if (is(S == Bson)) + Bson[] userPipeline = pipeline; + else { + import std.algorithm : map; + import std.array : array; + Bson[] userPipeline = pipeline.map!(stage => serializeToBson(stage)).array; + } + + auto open = (ChangeStreamOptions opts) @safe { + auto coll = MongoCollection(client, fullPath); + return coll.aggregate!R(buildChangeStreamPipeline(opts, userPipeline), AggregateOptions.init); + }; + return ChangeStream!R(open, options); + } + /** Returns an input range of all unique values for a certain field for records matching the given query. diff --git a/mongodb/vibe/db/mongo/database.d b/mongodb/vibe/db/mongo/database.d index 4abe54a784..46df0e0df2 100644 --- a/mongodb/vibe/db/mongo/database.d +++ b/mongodb/vibe/db/mongo/database.d @@ -15,6 +15,7 @@ import vibe.db.mongo.collection; import vibe.db.mongo.settings : ReadConcern, ReadPreference, readPreferenceBson; import vibe.db.mongo.impl.retryablewrites : isRetryableWriteCommand, applyRetryableWrite; import vibe.db.mongo.impl.serversession : ServerSession, MongoClientSession, inActiveTransaction; +import vibe.db.mongo.impl.changestream; import vibe.db.mongo.connection : MongoNetworkException; import vibe.data.bson; @@ -314,6 +315,37 @@ struct MongoDatabase return MongoCursor!R(m_client, cmd, batchSize, getMoreMaxTime, Nullable!ReadPreference(pref)); } + /** Opens a change stream over all collections in this database. + + Returns a ChangeStream input range that tracks resume tokens and resumes on + transient errors. Requires a replica set or sharded cluster. + + See_Also: $(LINK https://www.mongodb.com/docs/manual/changeStreams/) + */ + ChangeStream!R watch(R = Bson, S = Bson)(S[] pipeline = null, ChangeStreamOptions options = ChangeStreamOptions.init) @safe + { + auto client = m_client; + auto dbName = m_name; + + static if (is(S == Bson)) + Bson[] userPipeline = pipeline; + else { + import std.algorithm : map; + import std.array : array; + Bson[] userPipeline = pipeline.map!(stage => serializeToBson(stage)).array; + } + + auto open = (ChangeStreamOptions opts) @safe { + auto db = MongoDatabase(client, dbName); + Bson command = Bson.emptyObject; + command["aggregate"] = Bson(1); + command["pipeline"] = serializeToBson(buildChangeStreamPipeline(opts, userPipeline)); + command["cursor"] = Bson.emptyObject; + return db.runListCommand!R(command); + }; + return ChangeStream!R(open, options); + } + /// Normalizes a command argument into its Bson wire form: Bson passes through, /// anything else is serialized. private static Bson toCommandBson(T)(T command_and_options) diff --git a/mongodb/vibe/db/mongo/impl/changestream.d b/mongodb/vibe/db/mongo/impl/changestream.d new file mode 100644 index 0000000000..28463cf374 --- /dev/null +++ b/mongodb/vibe/db/mongo/impl/changestream.d @@ -0,0 +1,465 @@ +/** + MongoDB change stream spec helpers. + + A change stream is an aggregation pipeline whose first stage is `$changeStream`. + This module builds that pipeline from typed options, extracts resume tokens from + change events, recognises resumable server errors, and exposes `ChangeStream`, + an input range that transparently re-opens its cursor from the last seen token + when the server reports a resumable error. + + Copyright: © 2026 Szabo Bogdan + License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. + Authors: Szabo Bogdan +*/ +module vibe.db.mongo.impl.changestream; + +import vibe.data.bson; +import vibe.db.mongo.connection : MongoException; +import vibe.db.mongo.cursor : MongoCursor; +import std.typecons : Nullable; + +@safe: + +/// Controls which version of a document is returned for update events. +enum ChangeStreamFullDocument { + /// Omit the `fullDocument` field (server default). + default_, + /// Look up and return the current majority-committed document. + updateLookup, + /// Return the post-image when available. + whenAvailable, + /// Require the post-image, erroring when unavailable. + required, +} + +/// Options controlling how a change stream is opened. +struct ChangeStreamOptions { + /// Which document version to return for update events. + ChangeStreamFullDocument fullDocument; + /// Resume token to restart the stream after a previously seen event. + Nullable!Bson resumeAfter; + /// Resume token to restart the stream after the named event, excluding it. + Nullable!Bson startAfter; + /// Watch every collection in every database of the cluster (whole-cluster stream). + bool allChangesForCluster; +} + +/** Builds the `$changeStream` aggregation stage from the given options. + + Each option is only emitted when it deviates from the server default, so + `ChangeStreamOptions.init` yields an empty `$changeStream` stage. + + Params: + options = the typed options to encode into the stage. + + Returns: the BSON object `{ "$changeStream": { ... } }`. +*/ +Bson changeStreamStage(ChangeStreamOptions options) +{ + auto inner = Bson.emptyObject; + + const fullDocumentName = mongoName(options.fullDocument); + if (fullDocumentName !is null) + inner["fullDocument"] = Bson(fullDocumentName); + + setIfPresent(inner, "resumeAfter", options.resumeAfter); + setIfPresent(inner, "startAfter", options.startAfter); + + if (options.allChangesForCluster) + inner["allChangesForCluster"] = Bson(true); + + return Bson(["$changeStream": inner]); +} + +/// changeStreamStage with default options yields an empty $changeStream stage +unittest +{ + assert(changeStreamStage(ChangeStreamOptions.init) == Bson(["$changeStream": Bson.emptyObject])); +} + +/// changeStreamStage includes fullDocument when set to updateLookup +unittest +{ + ChangeStreamOptions options; + options.fullDocument = ChangeStreamFullDocument.updateLookup; + assert(changeStreamStage(options) == Bson(["$changeStream": Bson(["fullDocument": Bson("updateLookup")])])); +} + +/// changeStreamStage includes the resumeAfter token when set +unittest +{ + import std.typecons : nullable; + auto token = Bson(["_data": Bson("abc")]); + ChangeStreamOptions options; + options.resumeAfter = token.nullable; + assert(changeStreamStage(options) == Bson(["$changeStream": Bson(["resumeAfter": token])])); +} + +/// changeStreamStage includes the startAfter token when set +unittest +{ + import std.typecons : nullable; + auto token = Bson(["_data": Bson("xyz")]); + ChangeStreamOptions options; + options.startAfter = token.nullable; + assert(changeStreamStage(options) == Bson(["$changeStream": Bson(["startAfter": token])])); +} + +/// changeStreamStage includes allChangesForCluster when set +unittest +{ + ChangeStreamOptions options; + options.allChangesForCluster = true; + assert(changeStreamStage(options) == Bson(["$changeStream": Bson(["allChangesForCluster": Bson(true)])])); +} + +/** Builds the full aggregation pipeline for a change stream. + + The `$changeStream` stage built from `options` is prepended in front of the + caller-supplied stages. + + Params: + options = the typed options for the leading `$changeStream` stage. + userPipeline = the caller's downstream aggregation stages. + + Returns: the change-stream stage followed by `userPipeline`. +*/ +Bson[] buildChangeStreamPipeline(ChangeStreamOptions options, Bson[] userPipeline) +{ + return changeStreamStage(options) ~ userPipeline; +} + +/// buildChangeStreamPipeline prepends the changeStream stage before the user pipeline +unittest +{ + auto userStage = Bson(["$match": Bson(["operationType": Bson("insert")])]); + auto pipeline = buildChangeStreamPipeline(ChangeStreamOptions.init, [userStage]); + assert(pipeline.length == 2); + assert(pipeline[0] == Bson(["$changeStream": Bson.emptyObject])); + assert(pipeline[1] == userStage); +} + +/** Extracts the resume token (`_id`) from a change stream event document. + + Params: + event = a change event document as returned by the server. + + Returns: the event's `_id`, or null when the event carries no `_id`. +*/ +Nullable!Bson resumeToken(Bson event) +{ + import std.typecons : nullable; + auto id = event["_id"]; + if (id.isNull) + return Nullable!Bson.init; + return id.nullable; +} + +/// resumeToken returns the change event's _id +unittest +{ + import std.typecons : nullable; + auto event = Bson([ + "_id": Bson(["_data": Bson("826...")]), + "operationType": Bson("insert") + ]); + assert(resumeToken(event) == Bson(["_data": Bson("826...")]).nullable); +} + +/// resumeToken returns null when the event has no _id +unittest +{ + auto event = Bson(["operationType": Bson("insert")]); + assert(resumeToken(event).isNull); +} + +/** Whether a change event terminates the stream. + + An `invalidate` event (collection dropped/renamed, database dropped) is the + final event a change stream emits. The server rejects `resumeAfter` carrying an + invalidate token (only `startAfter` is legal there) and the spec forbids + auto-resuming past an invalidate, so the stream must end on it. + + Params: + event = a change event document as returned by the server. + + Returns: true when the event's `operationType` is `invalidate`. +*/ +bool isInvalidateEvent(Bson event) +{ + return event["operationType"].opt!string == "invalidate"; +} + +/// isInvalidateEvent is true for an invalidate event +unittest +{ + auto event = Bson([ + "_id": Bson(["_data": Bson("826...")]), + "operationType": Bson("invalidate") + ]); + assert(isInvalidateEvent(event)); +} + +/// isInvalidateEvent is false for non-invalidate events and missing operationType +unittest +{ + assert(!isInvalidateEvent(Bson(["operationType": Bson("insert")]))); + assert(!isInvalidateEvent(Bson(["_id": Bson(["_data": Bson("x")])]))); +} + +/** Whether a server error is a resumable change-stream error. + + Such errors carry the `ResumableChangeStreamError` label, meaning the stream + may safely re-open from the last seen resume token instead of failing. + + Params: + e = the server error to classify. + + Returns: true when the error carries the resumable label. +*/ +bool isResumableChangeStreamError(MongoException e) +{ + return e.hasErrorLabel("ResumableChangeStreamError"); +} + +/// isResumableChangeStreamError is true when the resumable label is present +unittest +{ + import vibe.db.mongo.connection : MongoException; + auto resumableError = new MongoException("getMore failed"); + resumableError.errorLabels = ["ResumableChangeStreamError"]; + assert(isResumableChangeStreamError(resumableError)); +} + +/// isResumableChangeStreamError is false without the resumable label +unittest +{ + import vibe.db.mongo.connection : MongoException; + auto other = new MongoException("network blip"); + other.errorLabels = ["TransientTransactionError"]; + assert(!isResumableChangeStreamError(other)); + + auto bare = new MongoException("no labels"); + assert(!isResumableChangeStreamError(bare)); +} + +/** Derives the change-stream options for a resume attempt. + + Once a resume token is known, resumption uses `resumeAfter` with that token and + drops `startAfter`. With no cached token the options are returned unchanged, so + the original `startAfter`/`resumeAfter` intent is preserved. + + Params: + original = the options the stream was originally opened with. + cachedToken = the last seen resume token, or null if none yet. + + Returns: the options to re-open the stream with. +*/ +ChangeStreamOptions optionsForResume(ChangeStreamOptions original, Nullable!Bson cachedToken) +{ + auto resumed = original; + if (!cachedToken.isNull) { + resumed.resumeAfter = cachedToken; + resumed.startAfter = Nullable!Bson.init; + } + return resumed; +} + +/// optionsForResume switches to resumeAfter with the cached token +unittest +{ + import std.typecons : nullable; + ChangeStreamOptions original; + original.startAfter = Bson(["_data": Bson("orig")]).nullable; + auto token = Bson(["_data": Bson("cached")]); + auto resumed = optionsForResume(original, token.nullable); + assert(resumed.resumeAfter == token.nullable); + assert(resumed.startAfter.isNull); +} + +/// The next resume point after consuming a change event. +struct ResumePoint { + /// The resume token to use on the next resume, or null to keep the previous one. + Nullable!Bson token; + /// Whether the consumed event ends the stream (an `invalidate`). + bool invalidated; +} + +/** Advances the resume point given the event about to be consumed. + + A normal event contributes its `_id` as the new resume token. An `invalidate` + event instead ends the stream and is deliberately NOT cached as the resume + token: a later resume would send `resumeAfter` with it, which the server + rejects (only `startAfter` is legal there) and the spec forbids anyway. + + Params: + previous = the resume token cached so far, or null if none yet. + event = the change event about to be consumed. + + Returns: the resume token to cache and whether the stream is now finished. +*/ +ResumePoint advanceResumePoint(Nullable!Bson previous, Bson event) +{ + if (isInvalidateEvent(event)) + return ResumePoint(previous, true); + auto token = resumeToken(event); + return ResumePoint(token.isNull ? previous : token, false); +} + +/// advanceResumePoint caches a normal event's token and stays live +unittest +{ + import std.typecons : nullable; + auto event = Bson([ + "_id": Bson(["_data": Bson("t1")]), + "operationType": Bson("insert") + ]); + auto point = advanceResumePoint(Nullable!Bson.init, event); + assert(point.token == Bson(["_data": Bson("t1")]).nullable); + assert(!point.invalidated); +} + +/// advanceResumePoint keeps the previous token for a tokenless event +unittest +{ + import std.typecons : nullable; + auto previous = Bson(["_data": Bson("prev")]).nullable; + auto point = advanceResumePoint(previous, Bson(["operationType": Bson("insert")])); + assert(point.token == previous); + assert(!point.invalidated); +} + +/// advanceResumePoint marks an invalidate event as finishing without caching its token +unittest +{ + import std.typecons : nullable; + auto previous = Bson(["_data": Bson("prev")]).nullable; + auto invalidate = Bson([ + "_id": Bson(["_data": Bson("inv")]), + "operationType": Bson("invalidate") + ]); + auto point = advanceResumePoint(previous, invalidate); + assert(point.invalidated); + // The invalidate token is NOT adopted as the resume point. + assert(point.token == previous); + assert(point.token != Bson(["_data": Bson("inv")]).nullable); +} + +/** An auto-resuming input range over a MongoDB change stream. + + It iterates change events like a normal cursor, but caches the latest resume + token and transparently re-opens the underlying cursor (via the supplied opener) + when the server reports a resumable error, so iteration survives transient + failures and elections. +*/ +struct ChangeStream(DocType = Bson) { + private { + MongoCursor!DocType delegate(ChangeStreamOptions) @safe m_open; + ChangeStreamOptions m_options; + MongoCursor!DocType m_cursor; + Nullable!Bson m_resumeToken; + bool m_started; + bool m_invalidated; + } + + /** Constructs a change stream from an opener delegate. + + Params: + open = runs the `$changeStream` aggregation for the given options and + returns its cursor; called again to resume after a resumable error. + options = the options the stream is first opened with. + */ + this(MongoCursor!DocType delegate(ChangeStreamOptions) @safe open, ChangeStreamOptions options) + { + m_open = open; + m_options = options; + } + + /** The most recent resume token observed. + + Usable to resume later via `ChangeStreamOptions.resumeAfter`. Null until the + first event is consumed. + */ + @property Nullable!Bson resumeToken() { return m_resumeToken; } + + /** Range primitive: whether no further change events are currently available. + + On a resumable server error the stream re-opens from the cached token and + retries before reporting emptiness. + + $(B Tailable semantics — important): a change stream is a tailable cursor, so + `empty` reflects only whether an event is available $(I right now). On an idle but + live stream the underlying getMore returns an empty batch and `empty` is `true`, + even though more events may still arrive — so `empty` is $(B non-monotonic): it can + return `true` now and `false` later. A plain `foreach (event; stream) {}` therefore + stops at the first idle moment rather than blocking for the next event. + + To follow a live stream, re-poll in a loop, e.g. + `while (true) { if (!stream.empty) { use(stream.front); stream.popFront(); } }`. + (A blocking `tryNext`/awaitData primitive is not yet provided.) + */ + @property bool empty() + { + // An invalidate event ends the stream; never auto-resume past it (the server + // rejects resumeAfter with an invalidate token and the spec forbids it). + if (m_invalidated) + return true; + ensureStarted(); + try + return m_cursor.empty; + catch (MongoException e) { + if (!isResumableChangeStreamError(e)) + throw e; + m_cursor = m_open(optionsForResume(m_options, m_resumeToken)); + return m_cursor.empty; + } + } + + /// Range primitive returning the current change event. + @property DocType front() { ensureStarted(); return m_cursor.front; } + + /// Range primitive that advances to the next change event, caching the consumed + /// event's resume token first. + void popFront() + { + ensureStarted(); + cacheResumeToken(); + m_cursor.popFront(); + } + + private void ensureStarted() + { + if (m_started) return; + m_cursor = m_open(m_options); + m_started = true; + } + + private void cacheResumeToken() + { + static if (is(DocType == Bson)) + auto eventBson = m_cursor.front; + else + auto eventBson = () @safe { return serializeToBson(m_cursor.front); }(); + auto point = advanceResumePoint(m_resumeToken, eventBson); + m_resumeToken = point.token; + m_invalidated = point.invalidated; + } +} + +/// The MongoDB wire name for a fullDocument mode, or null for the server default. +private string mongoName(ChangeStreamFullDocument fullDocument) +{ + final switch (fullDocument) { + case ChangeStreamFullDocument.default_: return null; + case ChangeStreamFullDocument.updateLookup: return "updateLookup"; + case ChangeStreamFullDocument.whenAvailable: return "whenAvailable"; + case ChangeStreamFullDocument.required: return "required"; + } +} + +/// Sets `obj[key]` to the token's value, but only when the token is present. +private void setIfPresent(ref Bson obj, string key, Nullable!Bson value) +{ + if (!value.isNull) + obj[key] = value.get; +} diff --git a/tests/mongodb/change-stream/dub.json b/tests/mongodb/change-stream/dub.json new file mode 100644 index 0000000000..9de0c8835f --- /dev/null +++ b/tests/mongodb/change-stream/dub.json @@ -0,0 +1,7 @@ +{ + "name": "change-stream-test", + "description": "MongoDB change stream (watch) integration test", + "dependencies": { + "vibe-d:mongodb": {"path": "../../../"} + } +} diff --git a/tests/mongodb/change-stream/run.sh b/tests/mongodb/change-stream/run.sh new file mode 100755 index 0000000000..fe8a062f86 --- /dev/null +++ b/tests/mongodb/change-stream/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Change-stream integration test. The app adapts to the deployment it is pointed at, +# so this harness runs it against both: +# Phase 1 (standalone): every watch() must be rejected with the replica-set topology +# error, proving the driver builds a well-formed $changeStream command. +# Phase 2 (single-node replica set): a real insert must be observed as an `insert` +# change event carrying a resume token. This positive path is dead on a +# standalone (the default CI harness), which is what this script fixes. +set -e + +STANDALONE_PORT=22840 +RS_PORT=22841 + +PIDS=() + +cleanup() { + echo "[INFO] Cleaning up mongod instances..." + for pid in "${PIDS[@]}"; do + if [ -n "$pid" ] && [ "$pid" != "0" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + fi + done + for pid in "${PIDS[@]}"; do + if [ -n "$pid" ] && [ "$pid" != "0" ]; then + while kill -0 "$pid" 2>/dev/null; do sleep 1; done + fi + done + rm -rf db + rm -f log*.txt +} +trap cleanup EXIT + +wait_for_primary() { + local port=$1 + for i in $(seq 1 30); do + local ok + ok=$($MONGO --quiet "mongodb://127.0.0.1:$port" \ + --eval "try { db.hello().isWritablePrimary } catch (e) { false }" 2>/dev/null || echo false) + if [ "$ok" = "true" ]; then + echo "[INFO] Primary ready on port $port" + return 0 + fi + sleep 2 + done + echo "[ERROR] No primary elected on port $port after 60s" + return 1 +} + +rm -f log*.txt +rm -rf db + +echo "========================================================" +echo " Phase 1: Standalone — every watch() rejected" +echo "========================================================" +mkdir -p db/standalone +PIDS[0]=$(mongod --logpath log0.txt --bind_ip 127.0.0.1 --port $STANDALONE_PORT \ + --dbpath db/standalone --fork | grep -Po 'forked process: \K\d+') +echo "[INFO] Started standalone mongod on $STANDALONE_PORT (PID ${PIDS[0]})" + +if ! eval $DUB_INVOKE -- $STANDALONE_PORT ; then + echo "[FAIL] Standalone change-stream test failed" + exit 1 +fi +echo "[PASS] Standalone change-stream test passed" + +echo "========================================================" +echo " Phase 2: Replica set — insert observed with resume token" +echo "========================================================" +mkdir -p db/rs +PIDS[1]=$(mongod --logpath log1.txt --bind_ip 127.0.0.1 --port $RS_PORT \ + --dbpath db/rs --replSet csrs0 --fork | grep -Po 'forked process: \K\d+') +echo "[INFO] Started replica-set mongod on $RS_PORT (PID ${PIDS[1]})" + +for attempt in $(seq 1 5); do + if $MONGO --quiet "mongodb://127.0.0.1:$RS_PORT" \ + --eval "rs.initiate({_id:'csrs0', members:[{_id:0, host:'127.0.0.1:$RS_PORT'}]})" 2>/dev/null; then + echo "[INFO] Replica set initiated" + break + fi + echo "[INFO] rs.initiate attempt $attempt failed, retrying in 2s..." + sleep 2 +done + +wait_for_primary $RS_PORT + +if ! eval $DUB_INVOKE -- $RS_PORT ; then + echo "[FAIL] Replica-set change-stream test failed" + exit 1 +fi +echo "[PASS] Replica-set change-stream test passed" + +echo "============================================" +echo "All change-stream tests passed!" +echo "============================================" diff --git a/tests/mongodb/change-stream/source/app.d b/tests/mongodb/change-stream/source/app.d new file mode 100644 index 0000000000..ab16ce0e77 --- /dev/null +++ b/tests/mongodb/change-stream/source/app.d @@ -0,0 +1,142 @@ +/// Requires a mongo service running on localhost; port via args[1]. +/// Verifies MongoDB change stream (watch) support end-to-end. +/// +/// Change streams are only available on replica sets and sharded clusters, so +/// the test adapts to the deployment it is pointed at: +/// * Against a STANDALONE server (the default CI harness) it confirms the +/// driver builds and transmits a valid `$changeStream` aggregation: the +/// server must reject it with the topology error ("only supported on replica +/// sets"), proving the command reached the server well-formed rather than +/// failing to parse. Collection-, database- and client-level watch() are all +/// exercised, with and without options/a user pipeline. +/// * Against a REPLICA SET it confirms watch() opens and a real insert is +/// observed as an `insert` change event carrying a resume token. + +module app; + +import vibe.data.bson; +import vibe.db.mongo.mongo; +import vibe.db.mongo.impl.changestream : ChangeStreamOptions, ChangeStreamFullDocument; + +import vibe.core.log; +import vibe.core.core : sleep; + +import core.time : MonoTime, seconds, msecs; +import std.algorithm : canFind; +import std.conv : to; + +int main(string[] args) +{ + ushort port = args.length > 1 + ? args[1].to!ushort + : MongoClientSettings.defaultPort; + runTest(port); + return 0; +} + +void runTest(ushort port) +{ + auto client = connectMongoDB("127.0.0.1", port); + + if (isChangeStreamCapable(client)) + runReplicaSetTest(client); + else + runStandaloneTest(client); +} + +/// Change streams need a replica set member or a mongos; a hello reply advertises +/// `setName` for a replica set and `msg == "isdbgrid"` for a sharded cluster. +bool isChangeStreamCapable(MongoClient client) +{ + auto hello = client.getDatabase("admin").runCommandChecked(Bson(["hello": Bson(1)])); + return hello["setName"].type != Bson.Type.null_ + || hello["msg"].opt!string == "isdbgrid"; +} + +/// On a standalone the server rejects any `$changeStream` aggregation. Opening a +/// watch and forcing the round-trip must raise that specific topology error, +/// which proves the driver transmitted a well-formed change-stream command. +void runStandaloneTest(MongoClient client) +{ + auto coll = client.getCollection("test.changestream"); + + assertChangeStreamUnsupported({ cast(void) coll.watch().empty; }, + "collection-level watch"); + + ChangeStreamOptions options; + options.fullDocument = ChangeStreamFullDocument.updateLookup; + auto userPipeline = [Bson(["$match": Bson(["operationType": Bson("insert")])])]; + assertChangeStreamUnsupported({ cast(void) coll.watch(userPipeline, options).empty; }, + "collection-level watch with options and a user pipeline"); + + assertChangeStreamUnsupported({ cast(void) client.getDatabase("test").watch().empty; }, + "database-level watch"); + + assertChangeStreamUnsupported({ cast(void) client.watch().empty; }, + "client-level (deployment) watch"); + + logInfo("Standalone change-stream test OK: the server rejected every watch() with the replica-set topology error."); +} + +/// Runs `body` and asserts it threw the change-stream topology error rather than +/// a malformed-command error (which would indicate the driver built it wrong). +void assertChangeStreamUnsupported(scope void delegate() body, string what) +{ + bool threw = false; + string message; + try + body(); + catch (Exception e) { + threw = true; + message = e.msg; + } + + assert(threw, what ~ " on a standalone must be rejected by the server"); + assert(message.canFind("replica") || message.canFind("changeStream") || message.canFind("$changeStream"), + what ~ " must fail with the change-stream topology error, got: " ~ message); +} + +/// On a replica set, a watch observes a subsequent insert as an `insert` event +/// carrying a resume token. +void runReplicaSetTest(MongoClient client) +{ + auto coll = client.getCollection("test.changestream"); + coll.drop(); + + // MongoDB 3.6 rejects opening a $changeStream on a non-existent database (newer servers + // tolerate a missing collection); create the collection explicitly before watching. + client.getDatabase("test").runCommandChecked(Bson(["create": Bson("changestream")])); + + auto stream = coll.watch(); + + // The change stream does not capture the aggregate's start point (postBatchResumeToken + // is untracked — see L14(B)), so a priming read is required to anchor the watch point + // before the write; otherwise the cursor effectively starts at the first getMore and + // never observes an insert that happened before it. + assert(stream.empty, "a freshly opened change stream has no buffered events yet"); + + coll.insertOne(["greeting": "hello change streams"]); + + // A change stream is a non-blocking tailable cursor: `empty` is non-monotonic and + // a getMore can return an empty batch before the event is visible. Poll until the + // insert is observed, per the documented usage pattern, rather than checking once. + auto deadline = MonoTime.currTime + 10.seconds; + while (stream.empty) { + assert(MonoTime.currTime < deadline, + "the change stream must observe the inserted document within the deadline"); + sleep(100.msecs); + } + + auto event = stream.front; + assert(event["operationType"].get!string == "insert", + "the observed change event must be an insert"); + + // The resume token is cached from the consumed event, so it is only available + // after popFront advances past it. + stream.popFront(); + assert(!stream.resumeToken.isNull, + "consuming an event must cache a resume token"); + + coll.drop(); + logInfo("Replica-set change-stream test OK: insert observed with resume token %s.", stream.resumeToken.get); +}