[WIP] feat: Basic multi-relay onboarding

deltachat-jsonrpc/src/api.rs

@@ -593,7 +593,9 @@ impl CommandApi {
     /// UI implementations must use [`Self::set_transport_unpublished`] instead.
     async fn delete_transport(&self, account_id: u32, addr: String) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        ctx.delete_transport(&addr).await
+        ctx.delete_transport(&addr)
+            .await
+            .context("delete_transport")
     }
 
     /// Change whether the transport is unpublished.

src/automatic_transport_management.rs

@@ -0,0 +1,142 @@
+use std::pin::Pin;
+
+use anyhow::Result;
+use deltachat_contact_tools::addr_normalize;
+use rand::distr::{Alphanumeric, SampleString};
+use rand::seq::IndexedRandom;
+
+use crate::config::Config;
+use crate::log::LogExt as _;
+use crate::login_param::{EnteredCertificateChecks, EnteredImapLoginParam};
+use crate::{configure::EnteredLoginParam, context::Context, tools::time};
+
+/// The target number of transports we try to reach.
+const NUM_TRANSPORTS_TARGET: usize = 3;
+/// How often we want to try adding new transports.
+const AUTOMATIC_ADDITION_DEBOUNCE_SECONDS: i64 = 60 * 60; // one hour
+/// How long we ignore a transport candidate after failing to create an account there:
+const BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT: i64 = 60 * 60 * 24 * 7; // one week
+
+pub(crate) fn maybe_add_additional_transports(
+    context: Context,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+    Box::pin(async move {
+        maybe_add_additional_transports_inner(&context)
+            .await
+            .log_err(&context)
+            .ok();
+    })
+}
+
+async fn maybe_add_additional_transports_inner(context: &Context) -> Result<()> {
+    let now = time();
+    let mut transport_added = false;
+    info!(context, "dbg maybe_add_additional_transports");
+
+    let Ok(_housekeeping_lock) = context.housekeeping_mutex.try_lock() else {
+        // Housekeeping or automatic relay management is already running in another thread, do nothing.
+        info!(context, "dbg skipping because of taken mutex");
+        return Ok(());
+    };
+    if context
+        .get_config_i64(Config::LastAutomaticTransportManagement)
+        .await?
+        > now.saturating_sub(AUTOMATIC_ADDITION_DEBOUNCE_SECONDS)
+    {
+        info!(context, "dbg already ran recently");
+        return Ok(());
+    }
+    // TODO uncomment this after I'm done with testing:
+    if context
+        .get_config_bool(Config::AutomaticTransportManagement)
+        .await?
+    {
+        info!(context, "dbg automatic transport management disabled");
+        return Ok(());
+    }
+    // Set the config at the beginning to avoid endless loops.
+    // Race conditions are not a concern because we locked the mutex.
+    context
+        .set_config_internal(
+            Config::LastAutomaticTransportManagement,
+            Some(&now.to_string()),
+        )
+        .await?;
+
+    // Using `for` instead of `while` to prevent infinite loop
+    for _ in 0..NUM_TRANSPORTS_TARGET {
+        if context.count_transports().await? >= NUM_TRANSPORTS_TARGET {
+            info!(context, "dbg target reached");
+            return Ok(());
+        }
+
+        // First, query all candidates that were not tried since `BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT` seconds.
+        // Domains that are already used are excluded.
+        let cutoff_timestamp = now.saturating_sub(BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT);
+        let candidates: Vec<String> = context
+            .sql
+            .query_map_vec(
+                "SELECT domain FROM relay_candidates WHERE last_tried<?
+                AND NOT EXISTS (
+                    SELECT 1
+                    FROM transports
+                    WHERE substr(addr, instr(addr, '@') + 1) = domain
+                )",
+                (cutoff_timestamp,),
+                |row| Ok(row.get::<_, String>(0)?),
+            )
+            .await?;
+
+        let Some(domain) = candidates.choose(&mut rand::rng()) else {
+            info!(
+                context,
+                "maybe_add_additional_relays: No suitable candidates"
+            );
+            return Ok(());
+        };
+        info!(context, "dbg from {candidates:?}, chose {domain}");
+
+        let param = login_param_from_domain(domain);
+        let res = crate::configure::configure(context, &param).await;
+        if res.is_err() {
+            info!(context, "dbg error {res:?}");
+            context
+                .sql
+                .execute("UPDATE relay_candidates SET last_tried=?", (now,))
+                .await?;
+        }
+
+        // TODO: Decide whether we want to immediately try again with another relay,
+        // if this one failed. If yes, remove the next line.
+        res?;
+
+        transport_added = true;
+        info!(context, "dbg success");
+    }
+    if transport_added {
+        info!(context, "dbg restarting");
+        context.restart_io_if_running().await;
+    }
+
+    Ok(())
+}
+
+pub(crate) fn login_param_from_domain(domain: &str) -> EnteredLoginParam {
+    let rng = &mut rand::rng();
+    let username = Alphanumeric.sample_string(rng, 9);
+    let addr = username + "@" + domain;
+    let addr = addr_normalize(&addr);
+    // 22 * log2(26 * 2 + 10) = 130 bits of entropy
+    let password = Alphanumeric.sample_string(rng, 22);
+
+    let param = EnteredLoginParam {
+        addr,
+        imap: EnteredImapLoginParam {
+            password,
+            ..Default::default()
+        },
+        smtp: Default::default(),
+        certificate_checks: EnteredCertificateChecks::Strict,
+    };
+    param
+}

src/config.rs

@@ -353,6 +353,13 @@ pub enum Config {
     /// Timestamp of the last `CantDecryptOutgoingMsgs` notification.
     LastCantDecryptOutgoingMsgs,
 
+    /// Timestamp of the last call to
+    /// [`crate::automatic_transport_management::maybe_add_additional_transports`]
+    LastAutomaticTransportManagement,
+
+    /// Whether to automatically add/remove transports
+    AutomaticTransportManagement,
+
     /// Whether to avoid using IMAP IDLE even if the server supports it.
     ///
     /// This is a developer option for testing "fake idle".

src/configure.rs

@@ -49,21 +49,6 @@ use crate::{chat, provider};
 /// See <https://github.com/chatmail/core/issues/7608>.
 pub(crate) const MAX_RELAYS: usize = 5;
 
-/// Hard-coded candidates for default relays.
-/// In the future, we want to use it during onboarding;
-/// note that before onboarding automatically on any of these,
-/// we need to ask the admins whether their relay is able to handle this.
-/// For now, this is just the first 6 relays from chatmail.at/relays.
-#[allow(unused)]
-const DEFAULT_RELAY_CANDIDATES: &[&str] = &[
-    "mehl.cloud",
-    "mailchat.pl",
-    "chatmail.woodpeckersnest.space",
-    "chatmail.culturanerd.it",
-    "tarpit.fun",
-    "d.gaufr.es",
-];
-
 macro_rules! progress {
     ($context:tt, $progress:expr, $comment:expr) => {
         assert!(
@@ -238,39 +223,45 @@ impl Context {
         let removed_transport_id = self
             .sql
             .transaction(|transaction| {
-                let primary_addr = transaction.query_row(
-                    "SELECT value FROM config WHERE keyname='configured_addr'",
-                    (),
-                    |row| {
-                        let addr: String = row.get(0)?;
-                        Ok(addr)
-                    },
-                )?;
+                let primary_addr = transaction
+                    .query_row(
+                        "SELECT value FROM config WHERE keyname='configured_addr'",
+                        (),
+                        |row| {
+                            let addr: String = row.get(0)?;
+                            Ok(addr)
+                        },
+                    )
+                    .context("querying addr")?;
 
                 if primary_addr == addr {
                     bail!("Cannot delete primary transport");
                 }
-                let (transport_id, add_timestamp) = transaction.query_row(
-                    "DELETE FROM transports WHERE addr=? RETURNING id, add_timestamp",
-                    (addr,),
-                    |row| {
-                        let id: u32 = row.get(0)?;
-                        let add_timestamp: i64 = row.get(1)?;
-                        Ok((id, add_timestamp))
-                    },
-                )?;
+                let (transport_id, add_timestamp) = transaction
+                    .query_row(
+                        "DELETE FROM transports WHERE addr=? RETURNING id, add_timestamp",
+                        (addr,),
+                        |row| {
+                            let id: u32 = row.get(0)?;
+                            let add_timestamp: i64 = row.get(1)?;
+                            Ok((id, add_timestamp))
+                        },
+                    )
+                    .with_context(|| format!("Deleting transport, addr: {addr}"))?;
 
                 // Removal timestamp should not be lower than addition timestamp
                 // to be accepted by other devices when synced.
                 let remove_timestamp = std::cmp::max(now, add_timestamp);
 
-                transaction.execute(
-                    "INSERT INTO removed_transports (addr, remove_timestamp)
+                transaction
+                    .execute(
+                        "INSERT INTO removed_transports (addr, remove_timestamp)
                      VALUES (?, ?)
                      ON CONFLICT (addr)
                      DO UPDATE SET remove_timestamp = excluded.remove_timestamp",
-                    (addr, remove_timestamp),
-                )?;
+                        (addr, remove_timestamp),
+                    )
+                    .context("Inserting into removed transport")?;
 
                 Ok(transport_id)
             })
@@ -604,7 +595,11 @@ async fn get_configured_param(
     Ok(configured_login_param)
 }
 
-async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Option<&'static Provider>> {
+// TODO maybe this function name should be changed
+pub(crate) async fn configure(
+    ctx: &Context,
+    param: &EnteredLoginParam,
+) -> Result<Option<&'static Provider>> {
     progress!(ctx, 1);
 
     let configured_param = get_configured_param(ctx, param).await?;

src/imap/idle.rs

@@ -8,7 +8,7 @@ use tokio::time::timeout;
 use super::Imap;
 use super::session::Session;
 use crate::context::Context;
-use crate::log::warn;
+use crate::log::{LogExt as _, warn};
 use crate::net::TIMEOUT;
 use crate::tools::{self, time_elapsed};
 
@@ -51,6 +51,13 @@ impl Session {
             return Ok(self);
         }
 
+        // we try to add additional relays right before going into IDLE mode,
+        // because apparently we are connected, but don't have anything important to do.
+        tokio::task::spawn(
+            crate::automatic_transport_management::maybe_add_additional_transports(context.clone()),
+        );
+        info!(context, "dbg spawned transports task");
+
         let mut handle = self.inner.idle();
         handle
             .init()

src/lib.rs

@@ -55,6 +55,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+mod automatic_transport_management;
 pub mod blob;
 pub mod calls;
 pub mod chat;

src/qr.rs

@@ -13,6 +13,7 @@ use rand::TryRngCore as _;
 use rand::distr::{Alphanumeric, SampleString};
 use serde::Deserialize;
 
+use crate::automatic_transport_management::login_param_from_domain;
 use crate::config::Config;
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
@@ -828,20 +829,7 @@ pub(crate) async fn login_param_from_account_qr(
         .context("Invalid DCACCOUNT scheme")?;
 
     if !payload.starts_with(HTTPS_SCHEME) {
-        let rng = &mut rand::rngs::OsRng.unwrap_err();
-        let username = Alphanumeric.sample_string(rng, 9);
-        let addr = username + "@" + payload;
-        let password = Alphanumeric.sample_string(rng, 50);
-
-        let param = EnteredLoginParam {
-            addr,
-            imap: EnteredImapLoginParam {
-                password,
-                ..Default::default()
-            },
-            smtp: Default::default(),
-            certificate_checks: EnteredCertificateChecks::Strict,
-        };
+        let param = login_param_from_domain(payload);
         return Ok(param);
     }
 

src/sql/migrations.rs

@@ -2522,6 +2522,40 @@ UPDATE msgs SET state=24 WHERE state=18; -- Change OutPreparing to OutFailed.
         .await?;
     }
 
+    // TODO should we call it relay_candidates or transport_candidates?
+    inc_and_check(&mut migration_version, 159)?;
+    if dbversion < migration_version {
+        // TODO put a better list here
+        const DEFAULT_RELAY_CANDIDATES: &[&str] = &[
+            "mehl.cloud",
+            "mailchat.pl",
+            "chatmail.woodpeckersnest.space",
+            "chatmail.culturanerd.it",
+            "tarpit.fun",
+            "d.gaufr.es",
+        ];
+
+        sql.execute_migration_transaction(
+            |transaction| {
+                transaction.execute(
+                    "CREATE TABLE relay_candidates(
+                        domain TEXT PRIMARY KEY NOT NULL,
+                        last_tried TEXT NOT NULL DEFAULT 0
+                    ) STRICT",
+                    (),
+                )?;
+                let mut statement =
+                    transaction.prepare("INSERT INTO relay_candidates(domain) VALUES (?)")?;
+                for domain in DEFAULT_RELAY_CANDIDATES {
+                    statement.execute((domain,))?;
+                }
+                Ok(())
+            },
+            migration_version,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?

src/transport.rs

@@ -831,7 +831,9 @@ pub(crate) async fn sync_transports(
 
 /// Same as `context.restart_io_if_running()`, but `Box::pin`ed and with a `+ Send` bound,
 /// so that it can be called recursively.
-fn restart_io_if_running_boxed(context: Context) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+pub(crate) fn restart_io_if_running_boxed(
+    context: Context,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
     Box::pin(async move { context.restart_io_if_running().await })
 }