From 2765bc92d9ae25523eb132ad20a077ba287b8c8e Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 27 May 2025 18:15:34 -0700 Subject: [PATCH 01/29] feat: support configuring the manifold URL I have a highly trivial Manifold implementation internally at Mercury (https://github.com/mercurytechnologies/locally-euclidean) for which I would like to set up for buck2 to receive rage and all that at. This allows for fully implementing the build logs feature mentioned in https://github.com/facebook/buck2/issues/441. There's still a buck2_core::facebook_only() or two that need inspected/removed and some more tidying. --- app/buck2_client_ctx/src/client_ctx.rs | 9 ++ .../src/persist_event_logs.rs | 23 ++- .../src/upload_re_logs.rs | 2 +- app/buck2_cmd_query_server/src/html.rs | 18 ++- app/buck2_cmd_rage_client/src/dice.rs | 6 +- app/buck2_cmd_rage_client/src/manifold.rs | 26 +++- app/buck2_cmd_rage_client/src/rage.rs | 4 +- app/buck2_common/src/init.rs | 5 + app/buck2_common/src/manifold.rs | 136 ++++++++++++++---- app/buck2_event_log/src/lib.rs | 3 - app/buck2_file_watcher/src/watchman/core.rs | 10 +- app/buck2_server_commands/src/install.rs | 23 ++- 12 files changed, 202 insertions(+), 63 deletions(-) diff --git a/app/buck2_client_ctx/src/client_ctx.rs b/app/buck2_client_ctx/src/client_ctx.rs index 40bb1d847d1ff..f57716e03694d 100644 --- a/app/buck2_client_ctx/src/client_ctx.rs +++ b/app/buck2_client_ctx/src/client_ctx.rs @@ -20,6 +20,7 @@ use buck2_common::argv::Argv; use buck2_common::init::LogDownloadMethod; use buck2_common::invocation_paths::InvocationPaths; use buck2_common::invocation_paths_result::InvocationPathsResult; +use buck2_common::manifold::BucketsConfig; use buck2_core::error::buck2_hard_error_env; use buck2_core::error::buck2_show_soft_errors_env; use buck2_error::BuckErrorContext; @@ -312,6 +313,14 @@ impl<'a> ClientCommandContext<'a> { .log_download_method .clone()) } + + pub fn buckets_config(&self) -> buck2_error::Result> { + Ok(self + .immediate_config + .daemon_startup_config()? + .buckets_config + .clone()) + } } /// Provides a common interface for buck subcommands that use event subscribers for logging. diff --git a/app/buck2_cmd_debug_client/src/persist_event_logs.rs b/app/buck2_cmd_debug_client/src/persist_event_logs.rs index 30d4b18b0bda7..5daf9159612ab 100644 --- a/app/buck2_cmd_debug_client/src/persist_event_logs.rs +++ b/app/buck2_cmd_debug_client/src/persist_event_logs.rs @@ -16,6 +16,7 @@ use buck2_client_ctx::events_ctx::EventsCtx; use buck2_client_ctx::exit_result::ExitResult; use buck2_common::chunk_reader::ChunkReader; use buck2_common::manifold; +use buck2_common::manifold::BucketsConfig; use buck2_common::manifold::ManifoldChunkedUploader; use buck2_common::manifold::ManifoldClient; use buck2_core::soft_error; @@ -81,9 +82,12 @@ impl PersistEventLogsCommand { events_ctx.log_invocation_record = false; let sink = create_scribe_sink(&ctx)?; let trace_id = self.trace_id.clone(); + let buckets_config = ctx.buckets_config()?; + ctx.with_runtime(|mut ctx| async move { let mut stdin = io::BufReader::new(ctx.stdin()); - let (local_result, remote_result) = self.write_and_upload(&mut stdin).await; + let (local_result, remote_result) = + self.write_and_upload(buckets_config, &mut stdin).await; let (local_error_messages, local_error_category, local_success) = status_from_result(local_result); @@ -106,6 +110,7 @@ impl PersistEventLogsCommand { async fn write_and_upload( self, + buckets_config: Option, stdin: impl io::AsyncBufRead + Unpin, ) -> (buck2_error::Result<()>, buck2_error::Result<()>) { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); @@ -122,7 +127,13 @@ impl PersistEventLogsCommand { } }; let write = write_task(&file, tx, stdin); - let upload = upload_task(&file, rx, self.manifold_name, self.no_upload); + let upload = upload_task( + &file, + rx, + buckets_config, + self.manifold_name, + self.no_upload, + ); // Wait for both tasks to finish. If the upload fails we want to keep writing to disk let (write_result, upload_result) = tokio::join!(write, upload); @@ -174,6 +185,7 @@ async fn create_log_file(local_path: String) -> Result, mut rx: tokio::sync::mpsc::UnboundedReceiver, + buckets_config: Option, manifold_name: String, no_upload: bool, ) -> buck2_error::Result<()> { @@ -181,7 +193,12 @@ async fn upload_task( return Ok(()); } - let manifold_client = ManifoldClient::new().await?; + let manifold_client = ManifoldClient::new_with_config(buckets_config).await?; + // No need to do more work if we don't have an endpoint configured (default + // in OSS) + if !manifold_client.will_upload() { + return Ok(()); + } let manifold_path = format!("flat/{manifold_name}"); let mut uploader = Uploader::new(file_mutex, &manifold_path, &manifold_client)?; diff --git a/app/buck2_cmd_debug_client/src/upload_re_logs.rs b/app/buck2_cmd_debug_client/src/upload_re_logs.rs index ca0f96615c37b..fad08490daa89 100644 --- a/app/buck2_cmd_debug_client/src/upload_re_logs.rs +++ b/app/buck2_cmd_debug_client/src/upload_re_logs.rs @@ -34,7 +34,7 @@ impl BuckSubcommand for UploadReLogsCommand { ) -> ExitResult { buck2_core::facebook_only(); events_ctx.log_invocation_record = false; - let manifold = ManifoldClient::new().await?; + let manifold = ManifoldClient::new_with_config(ctx.buckets_config()?).await?; // TODO: This should receive the path from the caller. let re_logs_dir = ctx.paths()?.re_logs_dir(); upload_re_logs( diff --git a/app/buck2_cmd_query_server/src/html.rs b/app/buck2_cmd_query_server/src/html.rs index 95ffa18a96e03..ff2d9ae46ed97 100644 --- a/app/buck2_cmd_query_server/src/html.rs +++ b/app/buck2_cmd_query_server/src/html.rs @@ -41,19 +41,17 @@ impl Html { let html_out = output_format(graph)?; let mut cursor = &mut Cursor::new(html_out.as_bytes()); let manifold_path = format!("flat/{trace_id}-graph.html"); - let manifold = ManifoldClient::new().await?; + // FIXME(jadel): thread configuration through + let manifold = ManifoldClient::new_with_config(None).await?; + let bucket = Bucket::EVENT_LOGS; manifold - .read_and_upload( - Bucket::EVENT_LOGS, - &manifold_path, - Default::default(), - &mut cursor, - ) + .read_and_upload(bucket, &manifold_path, Default::default(), &mut cursor) .await?; - res = format!( - "\nView html in your browser: https://interncache-all.fbcdn.net/manifold/buck2_logs/{manifold_path} (requires VPN/lighthouse)\n" - ); + let url = manifold + .file_view_url(&bucket, &manifold_path) + .unwrap_or_default(); + res = format!("\nView html in your browser: {url} (requires VPN/lighthouse)\n"); } #[cfg(not(fbcode_build))] { diff --git a/app/buck2_cmd_rage_client/src/dice.rs b/app/buck2_cmd_rage_client/src/dice.rs index 8ac0c26d1a971..bbefa71035f31 100644 --- a/app/buck2_cmd_rage_client/src/dice.rs +++ b/app/buck2_cmd_rage_client/src/dice.rs @@ -47,7 +47,11 @@ pub async fn upload_dice_dump( ) .await?; - Ok(manifold_leads(&manifold_bucket, manifold_filename)) + Ok(manifold_leads( + manifold, + &manifold_bucket, + manifold_filename, + )) } struct DiceDump { diff --git a/app/buck2_cmd_rage_client/src/manifold.rs b/app/buck2_cmd_rage_client/src/manifold.rs index 8e57c00ce8309..2369697101ada 100644 --- a/app/buck2_cmd_rage_client/src/manifold.rs +++ b/app/buck2_cmd_rage_client/src/manifold.rs @@ -16,11 +16,23 @@ use buck2_fs::async_fs_util; use buck2_fs::error::IoResultExt; use buck2_fs::paths::abs_path::AbsPath; -pub(crate) fn manifold_leads(bucket: &Bucket, filename: String) -> String { - let full_path = bucket.path(filename.as_str()); - let command = format!("manifold get {full_path}"); - let url = bucket.intern_url(filename.as_str()); - format!("{command}\n{url}") +pub(crate) fn manifold_leads( + manifold: &ManifoldClient, + bucket: &Bucket, + filename: String, +) -> String { + let url = manifold.file_view_url(bucket, &filename); + let command = manifold.file_dump_command(bucket, &filename); + let mut out = String::new(); + if let Some(url) = url { + out.push_str(&url); + } + if let Some(command) = command { + out.push('\n'); + out.push_str(&command); + } + + out } pub(crate) async fn file_to_manifold( @@ -37,7 +49,7 @@ pub(crate) async fn file_to_manifold( .read_and_upload(bucket, &filename, Default::default(), &mut file) .await?; - Ok(manifold_leads(&bucket, filename)) + Ok(manifold_leads(manifold, &bucket, filename)) } pub(crate) async fn buf_to_manifold( @@ -52,5 +64,5 @@ pub(crate) async fn buf_to_manifold( .read_and_upload(bucket, &filename, Default::default(), &mut cursor) .await?; - Ok(manifold_leads(&bucket, filename)) + Ok(manifold_leads(manifold, &bucket, filename)) } diff --git a/app/buck2_cmd_rage_client/src/rage.rs b/app/buck2_cmd_rage_client/src/rage.rs index 4afe4ae377bd4..29f8e465adad1 100644 --- a/app/buck2_cmd_rage_client/src/rage.rs +++ b/app/buck2_cmd_rage_client/src/rage.rs @@ -114,7 +114,7 @@ impl RageCommand { let client_ctx = ctx.empty_client_context("rage")?; // Don't fail the rage if you can't figure out whether to do vpnless. - let manifold = ManifoldClient::new().await?; + let manifold = ManifoldClient::new_with_config(ctx.buckets_config()?).await?; let rage_id = TraceId::new(); let mut manifold_id = format!("{rage_id}"); @@ -528,7 +528,7 @@ async fn upload_re_logs_impl( let filename = format!("flat/{}-re_logs.zst", &re_session_id); upload_re_logs(manifold, bucket, re_logs_dir, &re_session_id, &filename).await?; - Ok(manifold_leads(&bucket, filename)) + Ok(manifold_leads(manifold, &bucket, filename)) } async fn dispatch_result_event( diff --git a/app/buck2_common/src/init.rs b/app/buck2_common/src/init.rs index c9d2bb0ba7c3b..10cb654c67747 100644 --- a/app/buck2_common/src/init.rs +++ b/app/buck2_common/src/init.rs @@ -21,6 +21,7 @@ use serde::Serialize; use crate::legacy_configs::configs::LegacyBuckConfig; use crate::legacy_configs::key::BuckconfigKeyRef; +use crate::manifold::BucketsConfig; pub const DEFAULT_RETAINED_EVENT_LOGS: usize = 12; @@ -523,6 +524,7 @@ pub struct DaemonStartupConfig { pub http: HttpConfig, pub resource_control: ResourceControlConfig, pub log_download_method: LogDownloadMethod, + pub buckets_config: Option, pub health_check_config: HealthCheckConfig, pub retained_event_logs: usize, pub macos_qos_class: Option, @@ -600,6 +602,7 @@ impl DaemonStartupConfig { .map(ToOwned::to_owned), http: HttpConfig::from_config(config)?, resource_control: ResourceControlConfig::from_config(config)?, + buckets_config: BucketsConfig::from_config(config)?, log_download_method, health_check_config: HealthCheckConfig::from_config(config)?, retained_event_logs: config @@ -665,6 +668,8 @@ impl DaemonStartupConfig { } else { LogDownloadMethod::None }, + // TODO(jadel): is this a regression in test vs before? + buckets_config: None, health_check_config: HealthCheckConfig::default(), retained_event_logs: DEFAULT_RETAINED_EVENT_LOGS, macos_qos_class: None, diff --git a/app/buck2_common/src/manifold.rs b/app/buck2_common/src/manifold.rs index 996d57fe67667..640d3bd01442f 100644 --- a/app/buck2_common/src/manifold.rs +++ b/app/buck2_common/src/manifold.rs @@ -7,12 +7,13 @@ * of this source tree. You may select, at your option, one of the * above-listed licenses. */ - +//! Client to Manifold blob storage. use std::io; use std::time::Duration; use std::time::SystemTime; use std::time::UNIX_EPOCH; +use allocative::Allocative; use buck2_fs::paths::abs_path::AbsPath; use buck2_http::HttpClient; use buck2_http::HttpClientBuilder; @@ -25,10 +26,14 @@ use dupe::Dupe; use futures::stream::BoxStream; use futures::stream::StreamExt; use hyper::Response; +use serde::Deserialize; +use serde::Serialize; use tokio::fs::File; use tokio::io::AsyncRead; use crate::chunk_reader::ChunkReader; +use crate::legacy_configs::configs::LegacyBuckConfig; +use crate::legacy_configs::key::BuckconfigKeyRef; #[derive(Copy, Clone, Dupe)] pub struct Ttl { @@ -172,13 +177,68 @@ impl Bucket { } } -fn manifold_explorer_url(bucket: &Bucket, filename: String) -> String { - let full_path = format!("{}/{}", bucket.name, filename); - format!("https://www.internalfb.com/manifold/explorer/{full_path}") +/// Configuration for accessing a Manifold-like API for logs and other bucket-using features. +#[derive(Allocative, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct BucketsConfig { + /// Base URL for the uploads API. If not set, the uploads API is not used. + pub upload_url: String, + /// URL at which one can view a file `:bucketname/:filename` in a Web browser. + pub file_view_url: String, + /// Command that, when invoked, will retrieve a file from the given bucket, + /// for interactive use. + pub file_get_command: Option, +} + +impl BucketsConfig { + pub fn from_config(config: &LegacyBuckConfig) -> buck2_error::Result> { + let upload_url = config.parse(BuckconfigKeyRef { + section: "buckets", + property: "upload_url", + })?; + + let file_view_url: Option = config.parse(BuckconfigKeyRef { + section: "buckets", + property: "file_view_url", + })?; + + let file_get_command = config.parse(BuckconfigKeyRef { + section: "buckets", + property: "file_get_command", + })?; + + if upload_url.is_none() != file_view_url.is_none() { + return Err(buck2_error::buck2_error!( + buck2_error::ErrorTag::Input, + "Only one of buckets.upload_url and buckets.file_view_url is set" + )); + } + + if let Some(upload_url) = upload_url + && let Some(file_view_url) = file_view_url + { + Ok(Some(BucketsConfig { + file_view_url, + upload_url, + file_get_command, + })) + } else { + Ok(None) + } + } + + /// Infers an appropriate configuration at Meta. + fn infer_config(supports_vpnless: bool) -> Option { + let upload_url = internal_upload_url(supports_vpnless)?; + + Some(BucketsConfig { + upload_url: upload_url.to_owned(), + file_view_url: "https://interncache-all.fbcdn.net/manifold".to_owned(), + file_get_command: Some("manifold get".to_owned()), + }) + } } -/// Return the scheme+host manifold endpoint to upload to manifold, or None to not upload at all. -fn upload_endpoint_url(use_vpnless: bool) -> Option<&'static str> { +fn internal_upload_url(use_vpnless: bool) -> Option<&'static str> { #[cfg(fbcode_build)] if hostcaps::is_prod() { Some("https://manifold.facebook.net") @@ -196,18 +256,24 @@ fn upload_endpoint_url(use_vpnless: bool) -> Option<&'static str> { pub struct ManifoldClient { client: HttpClient, - manifold_url: Option, + config: Option, } impl ManifoldClient { - pub async fn new() -> buck2_error::Result { + pub async fn new_with_config(config: Option) -> buck2_error::Result { + #[cfg(fbcode_build)] let client = HttpClientBuilder::internal().await?.build(); - let manifold_url = upload_endpoint_url(client.supports_vpnless()).map(|s| s.to_owned()); + #[cfg(not(fbcode_build))] + let client = HttpClientBuilder::oss().await?.build(); - Ok(Self { - client, - manifold_url, - }) + let config = config.or_else(|| BucketsConfig::infer_config(client.supports_vpnless())); + + Ok(Self { client, config }) + } + + /// Whether the Manifold client has an endpoint and can upload. + pub fn will_upload(&self) -> bool { + self.config.is_some() } pub async fn write( @@ -217,13 +283,14 @@ impl ManifoldClient { buf: bytes::Bytes, ttl: Ttl, ) -> buck2_error::Result<()> { - let manifold_url = match &self.manifold_url { - None => return Ok(()), - Some(x) => x, + let Some(ref config) = self.config else { + return Ok(()); }; + let manifold_url = &config.upload_url; + let url = format!( - "{}/v0/write/{}?bucketName={}&apiKey={}&timeoutMsec=20000", - manifold_url, manifold_bucket_path, bucket.name, bucket.key + "{manifold_url}/v0/write/{manifold_bucket_path}?bucketName={}&apiKey={}&timeoutMsec=20000", + bucket.name, bucket.key ); let mut headers = vec![( @@ -261,13 +328,14 @@ impl ManifoldClient { buf: bytes::Bytes, offset: u64, ) -> buck2_error::Result<()> { - let manifold_url = match &self.manifold_url { - None => return Ok(()), - Some(x) => x, + let Some(ref config) = self.config else { + return Ok(()); }; + let manifold_url = &config.upload_url; + let url = format!( - "{}/v0/append/{}?bucketName={}&apiKey={}&timeoutMsec=20000&writeOffset={}", - manifold_url, manifold_bucket_path, bucket.name, bucket.key, offset + "{manifold_url}/v0/append/{manifold_bucket_path}?bucketName={}&apiKey={}&timeoutMsec=20000&writeOffset={offset}", + bucket.name, bucket.key ); let res = http_retry( @@ -336,7 +404,27 @@ impl ManifoldClient { self.read_and_upload(bucket, &filename, ttl, &mut file) .await?; - Ok(manifold_explorer_url(&bucket, filename)) + Ok(self.file_view_url(&bucket, &filename).unwrap_or_default()) + } + + /// Gets the URL for viewing an individual file. + pub fn file_view_url(&self, bucket: &Bucket, filename: &str) -> Option { + self.config + .as_ref() + .map(|config| format!("{}/{}/{}", config.file_view_url, bucket.name, filename)) + } + + /// Gets the command for getting an individual file, for interactive use. + pub fn file_dump_command(&self, bucket: &Bucket, filename: &str) -> Option { + // FIXME(jadel): This does overlap LogDownloadMethod::Curl, I am not + // sure what to do about that. + if let Some(ref config) = self.config + && let Some(ref command) = config.file_get_command + { + Some(format!("{} {}/{}", command, bucket.name, filename)) + } else { + None + } } } diff --git a/app/buck2_event_log/src/lib.rs b/app/buck2_event_log/src/lib.rs index 6a9661f750e4a..fc31a8dc866f3 100644 --- a/app/buck2_event_log/src/lib.rs +++ b/app/buck2_event_log/src/lib.rs @@ -31,9 +31,6 @@ pub mod write; pub mod writer; pub fn should_upload_log() -> buck2_error::Result { - if buck2_core::is_open_source() { - return Ok(false); - } Ok(!buck2_env!( "BUCK2_TEST_DISABLE_LOG_UPLOAD", bool, diff --git a/app/buck2_file_watcher/src/watchman/core.rs b/app/buck2_file_watcher/src/watchman/core.rs index 87925e20ad4a1..79bdb39595818 100644 --- a/app/buck2_file_watcher/src/watchman/core.rs +++ b/app/buck2_file_watcher/src/watchman/core.rs @@ -209,7 +209,8 @@ async fn with_timeout( } async fn write_to_manifold(buf: &[u8], name: &str) -> Option { - let manifold = ManifoldClient::new().await.ok()?; + // FIXME(jadel): thread configuration through + let manifold = ManifoldClient::new_with_config(None).await.ok()?; let filename = format!("flat/{}_{}_logs", uuid::Uuid::new_v4(), name); let ttl = Ttl::from_days(14); // 14 days should be plenty of time to take action @@ -222,12 +223,7 @@ async fn write_to_manifold(buf: &[u8], name: &str) -> Option { .await .ok()?; - let url = format!( - "https://interncache-all.fbcdn.net/manifold/{}/{}", - bucket.name, filename - ); - - Some(url) + manifold.file_view_url(&bucket, &filename) } async fn cmd_logs_to_manifold(cmd: &str, args: Vec<&str>) -> Option { diff --git a/app/buck2_server_commands/src/install.rs b/app/buck2_server_commands/src/install.rs index fc36e5c9d6431..4893c7694265c 100644 --- a/app/buck2_server_commands/src/install.rs +++ b/app/buck2_server_commands/src/install.rs @@ -829,17 +829,30 @@ async fn handle_install_request( } async fn upload_installer_logs(log_path: &AbsNormPathBuf) -> buck2_error::Result { - let manifold = ManifoldClient::new().await?; + // FIXME(jadel): thread configuration through + let manifold = ManifoldClient::new_with_config(None).await?; let trace_id: &str = &get_dispatcher().trace_id().to_string(); let manifold_filename = format!("flat/{trace_id}.log"); - manifold + let bucket = Bucket::INSTALLER_LOGS; + let url = manifold .upload_file( log_path, - manifold_filename, - Bucket::INSTALLER_LOGS, + manifold_filename.clone(), + bucket, Ttl::from_days(14), ) - .await + .await; + if cfg!(fbcode_build) { + let _unused = url?; + // We use the explorer url here rather than interncache since it is accessible off-vpn and + // we don't care about the truncation of long logs. + Ok(format!( + "https://www.internalfb.com/manifold/explorer/{}/{}", + bucket.name, manifold_filename, + )) + } else { + url + } } async fn build_launch_installer( From b2b16d8149ede518caa15e3600824475f4100b3e Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 28 Apr 2025 12:19:41 -0700 Subject: [PATCH 02/29] `//A/B/C` is a valid name for `//A/B/C:C` > Prefer using the short name when referring to an eponymous target (`//x` instead of `//x:x`). If > you are in the same package, prefer the local reference (`:x` instead of `//x`). See: https://bazel.build/build/style-guide#target-naming This syntax is supported on the command-line, but not in BUCK files. Unfortunately, the only(?) Starlark formatter (buildifier: https://github.com/bazelbuild/buildtools) automatically abbreviates targets on the assumption that this syntax is valid, which causes errors when using Buck2: ... 4: Error coercing "//src/Foo" 5: Invalid absolute target pattern `//src/Foo` is not allowed 6: Expected a `:`, a trailing `/...` or the literal `...`. We can adjust the parsing rules to allow this syntax in more places. --- app/buck2_core/src/pattern/pattern.rs | 18 ++++++++---------- .../src/attrs/coerce/attr_type/source.rs | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/app/buck2_core/src/pattern/pattern.rs b/app/buck2_core/src/pattern/pattern.rs index 21a10c5ed533f..526157c83ba1d 100644 --- a/app/buck2_core/src/pattern/pattern.rs +++ b/app/buck2_core/src/pattern/pattern.rs @@ -392,7 +392,7 @@ impl ParsedPattern { cell_alias_resolver, TargetParsingOptions { relative, - infer_target: false, + infer_target: true, strip_package_trailing_slash: false, }, pattern, @@ -1593,21 +1593,19 @@ mod tests { CellRelativePath::unchecked_new("package").to_owned(), ); - assert_matches!( + assert_eq!( + mk_target("root", "package/path", "path"), ParsedPattern::::parse_not_relaxed( "path", TargetParsingRel::AllowRelative( - &CellPathWithAllowedRelativeDir::backwards_relative_not_supported(package.clone()), + &CellPathWithAllowedRelativeDir::backwards_relative_not_supported( + package.clone() + ), Some(&NoAliases), ), &resolver(), &alias_resolver(), - ), - Err(e) => { - assert!( - format!("{e:?}").contains(&format!("{}", TargetPatternParseError::UnexpectedFormat)) - ); - } + )? ); assert_eq!( @@ -1775,7 +1773,7 @@ mod tests { ), Err(e) => { assert!( - format!("{e:?}").contains(&format!("{}", TargetPatternParseError::UnexpectedFormat)) + format!("{e:?}").contains(&format!("{}", TargetPatternParseError::AbsoluteRequired)) ); } ); diff --git a/app/buck2_interpreter_for_build/src/attrs/coerce/attr_type/source.rs b/app/buck2_interpreter_for_build/src/attrs/coerce/attr_type/source.rs index 5617b349fdb16..04dee6e8aa5e4 100644 --- a/app/buck2_interpreter_for_build/src/attrs/coerce/attr_type/source.rs +++ b/app/buck2_interpreter_for_build/src/attrs/coerce/attr_type/source.rs @@ -43,7 +43,7 @@ impl AttrTypeCoerce for SourceAttrType { ) -> buck2_error::Result { let source_label = value.unpack_str_err()?; - let label_err = if source_label.contains(':') { + let label_err = if source_label.contains(':') || source_label.contains("//") { match ctx.coerce_providers_label(source_label) { Ok(l) => return Ok(CoercedAttr::SourceLabel(l)), Err(e) => Some(e), From e73f56c22379f13107fbe0affef6b103f622a9b9 Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 02:02:23 -0400 Subject: [PATCH 03/29] test output flag patch Only show test output if errors. Can be overridden with: buck test --test-output=(all|none|error) ... --- app/buck2_cli_proto/daemon.proto | 12 ++++ app/buck2_client/src/commands/test.rs | 28 +++++++++ app/buck2_test/src/command.rs | 81 ++++++++++++++++++++++++++- app/buck2_test/src/orchestrator.rs | 35 ++++++++++-- 4 files changed, 149 insertions(+), 7 deletions(-) diff --git a/app/buck2_cli_proto/daemon.proto b/app/buck2_cli_proto/daemon.proto index fb26a4b6e1ad2..a0f2d1cefc2a5 100644 --- a/app/buck2_cli_proto/daemon.proto +++ b/app/buck2_cli_proto/daemon.proto @@ -570,6 +570,18 @@ message TestRequest { // Whether `RunInfo` providers for targets matching `target_patterns` // should be built instead of only `ExternalTestInfo` providers. bool build_run_info = 16; + + enum TestOutputMode { + // Show output from all tests (default). + ALL = 0; + // Only show output from failed tests. + ERRORS = 1; + // Show no test output, only the summary. + NONE = 2; + } + + // Controls how much test output is displayed. + TestOutputMode test_output_mode = 17; } message BxlRequest { diff --git a/app/buck2_client/src/commands/test.rs b/app/buck2_client/src/commands/test.rs index 26366f8916cbc..06d2014b1b8d6 100644 --- a/app/buck2_client/src/commands/test.rs +++ b/app/buck2_client/src/commands/test.rs @@ -45,6 +45,16 @@ use superconsole::Span; use crate::commands::build::print_buck_ui_and_rating; use crate::commands::build::print_build_result; +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +pub enum TestOutputMode { + /// Only show output from tests that failed + Errors, + /// Show output from all tests, passing and failing + All, + /// Show no output, only the summary + None, +} + fn forward_output_to_path( output: &str, path_arg: &PathArg, @@ -175,6 +185,13 @@ If include patterns are present, regardless of whether exclude patterns are pres #[clap(long, group = "run-info")] skip_run_info: bool, + /// Control test output display mode + /// + /// Note: The 'errors' mode requires test executors to provide per-test output. + /// If not available, no output will be shown to avoid displaying passing test output. + #[clap(long = "test-output", value_enum, default_value = "errors")] + test_output: TestOutputMode, + /// This option does nothing. It is here to keep compatibility with Buck1 and ci #[clap(long = "deep", hide = true)] _deep: bool, @@ -352,6 +369,17 @@ impl StreamingCommand for TestCommand { ignore_tests_attribute: self.ignore_tests_attribute, build_default_info: self.build_default_info, build_run_info: self.build_run_info, + test_output_mode: match self.test_output { + TestOutputMode::Errors => { + buck2_cli_proto::test_request::TestOutputMode::Errors.into() + } + TestOutputMode::All => { + buck2_cli_proto::test_request::TestOutputMode::All.into() + } + TestOutputMode::None => { + buck2_cli_proto::test_request::TestOutputMode::None.into() + } + }, }, events_ctx, ctx.console_interaction_stream(&self.common_opts.console_opts), diff --git a/app/buck2_test/src/command.rs b/app/buck2_test/src/command.rs index f28ed138667d1..90b620ca84d00 100644 --- a/app/buck2_test/src/command.rs +++ b/app/buck2_test/src/command.rs @@ -36,6 +36,7 @@ use buck2_cli_proto::HasClientContext; use buck2_cli_proto::TestRequest; use buck2_cli_proto::TestResponse; use buck2_cli_proto::representative_config_flag; +use buck2_cli_proto::test_request::TestOutputMode; use buck2_common::dice::cells::HasCellResolver; use buck2_common::events::HasEvents; use buck2_common::legacy_configs::dice::HasLegacyConfigs; @@ -123,6 +124,7 @@ struct TestOutcome { executor_stdout: String, executor_stderr: String, build_target_result: BuildTargetResult, + test_output_mode: TestOutputMode, } impl TestOutcome { @@ -138,6 +140,7 @@ struct ExecutorReport { exit_code: Option, statuses: TestStatuses, info_messages: Vec, + test_results: Vec, } impl ExecutorReport { @@ -145,6 +148,7 @@ impl ExecutorReport { match status { ExecutorMessage::TestResult(res) => { self.statuses.ingest(res); + self.test_results.push(res.clone()); } ExecutorMessage::ExitCode(exit_code) => { self.exit_code = Some(*exit_code); @@ -400,6 +404,11 @@ async fn test( let project_root = server_ctx.project_root(); let tpx_experiments = get_tpx_experiments(ctx.dupe(), project_root).await?; + let test_output_mode = request + .test_output_mode + .try_into() + .unwrap_or(TestOutputMode::All); + let test_outcome = test_targets( ctx.dupe(), resolved_pattern, @@ -422,6 +431,7 @@ async fn test( request.build_default_info, request.build_run_info, tpx_experiments, + test_output_mode, ) .await?; @@ -542,12 +552,76 @@ async fn test( .push(get_target_rule_type_name(&mut ctx, configured.target()).await?); } + // Filter executor output based on test output mode + let (filtered_stdout, filtered_stderr) = match test_outcome.test_output_mode { + TestOutputMode::None => { + // Show no output + (String::new(), String::new()) + } + TestOutputMode::Errors => { + // Only show output from failed tests + // Since most test executors don't provide per-test output details, + // and we cannot distinguish which output comes from which test when + // they're combined, we choose to show no output rather than risk + // showing output from passing tests. + + // Check if we have individual test outputs we can filter + let mut failed_test_output = String::new(); + let mut found_individual_outputs = false; + + for test_result in &test_outcome.executor_report.test_results { + match test_result.status { + TestStatus::FAIL + | TestStatus::FATAL + | TestStatus::TIMEOUT + | TestStatus::LISTING_FAILED => { + if !test_result.details.is_empty() { + found_individual_outputs = true; + failed_test_output + .push_str(&format!("\n=== {} ===\n", test_result.name)); + failed_test_output.push_str(&test_result.details); + if !test_result.details.ends_with('\n') { + failed_test_output.push('\n'); + } + } + } + _ => { + // Skip output from passing/skipped tests + } + } + } + + if found_individual_outputs { + // We have per-test details, use only the failed test outputs + (failed_test_output.clone(), failed_test_output) + } else { + // We don't have per-test details. Rather than showing all output + // (which would include passing tests), show nothing. + // This ensures we never show passing test output with --test-output=errors + let has_failures = test_statuses.failed.as_ref().map_or(false, |f| f.count > 0) + || test_statuses.fatals.as_ref().map_or(false, |f| f.count > 0); + + if has_failures { + // Add a note explaining why output is suppressed + let msg = "\nNote: Test output suppressed with --test-output=errors because the test executor does not provide per-test output details.\n"; + (msg.to_string(), String::new()) + } else { + (String::new(), String::new()) + } + } + } + TestOutputMode::All => { + // Show all output (default behavior) + (test_outcome.executor_stdout, test_outcome.executor_stderr) + } + }; + Ok(TestResponse { executor_exit_code, errors: test_outcome.errors, test_statuses: Some(test_statuses), - executor_stdout: test_outcome.executor_stdout, - executor_stderr: test_outcome.executor_stderr, + executor_stdout: filtered_stdout, + executor_stderr: filtered_stderr, executor_info_messages: test_outcome.executor_report.info_messages, serialized_build_report, target_rule_type_names, @@ -571,6 +645,7 @@ async fn test_targets( build_default_info: bool, build_run_info: bool, tpx_experiments: StdBuckHashSet, + test_output_mode: TestOutputMode, ) -> buck2_error::Result { let session = Arc::new(session); @@ -644,6 +719,7 @@ async fn test_targets( liveliness_observer.dupe(), test_status_sender, CancellationContext::never_cancelled(), // sending the orchestrator directly to be spawned by make_server, which never calls it. + test_output_mode, ) .await .buck_error_context("Failed to create a BuckTestOrchestrator")?; @@ -773,6 +849,7 @@ async fn test_targets( executor_stderr: executor_output.stderr, executor_report, build_target_result, + test_output_mode, }) } diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index 09975b76ec1f0..4614f6cbf48da 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -51,6 +51,7 @@ use buck2_build_api::interpreter::rule_defs::provider::builtin::local_resource_i use buck2_build_api::keep_going::KeepGoing; use buck2_build_signals::env::NodeDuration; use buck2_build_signals::env::WaitingData; +use buck2_cli_proto::test_request::TestOutputMode; use buck2_common::dice::cells::HasCellResolver; use buck2_common::events::HasEvents; use buck2_common::liveliness_observer::LivelinessObserver; @@ -149,6 +150,7 @@ use buck2_test_api::data::PrepareForLocalExecutionResult; use buck2_test_api::data::RequiredLocalResources; use buck2_test_api::data::TestResult; use buck2_test_api::data::TestStage; +use buck2_test_api::data::TestStatus; use buck2_test_api::data::convert::host_sharing_requirements_to_grpc; use buck2_test_api::protocol::TestOrchestrator; use derive_more::From; @@ -198,6 +200,7 @@ pub struct BuckTestOrchestrator<'a: 'static> { liveliness_observer: Arc, cancellations: &'a CancellationContext, re_client: Arc, + test_output_mode: TestOutputMode, } impl<'a> BuckTestOrchestrator<'a> { @@ -207,6 +210,7 @@ impl<'a> BuckTestOrchestrator<'a> { liveliness_observer: Arc, results_channel: UnboundedSender>, cancellations: &'a CancellationContext, + test_output_mode: TestOutputMode, ) -> buck2_error::Result> { let events = dice.per_transaction_data().get_dispatcher().dupe(); let re_client = Arc::new(remote_storage::ReClientWithCache::new( @@ -220,6 +224,7 @@ impl<'a> BuckTestOrchestrator<'a> { events, cancellations, re_client, + test_output_mode, )) } @@ -231,6 +236,7 @@ impl<'a> BuckTestOrchestrator<'a> { events: EventDispatcher, cancellations: &'a CancellationContext, re_client: Arc, + test_output_mode: TestOutputMode, ) -> BuckTestOrchestrator<'a> { Self { dice, @@ -240,6 +246,7 @@ impl<'a> BuckTestOrchestrator<'a> { liveliness_observer, cancellations, re_client, + test_output_mode, } } @@ -745,11 +752,28 @@ impl TestOrchestrator for BuckTestOrchestrator<'_> { } async fn report_test_result(&self, r: TestResult) -> buck2_error::Result<()> { - let event = buck2_data::instant_event::Data::TestResult(translations::convert_test_result( - r.clone(), - &self.session, - )?); - self.events.instant_event(event); + // Only send test result events based on the output mode + let should_send_event = match self.test_output_mode { + TestOutputMode::All => true, // Show all test results + TestOutputMode::Errors => { + // Only show failed test results + matches!( + r.status, + TestStatus::FAIL + | TestStatus::FATAL + | TestStatus::TIMEOUT + | TestStatus::LISTING_FAILED + ) + } + TestOutputMode::None => false, // Don't show any test results + }; + + if should_send_event { + let event = buck2_data::instant_event::Data::TestResult( + translations::convert_test_result(r.clone(), &self.session)?, + ); + self.events.instant_event(event); + } self.results_channel .unbounded_send(Ok(ExecutorMessage::TestResult(r))) .map_err(|_| { @@ -2415,6 +2439,7 @@ mod tests { EventDispatcher::null(), CancellationContext::testing(), re_client, + TestOutputMode::All, // Default to showing all output in tests ), receiver, )) From 0c40b5382460ec27ea63bee5410c7a6950cf4b5b Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 02:03:27 -0400 Subject: [PATCH 04/29] test caching dice patch Avoids re-running passed tests. Only compatible with DICE (local only). --- app/buck2_test/src/orchestrator.rs | 94 +++++++++++++----------------- 1 file changed, 40 insertions(+), 54 deletions(-) diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index 4614f6cbf48da..cc288c6527198 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -622,7 +622,7 @@ async fn prepare_and_execute( ) -> Result { let execute_on_dice = match key.stage.as_ref() { TestStage::Listing { cacheable, .. } => *cacheable, - TestStage::Testing { .. } => false, + TestStage::Testing { .. } => true, }; if execute_on_dice { let result = tokio::select! { @@ -1044,7 +1044,7 @@ impl BuckTestOrchestrator<'_> { request: CommandExecutionRequest, liveliness_observer: Arc, re_cache_enabled: bool, - supports_test_execution_caching: bool, + _supports_test_execution_caching: bool, ) -> Result { let events = dice.per_transaction_data().get_dispatcher().dupe(); let manager = CommandExecutionManager::new( @@ -1060,8 +1060,6 @@ impl BuckTestOrchestrator<'_> { action_key_suffix: create_action_key_suffix(stage), }; - // For test execution, we currently do not do any cache queries - let prepared_action = match executor.prepare_action(&request, digest_config, false) { Ok(prepared_action) => prepared_action, Err(e) => return Err(ExecuteError::Error(e)), @@ -1154,24 +1152,26 @@ impl BuckTestOrchestrator<'_> { let start = TestRunStart { suite: test_suite.clone(), }; - events + let (result, cached) = events .span_async(start, async move { - let result = if supports_test_execution_caching { + let (result, cached) = if re_cache_enabled { match executor .action_cache(manager, &prepared_command, cancellation) .await { ControlFlow::Continue(manager) => { - executor + let result = executor .exec_cmd(manager, &prepared_command, cancellation) - .await + .await; + (result, false) } - ControlFlow::Break(result) => result, + ControlFlow::Break(result) => (result, true), } } else { - executor + let result = executor .exec_cmd(manager, &prepared_command, cancellation) - .await + .await; + (result, false) }; let end = TestRunEnd { suite: test_suite, @@ -1186,9 +1186,31 @@ impl BuckTestOrchestrator<'_> { ) .ok(), }; - (result, end) + ((result, cached), end) }) - .await + .await; + if !cached && re_cache_enabled { + let info = CacheUploadInfo { + target: &test_target as _, + digest_config, + mergebase: &None, + re_platform: executor.re_platform(), + }; + let _result = match executor + .cache_upload( + &info, + &result, + None, + None, + &prepared_action.action_and_blobs, + ) + .await + { + Ok(result) => result, + Err(e) => return Err(ExecuteError::Error(e.into())), + }; + } + result } }; @@ -1285,8 +1307,8 @@ impl BuckTestOrchestrator<'_> { fn executor_config_with_remote_cache_override<'a>( test_target_node: &'a ConfiguredTargetNode, executor_override: Option<&'a CommandExecutorConfig>, - stage: &TestStage, - supports_test_execution_caching: bool, + _stage: &TestStage, + _supports_test_execution_caching: bool, ) -> buck2_error::Result> { let executor_config = match executor_override { Some(o) => o, @@ -1296,36 +1318,15 @@ impl BuckTestOrchestrator<'_> { .buck_error_context("Error accessing executor config")?, }; - if let TestStage::Listing { .. } = &stage { - return Ok(Cow::Borrowed(executor_config)); - } - - if supports_test_execution_caching { - return Ok(Cow::Borrowed(executor_config)); - } - - match &executor_config.executor { - Executor::RemoteEnabled(options) if options.remote_cache_enabled => { - let mut exec_options = options.clone(); - exec_options.remote_cache_enabled = false; - let executor_config = CommandExecutorConfig { - executor: Executor::RemoteEnabled(exec_options), - options: executor_config.options.dupe(), - }; - Ok(Cow::Owned(executor_config)) - } - Executor::Local(_) | Executor::RemoteEnabled(_) | Executor::None => { - Ok(Cow::Borrowed(executor_config)) - } - } + Ok(Cow::Borrowed(executor_config)) } async fn get_command_executor( dice: &mut DiceComputations<'_>, fs: &ArtifactFs, executor_config: &CommandExecutorConfig, - stage: &TestStage, - supports_test_execution_caching: bool, + _stage: &TestStage, + _supports_test_execution_caching: bool, ) -> buck2_error::Result { let CommandExecutorResponse { executor, @@ -1336,21 +1337,6 @@ impl BuckTestOrchestrator<'_> { output_trees_download_config: _, } = dice.get_command_executor_from_dice(executor_config).await?; - let (cache_uploader, action_cache_checker) = match stage { - TestStage::Listing { .. } => (cache_uploader, action_cache_checker), - TestStage::Testing { .. } => { - ( - // We never upload local test executions - Arc::new(NoOpCacheUploader {}) as _, - if supports_test_execution_caching { - action_cache_checker - } else { - Arc::new(NoOpCommandOptionalExecutor {}) as _ - }, - ) - } - }; - let executor = CommandExecutor::new( executor, action_cache_checker, From 331d8c7ea91b354cc8ce73ca2109a8ba4dc6e976 Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 02:07:59 -0400 Subject: [PATCH 05/29] UX: warn on misuse of include/exclude https://github.com/facebook/buck2/pull/1244 --- app/buck2_client/src/commands/test.rs | 113 ++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/app/buck2_client/src/commands/test.rs b/app/buck2_client/src/commands/test.rs index 06d2014b1b8d6..2eb3d5c64957d 100644 --- a/app/buck2_client/src/commands/test.rs +++ b/app/buck2_client/src/commands/test.rs @@ -8,6 +8,8 @@ * above-listed licenses. */ +use std::fmt::Write; + use async_trait::async_trait; use buck2_cli_proto::CounterWithExamples; use buck2_cli_proto::TestRequest; @@ -86,6 +88,29 @@ fn print_error_counter( } Ok(()) } + +/// Check if we should warn about potentially misplaced --include/--exclude flags. +/// Returns Some(suspicious_labels) if warning should be shown, where suspicious_labels +/// are label values that look like they might be target patterns (contain '/' or ':'). +fn should_warn_about_flag_position( + patterns: &[String], + include: &[String], + exclude: &[String], +) -> Option> { + if patterns.is_empty() && (!include.is_empty() || !exclude.is_empty()) { + let suspicious_labels: Vec = include + .iter() + .chain(exclude.iter()) + .filter(|label| label.contains('/') || label.contains(':')) + .cloned() + .collect(); + + Some(suspicious_labels) + } else { + None + } +} + #[derive(Debug, clap::Parser)] #[clap(name = "test", about = "Build and test the specified targets")] pub struct TestCommand { @@ -343,6 +368,42 @@ impl StreamingCommand for TestCommand { ctx: &mut ClientCommandContext<'_>, events_ctx: &mut EventsCtx, ) -> ExitResult { + // Warn if no target patterns but label filters are set + // This usually means the patterns were accidentally consumed as label values + // NOTE: maybe these should accept just one arg, but that probably breaks users who + // are doing buck test //... --include myproject myproject2 + if let Some(suspicious_labels) = + should_warn_about_flag_position(&self.patterns, &self.include, &self.exclude) + { + let console = self.common_opts.console_opts.final_console(); + + let mut message = String::new(); + writeln!( + &mut message, + "No target patterns specified, but --include/--exclude flags are set." + ) + .unwrap(); + writeln!( + &mut message, + "This is likely a mistake: put targets first, then --include/--exclude, since include/exclude consume all remaining args" + ).unwrap(); + if !suspicious_labels.is_empty() { + writeln!( + &mut message, + "hint: The following requested labels look like target patterns: {sus}\n\ + hint: Try putting them before --include/--exclude.\n\ + hint: For example: buck2 test //foo --include mylabel", + sus = suspicious_labels + .iter() + .map(|s| format!("'{}'", s)) + .collect::>() + .join(", ") + ) + .unwrap(); + } + console.print_warning(&message)?; + } + let context = ctx.client_context(matches, &self)?; let response = buckd .with_flushing() @@ -548,3 +609,55 @@ impl StreamingCommand for TestCommand { &self.write_test_id } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_warn_when_no_patterns_with_include() { + let result = should_warn_about_flag_position(&[], &["some_label".to_owned()], &[]) + .expect("should warn"); + assert!(result.is_empty()); + } + + #[test] + fn test_warn_when_no_patterns_with_exclude() { + let result = should_warn_about_flag_position(&[], &[], &["some_label".to_owned()]) + .expect("should warn"); + assert!(result.is_empty()); + } + + #[test] + fn test_warn_detects_suspicious_target_pattern_with_slashes() { + let result = should_warn_about_flag_position( + &[], + &["some_label".to_owned(), "//foobar/...".to_owned()], + &["//foobar2/...".to_owned(), "//foobar2:".to_owned()], + ) + .expect("should warn"); + + assert_eq!(result, vec!["//foobar/...", "//foobar2/...", "//foobar2:"]); + } + + #[test] + fn test_no_warn_when_patterns_present() { + let result = should_warn_about_flag_position( + &["//target/...".to_owned()], + &["some_label".to_owned()], + &["some_other_label".to_owned()], + ); + + assert!( + result.is_none(), + "Should not warn when patterns are present" + ); + } + + #[test] + fn test_no_warn_when_nothing_specified() { + let result = should_warn_about_flag_position(&[], &[], &[]); + + assert!(result.is_none(), "Should not warn when nothing specified"); + } +} From 6231a9fef2c55841c1357448da1892a3a5946bee Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:18:49 -0400 Subject: [PATCH 06/29] Allow BXLs to aquery actions directly; used for action debugging. https://github.com/facebook/buck2/pull/1250 --- app/buck2_build_api/src/query/bxl.rs | 6 ++++ app/buck2_bxl/src/bxl/starlark_defs/aquery.rs | 16 +++++++++ app/buck2_query_impls/src/aquery/bxl.rs | 19 ++++++++++ tests/core/query/aquery/test_aquery.py | 13 +++++++ .../query/aquery/test_aquery_data/aquery.bxl | 35 +++++++++++++++++++ .../query/aquery/test_aquery_data/prelude.bzl | 3 +- 6 files changed, 91 insertions(+), 1 deletion(-) diff --git a/app/buck2_build_api/src/query/bxl.rs b/app/buck2_build_api/src/query/bxl.rs index dabda4d754118..30f7600b85745 100644 --- a/app/buck2_build_api/src/query/bxl.rs +++ b/app/buck2_build_api/src/query/bxl.rs @@ -12,6 +12,7 @@ use std::future::Future; use std::pin::Pin; use async_trait::async_trait; +use buck2_artifact::actions::key::ActionKey; use buck2_core::cells::CellResolver; use buck2_core::cells::name::CellName; use buck2_core::configuration::compatibility::MaybeCompatible; @@ -183,6 +184,11 @@ pub trait BxlAqueryFunctions: Send { dice: &mut DiceComputations<'_>, targets: &TargetSet, ) -> buck2_error::Result>; + async fn get_action_nodes( + &self, + dice: &mut DiceComputations<'_>, + action_keys: Vec, + ) -> buck2_error::Result>; } pub static NEW_BXL_CQUERY_FUNCTIONS: LateBinding< diff --git a/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs b/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs index 1fae20b093562..df1b7e558f70c 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/aquery.rs @@ -9,6 +9,7 @@ */ use allocative::Allocative; +use buck2_artifact::actions::key::ActionKey; use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::query::bxl::BxlAqueryFunctions; use buck2_build_api::query::bxl::NEW_BXL_AQUERY_FUNCTIONS; @@ -46,6 +47,7 @@ use starlark::values::type_repr::StarlarkTypeRepr; use crate::bxl::starlark_defs::context::BxlContext; use crate::bxl::starlark_defs::context::ErrorPrinter; +use crate::bxl::starlark_defs::nodes::action::StarlarkAction; use crate::bxl::starlark_defs::nodes::action::StarlarkActionQueryNode; use crate::bxl::starlark_defs::providers_expr::AnyProvidersExprArg; use crate::bxl::starlark_defs::providers_expr::ProvidersExpr; @@ -122,6 +124,7 @@ enum UnpackActionNodes<'v> { ActionQueryNodesSet(&'v StarlarkTargetSet), ConfiguredProviders(AnyProvidersExprArg<'v>), ConfiguredTargets(ConfiguredTargetListExprArg<'v>), + StarlarkActions(UnpackList), } // Aquery operates on `ActionQueryNode`s. Under the hood, the target set of action query nodes is obtained @@ -140,6 +143,14 @@ async fn unpack_action_nodes<'v>( return Ok(action_nodes.into_iter().map(|v| v.0).collect()); } UnpackActionNodes::ActionQueryNodesSet(action_nodes) => return Ok(action_nodes.0.clone()), + UnpackActionNodes::StarlarkActions(actions) => { + let action_keys: Vec = actions + .into_iter() + .map(|starlark_action| starlark_action.0.key().dupe()) + .collect(); + + return aquery_env.get_action_nodes(dice, action_keys).await; + } UnpackActionNodes::ConfiguredProviders(arg) => { ProvidersExpr::::unpack( arg, @@ -183,6 +194,11 @@ async fn unpack_action_nodes<'v>( /// /// Query results are `target_set`s of `action_query_node`s, which supports iteration, /// indexing, `len()`, set addition/subtraction, and `equals()`. +/// +/// Actions can be specified as: +/// - Target expressions (configured targets/providers) +/// - Existing action query nodes or target sets +/// - `bxl.Action` objects (obtained from `ctx.audit().output()`) #[starlark_module] fn aquery_methods(builder: &mut MethodsBuilder) { /// The deps query for finding the transitive closure of dependencies. diff --git a/app/buck2_query_impls/src/aquery/bxl.rs b/app/buck2_query_impls/src/aquery/bxl.rs index 28cc4a7ac78f9..b51b3f9d32a80 100644 --- a/app/buck2_query_impls/src/aquery/bxl.rs +++ b/app/buck2_query_impls/src/aquery/bxl.rs @@ -12,6 +12,7 @@ use std::marker::PhantomData; use std::sync::Arc; use async_trait::async_trait; +use buck2_artifact::actions::key::ActionKey; use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::analysis::calculation::RuleAnalysisCalculation; use buck2_build_api::query::bxl::BxlAqueryFunctions; @@ -303,6 +304,24 @@ impl BxlAqueryFunctions for BxlAqueryFunctionsImpl { }) .await } + + async fn get_action_nodes( + &self, + dice: &mut DiceComputations<'_>, + action_keys: Vec, + ) -> buck2_error::Result> { + dice.with_linear_recompute(|dice| async move { + let delegate = self.aquery_delegate(&dice).await?; + let mut result = TargetSet::new(); + let nodes = buck2_util::future::try_join_all( + action_keys.iter().map(|key| delegate.get_action_node(&key)), + ) + .await?; + result.extend(nodes); + Ok(result) + }) + .await + } } pub(crate) fn init_new_bxl_aquery_functions() { diff --git a/tests/core/query/aquery/test_aquery.py b/tests/core/query/aquery/test_aquery.py index f8aa4842dfa45..9a46670bac0c0 100644 --- a/tests/core/query/aquery/test_aquery.py +++ b/tests/core/query/aquery/test_aquery.py @@ -154,3 +154,16 @@ async def test_bxl_aquery_eval(buck: Buck) -> None: @buck_test() async def test_bxl_aquery_action_query_node(buck: Buck) -> None: await buck.bxl("//:aquery.bxl:action_query_node") + + +# Tests for bxl.Action support in aquery operations +@buck_test() +async def test_bxl_action_deps_0(buck: Buck) -> None: + """Test passing a bxl.Action to aquery.deps() with depth=0 returns itself""" + await buck.bxl("//:aquery.bxl:action_deps_0") + + +@buck_test() +async def test_bxl_action_deps(buck: Buck) -> None: + """Test that you can isolate the deps of one action by itself""" + await buck.bxl("//:aquery.bxl:action_deps") diff --git a/tests/core/query/aquery/test_aquery_data/aquery.bxl b/tests/core/query/aquery/test_aquery_data/aquery.bxl index 6033afef00a02..b28c7b4711da4 100644 --- a/tests/core/query/aquery/test_aquery_data/aquery.bxl +++ b/tests/core/query/aquery/test_aquery_data/aquery.bxl @@ -141,3 +141,38 @@ action_query_node = bxl_main( impl = _impl_action_query_node, cli_args = {}, ) + +def _impl_action_deps_0(ctx): + """Test passing a bxl.Action to aquery.deps() returns itself""" + nodes = ctx.aquery().eval("//:test") + + action = nodes[0].action() + + # Pass the bxl.Action rather than a target to aquery.deps() + result = ctx.aquery().deps([action], depth = 0) + _assert_eq(action.outputs(), result[0].action().outputs()) + +action_deps_0 = bxl_main( + impl = _impl_action_deps_0, + cli_args = {}, +) + +def _impl_action_deps(ctx): + """Test that you can isolate the deps of one action by itself""" + nodes = ctx.aquery().eval("//:test") + + action = nodes[0].action() + + # Pass the bxl.Action rather than a target to aquery.deps() + result = ctx.aquery().deps([action], depth = 1) + + _assert_eq(len(result), 2) + # First should be itself + _assert_eq(action.outputs(), result[0].action().outputs()) + dep = result[1] + _assert_eq(dep.action().outputs()[0].short_path, "dep") + +action_deps = bxl_main( + impl = _impl_action_deps, + cli_args = {}, +) diff --git a/tests/core/query/aquery/test_aquery_data/prelude.bzl b/tests/core/query/aquery/test_aquery_data/prelude.bzl index 1ae02abab3595..aca6366da9bad 100644 --- a/tests/core/query/aquery/test_aquery_data/prelude.bzl +++ b/tests/core/query/aquery/test_aquery_data/prelude.bzl @@ -10,6 +10,7 @@ def _test(ctx: AnalysisContext): dep = ctx.actions.write("dep", "", has_content_based_path = False) default = ctx.actions.copy_file("default", dep, has_content_based_path = False) other = ctx.actions.write("other", "", has_content_based_path = False) + other_default = ctx.actions.copy_file("other_default", dep, has_content_based_path = False) sub_default = ctx.actions.write("sub_default", "", has_content_based_path = False) sub_other = ctx.actions.write("sub_other", "", has_content_based_path = False) @@ -18,7 +19,7 @@ def _test(ctx: AnalysisContext): ctx.actions.write("unused", "", has_content_based_path = False) return [DefaultInfo( - default_outputs = [default], + default_outputs = [default, other_default], other_outputs = [other], sub_targets = { "sub": [ From aa71644cde6b2fb00ed5cad8510b663893d6a2fa Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:18:55 -0400 Subject: [PATCH 07/29] Allow BXLs to make atarget_set. For symmetry with the other target set types. https://github.com/facebook/buck2/pull/1251 --- .../src/bxl/starlark_defs/analysis_result.rs | 17 ++++++ .../src/bxl/starlark_defs/functions.rs | 26 +++++++++ tests/core/query/aquery/test_aquery.py | 12 +++++ .../query/aquery/test_aquery_data/aquery.bxl | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+) diff --git a/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs b/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs index 58da30b85e6e0..f1d119e14e162 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/analysis_result.rs @@ -15,6 +15,7 @@ use buck2_build_api::analysis::AnalysisResult; use buck2_build_api::interpreter::rule_defs::provider::collection::FrozenProviderCollection; use buck2_build_api::interpreter::rule_defs::provider::dependency::Dependency; use buck2_core::provider::label::ConfiguredProvidersLabel; +use buck2_interpreter::types::configured_providers_label::StarlarkConfiguredProvidersLabel; use dupe::Dupe; use starlark::any::ProvidesStaticType; use starlark::environment::Methods; @@ -149,6 +150,22 @@ fn starlark_analysis_result_methods(builder: &mut MethodsBuilder) { Ok(heap.alloc(result)) } + /// Gets the configured providers label for this analysis result. + /// + /// Sample usage: + /// ```python + /// def _impl_label(ctx): + /// actions = ctx.aquery().all_actions("//target") + /// for node in actions: + /// if analysis := node.analysis(): + /// ctx.output.print(analysis.label()) + /// ``` + fn label<'v>( + this: &StarlarkAnalysisResult, + ) -> starlark::Result { + Ok(StarlarkConfiguredProvidersLabel::new(this.label.dupe())) + } + /// Converts the analysis result into a `Dependency`. Currently, you can only get a `Dependency` without any /// transitions. This means that you cannot create an exec dep or toolchain from an analysis result. /// diff --git a/app/buck2_bxl/src/bxl/starlark_defs/functions.rs b/app/buck2_bxl/src/bxl/starlark_defs/functions.rs index 305c99d403ea7..e690b9d877830 100644 --- a/app/buck2_bxl/src/bxl/starlark_defs/functions.rs +++ b/app/buck2_bxl/src/bxl/starlark_defs/functions.rs @@ -10,6 +10,7 @@ use std::time::Instant; +use buck2_build_api::actions::query::ActionQueryNode; use buck2_build_api::interpreter::rule_defs::artifact::starlark_artifact_like::ValueAsInputArtifactLikeUnpack; use buck2_build_api::interpreter::rule_defs::cmd_args::value_as::ValueAsCommandLineLike; use buck2_core::cells::CellAliasResolver; @@ -42,6 +43,7 @@ use super::context::output::get_cmd_line_inputs; use super::nodes::unconfigured::StarlarkTargetNode; use crate::bxl::starlark_defs::context::BxlContext; use crate::bxl::starlark_defs::eval_extra::BxlEvalExtra; +use crate::bxl::starlark_defs::nodes::action::StarlarkActionQueryNode; use crate::bxl::starlark_defs::nodes::configured::StarlarkConfiguredTargetNode; use crate::bxl::starlark_defs::targetset::StarlarkTargetSet; use crate::bxl::starlark_defs::time::StarlarkInstant; @@ -92,6 +94,30 @@ pub(crate) fn register_target_function(builder: &mut GlobalsBuilder) { .map(|node| node.0), )) } + + /// Creates a target set from a list of action query nodes. + /// + /// Sample usage: + /// ```python + /// def _impl_atarget_set(ctx): + /// actions = ctx.aquery().all_actions("//target") + /// action_a = actions[0] + /// action_b = actions[1] + /// action_set = bxl.atarget_set([action_a, action_b]) + /// # Now can use in further queries + /// deps = ctx.aquery().deps(action_set) + /// ``` + fn atarget_set( + nodes: Option>, + ) -> starlark::Result> { + Ok(StarlarkTargetSet::from_iter( + nodes + .unwrap_or(UnpackList::default()) + .items + .into_iter() + .map(|node| node.0), + )) + } } #[derive(Debug, buck2_error::Error, Clone)] diff --git a/tests/core/query/aquery/test_aquery.py b/tests/core/query/aquery/test_aquery.py index 9a46670bac0c0..59c5c69f3efed 100644 --- a/tests/core/query/aquery/test_aquery.py +++ b/tests/core/query/aquery/test_aquery.py @@ -167,3 +167,15 @@ async def test_bxl_action_deps_0(buck: Buck) -> None: async def test_bxl_action_deps(buck: Buck) -> None: """Test that you can isolate the deps of one action by itself""" await buck.bxl("//:aquery.bxl:action_deps") + + +@buck_test() +async def test_bxl_atarget_set(buck: Buck) -> None: + """Test bxl.atarget_set() creates a target set from ActionQueryNodes""" + await buck.bxl("//:aquery.bxl:atarget_set_test") + + +@buck_test() +async def test_bxl_analysis_label(buck: Buck) -> None: + """Test analysis.label() returns the configured providers label""" + await buck.bxl("//:aquery.bxl:analysis_label_test") diff --git a/tests/core/query/aquery/test_aquery_data/aquery.bxl b/tests/core/query/aquery/test_aquery_data/aquery.bxl index b28c7b4711da4..ed5aa82a115f1 100644 --- a/tests/core/query/aquery/test_aquery_data/aquery.bxl +++ b/tests/core/query/aquery/test_aquery_data/aquery.bxl @@ -176,3 +176,56 @@ action_deps = bxl_main( impl = _impl_action_deps, cli_args = {}, ) + +def _impl_atarget_set_test(ctx): + """Test bxl.atarget_set() creates a target set from ActionQueryNodes""" + nodes = ctx.aquery().eval("//:test") + + # Get two action query nodes + node_a = nodes[0] + node_b = nodes[1] + + # Create a target set from action query nodes + action_set = bxl.atarget_set([node_a, node_b]) + + # Verify it's a target_set type + _assert_eq(type(action_set), "target_set") + + # Verify we can use it in further aquery operations + deps = ctx.aquery().deps(action_set, depth = 0) + _assert_eq(len(deps), 2) + +atarget_set_test = bxl_main( + impl = _impl_atarget_set_test, + cli_args = {}, +) + +def _impl_analysis_label_test(ctx): + """Test analysis.label() returns the configured providers label""" + nodes = ctx.aquery().eval("//:test") + + # Find an analysis node by iterating through results + analysis = None + for node in nodes: + if node.rule_type == "analysis": + analysis = node.analysis() + break + + if not analysis: + fail("No analysis node found in results") + + _assert_eq(type(analysis), "bxl.AnalysisResult") + + # Test the label() method + label = analysis.label() + _assert_eq(type(label), "Label") + + # Verify it's a valid label (should contain the target) + label_str = str(label) + if "root//:test" not in label_str: + fail("Expected label to contain 'root//:test', got: {}".format(label_str)) + +analysis_label_test = bxl_main( + impl = _impl_analysis_label_test, + cli_args = {}, +) From eaebab5a35e6a3d8e5b2149e723034109d8177dd Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:19:03 -0400 Subject: [PATCH 08/29] bug fix for https://linear.app/mercury/issue/DUX-4404/error-during-pre-commit-due-to-buck-starlark-lint-failure-concurrency make buck external cells more reliable https://github.com/facebook/buck2/pull/1259 --- app/buck2_external_cells/src/git.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/app/buck2_external_cells/src/git.rs b/app/buck2_external_cells/src/git.rs index 2514040decdab..25e110b504784 100644 --- a/app/buck2_external_cells/src/git.rs +++ b/app/buck2_external_cells/src/git.rs @@ -122,14 +122,9 @@ impl IoRequest for GitFetchIoRequest { })?; run_git(&path, |c| { - c.arg("remote") - .arg("add") - .arg("origin") - .arg(self.setup.git_origin.as_ref()); - })?; - - run_git(&path, |c| { - c.arg("fetch").arg("origin").arg(self.setup.commit.as_ref()); + c.arg("fetch") + .arg(self.setup.git_origin.as_ref()) + .arg(self.setup.commit.as_ref()); })?; run_git(&path, |c| { From b5def9236f6035e43a1ac01c1ecde16814f54f2c Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:19:09 -0400 Subject: [PATCH 09/29] remote_execution: Retry BatchReadBlobs requests See: https://github.com/facebook/buck2/pull/1076 --- app/buck2_re_configuration/src/lib.rs | 8 + remote_execution/oss/re_grpc/src/client.rs | 220 +++++++++++++-------- 2 files changed, 144 insertions(+), 84 deletions(-) diff --git a/app/buck2_re_configuration/src/lib.rs b/app/buck2_re_configuration/src/lib.rs index e36e42d2442a6..56e1d32213c87 100644 --- a/app/buck2_re_configuration/src/lib.rs +++ b/app/buck2_re_configuration/src/lib.rs @@ -468,6 +468,8 @@ pub struct Buck2OssReConfiguration { pub grpc_keepalive_while_idle: Option, /// Maximum number of concurrent execution requests. pub execution_concurrency_limit: Option, + /// Maximum retries for network requests. + pub max_retries: usize, } #[derive(Clone, Debug, Default, Allocative)] @@ -589,6 +591,12 @@ impl Buck2OssReConfiguration { section: BUCK2_RE_CLIENT_CFG_SECTION, property: "execution_concurrency_limit", })?, + max_retries: legacy_config + .parse(BuckconfigKeyRef { + section: BUCK2_RE_CLIENT_CFG_SECTION, + property: "max_retries", + })? + .unwrap_or(0), }) } } diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index dc7dac9b15eeb..7dcb51a7b5794 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -246,6 +246,8 @@ pub struct RERuntimeOpts { max_concurrent_uploads_per_action: Option, /// Time that digests are assumed to live in CAS after being touched. cas_ttl_secs: i64, + /// Maximum retries for network requests. + max_retries: usize, } struct InstanceName(Option); @@ -440,6 +442,7 @@ impl REClientBuilder { // NOTE: This is an arbitrary number because RBE does not return information // on the TTL of the remote blob. cas_ttl_secs: opts.cas_ttl_secs.unwrap_or(3 * 60 * 60), + max_retries: opts.max_retries, }, grpc_clients, capabilities, @@ -931,6 +934,7 @@ impl REClient { request: DownloadRequest, ) -> anyhow::Result { download_impl( + &self.runtime_opts, &self.instance_name, request, self.bystream_compressor, @@ -1260,11 +1264,12 @@ fn convert_t_action_result2(t_action_result: TActionResult2) -> anyhow::Result( + opts: &RERuntimeOpts, instance_name: &InstanceName, request: DownloadRequest, bystream_compressor: Option, max_total_batch_size: usize, - cas_f: impl Fn(BatchReadBlobsRequest) -> Cas, + cas_f: impl Clone + Fn(BatchReadBlobsRequest) -> Cas, bystream_fut: impl Fn(ReadRequest) -> Byt + Sync + Send + Copy, ) -> anyhow::Result where @@ -1378,9 +1383,10 @@ where let mut batched_blobs_response = HashMap::new(); for read_blob_req in requests { - let resp = cas_f(read_blob_req) + let resp = batch_read_blobs(opts, read_blob_req, cas_f.clone()) .await .context("Failed to make BatchReadBlobs request")?; + for r in resp.responses.into_iter() { let digest = tdigest_from(r.digest.context("Response digest not found.")?); check_status(r.status.unwrap_or_default())?; @@ -1470,6 +1476,35 @@ where }) } +async fn batch_read_blobs( + opts: &RERuntimeOpts, + read_blobs_request: BatchReadBlobsRequest, + cas_f: impl Clone + Fn(BatchReadBlobsRequest) -> Cas, +) -> anyhow::Result +where + Cas: Future>, +{ + for i in 1..=opts.max_retries + 1 { + // TODO: Hopefully this isn't too expensive? Can we take a reference to the request + // instead? + match cas_f(read_blobs_request.clone()).await { + Ok(resp) => { + return Ok(resp); + } + Err(e) => { + tracing::debug!( + "Failed to make BatchReadBlobs request, retrying after sleeping {} seconds: {:#?}", + i, + e + ); + tokio::time::sleep(Duration::from_secs(i as u64)).await; + } + } + } + + cas_f(read_blobs_request).await +} + async fn upload_impl( instance_name: &InstanceName, request: UploadRequest, @@ -1815,6 +1850,15 @@ mod tests { use super::*; + fn test_re_runtime_opts() -> RERuntimeOpts { + RERuntimeOpts { + use_fbcode_metadata: false, + max_concurrent_uploads_per_action: None, + cas_ttl_secs: 0, + max_retries: 0, + } + } + #[tokio::test] async fn test_download_named() -> anyhow::Result<()> { let work = tempfile::tempdir()?; @@ -1878,6 +1922,7 @@ mod tests { }; download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -1985,6 +2030,7 @@ mod tests { }; download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -2067,6 +2113,7 @@ mod tests { }; let res = download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -2154,6 +2201,7 @@ mod tests { let counter = AtomicU16::new(0); let res = download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -2223,6 +2271,7 @@ mod tests { }; let res = download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -2279,6 +2328,7 @@ mod tests { let res = BatchReadBlobsResponse { responses: vec![] }; let res = download_impl( + &test_re_runtime_opts(), &InstanceName(None), req, None, @@ -2318,6 +2368,7 @@ mod tests { }; download_impl( + &test_re_runtime_opts(), &InstanceName(Some("instance".to_owned())), req, None, @@ -2885,94 +2936,95 @@ mod tests { assert_eq!(substitute_env_vars_impl("FOO", getter).unwrap(), "FOO"); assert!(substitute_env_vars_impl("$FOO$BAZ", getter).is_err()); } -} -#[tokio::test] -async fn test_upload_compressed() -> anyhow::Result<()> { - let blob_data = vec![1; 10 * 1024 * 1024]; - let digest1 = TDigest { - hash: "aa".to_owned(), - size_in_bytes: blob_data.len() as i64, - ..Default::default() - }; + #[tokio::test] + async fn test_upload_compressed() -> anyhow::Result<()> { + let blob_data = vec![1; 10 * 1024 * 1024]; + let digest1 = TDigest { + hash: "aa".to_owned(), + size_in_bytes: blob_data.len() as i64, + ..Default::default() + }; - let req = UploadRequest { - inlined_blobs_with_digest: Some(vec![InlinedBlobWithDigest { - digest: digest1.clone(), - blob: blob_data.clone(), + let req = UploadRequest { + inlined_blobs_with_digest: Some(vec![InlinedBlobWithDigest { + digest: digest1.clone(), + blob: blob_data.clone(), + ..Default::default() + }]), ..Default::default() - }]), - ..Default::default() - }; + }; - let blob_data_ref = &blob_data; - upload_impl( - &InstanceName(Some("instance".to_owned())), - req, - Some(Compressor::Zstd), - 1, - None, - |_req| async move { - panic!("Not called"); - }, - { - |write_reqs| async move { - let compressed_data: Vec = - write_reqs.iter().flat_map(|wr| wr.data.clone()).collect(); - let mut data = vec![]; - ZstdDecoder::new(Cursor::new(compressed_data)) - .read_to_end(&mut data) - .await - .unwrap(); - assert_eq!(&data, blob_data_ref); - anyhow::Ok(WriteResponse { committed_size: -1 }) - } - }, - ) - .await?; + let blob_data_ref = &blob_data; + upload_impl( + &InstanceName(Some("instance".to_owned())), + req, + Some(Compressor::Zstd), + 1, + None, + |_req| async move { + panic!("Not called"); + }, + { + |write_reqs| async move { + let compressed_data: Vec = + write_reqs.iter().flat_map(|wr| wr.data.clone()).collect(); + let mut data = vec![]; + ZstdDecoder::new(Cursor::new(compressed_data)) + .read_to_end(&mut data) + .await + .unwrap(); + assert_eq!(&data, blob_data_ref); + anyhow::Ok(WriteResponse { committed_size: -1 }) + } + }, + ) + .await?; - Ok(()) -} + Ok(()) + } -#[tokio::test] -async fn test_download_compressed() -> anyhow::Result<()> { - let blob_data = vec![1; 1024]; + #[tokio::test] + async fn test_download_compressed() -> anyhow::Result<()> { + let blob_data = vec![1; 1024]; - let mut compressed_data = vec![]; - ZstdEncoder::new(Cursor::new(blob_data.clone())) - .read_to_end(&mut compressed_data) - .await - .unwrap(); - let compressed_data_ref = &compressed_data; - - let d_resp = download_impl( - &InstanceName(None), - DownloadRequest { - inlined_digests: Some(vec![TDigest { - hash: "aa".to_owned(), - size_in_bytes: blob_data.len() as i64, + let mut compressed_data = vec![]; + ZstdEncoder::new(Cursor::new(blob_data.clone())) + .read_to_end(&mut compressed_data) + .await + .unwrap(); + let compressed_data_ref = &compressed_data; + + let d_resp = download_impl( + &test_re_runtime_opts(), + &InstanceName(None), + DownloadRequest { + inlined_digests: Some(vec![TDigest { + hash: "aa".to_owned(), + size_in_bytes: blob_data.len() as i64, + ..Default::default() + }]), + file_digests: None, ..Default::default() - }]), - file_digests: None, - ..Default::default() - }, - Some(Compressor::Zstd), - 10, - |_req| async { panic!("not called") }, - |_req| async move { - Ok(Box::pin(futures::stream::iter( - compressed_data_ref - .chunks(10) - .map(|d| Result::Ok(ReadResponse { data: d.to_vec() })), - ))) - }, - ) - .await?; - - assert_eq!( - d_resp.inlined_blobs.as_ref().unwrap()[0].blob.len(), - blob_data.len() - ); - assert_eq!(d_resp.inlined_blobs.unwrap()[0].blob, blob_data); - Ok(()) + }, + Some(Compressor::Zstd), + 10, + |_req| async { panic!("not called") }, + |_req| async move { + Ok(Box::pin(futures::stream::iter( + compressed_data_ref + .chunks(10) + .map(|d| Result::Ok(ReadResponse { data: d.to_vec() })), + ))) + }, + ) + .await?; + + assert_eq!( + d_resp.inlined_blobs.as_ref().unwrap()[0].blob.len(), + blob_data.len() + ); + assert_eq!(d_resp.inlined_blobs.unwrap()[0].blob, blob_data); + Ok(()) + } } From 974bc41f4de89a0ba71ba7b355373b899e16273f Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:19:20 -0400 Subject: [PATCH 10/29] Disable cell segmentation indiscriminately as a workaround. Avoids needing to rebase the entire patch of https://github.com/facebook/buck2/pull/1100, since we don't care about supporting cell segmentation at Mercury anyway. --- app/buck2_core/src/bzl.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/buck2_core/src/bzl.rs b/app/buck2_core/src/bzl.rs index b0848292f8f06..2ca5d8e3659e4 100644 --- a/app/buck2_core/src/bzl.rs +++ b/app/buck2_core/src/bzl.rs @@ -53,7 +53,7 @@ impl ImportPath { pub fn new_with_build_file_cells( path: CellPath, - build_file_cell: BuildFileCell, + mut build_file_cell: BuildFileCell, ) -> buck2_error::Result { if path.parent().is_none() { return Err(ImportPathError::Invalid(path).into()); @@ -67,6 +67,8 @@ impl ImportPath { return Err(ImportPathError::Suffix(path).into()); } + build_file_cell = BuildFileCell::new(path.cell()); + Ok(Self { path, build_file_cell, @@ -76,7 +78,7 @@ impl ImportPath { /// LSP creates imports for non-bzl files. pub fn new_hack_for_lsp( path: CellPath, - build_file_cell: BuildFileCell, + mut build_file_cell: BuildFileCell, ) -> buck2_error::Result { if path.parent().is_none() { return Err(ImportPathError::Invalid(path).into()); @@ -86,6 +88,8 @@ impl ImportPath { return Err(ImportPathError::Invalid(path).into()); } + build_file_cell = BuildFileCell::new(path.cell()); + Ok(Self { path, build_file_cell, From 628367cdc561c09782413578f330ace18f4b7cfc Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:47:51 -0400 Subject: [PATCH 11/29] RPC retry and timeout improvements for remote execution https://github.com/facebook/buck2/pull/1221 Note: modified during rebase to reconcile with prior patches (pr1076) which partially overlapped with this PR. Also adds execution_concurrency_limit and grpc_timeout config fields which were assumed present by the upstream diff. --- app/buck2_execute/src/re/client.rs | 2 - app/buck2_re_configuration/src/lib.rs | 14 +- remote_execution/oss/re_grpc/src/client.rs | 358 +++++++++++++------- remote_execution/oss/re_grpc/src/lib.rs | 2 + remote_execution/oss/re_grpc/src/request.rs | 20 +- remote_execution/oss/re_grpc/src/retry.rs | 108 ++++++ 6 files changed, 370 insertions(+), 134 deletions(-) create mode 100644 remote_execution/oss/re_grpc/src/retry.rs diff --git a/app/buck2_execute/src/re/client.rs b/app/buck2_execute/src/re/client.rs index 81a3838c29a4f..f0f621ac302ef 100644 --- a/app/buck2_execute/src/re/client.rs +++ b/app/buck2_execute/src/re/client.rs @@ -535,9 +535,7 @@ fn re_platform(x: &RE::Platform) -> remote_execution::TPlatform { properties: x.properties.map(|x| remote_execution::TProperty { name: x.name.clone(), value: x.value.clone(), - ..Default::default() }), - ..Default::default() } } diff --git a/app/buck2_re_configuration/src/lib.rs b/app/buck2_re_configuration/src/lib.rs index 56e1d32213c87..a75b202802c7b 100644 --- a/app/buck2_re_configuration/src/lib.rs +++ b/app/buck2_re_configuration/src/lib.rs @@ -19,6 +19,8 @@ use buck2_core::rollout_percentage::RolloutPercentage; static BUCK2_RE_CLIENT_CFG_SECTION: &str = "buck2_re_client"; +const DEFAULT_MAX_RETRIES: usize = 5; + /// We put functions here that both things need to implement for code that isn't gated behind a /// fbcode_build or not(fbcode_build) pub trait RemoteExecutionStaticMetadataImpl: Sized { @@ -468,8 +470,10 @@ pub struct Buck2OssReConfiguration { pub grpc_keepalive_while_idle: Option, /// Maximum number of concurrent execution requests. pub execution_concurrency_limit: Option, - /// Maximum retries for network requests. + /// Maximum retries for RPC requests. Defaults to 5. pub max_retries: usize, + /// Timeout for RPC requests in seconds. Defaults to 60s. + pub grpc_timeout: u64, } #[derive(Clone, Debug, Default, Allocative)] @@ -596,7 +600,13 @@ impl Buck2OssReConfiguration { section: BUCK2_RE_CLIENT_CFG_SECTION, property: "max_retries", })? - .unwrap_or(0), + .unwrap_or(DEFAULT_MAX_RETRIES), + grpc_timeout: legacy_config + .parse(BuckconfigKeyRef { + section: BUCK2_RE_CLIENT_CFG_SECTION, + property: "grpc_timeout", + })? + .unwrap_or(60), }) } } diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index 7dcb51a7b5794..4f76e608d8f8c 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -100,9 +100,12 @@ use crate::error::*; use crate::metadata::*; use crate::request::*; use crate::response::*; +use crate::retry::retry; use crate::stats::CountingConnector; const DEFAULT_MAX_TOTAL_BATCH_SIZE: usize = 4 * 1000 * 1000; +const INITIAL_DELAY: Duration = Duration::from_millis(100); +const MAX_DELAY: Duration = Duration::from_secs(10); fn tdigest_to(tdigest: TDigest) -> Digest { Digest { @@ -239,6 +242,7 @@ pub struct RECapabilities { } /// Contains runtime options for the remote execution client as set under `buck2_re_client` +#[derive(Clone, Copy)] pub struct RERuntimeOpts { /// Use the Meta version of the request metadata use_fbcode_metadata: bool, @@ -248,8 +252,11 @@ pub struct RERuntimeOpts { cas_ttl_secs: i64, /// Maximum retries for network requests. max_retries: usize, + /// Timeout for RPC requests. + rpc_timeout: Duration, } +#[derive(Clone)] struct InstanceName(Option); impl InstanceName { @@ -337,8 +344,11 @@ impl REClientBuilder { // be set here instead of on the endpoint let mut http = HttpConnector::new(); http.enforce_http(false); + http.set_keepalive(Some(Duration::from_secs(180))); let connector = CountingConnector::new(http); + endpoint = endpoint.timeout(Duration::from_secs(opts.grpc_timeout)); + anyhow::Ok( endpoint .connect_with_connector(connector) @@ -443,6 +453,7 @@ impl REClientBuilder { // on the TTL of the remote blob. cas_ttl_secs: opts.cas_ttl_secs.unwrap_or(3 * 60 * 60), max_retries: opts.max_retries, + rpc_timeout: Duration::from_secs(opts.grpc_timeout), }, grpc_clients, capabilities, @@ -686,24 +697,35 @@ impl REClient { metadata: RemoteExecutionMetadata, request: ActionResultRequest, ) -> anyhow::Result { - let mut client = self.grpc_clients.action_cache_client.clone(); + retry( + || async { + let mut client = self.grpc_clients.action_cache_client.clone(); + let request = request.clone(); + let metadata = metadata.clone(); - let res = client - .get_action_result(with_re_metadata( - GetActionResultRequest { - instance_name: self.instance_name.as_str().to_owned(), - action_digest: Some(tdigest_to(request.digest)), - ..Default::default() - }, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await?; + let res = client + .get_action_result(with_re_metadata( + GetActionResultRequest { + instance_name: self.instance_name.as_str().to_owned(), + action_digest: Some(tdigest_to(request.digest)), + ..Default::default() + }, + metadata, + self.runtime_opts, + )) + .await?; - Ok(ActionResultResponse { - action_result: convert_action_result(res.into_inner())?, - ttl: 0, - }) + Ok(ActionResultResponse { + action_result: convert_action_result(res.into_inner())?, + ttl: 0, + }) + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await } pub async fn write_action_result( @@ -711,26 +733,37 @@ impl REClient { metadata: RemoteExecutionMetadata, request: WriteActionResultRequest, ) -> anyhow::Result { - let mut client = self.grpc_clients.action_cache_client.clone(); - - let res = client - .update_action_result(with_re_metadata( - UpdateActionResultRequest { - instance_name: self.instance_name.as_str().to_owned(), - action_digest: Some(tdigest_to(request.action_digest)), - action_result: Some(convert_t_action_result2(request.action_result)?), - results_cache_policy: None, - ..Default::default() - }, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await?; - - Ok(WriteActionResultResponse { - actual_action_result: convert_action_result(res.into_inner())?, - ttl_seconds: 0, - }) + retry( + || async { + let mut client = self.grpc_clients.action_cache_client.clone(); + let request = request.clone(); + let metadata = metadata.clone(); + + let res = client + .update_action_result(with_re_metadata( + UpdateActionResultRequest { + instance_name: self.instance_name.as_str().to_owned(), + action_digest: Some(tdigest_to(request.action_digest)), + action_result: Some(convert_t_action_result2(request.action_result)?), + results_cache_policy: None, + ..Default::default() + }, + metadata, + self.runtime_opts, + )) + .await?; + + Ok(WriteActionResultResponse { + actual_action_result: convert_action_result(res.into_inner())?, + ttl_seconds: 0, + }) + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await } pub async fn execute_with_progress( @@ -741,8 +774,6 @@ impl REClient { // TODO(aloiscochard): Map those properly in the request // use crate::proto::build::bazel::remote::execution::v2::ExecutionPolicy; - let mut client = self.grpc_clients.execution_client.clone(); - let action_digest = tdigest_to(execute_request.action_digest.clone()); let request = GExecuteRequest { @@ -759,14 +790,24 @@ impl REClient { ..Default::default() }; - let stream = client - .execute(with_re_metadata( - request, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await? - .into_inner(); + let stream = retry( + || async { + let mut client = self.grpc_clients.execution_client.clone(); + let request = request.clone(); + let metadata = metadata.clone(); + + let stream = client + .execute(with_re_metadata(request, metadata, self.runtime_opts)) + .await? + .into_inner(); + Ok(stream) + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + true, + ) + .await?; let stream = futures::stream::try_unfold(stream, move |mut stream| async { let msg = match stream.try_next().await.context("RE channel error")? { @@ -875,29 +916,56 @@ impl REClient { self.runtime_opts.max_concurrent_uploads_per_action, |re_request| async { let metadata = metadata.clone(); - let mut cas_client = self.grpc_clients.cas_client.clone(); - let resp = cas_client - .batch_update_blobs(with_re_metadata( - re_request, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await?; - Ok(resp.into_inner()) + let cas_client = self.grpc_clients.cas_client.clone(); + let runtime_opts = self.runtime_opts; + + retry( + move || { + let metadata = metadata.clone(); + let mut cas_client = cas_client.clone(); + let re_request = re_request.clone(); + async move { + let resp = cas_client + .batch_update_blobs(with_re_metadata( + re_request, + metadata, + runtime_opts, + )) + .await?; + Ok(resp.into_inner()) + } + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await }, |segments| async { let metadata = metadata.clone(); - let mut bytestream_client = self.grpc_clients.bytestream_client.clone(); - let requests = futures::stream::iter(segments); - let resp = bytestream_client - .write(with_re_metadata( - requests, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await?; - - Ok(resp.into_inner()) + let bytestream_client = self.grpc_clients.bytestream_client.clone(); + let runtime_opts = self.runtime_opts; + + retry( + move || { + let metadata = metadata.clone(); + let mut bytestream_client = bytestream_client.clone(); + let requests = futures::stream::iter(segments.clone()); + async move { + let resp = bytestream_client + .write(with_re_metadata(requests, metadata, runtime_opts)) + .await?; + + Ok(resp.into_inner()) + } + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await }, ) .await @@ -941,29 +1009,70 @@ impl REClient { self.capabilities.max_total_batch_size, |re_request| async { let metadata = metadata.clone(); - let mut client = self.grpc_clients.cas_client.clone(); - Ok(client - .batch_read_blobs(with_re_metadata( - re_request, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await? - .into_inner()) + let client = self.grpc_clients.cas_client.clone(); + let runtime_opts = self.runtime_opts; + + retry( + move || { + let metadata = metadata.clone(); + let mut client = client.clone(); + let re_request = re_request.clone(); + async move { + Ok(client + .batch_read_blobs(with_re_metadata( + re_request, + metadata, + runtime_opts, + )) + .await? + .into_inner()) + } + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await }, |read_request| { let metadata = metadata.clone(); + let runtime_opts = self.runtime_opts; async move { - let mut client = self.grpc_clients.bytestream_client.clone(); - let response = client - .read(with_re_metadata( - read_request, - metadata, - self.runtime_opts.use_fbcode_metadata, - )) - .await? - .into_inner(); - Ok(Box::pin(response.into_stream())) + let client = self.grpc_clients.bytestream_client.clone(); + retry( + move || { + let metadata = metadata.clone(); + let mut client = client.clone(); + let read_request = read_request.clone(); + async move { + let response = client + .read(with_re_metadata( + read_request, + metadata, + runtime_opts, + )) + .await? + .into_inner(); + Ok(Box::pin(response.into_stream()) + as Pin< + Box< + dyn Stream< + Item = Result< + re_grpc_proto::google::bytestream::ReadResponse, + tonic::Status, + >, + > + Send, + >, + >) + } + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await } }, ) @@ -975,7 +1084,7 @@ impl REClient { metadata: RemoteExecutionMetadata, request: GetDigestsTtlRequest, ) -> anyhow::Result { - let mut cas_client = self.grpc_clients.cas_client.clone(); + let cas_client = self.grpc_clients.cas_client.clone(); let mut remote_results: HashMap = HashMap::new(); let mut digests_to_check: Vec = Vec::new(); @@ -1001,18 +1110,36 @@ impl REClient { // Send a request and notify others of the result if !digests_to_check.is_empty() { tracing::debug!(num_digests = digests_to_check.len(), "FindMissingBlobs"); - let missing_blobs = cas_client - .find_missing_blobs(with_re_metadata( - FindMissingBlobsRequest { - instance_name: self.instance_name.as_str().to_owned(), - blob_digests: digests_to_check.map(|b| tdigest_to(b.clone())), - ..Default::default() - }, - metadata.clone(), - self.runtime_opts.use_fbcode_metadata, - )) - .await - .context("Failed to request what blobs are not present on remote")?; + let runtime_opts = self.runtime_opts; + let missing_blobs = retry( + || { + let mut cas_client = cas_client.clone(); + let metadata = metadata.clone(); + let digests_to_check = digests_to_check.clone(); + let instance_name = self.instance_name.as_str().to_owned(); + + async move { + cas_client + .find_missing_blobs(with_re_metadata( + FindMissingBlobsRequest { + instance_name, + blob_digests: digests_to_check + .map(|b| tdigest_to(b.clone())), + ..Default::default() + }, + metadata, + runtime_opts, + )) + .await + .context("Failed to request what blobs are not present on remote") + } + }, + self.runtime_opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + false, + ) + .await?; let resp: FindMissingBlobsResponse = missing_blobs.into_inner(); // Update the results and the cache @@ -1274,7 +1401,7 @@ async fn download_impl( ) -> anyhow::Result where Byt: Future>>>, - BytRet: Stream> + Send, + BytRet: Stream> + Send + ?Sized, Cas: Future>, { fn resource_name( @@ -1484,25 +1611,14 @@ async fn batch_read_blobs( where Cas: Future>, { - for i in 1..=opts.max_retries + 1 { - // TODO: Hopefully this isn't too expensive? Can we take a reference to the request - // instead? - match cas_f(read_blobs_request.clone()).await { - Ok(resp) => { - return Ok(resp); - } - Err(e) => { - tracing::debug!( - "Failed to make BatchReadBlobs request, retrying after sleeping {} seconds: {:#?}", - i, - e - ); - tokio::time::sleep(Duration::from_secs(i as u64)).await; - } - } - } - - cas_f(read_blobs_request).await + retry( + || async { cas_f(read_blobs_request.clone()).await }, + opts.max_retries, + INITIAL_DELAY, + MAX_DELAY, + true, + ) + .await } async fn upload_impl( @@ -1734,7 +1850,7 @@ where fn with_re_metadata( t: T, metadata: RemoteExecutionMetadata, - use_fbcode_metadata: bool, + runtime_opts: RERuntimeOpts, ) -> tonic::Request { // This creates a new Tonic request with attached metadata for the RE // backend. There are two cases here we need to support: @@ -1756,8 +1872,9 @@ fn with_re_metadata( // Meta builds catch those issues earlier. let mut msg = tonic::Request::new(t); + msg.set_timeout(runtime_opts.rpc_timeout); - if use_fbcode_metadata { + if runtime_opts.use_fbcode_metadata { // This is pretty ugly, but the protobuf spec that defines this is // internal, so considering field numbers need to be stable anyway (= // low risk), and this is not used in prod (= low impact if this goes @@ -1856,6 +1973,7 @@ mod tests { max_concurrent_uploads_per_action: None, cas_ttl_secs: 0, max_retries: 0, + rpc_timeout: Duration::from_secs(60), } } diff --git a/remote_execution/oss/re_grpc/src/lib.rs b/remote_execution/oss/re_grpc/src/lib.rs index 1a7aac1e613d8..71c8341f7eadb 100644 --- a/remote_execution/oss/re_grpc/src/lib.rs +++ b/remote_execution/oss/re_grpc/src/lib.rs @@ -15,6 +15,7 @@ mod grpc; mod metadata; mod request; mod response; +mod retry; mod stats; use std::sync::Arc; @@ -28,6 +29,7 @@ pub use grpc::*; pub use metadata::*; pub use request::*; pub use response::*; +pub use retry::*; /// The global version of the network stats full of atomics #[derive(Default, Debug)] diff --git a/remote_execution/oss/re_grpc/src/request.rs b/remote_execution/oss/re_grpc/src/request.rs index 17f24ddae6597..c60e291d4c8ce 100644 --- a/remote_execution/oss/re_grpc/src/request.rs +++ b/remote_execution/oss/re_grpc/src/request.rs @@ -12,35 +12,35 @@ pub use crate::digest::*; use crate::grpc::Platform as TPlatform; use crate::response::TActionResult2; -#[derive(Default)] +#[derive(Clone, Default)] pub struct ActionResultRequest { pub digest: TDigest, pub platform: Option, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct DownloadRequest { pub inlined_digests: Option>, pub file_digests: Option>, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct NamedDigestWithPermissions { pub named_digest: NamedDigest, pub is_executable: bool, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct NamedDigest { pub name: String, pub digest: TDigest, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct UploadRequest { pub files_with_digest: Option>, pub inlined_blobs_with_digest: Option>, @@ -49,7 +49,7 @@ pub struct UploadRequest { pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct Path { pub path: String, pub follow_symlinks: bool, @@ -57,26 +57,26 @@ pub struct Path { pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct InlinedBlobWithDigest { pub blob: Vec, pub digest: TDigest, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct FindMissingBlobsRequest { pub digests: Vec, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct GetDigestsTtlRequest { pub digests: Vec, pub _dot_dot: (), } -#[derive(Default)] +#[derive(Clone, Default)] pub struct ExtendDigestsTtlRequest { pub digests: Vec, pub ttl: i64, diff --git a/remote_execution/oss/re_grpc/src/retry.rs b/remote_execution/oss/re_grpc/src/retry.rs new file mode 100644 index 0000000000000..718d08f8af461 --- /dev/null +++ b/remote_execution/oss/re_grpc/src/retry.rs @@ -0,0 +1,108 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under both the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree and the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. + */ + +use std::time::Duration; +use std::future::Future; +use crate::error::{REClientError, TCode}; +use tracing::warn; + +pub async fn retry( + mut f: F, + max_retries: usize, + initial_delay: Duration, + max_delay: Duration, + retry_not_found: bool, +) -> anyhow::Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut retries = 0; + let mut delay = initial_delay; + + loop { + match f().await { + Ok(val) => return Ok(val), + Err(err) => { + if retries >= max_retries { + return Err(err); + } + let retryable = is_retryable(&err, retry_not_found); + + if retryable == Retryable::No { + return Err(err); + } + + retries += 1; + // Try to downcast to REClientError to get a better message, otherwise just print the error + let msg = if let Some(re_err) = err.downcast_ref::() { + re_err.message.as_str() + } else { + // Try tonic::Status + if let Some(status) = err.downcast_ref::() { + status.message() + } else { + "unknown error" + } + }; + + if retryable == Retryable::Wait { + warn!("Retrying request after error: {}. Attempt {}/{} (waiting {:?})", msg, retries, max_retries, delay); + tokio::time::sleep(delay).await; + + delay *= 2; + if delay > max_delay { + delay = max_delay; + } + } else { + warn!("Retrying request after error: {}. Attempt {}/{}", msg, retries, max_retries); + } + } + } + } +} + +#[derive(PartialEq)] +enum Retryable { + Immediately, + Wait, + No, +} + +fn is_retryable(err: &anyhow::Error, retry_not_found: bool) -> Retryable { + for cause in err.chain() { + if let Some(re_err) = cause.downcast_ref::() { + return match re_err.code { + TCode::DEADLINE_EXCEEDED => Retryable::Immediately, + TCode::UNKNOWN + | TCode::CANCELLED + | TCode::ABORTED + | TCode::INTERNAL + | TCode::UNAVAILABLE + | TCode::RESOURCE_EXHAUSTED => Retryable::Wait, + TCode::NOT_FOUND if retry_not_found => Retryable::Wait, + _ => Retryable::No, + }; + } + if let Some(status) = cause.downcast_ref::() { + return match status.code() { + tonic::Code::DeadlineExceeded => Retryable::Immediately, + tonic::Code::Unknown + | tonic::Code::Cancelled + | tonic::Code::Aborted + | tonic::Code::Internal + | tonic::Code::Unavailable + | tonic::Code::ResourceExhausted => Retryable::Wait, + tonic::Code::NotFound if retry_not_found => Retryable::Wait, + _ => Retryable::No, + }; + } + } + Retryable::No +} From 89deb83ddb5011b72ae61f29e758b98cd59b5ffa Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 11 Mar 2026 07:56:39 -0400 Subject: [PATCH 12/29] More semaphores for grpc experimental patch by joseph which we may try to upstream later if it works --- app/buck2_execute/src/re/client.rs | 75 ++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/app/buck2_execute/src/re/client.rs b/app/buck2_execute/src/re/client.rs index f0f621ac302ef..1943d1a579b1b 100644 --- a/app/buck2_execute/src/re/client.rs +++ b/app/buck2_execute/src/re/client.rs @@ -257,6 +257,13 @@ impl RemoteExecutionClient { use_case: RemoteExecutorUseCase, platform: &RE::Platform, ) -> buck2_error::Result> { + let _permit = self + .data + .client + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); self.data .action_cache .op(self @@ -305,6 +312,13 @@ impl RemoteExecutionClient { inlined_blobs_with_digest: Vec, use_case: RemoteExecutorUseCase, ) -> buck2_error::Result<()> { + let _permit = self + .data + .client + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); self.data .uploads .op(self.data.client.upload_files_and_directories( @@ -380,6 +394,13 @@ impl RemoteExecutionClient { digests: Vec, use_case: RemoteExecutorUseCase, ) -> buck2_error::Result> { + let _permit = self + .data + .client + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); self.data .downloads .op(self @@ -398,6 +419,13 @@ impl RemoteExecutionClient { digest: &TDigest, use_case: RemoteExecutorUseCase, ) -> buck2_error::Result> { + let _permit = self + .data + .client + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); self.data .downloads .op(self.data.client.download_blob(digest, use_case)) @@ -464,6 +492,13 @@ impl RemoteExecutionClient { platform: &RE::Platform, write_type: ActionCacheWriteType, ) -> buck2_error::Result { + let _permit = self + .data + .client + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); self.data .write_action_results .op(self @@ -521,6 +556,9 @@ struct RemoteExecutionClientImpl { /// How many files we can be downloading concurrently. #[allocative(skip)] download_files_semapore: Arc, + /// Global limit on all concurrent gRPC calls (action cache, downloads, etc) + #[allocative(skip)] + grpc_semaphore: Arc, /// How many files to kick off downloading concurrently for one request. This should be smaller /// than the files semaphore to ensure we can actually *acquire* that semaphore. download_chunk_size: usize, @@ -1034,6 +1072,7 @@ impl RemoteExecutionClientImpl { "buck2-download-priority-sem", download_concurrency, )), + grpc_semaphore: Arc::new(Semaphore::new(400)), // TODO: make this configurable download_chunk_size, respect_file_symlinks, persistent_cache_mode, @@ -1380,15 +1419,24 @@ impl RemoteExecutionClientImpl { // 4. We get Ok(0 everywhere), which means it worked // Obtain a stream of events from RE. If this fails then that is case #1 above so we - // bail. - let mut receiver = self - .client() - .get_execution_client() - .execute_with_progress(metadata, request) - // boxed() to segment the future - .boxed() - .await - .context("Failed to start remote execution")?; + // bail. The semaphore is held only for the initial gRPC call that establishes the stream, + // then released before entering the streaming loop so it doesn't block other calls for + // the entire (potentially long) duration of remote execution. + let mut receiver = { + let _permit = self + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); + self.client() + .get_execution_client() + .execute_with_progress(metadata, request) + // boxed() to segment the future + .boxed() + .await + .map_err(anyhow::Error::from) + .context("Failed to start remote execution")? + }; // Now we wait until the ExecuteResponse shows up, and produce events accordingly. If // this doesn't give us an ExecuteResponse then this is case #1 again so we also fail. @@ -1884,7 +1932,14 @@ impl RemoteExecutionClientImpl { ) .await, ) - .await?; + .await + .buck_error_context("Failed to acquire download_files_semapore")?; + + let _grpc_permit = self + .grpc_semaphore + .acquire() + .await + .expect("Semaphore should never be closed"); buck2_error::Ok(ChunkDownloadResult::Downloaded(response.local_cache_stats)) } From d3c7943e07d670fe059093c9c50f18e96e79e29d Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Wed, 25 Mar 2026 12:18:57 -0400 Subject: [PATCH 13/29] Add contribution guidelines --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..9f3505de0252e --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +# Contributing to Mercury's Buck2 fork (mercury-head) + +This repo is Mercury's internal fork of [Buck2](https://github.com/facebook/buck2). The `mercury-head` branch is our maintained head. + +## Workflow + +### 1. Branch from mercury-head + +```bash +git fetch origin +git checkout -b my-feature origin/mercury-head +``` + +### 2. Make a PR to mercury-head + +Open a pull request targeting the `mercury-head` branch (not `main`). + +### 3. After merging, create a tag + +Once your PR is merged, tag the new `mercury-head` HEAD using the format: + +``` +mwb--base- +``` + +Where: +- `` is today's date in `YYYY-MM-DD` format +- `` is the date tag of the FB upstream commit that `mercury` is currently based on (e.g. `2026-01-19`) + +Example: + +```bash +# Find the current mercury HEAD +git fetch origin +git rev-parse origin/mercury-head + +# Create and push the tag +git tag mwb-2026-03-25-base-2026-01-19 origin/mercury +git push origin mwb-2026-03-25-base-2026-01-19 +``` + +To find the correct ``: look at the FB upstream tags in this repo (e.g. `2026-01-19`, `2026-03-15`) and identify which one the current `mercury-head` branch is based on. + +### 4. Update buck2-source in mwb + +In the [mercury-web-backend](https://github.com/MercuryTechnologies/mercury-web-backend) repo, update the `buck2-source` reference to point to your new tag. From 067b67f8623fe37db9982a882e6785295d4317a3 Mon Sep 17 00:00:00 2001 From: Ian-Woo Kim Date: Fri, 3 Apr 2026 11:59:25 -0700 Subject: [PATCH 14/29] Fix regression after DirectIoExecutor by default on macOS (#4) use BlockingExecuter = BuckBlockingExecutor::default_concurrency(fs.dupe()) for macos --- app/buck2_server/src/daemon/state.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/buck2_server/src/daemon/state.rs b/app/buck2_server/src/daemon/state.rs index 9edcdc6bfd702..d5c41fca34131 100644 --- a/app/buck2_server/src/daemon/state.rs +++ b/app/buck2_server/src/daemon/state.rs @@ -407,9 +407,13 @@ impl DaemonState { let disk_state_options = DiskStateOptions::new(root_config, materializations.dupe())?; let blocking_executor: Arc = - if cfg!(any(target_os = "macos", target_os = "windows")) { - Arc::new(DirectIoExecutor::new(fs.dupe())?) - } else { + // NOTE: Due to this issue: https://github.com/facebook/buck2/issues/1282 + // We need to revert DirectIoExecutor as a default to BuckBlockingExecutor on macOS. + // + //if cfg!(any(target_os = "macos", target_os = "windows")) { + // Arc::new(DirectIoExecutor::new(fs.dupe())?) + //} else + { Arc::new(BuckBlockingExecutor::default_concurrency(fs.dupe())?) }; From f31d8a7a28a5bc0544fce38a8fd83d58190382c9 Mon Sep 17 00:00:00 2001 From: Joseph Sumabat Date: Tue, 14 Apr 2026 14:41:35 -0400 Subject: [PATCH 15/29] Use fork patches for hyper and tonic Patch hyper with arianvp's fork and tonic/tonic-prost/tonic-prost-build with edef1c's fork until these changes make it upstream. --- Cargo.toml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4935fdeff5819..7c172e18e931f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,7 +213,7 @@ http-body-util = "0.1.0" httparse = "1.7.1" httptest = "0.16" humantime = "2.1" -hyper = { version = "1.8.1", features = ["client", "http1", "http2"] } +hyper = { version = "1.9", features = ["client", "http1", "http2"] } hyper-http-proxy = "1.0" hyper-rustls = { version = "0.27.0", features = ["http2"] } hyper-timeout = "0.5" @@ -271,10 +271,10 @@ postcard = { version = "1.0.8", features = ["use-crc", "use-std"] } pretty_assertions = "1.2.1" probminhash = { git = "https://github.com/krallin/probminhash", rev = "3d6b7a5315305c2893a7a04e0de9e23b305992c5", default-features = false } # WAITING_FOR_PR https://github.com/jean-pierreBoth/probminhash/pull/4 proc-macro2 = "1.0.106" -prost = "0.14.3" -prost-build = "0.14.3" -prost-derive = "0.14.3" -prost-types = "0.14.3" +prost = "0.14" +prost-build = "0.14" +prost-derive = "0.14" +prost-types = "0.14" protoc-bin-vendored = "3.0.0" psutil = "3.2" quick-xml = { version = "0.37", features = ["overlapped-lists", "serialize"] } @@ -510,3 +510,9 @@ incremental = true # Base on the comment in https://github.com/jimblandy/perf-event/pull/53, we will no long need this patch in future. perf-event = { git = "https://github.com/Nero5023/perf-event.git", rev = "6dae86b6d4807acec081e6dc0a53167f57f8c0f4", version = "0.4" } perf-event-open-sys = { git = "https://github.com/Nero5023/perf-event.git", rev = "6dae86b6d4807acec081e6dc0a53167f57f8c0f4", version = "5.0" } +hyper = { git = "https://github.com/arianvp/hyper.git", branch = "push-ktssyytnyrru" } +tonic = { git = "https://github.com/edef1c/tonic.git", branch = "push-rosuyzxnysvw" } +tonic-health = { git = "https://github.com/edef1c/tonic.git", branch = "push-rosuyzxnysvw" } +tonic-reflection = { git = "https://github.com/edef1c/tonic.git", branch = "push-rosuyzxnysvw" } +tonic-prost = { git = "https://github.com/edef1c/tonic.git", branch = "push-rosuyzxnysvw" } +tonic-prost-build = { git = "https://github.com/edef1c/tonic.git", branch = "push-rosuyzxnysvw" } From eebc1b01cc472439ae2a61f4ae3ea124de6481e8 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 17 Apr 2026 20:01:18 +0200 Subject: [PATCH 16/29] Improve reliability and observability of remote exeution gRPC client (#5) * Add debugging output for RE actions * Print method for RPC retries * Track and retry message received from stream During stream reading, there was no retry mechanism in place, so seeing many `ResourceExhausted` errors would lead h2 issuing a GOWAWY to the server, effectively closing the connection. Now, stream responses are also retried by wrapping both, stream establishment and stream reading in a retry loop. * Do not set a timeout on the gRPC message itself This is otherwise enforced by the server which cancels the action, which wo don't want. * Do not enforce a timeout for Execute requests --- remote_execution/oss/re_grpc/src/client.rs | 214 +++++++++++-------- remote_execution/oss/re_grpc/src/metadata.rs | 8 +- remote_execution/oss/re_grpc/src/retry.rs | 119 ++++++++++- 3 files changed, 243 insertions(+), 98 deletions(-) diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index 4f76e608d8f8c..b79105698845f 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -74,6 +74,7 @@ use re_grpc_proto::google::bytestream::ReadResponse; use re_grpc_proto::google::bytestream::WriteRequest; use re_grpc_proto::google::bytestream::WriteResponse; use re_grpc_proto::google::bytestream::byte_stream_client::ByteStreamClient; +use re_grpc_proto::google::longrunning::Operation; use re_grpc_proto::google::longrunning::operation::Result as OpResult; use re_grpc_proto::google::rpc::Code; use re_grpc_proto::google::rpc::Status; @@ -95,12 +96,14 @@ use tonic::transport::Channel; use tonic::transport::Identity; use tonic::transport::Uri; use tonic::transport::channel::ClientTlsConfig; +use tracing::debug; use crate::error::*; use crate::metadata::*; use crate::request::*; use crate::response::*; use crate::retry::retry; +use crate::retry::retrying_stream; use crate::stats::CountingConnector; const DEFAULT_MAX_TOTAL_BATCH_SIZE: usize = 4 * 1000 * 1000; @@ -316,7 +319,7 @@ impl REClientBuilder { let tls_config = &tls_config; - let create_channel = |address: Option| async move { + let create_channel = |address: Option, with_timeout: bool| async move { let address = address.as_ref().context("No address")?; let address = substitute_env_vars(address).context("Invalid address")?; let uri = address.parse().context("Invalid address")?; @@ -347,7 +350,14 @@ impl REClientBuilder { http.set_keepalive(Some(Duration::from_secs(180))); let connector = CountingConnector::new(http); - endpoint = endpoint.timeout(Duration::from_secs(opts.grpc_timeout)); + // Apply a per-RPC deadline to all channels except the execution channel. + // The Execute RPC is a long-lived stream that stays open for the entire + // duration of the remote action (queued + executing), so a fixed short + // timeout would spuriously cancel it. All other RPCs are short unary calls + // where a deadline is appropriate. + if with_timeout { + endpoint = endpoint.timeout(Duration::from_secs(opts.grpc_timeout)); + } anyhow::Ok( endpoint @@ -358,11 +368,11 @@ impl REClientBuilder { }; let (cas, execution, action_cache, bytestream, capabilities) = futures::future::join5( - create_channel(opts.cas_address.clone()), - create_channel(opts.engine_address.clone()), - create_channel(opts.action_cache_address.clone()), - create_channel(opts.cas_address.clone()), - create_channel(opts.engine_address.clone()), + create_channel(opts.cas_address.clone(), true), + create_channel(opts.engine_address.clone(), false), // Execute streams are long-lived; no channel timeout + create_channel(opts.action_cache_address.clone(), true), + create_channel(opts.cas_address.clone(), true), + create_channel(opts.engine_address.clone(), true), ) .await; @@ -670,6 +680,82 @@ impl BatchUploadReqAggregator { } } +/// Converts a longrunning `Operation` message received from the Execute streaming RPC into a +/// `ExecuteWithProgressResponse`. This is extracted so it can be used with `retrying_stream`. +fn process_operation_message(msg: Operation) -> anyhow::Result { + debug!(?msg, "RE ACTION RESPONSE"); + + if let Some(metadata) = &msg.metadata { + if let Ok(meta) = ExecuteOperationMetadata::decode(&metadata.value[..]) { + debug!(?meta, "RE ACTION RESPONSE METADATA"); + } + } + + if msg.done { + match msg + .result + .context("Missing `result` when message was `done`")? + { + OpResult::Error(rpc_status) => Err(REClientError { + code: TCode(rpc_status.code), + message: rpc_status.message, + group: TCodeReasonGroup::UNKNOWN, + } + .into()), + OpResult::Response(any) => { + let execute_response_grpc: GExecuteResponse = + GExecuteResponse::decode(&any.value[..])?; + + check_status(execute_response_grpc.status.unwrap_or_default())?; + + let action_result = execute_response_grpc + .result + .with_context(|| "The action result is not defined.")?; + + debug!(?action_result, "BUCK2 ACTION RESULT"); + + let action_result = convert_action_result(action_result)?; + + let execute_response = ExecuteResponse { + action_result, + action_result_digest: TDigest::default(), + action_result_ttl: 0, + status: TStatus { + code: TCode::OK, + message: execute_response_grpc.message, + ..Default::default() + }, + cached_result: execute_response_grpc.cached_result, + action_digest: Default::default(), // Filled in below. + }; + + Ok(ExecuteWithProgressResponse { + stage: Stage::COMPLETED, + execute_response: Some(execute_response), + ..Default::default() + }) + } + } + } else { + let meta = ExecuteOperationMetadata::decode(&msg.metadata.unwrap_or_default().value[..])?; + + let stage = match execution_stage::Value::try_from(meta.stage) { + Ok(execution_stage::Value::Unknown) => Stage::UNKNOWN, + Ok(execution_stage::Value::CacheCheck) => Stage::CACHE_CHECK, + Ok(execution_stage::Value::Queued) => Stage::QUEUED, + Ok(execution_stage::Value::Executing) => Stage::EXECUTING, + Ok(execution_stage::Value::Completed) => Stage::COMPLETED, + _ => Stage::UNKNOWN, + }; + + Ok(ExecuteWithProgressResponse { + stage, + execute_response: None, + ..Default::default() + }) + } +} + impl REClient { fn new( runtime_opts: RERuntimeOpts, @@ -698,6 +784,7 @@ impl REClient { request: ActionResultRequest, ) -> anyhow::Result { retry( + "GetActionResultRequest", || async { let mut client = self.grpc_clients.action_cache_client.clone(); let request = request.clone(); @@ -734,6 +821,7 @@ impl REClient { request: WriteActionResultRequest, ) -> anyhow::Result { retry( + "UpdateActionResult", || async { let mut client = self.grpc_clients.action_cache_client.clone(); let request = request.clone(); @@ -790,98 +878,35 @@ impl REClient { ..Default::default() }; - let stream = retry( - || async { - let mut client = self.grpc_clients.execution_client.clone(); + debug!(?request, "RE ACTION REQUEST"); + debug!(?metadata, "RE ACTION REQUEST METADATA"); + + let execution_client = self.grpc_clients.execution_client.clone(); + let runtime_opts = self.runtime_opts; + + // retrying_stream covers both stream establishment and stream reading, so errors like + // h2 GOAWAY (ENHANCE_YOUR_CALM / "too_many_internal_resets") that surface during + // stream reading are retried and cause tonic to reconnect with a fresh h2 connection. + let stream = retrying_stream( + "Execute", + move || { + let mut client = execution_client.clone(); let request = request.clone(); let metadata = metadata.clone(); - - let stream = client - .execute(with_re_metadata(request, metadata, self.runtime_opts)) - .await? - .into_inner(); - Ok(stream) + async move { + Ok(client + .execute(with_re_metadata(request, metadata, runtime_opts)) + .await? + .into_inner()) + } }, self.runtime_opts.max_retries, INITIAL_DELAY, MAX_DELAY, true, - ) - .await?; - - let stream = futures::stream::try_unfold(stream, move |mut stream| async { - let msg = match stream.try_next().await.context("RE channel error")? { - Some(msg) => msg, - None => return Ok(None), - }; - - let status = if msg.done { - match msg - .result - .context("Missing `result` when message was `done`")? - { - OpResult::Error(rpc_status) => { - return Err(REClientError { - code: TCode(rpc_status.code), - message: rpc_status.message, - group: TCodeReasonGroup::UNKNOWN, - } - .into()); - } - OpResult::Response(any) => { - let execute_response_grpc: GExecuteResponse = - GExecuteResponse::decode(&any.value[..])?; - - check_status(execute_response_grpc.status.unwrap_or_default())?; - - let action_result = execute_response_grpc - .result - .with_context(|| "The action result is not defined.")?; - - let action_result = convert_action_result(action_result)?; - - let execute_response = ExecuteResponse { - action_result, - action_result_digest: TDigest::default(), - action_result_ttl: 0, - status: TStatus { - code: TCode::OK, - message: execute_response_grpc.message, - ..Default::default() - }, - cached_result: execute_response_grpc.cached_result, - action_digest: Default::default(), // Filled in below. - }; - - ExecuteWithProgressResponse { - stage: Stage::COMPLETED, - execute_response: Some(execute_response), - ..Default::default() - } - } - } - } else { - let meta = - ExecuteOperationMetadata::decode(&msg.metadata.unwrap_or_default().value[..])?; - - let stage = match execution_stage::Value::try_from(meta.stage) { - Ok(execution_stage::Value::Unknown) => Stage::UNKNOWN, - Ok(execution_stage::Value::CacheCheck) => Stage::CACHE_CHECK, - Ok(execution_stage::Value::Queued) => Stage::QUEUED, - Ok(execution_stage::Value::Executing) => Stage::EXECUTING, - Ok(execution_stage::Value::Completed) => Stage::COMPLETED, - _ => Stage::UNKNOWN, - }; - - ExecuteWithProgressResponse { - stage, - execute_response: None, - ..Default::default() - } - }; + ); - anyhow::Ok(Some((status, stream))) - }); + let stream = stream.and_then(|msg| async move { process_operation_message(msg) }); // We fill in the action digest a little later here. We do it this way so we don't have to // clone the execute_request into every future we create above. @@ -920,6 +945,7 @@ impl REClient { let runtime_opts = self.runtime_opts; retry( + "BatchUpdateBlobs", move || { let metadata = metadata.clone(); let mut cas_client = cas_client.clone(); @@ -948,6 +974,7 @@ impl REClient { let runtime_opts = self.runtime_opts; retry( + "BS.write", move || { let metadata = metadata.clone(); let mut bytestream_client = bytestream_client.clone(); @@ -1013,6 +1040,7 @@ impl REClient { let runtime_opts = self.runtime_opts; retry( + "BatchReadBlobs", move || { let metadata = metadata.clone(); let mut client = client.clone(); @@ -1041,6 +1069,7 @@ impl REClient { async move { let client = self.grpc_clients.bytestream_client.clone(); retry( + "Read", move || { let metadata = metadata.clone(); let mut client = client.clone(); @@ -1112,6 +1141,7 @@ impl REClient { tracing::debug!(num_digests = digests_to_check.len(), "FindMissingBlobs"); let runtime_opts = self.runtime_opts; let missing_blobs = retry( + "FindMissingBlobs", || { let mut cas_client = cas_client.clone(); let metadata = metadata.clone(); @@ -1612,6 +1642,7 @@ where Cas: Future>, { retry( + "BatchReadBlobs", || async { cas_f(read_blobs_request.clone()).await }, opts.max_retries, INITIAL_DELAY, @@ -1872,7 +1903,6 @@ fn with_re_metadata( // Meta builds catch those issues earlier. let mut msg = tonic::Request::new(t); - msg.set_timeout(runtime_opts.rpc_timeout); if runtime_opts.use_fbcode_metadata { // This is pretty ugly, but the protobuf spec that defines this is diff --git a/remote_execution/oss/re_grpc/src/metadata.rs b/remote_execution/oss/re_grpc/src/metadata.rs index a4981249107a6..f96e088993437 100644 --- a/remote_execution/oss/re_grpc/src/metadata.rs +++ b/remote_execution/oss/re_grpc/src/metadata.rs @@ -13,27 +13,27 @@ use std::collections::BTreeMap; pub type TPlatform = crate::grpc::Platform; pub type TProperty = crate::grpc::Property; -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct ActionHistoryInfo { pub action_key: String, pub disable_retry_on_oom: bool, pub _dot_dot: (), } -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct BuckInfo { pub build_id: String, pub version: String, pub _dot_dot: (), } -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct TClientContextMetadata { pub attributes: BTreeMap, pub _dot_dot: (), } -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct RemoteExecutionMetadata { pub action_history_info: Option, pub buck_info: Option, diff --git a/remote_execution/oss/re_grpc/src/retry.rs b/remote_execution/oss/re_grpc/src/retry.rs index 718d08f8af461..14f3845b560d0 100644 --- a/remote_execution/oss/re_grpc/src/retry.rs +++ b/remote_execution/oss/re_grpc/src/retry.rs @@ -7,12 +7,16 @@ * of this source tree. */ +use std::sync::Arc; use std::time::Duration; use std::future::Future; +use futures::Stream; +use futures::TryStreamExt; use crate::error::{REClientError, TCode}; use tracing::warn; pub async fn retry( + method: &str, mut f: F, max_retries: usize, initial_delay: Duration, @@ -53,7 +57,10 @@ where }; if retryable == Retryable::Wait { - warn!("Retrying request after error: {}. Attempt {}/{} (waiting {:?})", msg, retries, max_retries, delay); + warn!( + "Retrying {} request after error: {}. Attempt {}/{} (waiting {:?})", + method, msg, retries, max_retries, delay + ); tokio::time::sleep(delay).await; delay *= 2; @@ -61,7 +68,10 @@ where delay = max_delay; } } else { - warn!("Retrying request after error: {}. Attempt {}/{}", msg, retries, max_retries); + warn!( + "Retrying {} request after error: {}. Attempt {}/{}", + method, msg, retries, max_retries + ); } } } @@ -106,3 +116,108 @@ fn is_retryable(err: &anyhow::Error, retry_not_found: bool) -> Retryable { } Retryable::No } + +/// Like [`retry`], but covers both stream establishment and stream reading. +/// +/// `make_stream` is called to open the gRPC streaming call. If the call itself fails, or if +/// reading a message from the resulting stream fails, and the error is retryable, `make_stream` +/// is called again to establish a fresh stream (which causes tonic to obtain a new h2 connection +/// when the old one was closed by a GOAWAY frame, e.g. `ENHANCE_YOUR_CALM` / +/// `"too_many_internal_resets"`). +/// +/// Items from the stream are yielded progressively to the caller (preserving streaming progress +/// updates), so this is a drop-in replacement for `retry(make_stream) + try_unfold(try_next)`. +pub fn retrying_stream( + method: &'static str, + make_stream: F, + max_retries: usize, + initial_delay: Duration, + max_delay: Duration, + retry_not_found: bool, +) -> impl Stream> + Send + 'static +where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + S: futures::TryStream + Send + Unpin + 'static, + T: Send + 'static, +{ + struct RetryState { + make_stream: Arc, + stream: Option, + retries_left: usize, + delay: Duration, + max_delay: Duration, + retry_not_found: bool, + } + + let state = RetryState { + make_stream: Arc::new(make_stream), + stream: None, + retries_left: max_retries, + delay: initial_delay, + max_delay, + retry_not_found, + }; + + futures::stream::try_unfold(state, move |mut state| async move { + loop { + // Establish a stream if we don't have one (first call or after a retry). + if state.stream.is_none() { + match (state.make_stream)().await { + Ok(s) => { + state.stream = Some(s); + } + Err(err) => { + if is_retryable(&err, state.retry_not_found) != Retryable::No + && state.retries_left > 0 + { + warn!( + "Retrying {} stream open after error: {}. Attempts remaining: {}", + method, + err, + state.retries_left, + ); + tokio::time::sleep(state.delay).await; + state.delay *= 2; + if state.delay > state.max_delay { + state.delay = state.max_delay; + } + state.retries_left -= 1; + continue; + } + return Err(err); + } + } + } + + // Read the next message from the stream. + match state.stream.as_mut().unwrap().try_next().await { + Ok(Some(item)) => return Ok(Some((item, state))), + Ok(None) => return Ok(None), + Err(status) => { + let err = anyhow::Error::from(status); + if is_retryable(&err, state.retry_not_found) != Retryable::No + && state.retries_left > 0 + { + warn!( + "Retrying {} stream read after error: {}. Attempts remaining: {}", + method, + err, + state.retries_left, + ); + tokio::time::sleep(state.delay).await; + state.delay *= 2; + if state.delay > state.max_delay { + state.delay = state.max_delay; + } + state.retries_left -= 1; + // Drop the broken stream; next iteration re-establishes it. + state.stream = None; + continue; + } + return Err(err); + } + } + } + }) +} From 3e9e553f454715fecb170f27f41364940efda9c0 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 10 Jun 2026 15:22:07 -0700 Subject: [PATCH 17/29] buck2_external_cells: Don't inherit Git environment variables If you run `buck2` in a `pre-commit` hook, then the inherited `GIT_DIR` and similar can cause `buck2` to run destructive commands like `git reset --hard` in the wrong repository! --- app/buck2_external_cells/src/git.rs | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/app/buck2_external_cells/src/git.rs b/app/buck2_external_cells/src/git.rs index 25e110b504784..34074b7ef3a57 100644 --- a/app/buck2_external_cells/src/git.rs +++ b/app/buck2_external_cells/src/git.rs @@ -13,6 +13,7 @@ use std::process::Command; use std::process::ExitStatus; use std::process::Stdio; use std::sync::Arc; +use std::sync::LazyLock; use std::sync::Mutex; use std::sync::OnceLock; @@ -87,12 +88,48 @@ impl IoRequest for GitFetchIoRequest { let path = project_fs.resolve(&self.path); fs_util::create_dir_all(path.clone())?; + /// Remove some `GIT_` environment variables exposed by `git`. + /// + /// From `prek`. MIT-licensed, Copyright (c) 2024 j178. + /// + /// See: + static GIT_ENV_TO_REMOVE: LazyLock> = LazyLock::new(|| { + let keep = &[ + "GIT_EXEC_PATH", + "GIT_SSH", + "GIT_SSH_COMMAND", + "GIT_SSL_CAINFO", + "GIT_SSL_NO_VERIFY", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_PARAMETERS", + "GIT_HTTP_PROXY_AUTHMETHOD", + "GIT_ALLOW_PROTOCOL", + "GIT_ASKPASS", + ]; + + std::env::vars() + .map(|(key, _value)| key) + .filter(|key| { + key.starts_with("GIT_") + && !key.starts_with("GIT_CONFIG_KEY_") + && !key.starts_with("GIT_CONFIG_VALUE_") + && !keep.contains(&key.as_str()) + }) + .collect() + }); + // FIXME(JakobDegen): Ideally we'd use libgit2 directly here instead of shelling out, but // unfortunately the third party situation for that library in fbsource isn't great, so // let's do this for now fn run_git(cwd: &AbsNormPath, f: impl FnOnce(&mut Command)) -> buck2_error::Result<()> { let mut cmd = background_command("git"); f(&mut cmd); + // If the user has Git environment variables set, they can cause this Git command to + // operate on the wrong repo. + for name in &*GIT_ENV_TO_REMOVE { + cmd.env_remove(name); + } + let output = cmd .current_dir(cwd) .stderr(Stdio::piped()) From 9fb4f3a2feee6e1a8c82f66fa4ef775b44cd3c3b Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 29 Jun 2026 15:12:51 -0400 Subject: [PATCH 18/29] buck2_server: Implement Git revision data (#15) * Merge pull request #13 from MercuryTechnologies/hdgarrood/push-rlpkyxwyzskk Allow source labels to omit eponymous target names * buck2_server: Implement Git revision data What it says on the tin, closes a long-standing TODO and helps with our telemetry. TODO: Should we just reuse the `hg_revision` field for Git revisions? They _are_ also 40-character hashes... Upstream: - https://github.com/facebook/buck2/pull/1346 --------- Co-authored-by: Harry Garrood --- .../src/subscribers/recorder.rs | 4 + app/buck2_cmd_log_client/src/summary.rs | 18 ++- app/buck2_data/data.proto | 4 + .../src/version_control_revision.rs | 117 ++++++++++++++++-- 4 files changed, 128 insertions(+), 15 deletions(-) diff --git a/app/buck2_client_ctx/src/subscribers/recorder.rs b/app/buck2_client_ctx/src/subscribers/recorder.rs index e6d1ded3012cb..6dac56355b7e7 100644 --- a/app/buck2_client_ctx/src/subscribers/recorder.rs +++ b/app/buck2_client_ctx/src/subscribers/recorder.rs @@ -235,6 +235,7 @@ pub struct InvocationRecorder { active_networks_kinds: StdBuckHashSet, target_cfg: Option, hg_revision: Option, + git_revision: Option, has_local_changes: Option, version_control_errors: Vec, concurrent_commands: bool, @@ -456,6 +457,7 @@ impl InvocationRecorder { active_networks_kinds: StdBuckHashSet::default(), target_cfg: None, hg_revision: None, + git_revision: None, has_local_changes: None, version_control_errors: Vec::new(), concurrent_commands: false, @@ -1166,6 +1168,7 @@ impl InvocationRecorder { .collect(), target_cfg: self.target_cfg.take(), hg_revision: self.hg_revision.take(), + git_revision: self.git_revision.take(), has_local_changes: self.has_local_changes.take(), version_control_errors: self.version_control_errors.drain(..).collect(), version_control_revision: None, @@ -2200,6 +2203,7 @@ impl InvocationRecorder { revision: &buck2_data::VersionControlRevision, ) -> buck2_error::Result<()> { self.hg_revision = revision.hg_revision.clone().or(self.hg_revision.clone()); + self.git_revision = revision.git_revision.clone().or(self.git_revision.clone()); self.has_local_changes = revision.has_local_changes.or(self.has_local_changes); self.version_control_errors .extend(revision.command_error.clone()); diff --git a/app/buck2_cmd_log_client/src/summary.rs b/app/buck2_cmd_log_client/src/summary.rs index fb5058ff5bed4..3e50ce23a25b9 100644 --- a/app/buck2_cmd_log_client/src/summary.rs +++ b/app/buck2_cmd_log_client/src/summary.rs @@ -54,6 +54,7 @@ struct Stats { re_max_download_speeds: Vec, re_max_upload_speeds: Vec, hg_revision: Option, + git_revision: Option, has_local_changes: Option, } @@ -120,8 +121,17 @@ impl Stats { if let Some(ref revision) = vcs.hg_revision { self.hg_revision = Some(revision.clone()); } - if let Some(ref has_local_changes) = vcs.has_local_changes { - self.has_local_changes = Some(*has_local_changes); + match vcs.git_revision { + Some(ref revision) => { + self.git_revision = Some(revision.clone()); + } + None => {} + } + match vcs.has_local_changes { + Some(ref has_local_changes) => { + self.has_local_changes = Some(*has_local_changes); + } + None => {} } } _ => {} @@ -154,6 +164,10 @@ impl Display for Stats { writeln!(f, "- HG Revision: {hg_revision}")?; } + if let Some(git_revision) = &self.git_revision { + writeln!(f, "- Git Revision: {git_revision}")?; + } + if let Some(has_local_changes) = self.has_local_changes { writeln!(f, "- Has Local Changes: {has_local_changes}")?; } else { diff --git a/app/buck2_data/data.proto b/app/buck2_data/data.proto index 55b3ba793c80c..1d30d8527aa2c 100644 --- a/app/buck2_data/data.proto +++ b/app/buck2_data/data.proto @@ -876,6 +876,8 @@ message VersionControlRevision { // Unset: Unknown state. optional bool has_local_changes = 2; optional string command_error = 3; + // 40 characters hash for git revision. + optional string git_revision = 4; } // Event sent during build commands @@ -2680,6 +2682,8 @@ message InvocationRecord { optional bool has_local_changes = 107; // Errors encountered during version control operations. repeated string version_control_errors = 108; + // 40 characters hash for git revision. + optional string git_revision = 109; // Detailed per-backend RE stats. optional uint64 zdb_download_queries = 200; diff --git a/app/buck2_server/src/version_control_revision.rs b/app/buck2_server/src/version_control_revision.rs index 7f8512402672c..1b61efd4a337c 100644 --- a/app/buck2_server/src/version_control_revision.rs +++ b/app/buck2_server/src/version_control_revision.rs @@ -85,19 +85,13 @@ async fn create_revision_data( ) -> buck2_data::VersionControlRevision { let mut revision = buck2_data::VersionControlRevision::default(); match repo_type(repo_root).await { - Ok(repo_vcs) => { - match repo_vcs { - RepoVcs::Hg => create_hg_data(&mut revision, revision_type, repo_root).await, - RepoVcs::Git => { - // TODO(rajneeshl): Implement the git data - // Add a message for now so we can actually tell if revision is null due to git - revision.command_error = Some("Git revision data not implemented".to_owned()); - } - RepoVcs::Unknown => { - revision.command_error = Some("Unknown repository type".to_owned()); - } + Ok(repo_vcs) => match repo_vcs { + RepoVcs::Hg => create_hg_data(&mut revision, revision_type, repo_root).await, + RepoVcs::Git => create_git_data(&mut revision, revision_type, repo_root).await, + RepoVcs::Unknown => { + revision.command_error = Some("Unknown repository type".to_owned()); } - } + }, Err(e) => { revision.command_error = Some(format!("Failed to get repository type: {e:#}")); } @@ -183,6 +177,100 @@ async fn get_hg_status(revision: &mut buck2_data::VersionControlRevision) { }; } +async fn create_git_data( + revision: &mut buck2_data::VersionControlRevision, + revision_type: RevisionDataType, + repo_root: &AbsNormPathBuf, +) { + match revision_type { + RevisionDataType::CurrentRevision => get_git_revision(revision, repo_root).await, + RevisionDataType::Status => get_git_status(revision, repo_root).await, + } +} + +async fn get_git_revision( + revision: &mut buck2_data::VersionControlRevision, + repo_root: &AbsNormPathBuf, +) { + // `git rev-parse HEAD` resolves the current commit hash. This handles + // packed refs, detached HEADs, etc. without us having to parse `.git`. + match run_git(repo_root, &["rev-parse", "HEAD"]).await { + Ok(stdout) => revision.git_revision = Some(stdout.trim().to_owned()), + Err(e) => revision.command_error = Some(e), + } +} + +async fn get_git_status( + revision: &mut buck2_data::VersionControlRevision, + repo_root: &AbsNormPathBuf, +) { + // `git status --porcelain` prints one line per change; empty output means + // there are no local changes. Note that this counts untracked files as local + // changes (they show up as `??` lines), matching the behavior of `hg status`. + match run_git(repo_root, &["status", "--porcelain"]).await { + Ok(stdout) => revision.has_local_changes = Some(!stdout.trim().is_empty()), + Err(e) => revision.command_error = Some(e), + } +} + +/// Run a `git` command rooted at `repo_root`, returning its stdout on success or +/// an error message describing the failure. +async fn run_git(repo_root: &AbsNormPathBuf, args: &[&str]) -> Result { + let Some(repo_root) = repo_root.as_path().to_str() else { + return Err(format!( + "Repository root is not valid utf8: {}", + repo_root.as_path().display() + )); + }; + + // `-C ` makes git operate on the repository regardless of the + // daemon's current working directory. + let mut full_args = vec!["-C", repo_root]; + full_args.extend_from_slice(args); + + // `GIT_OPTIONAL_LOCKS=0` prevents git from taking the index lock or + // refreshing the index on disk, so we don't contend with a concurrent user + // `git` invocation (e.g. for `git status`). + let output = match reap_on_drop_command("git", &full_args, Some(&[("GIT_OPTIONAL_LOCKS", "0")])) + { + Ok(command) => command.output().await, + Err(e) => { + return Err(format!( + "reap_on_drop_command for `git {}` failed: {e}", + args.join(" ") + )); + } + }; + + let result = match output { + Ok(result) => result, + Err(e) => { + return Err(format!( + "Command `git {}` failed with error: {e:?}", + args.join(" ") + )); + } + }; + + if !result.status.success() { + let stderr = match std::str::from_utf8(&result.stderr) { + Ok(s) => s, + Err(e) => return Err(format!("git {} stderr is not utf8: {e}", args.join(" "))), + }; + return Err(format!( + "Command `git {}` failed with error code {}; stderr: {}", + args.join(" "), + result.status, + stderr + )); + } + + match std::str::from_utf8(&result.stdout) { + Ok(s) => Ok(s.to_owned()), + Err(e) => Err(format!("git {} stdout is not utf8: {e}", args.join(" "))), + } +} + async fn repo_type(repo_root: &AbsNormPathBuf) -> buck2_error::Result<&'static RepoVcs> { static REPO_TYPE: OnceCell> = OnceCell::const_new(); async fn repo_type_impl(repo_root: &AbsNormPathBuf) -> buck2_error::Result { @@ -192,7 +280,10 @@ async fn repo_type(repo_root: &AbsNormPathBuf) -> buck2_error::Result<&'static R ); let is_hg = hg_metadata.is_ok_and(|output| output.is_dir()); - let is_git = git_metadata.is_ok_and(|output| output.is_dir()); + // `.git` can be a symlink or a file with contents like: + // + // gitdir: /home/dog/buck2/.git/worktrees/buck3 + let is_git = git_metadata.is_ok(); if is_hg { Ok(RepoVcs::Hg) From 41e30f26bf3694b52fdbec5333ba1bf8d0375c9a Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 2 Jul 2026 15:35:44 -0400 Subject: [PATCH 19/29] buck2_events: Add OpenTelemetry `EventSink` (#14) This adds a new `OtelEventSink` which sends `InvocationRecord` wide events to an OpenTelemetry collector, if configured. --- Cargo.toml | 4 + app/buck2/src/lib.rs | 17 + app/buck2_client_ctx/src/exit_result.rs | 6 + .../src/subscribers/recorder.rs | 18 +- app/buck2_core/Cargo.toml | 5 + app/buck2_core/src/logging.rs | 8 + app/buck2_core/src/logging/otel.rs | 328 ++++ app/buck2_core/src/logging/otel_stub.rs | 12 + app/buck2_events/Cargo.toml | 3 + app/buck2_events/src/sink.rs | 4 + app/buck2_events/src/sink/otel.rs | 138 ++ app/buck2_events/src/sink/otel_record.rs | 1570 +++++++++++++++++ 12 files changed, 2112 insertions(+), 1 deletion(-) create mode 100644 app/buck2_core/src/logging/otel.rs create mode 100644 app/buck2_core/src/logging/otel_stub.rs create mode 100644 app/buck2_events/src/sink/otel.rs create mode 100644 app/buck2_events/src/sink/otel_record.rs diff --git a/Cargo.toml b/Cargo.toml index 7c172e18e931f..9722fefba14c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -259,6 +259,10 @@ num_cpus = "1.16" num_enum = "0.7.4" object = "0.36.7" once_cell = "1.21" +opentelemetry = "0.32.0" +opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] } +opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } +opentelemetry_sdk = "0.32.1" os_str_bytes = { version = "6.6.0", features = ["conversions"] } parking_lot = { version = "0.12.1", features = ["send_guard"] } paste = "1.0.15" diff --git a/app/buck2/src/lib.rs b/app/buck2/src/lib.rs index 6c4adfa703143..e65a2424c5707 100644 --- a/app/buck2/src/lib.rs +++ b/app/buck2/src/lib.rs @@ -503,6 +503,23 @@ impl CommandKind { recorder.update_for_client_ctx(&command_ctx, self.command_name()); } + // Build the client's OTLP exporter now. The client process emits the invocation record (and, + // when `tracing`-span export is enabled, the bulk of the spans), and unlike the daemon it + // never `fork()`s without an immediate `exec()`, so it is safe to spawn the exporter's + // background threads here. Only the client activates today; a future daemon exporter would + // have to activate separately, after daemonizing. Skip the forking helpers, which fork+exec + // constantly and have nothing to export. + #[cfg(not(client_only))] + let enable_telemetry = !matches!( + self, + CommandKind::Forkserver(..) | CommandKind::InternalTestRunner(..) + ); + #[cfg(client_only)] + let enable_telemetry = true; + if enable_telemetry { + buck2_core::logging::otel::activate(BuckVersion::get_version_for_clap()); + } + match self { #[cfg(not(client_only))] CommandKind::Daemon(..) => unreachable!("Checked earlier"), diff --git a/app/buck2_client_ctx/src/exit_result.rs b/app/buck2_client_ctx/src/exit_result.rs index 74e64e0384062..6335e61d3dd69 100644 --- a/app/buck2_client_ctx/src/exit_result.rs +++ b/app/buck2_client_ctx/src/exit_result.rs @@ -342,6 +342,12 @@ impl ExitResultVariant { pub fn report(self) -> ! { // Log the exit timestamp tracing::debug!("Client exiting"); + + // Drain any buffered OTLP spans before we leave. Every variant below either calls + // `libc::_exit` or `exec`s a new process image -- neither runs destructors, so the batch + // exporter would otherwise drop its final spans on the floor. + // See https://github.com/open-telemetry/opentelemetry-rust/issues/1961. + buck2_core::logging::otel::shutdown(); // NOTE: We use writeln instead of println so we don't panic if stderr is closed. This // ensures we get the desired exit code printed instead of potentially a panic. let mut exit_code = match self { diff --git a/app/buck2_client_ctx/src/subscribers/recorder.rs b/app/buck2_client_ctx/src/subscribers/recorder.rs index 6dac56355b7e7..fdb6a58f843e4 100644 --- a/app/buck2_client_ctx/src/subscribers/recorder.rs +++ b/app/buck2_client_ctx/src/subscribers/recorder.rs @@ -53,6 +53,8 @@ use buck2_event_observer::last_command_execution_kind::LastCommandExecutionKind; use buck2_event_observer::last_command_execution_kind::get_last_command_execution_time; use buck2_events::BuckEvent; use buck2_events::daemon_id::DaemonId; +#[cfg(not(fbcode_build))] +use buck2_events::sink::otel::new_otel_event_sink_if_enabled; use buck2_events::sink::remote::ScribeConfig; use buck2_events::sink::remote::new_remote_event_sink_if_enabled; use buck2_fs::error::IoResultExt; @@ -2531,6 +2533,20 @@ impl EventSubscriber for InvocationRecorder { // Typically initialized already unless the command failed early. let fb = buck2_common::fbinit::get_or_init_fbcode_globals(); let event = self.create_record_event(); + + // Send the record to every enabled remote sink. The OTLP sink has the same interface as the + // Scribe sink and is not Meta-specific, so it is the export path in OSS builds where the + // Scribe sink is a compile-time no-op. Telemetry must never fail a command, so an OTLP + // failure is logged rather than propagated. + #[cfg(not(fbcode_build))] + if let Some(otel_sink) = new_otel_event_sink_if_enabled() { + let span = tracing::info_span!("Recording invocation to OpenTelemetry"); + let _guard = span.enter(); + if let Err(e) = otel_sink.send_now(event.clone()).await { + tracing::warn!("Failed to export invocation record via OTLP: {e:#}"); + } + } + if let Some(scribe_sink) = new_remote_event_sink_if_enabled( fb, ScribeConfig { @@ -2545,7 +2561,7 @@ impl EventSubscriber for InvocationRecorder { scribe_sink.send_now(event).await } else { tracing::info!("Invocation record is not sent to Scribe: {:?}", &event); - Err(internal_error!("Scribe sink not enabled")) + Ok(()) } } diff --git a/app/buck2_core/Cargo.toml b/app/buck2_core/Cargo.toml index 074cec795299d..179783d52e8bf 100644 --- a/app/buck2_core/Cargo.toml +++ b/app/buck2_core/Cargo.toml @@ -17,6 +17,10 @@ indent_write = { workspace = true } itertools = { workspace = true } memchr = { workspace = true } once_cell = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +opentelemetry_sdk = { workspace = true } os_str_bytes = { workspace = true } pagable = { workspace = true } pin-project = { workspace = true } @@ -35,6 +39,7 @@ tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } triomphe = { workspace = true } +uuid = { workspace = true } allocative = { workspace = true } cmp_any = { workspace = true } diff --git a/app/buck2_core/src/logging.rs b/app/buck2_core/src/logging.rs index a7d041ca2eb16..10ff02b2204f1 100644 --- a/app/buck2_core/src/logging.rs +++ b/app/buck2_core/src/logging.rs @@ -23,6 +23,14 @@ use crate::buck2_env; pub mod log_file; +#[cfg(not(fbcode_build))] +pub mod otel; + +#[cfg(fbcode_build)] +mod otel_stub; +#[cfg(fbcode_build)] +pub use otel_stub as otel; + pub trait LogConfigurationReloadHandle: Send + Sync + 'static { fn update_log_filter(&self, format: &str) -> buck2_error::Result<()>; } diff --git a/app/buck2_core/src/logging/otel.rs b/app/buck2_core/src/logging/otel.rs new file mode 100644 index 0000000000000..0f838f4b7a546 --- /dev/null +++ b/app/buck2_core/src/logging/otel.rs @@ -0,0 +1,328 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +//! Optional OpenTelemetry (OTLP) export. +//! +//! When an OTLP endpoint is configured via the standard `OTEL_EXPORTER_OTLP_*` environment +//! variables, this module builds an OTLP exporter and tracer provider. Unlike a server, buck2 is a +//! short-lived CLI process, so the exported resource attributes describe *this invocation* (a fresh +//! `service.instance.id` per run, the compiled-in `service.version`, the host and pid) rather than a +//! long-running deployment. +//! +//! This module owns only the exporter lifecycle (build, activate, flush). What gets exported is up +//! to callers: anything that reaches the global provider stored here -- a `tracing` layer bridged on +//! top, or an out-of-band "wide event" span -- rides the same exporter. +//! +//! ## Deferred activation (important) +//! +//! The OTLP batch exporter spawns background threads (a batch-processor thread plus an HTTP client +//! runtime). The buck2 daemon daemonizes via `fork()` *without* a following `exec()`, and `fork()` +//! only copies the calling thread -- any other thread vanishes in the child but leaves the locks and +//! state it held (allocator, TLS/crypto, exporter queues) permanently wedged. A daemon that spawned +//! these threads pre-fork therefore deadlocks or aborts shortly after start. See the +//! "Do not create any threads before this point" invariant in `buck2_daemon::daemon`. +//! +//! So we do *not* build the exporter when the subscriber is installed. [`activate`] builds it later, +//! once the process is past any such `fork()`. Today only the client calls [`activate`] -- it emits +//! the invocation record and never `fork()`s-without-`exec()`. If the daemon ever exports its own +//! spans it must call [`activate`] only after it has finished daemonizing, never before. +//! +//! ## Flushing +//! +//! Spans are buffered and only flushed periodically. buck2 exits via `libc::_exit` (see +//! `ExitResult::report`), which runs no destructors, so the buffered batch would be dropped unless +//! we drain it explicitly. [`shutdown`] does that drain and must be called on every exit path. See +//! . + +use std::sync::OnceLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::SystemTime; + +use buck2_error::conversion::from_any_with_tag; +use opentelemetry::KeyValue; +use opentelemetry::trace::Span as _; +use opentelemetry::trace::Tracer as _; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_otlp::Protocol; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::BatchConfigBuilder; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::trace::SpanLimits; +use opentelemetry_semantic_conventions::resource::HOST_ARCH; +use opentelemetry_semantic_conventions::resource::HOST_NAME; +use opentelemetry_semantic_conventions::resource::OS_TYPE; +use opentelemetry_semantic_conventions::resource::PROCESS_PID; +use opentelemetry_semantic_conventions::resource::SERVICE_INSTANCE_ID; +use opentelemetry_semantic_conventions::resource::SERVICE_NAME; +use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; +use uuid::Uuid; + +/// The active OTLP tracer provider, if any. Stored globally so that [`shutdown`] can reach it from +/// the process exit path (which lives in a different crate, and runs after any layer's type has been +/// erased into the global subscriber) and so out-of-band emitters can reuse the same exporter. +/// +/// We can't use [`opentelemetry::global::tracer_provider`] here because that produces a +/// [`opentelemetry::global::GlobalTracerProvider`], which lacks the +/// [`opentelemetry_sdk::trace::SdkTracerProvider::shutdown`] method we actually need to call. +static PROVIDER: OnceLock = OnceLock::new(); + +/// Guards [`activate`] so the exporter is built at most once, even if it is called from more than one +/// entry point (today just the client, before running a command; a future daemon exporter, which +/// would activate after daemonizing, would be a second). +static ACTIVATED: AtomicBool = AtomicBool::new(false); + +/// We only enable OTLP export when an endpoint is explicitly configured. This keeps the common case +/// (no telemetry) free of overhead and avoids futile connection attempts to the default +/// `localhost:4318`. These are standard OpenTelemetry variables read by the exporter itself, not +/// buck2-owned configuration, so we check them directly rather than registering them via +/// `buck2_env!` (which would surface them misleadingly in `buck2 help-env`). +fn otlp_endpoint_configured() -> bool { + [ + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + ] + .iter() + .any(|var| std::env::var_os(var).is_some_and(|v| !v.is_empty())) +} + +/// Map Rust's [`std::env::consts::OS`] to the OpenTelemetry `os.type` value set, passing through any +/// value without a standardized equivalent (e.g. `ios`, `android`) as-is. Most names already match; +/// only a couple are spelled differently. +fn otel_os_type(os: &str) -> &str { + match os { + "macos" => "darwin", + "dragonfly" => "dragonflybsd", + // `linux`, `windows`, `freebsd`, `netbsd`, `openbsd`, `solaris`, `aix` already match. + other => other, + } +} + +/// Map Rust's [`std::env::consts::ARCH`] to the OpenTelemetry `host.arch` value set, passing through +/// any value without a standardized equivalent (e.g. `riscv64`, `loongarch64`) as-is. +fn otel_host_arch(arch: &str) -> &str { + match arch { + "x86_64" => "amd64", + "aarch64" => "arm64", + "arm" => "arm32", + "powerpc" => "ppc32", + "powerpc64" => "ppc64", + // `x86` and `s390x` already match. + other => other, + } +} + +/// Resource attributes identifying this build invocation, following OpenTelemetry semantic +/// conventions (). +fn resource(version: &'static str) -> Resource { + let mut attributes = vec![ + KeyValue::new(SERVICE_NAME, "buck2"), + // buck2 is not a deployed service, so `service.version` is just this binary's build version. + // The caller passes it in (`BuckVersion::get_version()`, the same string `buck2 --version` + // prints) because the richer version -- the source revision stamped at build time via + // `BUCK2_SET_EXPLICIT_VERSION` -- is only resolvable in the `buck2` bin crate, not here. It + // falls back to the binary's build-id when no revision is stamped. + KeyValue::new(SERVICE_VERSION, version), + // Every buck2 process is its own "instance"; a fresh v4 UUID keeps invocations distinct. + // + // NB: This is not the build ID / trace ID, which is _also_ written as a v4 UUID. + KeyValue::new(SERVICE_INSTANCE_ID, Uuid::new_v4().to_string()), + KeyValue::new(HOST_ARCH, otel_host_arch(std::env::consts::ARCH)), + KeyValue::new(OS_TYPE, otel_os_type(std::env::consts::OS)), + KeyValue::new(PROCESS_PID, i64::from(std::process::id())), + ]; + if let Ok(Some(hostname)) = hostname::get().map(|h| h.into_string().ok()) { + attributes.push(KeyValue::new(HOST_NAME, hostname)); + } + Resource::builder().with_attributes(attributes).build() +} + +/// Build the OTLP exporter and tracer provider (spawning the exporter's background threads) and store +/// the provider in [`PROVIDER`]. No-op when no endpoint is configured. +fn build_provider(version: &'static str) -> buck2_error::Result<()> { + if !otlp_endpoint_configured() { + return Ok(()); + } + + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .with_protocol(Protocol::HttpBinary) + .build() + .map_err(|e| from_any_with_tag(e, buck2_error::ErrorTag::Tier0))?; + + let batch_config = BatchConfigBuilder::default().build(); + let processor = BatchSpanProcessor::builder(exporter) + .with_batch_config(batch_config) + .build(); + + let provider = SdkTracerProvider::builder() + // The default settings limit all of these values at 128, which is small enough to start + // dropping data on our `InvocationRecord`s! + .with_span_limits(SpanLimits { + max_events_per_span: 1024, + max_attributes_per_span: 2048, + max_links_per_span: 512, + max_attributes_per_event: 2048, + max_attributes_per_link: 1024, + }) + .with_resource(resource(version)) + .with_span_processor(processor) + .build(); + + // `activate` runs once, so ignore an already-set slot. + let _ = PROVIDER.set(provider); + + Ok(()) +} + +/// Emit a fully-assembled "wide event" as a single span, out-of-band from the `tracing` subscriber. +/// +/// Most spans are accumulated from nested `tracing` spans, but some records -- notably the +/// end-of-invocation `InvocationRecord` -- are assembled once as a flat field set. OTLP backends +/// (Honeycomb and friends) ingest a span as a single wide row, so we ship such a record as one span +/// whose `attributes` are its fields and whose start/end bracket the invocation. +/// +/// This is intentionally dumb: the caller hands us the finished attribute set (it owns the mapping +/// from its domain type to keys/values), and we just attach it to a span. The span is enqueued on +/// the same batch processor as every other span and flushed by [`shutdown`], so it must be emitted +/// before the process exits. No-op when telemetry was never activated (no endpoint configured, or +/// [`activate`] not yet called). Because it reuses the global provider, it must only be called after +/// any `fork()`-without-`exec()` -- in practice it is only emitted by the client at +/// end-of-invocation, which never forks that way. +pub fn export_span( + name: &'static str, + start: SystemTime, + end: SystemTime, + attributes: Vec, +) { + let Some(provider) = PROVIDER.get() else { + return; + }; + + let tracer = provider.tracer("buck2"); + let mut span = tracer + .span_builder(name) + .with_start_time(start) + .with_attributes(attributes) + .start(&tracer); + span.end_with_timestamp(end); +} + +/// Build the OTLP exporter and start exporting, if telemetry is configured. +/// +/// This spawns the exporter's background threads, so it MUST be called only after the process has +/// finished any `fork()`-without-`exec()`. Today only the client calls it, before running a command +/// (the client never `fork()`s-without-`exec()`, so any time is fine); a future daemon exporter would +/// have to call it only after daemonizing. Idempotent: the exporter is built at most once. No-op if +/// telemetry is not configured. +/// Errors are logged rather than propagated -- telemetry must never fail a command. +/// +/// `version` is recorded as the `service.version` resource attribute; pass +/// `BuckVersion::get_version()` (the string `buck2 --version` prints). +pub fn activate(version: &'static str) { + if ACTIVATED.swap(true, Ordering::SeqCst) { + return; + } + if let Err(e) = build_provider(version) { + tracing::warn!("Failed to start OpenTelemetry exporter: {e}"); + } +} + +/// Whether the OTLP exporter is built and exporting (i.e. [`activate`] ran and an endpoint was +/// configured). Lets callers skip building span attributes when nothing would consume them -- +/// [`export_span`] is itself a no-op in that case, but assembling its attributes is not free. +pub fn is_active() -> bool { + PROVIDER.get().is_some() +} + +/// Flush and shut down the OTLP exporter, draining any spans still buffered in the batch processor. +/// +/// This must be called before the process exits. buck2 exits via `libc::_exit`, which runs no +/// destructors, so without this the final batch of spans is silently lost. No-op when OTLP export +/// was never activated. +pub fn shutdown() { + if let Some(provider) = PROVIDER.get() { + // Best-effort: we are on the way out regardless, so a failed flush only costs us the last + // batch of spans. + if let Err(e) = provider.shutdown() { + tracing::warn!("Failed to shut down OpenTelemetry exporter on exit: {e}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn os_type_remaps_and_passes_through() { + // Rust spellings that differ from the OpenTelemetry value set. + assert_eq!(otel_os_type("macos"), "darwin"); + assert_eq!(otel_os_type("dragonfly"), "dragonflybsd"); + // Already-conformant values are unchanged. + assert_eq!(otel_os_type("linux"), "linux"); + assert_eq!(otel_os_type("windows"), "windows"); + // No standardized equivalent: passed through as-is. + assert_eq!(otel_os_type("ios"), "ios"); + assert_eq!(otel_os_type("android"), "android"); + } + + #[test] + fn host_arch_remaps_and_passes_through() { + assert_eq!(otel_host_arch("x86_64"), "amd64"); + assert_eq!(otel_host_arch("aarch64"), "arm64"); + assert_eq!(otel_host_arch("arm"), "arm32"); + assert_eq!(otel_host_arch("powerpc"), "ppc32"); + assert_eq!(otel_host_arch("powerpc64"), "ppc64"); + // Already-conformant values are unchanged. + assert_eq!(otel_host_arch("x86"), "x86"); + assert_eq!(otel_host_arch("s390x"), "s390x"); + // No standardized equivalent: passed through as-is. + assert_eq!(otel_host_arch("riscv64"), "riscv64"); + } + + /// The live host's arch/os must map to a value the OpenTelemetry spec actually defines (this + /// catches a Rust target whose spelling we have not remapped). + #[test] + fn current_host_is_conformant() { + const OS_TYPES: &[&str] = &[ + "windows", + "linux", + "darwin", + "freebsd", + "netbsd", + "openbsd", + "dragonflybsd", + "hpux", + "aix", + "solaris", + "z_os", + "zos", + ]; + const HOST_ARCHES: &[&str] = &[ + "amd64", "arm32", "arm64", "ia64", "ppc32", "ppc64", "s390x", "x86", + ]; + + // These hold on every platform buck2 currently builds for; a new target that needs a remap + // would trip one of them rather than silently emitting a non-conformant value. + assert!( + OS_TYPES.contains(&otel_os_type(std::env::consts::OS)), + "os.type {:?} is not an OpenTelemetry-defined value", + otel_os_type(std::env::consts::OS) + ); + assert!( + HOST_ARCHES.contains(&otel_host_arch(std::env::consts::ARCH)), + "host.arch {:?} is not an OpenTelemetry-defined value", + otel_host_arch(std::env::consts::ARCH) + ); + } +} diff --git a/app/buck2_core/src/logging/otel_stub.rs b/app/buck2_core/src/logging/otel_stub.rs new file mode 100644 index 0000000000000..037b10568bb35 --- /dev/null +++ b/app/buck2_core/src/logging/otel_stub.rs @@ -0,0 +1,12 @@ +//! Stub of [`crate::logging::otel`] for Meta, with empty implementations. +//! +//! Note that this module provides no `export_span` equivalent, because that function takes an +//! [`opentelemetry::KeyValue`] as a parameter. + +pub fn activate(version: &'static str) {} + +pub fn is_active() -> bool { + false +} + +pub fn shutdown() {} diff --git a/app/buck2_events/Cargo.toml b/app/buck2_events/Cargo.toml index b9739286511f0..baeaf03a6813a 100644 --- a/app/buck2_events/Cargo.toml +++ b/app/buck2_events/Cargo.toml @@ -13,7 +13,10 @@ derive_more = { workspace = true } futures = { workspace = true } hostname = { workspace = true } is_proc_translated = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } pin-project = { workspace = true } +prost-types = { workspace = true } serde = { workspace = true } smallvec = { workspace = true } sys-info = { workspace = true } diff --git a/app/buck2_events/src/sink.rs b/app/buck2_events/src/sink.rs index d75000d2c3a1b..99a320043f4ca 100644 --- a/app/buck2_events/src/sink.rs +++ b/app/buck2_events/src/sink.rs @@ -13,6 +13,10 @@ pub(crate) mod channel; pub mod error_on_event; pub mod null; +#[cfg(not(fbcode_build))] +pub mod otel; +#[cfg(not(fbcode_build))] +pub(crate) mod otel_record; pub mod remote; #[cfg(fbcode_build)] pub(crate) mod scribe; diff --git a/app/buck2_events/src/sink/otel.rs b/app/buck2_events/src/sink/otel.rs new file mode 100644 index 0000000000000..6d12c6814b12b --- /dev/null +++ b/app/buck2_events/src/sink/otel.rs @@ -0,0 +1,138 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +//! An event sink that exports the end-of-invocation [`InvocationRecord`](buck2_data::InvocationRecord) +//! to OpenTelemetry (OTLP) as a single "wide event" span. +//! +//! This is a peer of the Scribe [`RemoteEventSink`](crate::sink::remote::RemoteEventSink): it has +//! the same shape (`send_now` / `send_messages_now` / `offer` / [`EventSink`]) so callers route the +//! record to it exactly as they route to Scribe, with no special-casing. Unlike Scribe it is not +//! Meta-specific, so it is the wide-event export path in open-source builds too. +//! +//! The actual proto -> attribute mapping lives in [`crate::sink::otel_record`]; spans are emitted +//! through [`buck2_core::logging::otel`], which owns the exporter lifecycle (activation after the +//! daemon fork, flushing on exit). + +use std::sync::Arc; +use std::time::Duration; + +use opentelemetry::KeyValue; + +use crate::BuckEvent; +use crate::Event; +use crate::EventSink; +use crate::EventSinkStats; +use crate::EventSinkWithStats; +use crate::sink::otel_record::invocation_record_attributes; + +/// Forwards the invocation record to the process-wide OTLP exporter as a span. +pub struct OtelEventSink; + +impl OtelEventSink { + pub fn new() -> OtelEventSink { + OtelEventSink + } + + /// Export the event now. The OTLP batch processor accepts spans synchronously and is drained by + /// `buck2_core::logging::otel::shutdown` on exit, so there is nothing to await -- the `async` + /// signature exists only to match [`RemoteEventSink`](crate::sink::remote::RemoteEventSink). + pub async fn send_now(&self, event: BuckEvent) -> buck2_error::Result<()> { + Self::export(&event); + Ok(()) + } + + pub async fn send_messages_now(&self, events: Vec) -> buck2_error::Result<()> { + for event in &events { + Self::export(event); + } + Ok(()) + } + + pub fn offer(&self, event: BuckEvent) { + Self::export(&event); + } + + /// Translate an invocation-record event into an OTLP span. Events that are not an + /// `InvocationRecord` are ignored -- this sink only produces the wide event. + fn export(event: &BuckEvent) { + let buck2_data::buck_event::Data::Record(record_event) = event.data() else { + return; + }; + let Some(buck2_data::record_event::Data::InvocationRecord(record)) = &record_event.data + else { + return; + }; + + // The span brackets the invocation: it ends when the record was emitted and started + // `client_walltime` earlier (the walltime the client measured for the whole command). + let end = event.timestamp(); + let walltime = record + .client_walltime + .as_ref() + .map(|d| Duration::new(d.seconds.max(0) as u64, d.nanos.max(0) as u32)) + .unwrap_or_default(); + let start = end.checked_sub(walltime).unwrap_or(end); + + let mut attributes = invocation_record_attributes(record); + // The invocation's trace UUID lives on the event envelope, not in the record payload, but + // it is the primary correlation key (it ties this wide event to the event log and to any + // other telemetry for the same invocation), so attach it explicitly. + if let Ok(trace_id) = event.trace_id() { + attributes.push(KeyValue::new("buck2.trace_id", trace_id.to_string())); + } + // NB: The event isn't necessarily a 'build' in the strict sense (this also fires for tests, + // queries, etc.), but this matches Meta's `buck2_builds` table upstream. + buck2_core::logging::otel::export_span("buck2_build", start, end, attributes); + } +} + +impl EventSink for OtelEventSink { + fn send(&self, event: Event) { + if let Event::Buck(event) = event { + Self::export(&event); + } + } +} + +impl EventSinkWithStats for OtelEventSink { + fn to_event_sync(self: Arc) -> Arc { + self as _ + } + + fn stats(&self) -> EventSinkStats { + // The exporter owns its own buffering/retry accounting; this sink keeps no counters. + EventSinkStats { + successes: 0, + failures_invalid_request: 0, + failures_unauthorized: 0, + failures_rate_limited: 0, + failures_pushed_back: 0, + failures_enqueue_failed: 0, + failures_internal_error: 0, + failures_timed_out: 0, + failures_unknown: 0, + buffered: 0, + dropped: 0, + bytes_written: 0, + } + } +} + +/// Construct an OTLP sink, or `None` when OTLP export is not active (no endpoint configured, or the +/// exporter was never activated). Mirrors +/// [`new_remote_event_sink_if_enabled`](crate::sink::remote::new_remote_event_sink_if_enabled) so +/// the two remote sinks are constructed and used identically. +pub fn new_otel_event_sink_if_enabled() -> Option { + if buck2_core::logging::otel::is_active() { + Some(OtelEventSink::new()) + } else { + None + } +} diff --git a/app/buck2_events/src/sink/otel_record.rs b/app/buck2_events/src/sink/otel_record.rs new file mode 100644 index 0000000000000..7dd612b6c59c4 --- /dev/null +++ b/app/buck2_events/src/sink/otel_record.rs @@ -0,0 +1,1570 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +//! Maps an [`InvocationRecord`](buck2_data::InvocationRecord) to OTLP span attributes for the +//! end-of-invocation "wide event". +//! +//! Every field is mapped *explicitly*, by hand, rather than by serializing the proto and flattening +//! it generically. This matches how the upstream Scuba pipeline curates its schema: the shape of +//! each column (key name, unit, how repeated/nested values are projected) is a deliberate choice, +//! not a mechanical consequence of the proto layout. Attribute keys follow the proto's own +//! serde field names so they line up with the event-log JSON (durations as `_us` micros, etc.). +//! +//! Repeated scalar fields become a single multi-valued attribute (the OTLP equivalent of a Scuba +//! normvector). Repeated *messages* are projected field-wise into parallel multi-valued attributes +//! (`errors.category`, `errors.message`, ...), since a list of structs has no single-column form. +//! +//! ## Attribute namespacing +//! +//! Every attribute is emitted under the `buck2.` namespace (e.g. `re_session_id` becomes +//! `buck2.re_session_id`), *except* for the handful of fields that correspond to an existing +//! OpenTelemetry [semantic convention], which are emitted under the standardized attribute name +//! (taken from the [`opentelemetry_semantic_conventions`] crate) so backends can interpret them +//! generically. The prefixing is centralized in [`Attrs::push`]; the standardized fields opt out +//! via the `*_std` helpers and the [`opentelemetry_semantic_conventions::attribute`] constants. +//! +//! The mapped fields are `cli_args` -> `process.command_args`, `exit_code` -> `process.exit.code`, +//! `git_revision`/`hg_revision` -> `vcs.ref.head.revision` (the current checkout), and +//! `file_watcher_stats.branched_from_revision` -> `vcs.ref.base.revision` (the mergebase it +//! branched from). The structured `version_control_revision` sub-message carries the same revision +//! as the top-level `git_revision`/`hg_revision`, so it stays under `buck2.` to avoid emitting +//! `vcs.ref.head.revision` twice. +//! +//! [semantic convention]: https://opentelemetry.io/docs/specs/semconv/ + +use std::collections::HashMap; + +use opentelemetry::Array; +use opentelemetry::Key; +use opentelemetry::KeyValue; +use opentelemetry::StringValue; +use opentelemetry::Value; +use opentelemetry_semantic_conventions::attribute::PROCESS_COMMAND_ARGS; +use opentelemetry_semantic_conventions::attribute::PROCESS_EXIT_CODE; +use opentelemetry_semantic_conventions::attribute::VCS_REF_BASE_REVISION; +use opentelemetry_semantic_conventions::attribute::VCS_REF_HEAD_REVISION; + +/// Accumulates OTLP attributes while keeping each field mapping to a single readable line. The +/// helpers centralise the `Option`/unit/empty-skipping conventions so the mapping below reads as a +/// flat list of "field -> key" decisions. +#[derive(Default)] +struct Attrs { + attrs: Vec, +} + +impl Attrs { + /// Push an attribute under the `buck2.` namespace. All buck2-specific keys funnel through here + /// so the prefixing is applied in exactly one place; fields that map onto an OpenTelemetry + /// semantic convention bypass it via [`Attrs::push_std`]. + fn push(&mut self, key: impl Into, value: impl Into) { + let key = key.into(); + self.attrs.push(KeyValue::new( + Key::new(format!("buck2.{}", key.as_str())), + value.into(), + )); + } + + /// Push an attribute under a key taken verbatim from the OpenTelemetry semantic conventions (no + /// `buck2.` prefix). + fn push_std(&mut self, key: impl Into, value: impl Into) { + self.attrs.push(KeyValue::new(key, value.into())); + } + + /// Resolve the accumulated attributes into the final list, deduplicating by key with last-write + /// wins. OTLP attributes are keyed by name, so a duplicate key is invalid; the dynamic + /// key-value maps (`metadata`, `client_metadata`, `install_device_metadata`) can in principle + /// produce one (e.g. two install devices sharing an entry key), and overwriting keeps the event + /// usable rather than emitting an ambiguous pair. First-seen position is preserved. + fn into_attrs(self) -> Vec { + let mut index: HashMap = HashMap::new(); + let mut deduped: Vec = Vec::with_capacity(self.attrs.len()); + for kv in self.attrs { + match index.get(kv.key.as_str()) { + Some(&i) => deduped[i] = kv, + None => { + index.insert(kv.key.as_str().to_owned(), deduped.len()); + deduped.push(kv); + } + } + } + deduped + } + + fn string(&mut self, key: impl Into, v: impl Into) { + self.push(key, v.into()); + } + + fn opt_string(&mut self, key: impl Into, v: Option>) { + if let Some(v) = v { + self.string(key, v); + } + } + + /// As [`Attrs::opt_string`], but emitted under a verbatim semantic-convention key. + fn opt_string_std(&mut self, key: impl Into, v: Option>) { + if let Some(v) = v { + self.push_std(key, v.into()); + } + } + + /// Integer-valued attribute. OTLP only has `i64`, so anything that does not fit (a `u64` past + /// `i64::MAX`) is dropped rather than silently wrapping; in practice these are counts/sizes that + /// never approach the limit. + fn int(&mut self, key: impl Into, v: impl TryInto) { + if let Ok(v) = v.try_into() { + self.push(key, v); + } + } + + fn opt_int>(&mut self, key: impl Into, v: Option) { + if let Some(v) = v { + self.int(key, v); + } + } + + /// As [`Attrs::opt_int`], but emitted under a verbatim semantic-convention key. + fn opt_int_std>(&mut self, key: impl Into, v: Option) { + if let Some(v) = v + && let Ok(v) = v.try_into() + { + self.push_std(key, v); + } + } + + fn float(&mut self, key: impl Into, v: f64) { + self.push(key, v); + } + + fn opt_float(&mut self, key: impl Into, v: Option) { + if let Some(v) = v { + self.float(key, v); + } + } + + fn bool(&mut self, key: impl Into, v: bool) { + self.push(key, v); + } + + fn opt_bool(&mut self, key: impl Into, v: Option) { + if let Some(v) = v { + self.bool(key, v); + } + } + + /// A `google.protobuf.Duration` as integer microseconds, matching the `_us` serde convention. + fn opt_duration_us(&mut self, key: impl Into, v: Option<&::prost_types::Duration>) { + if let Some(d) = v { + let micros = d.seconds * 1_000_000 + i64::from(d.nanos) / 1_000; + self.int(key, micros); + } + } + + /// A repeated scalar field as a single homogeneous string array. Skipped when empty (an empty + /// array carries no information and just adds a column). + fn strings(&mut self, key: impl Into, vs: I) + where + I: IntoIterator, + S: Into, + { + let vs: Vec = vs + .into_iter() + .map(|s| StringValue::from(s.into())) + .collect(); + if !vs.is_empty() { + self.push(key, Value::Array(Array::String(vs))); + } + } + + /// As [`Attrs::strings`], but emitted under a verbatim semantic-convention key. + fn strings_std(&mut self, key: impl Into, vs: I) + where + I: IntoIterator, + S: Into, + { + let vs: Vec = vs + .into_iter() + .map(|s| StringValue::from(s.into())) + .collect(); + if !vs.is_empty() { + self.push_std(key, Value::Array(Array::String(vs))); + } + } + + /// A repeated bool field as a single homogeneous bool array. Skipped when empty. + fn bools(&mut self, key: impl Into, vs: I) + where + I: IntoIterator, + { + let vs: Vec = vs.into_iter().collect(); + if !vs.is_empty() { + self.push(key, Value::Array(Array::Bool(vs))); + } + } +} + +/// Map every field of an [`InvocationRecord`](buck2_data::InvocationRecord) to OTLP span attributes. +pub(crate) fn invocation_record_attributes(record: &buck2_data::InvocationRecord) -> Vec { + let mut a = Attrs::default(); + + // Marker so backends can segment this wide event from buck2's fine-grained action/analysis + // spans (e.g. into a dedicated Honeycomb dataset/view). + a.string("event_type", "invocation_record"); + + // -- Identity / command ------------------------------------------------------------------ + a.string("re_session_id", record.re_session_id.clone()); + // `cli_args` is the full argv as received by the process -- OTel `process.command_args`. + a.strings_std(PROCESS_COMMAND_ARGS, record.cli_args.iter().cloned()); + a.string("filesystem", record.filesystem.clone()); + a.opt_string("isolation_dir", record.isolation_dir.clone()); + a.opt_string("command_name", record.command_name.clone()); + a.opt_string("test_info", record.test_info.clone()); + a.strings("tags", record.tags.iter().cloned()); + a.string("re_experiment_name", record.re_experiment_name.clone()); + a.opt_string("restarted_trace_id", record.restarted_trace_id.clone()); + a.strings( + "concurrent_command_ids", + record.concurrent_command_ids.iter().cloned(), + ); + a.opt_string("preemptible", record.preemptible.clone()); + a.opt_string( + "previous_uuid_with_mismatched_config", + record.previous_uuid_with_mismatched_config.clone(), + ); + + // -- Durations (microseconds) ------------------------------------------------------------ + a.opt_duration_us("command_duration_us", record.command_duration.as_ref()); + a.opt_duration_us("client_walltime_us", record.client_walltime.as_ref()); + a.opt_duration_us( + "critical_path_duration_us", + record.critical_path_duration.as_ref(), + ); + a.opt_duration_us( + "concurrent_command_blocking_duration_us", + record.concurrent_command_blocking_duration.as_ref(), + ); + a.opt_duration_us( + "bxl_ensure_artifacts_duration_us", + record.bxl_ensure_artifacts_duration.as_ref(), + ); + a.opt_duration_us("install_duration_us", record.install_duration.as_ref()); + + // -- Action execution counts ------------------------------------------------------------- + a.int("run_local_count", record.run_local_count); + a.int("run_remote_count", record.run_remote_count); + a.int("run_action_cache_count", record.run_action_cache_count); + a.int("run_skipped_count", record.run_skipped_count); + a.opt_int("run_fallback_count", record.run_fallback_count); + a.opt_int( + "run_fallback_re_queue_count", + record.run_fallback_re_queue_count, + ); + a.opt_int("run_local_only_count", record.run_local_only_count); + a.opt_int( + "local_actions_executed_via_worker", + record.local_actions_executed_via_worker, + ); + a.int( + "run_remote_dep_file_cache_count", + record.run_remote_dep_file_cache_count, + ); + a.opt_int( + "run_command_failure_count", + record.run_command_failure_count, + ); + + // -- Cache / uploads --------------------------------------------------------------------- + a.int("cache_upload_count", record.cache_upload_count); + a.int( + "cache_upload_attempt_count", + record.cache_upload_attempt_count, + ); + a.int("dep_file_upload_count", record.dep_file_upload_count); + a.int( + "dep_file_upload_attempt_count", + record.dep_file_upload_attempt_count, + ); + a.float("cache_hit_rate", f64::from(record.cache_hit_rate)); + a.int( + "min_build_count_since_rebase", + record.min_build_count_since_rebase, + ); + a.int( + "min_attempted_build_count_since_rebase", + record.min_attempted_build_count_since_rebase, + ); + + // -- Build graph / analysis -------------------------------------------------------------- + a.opt_int("analysis_count", record.analysis_count); + a.opt_int("load_count", record.load_count); + a.opt_int("event_count", record.event_count); + a.opt_int( + "materialization_output_size", + record.materialization_output_size, + ); + a.opt_int("materialization_files", record.materialization_files); + a.strings( + "target_rule_type_names", + record.target_rule_type_names.iter().cloned(), + ); + + // -- "Time to X" latency milestones (milliseconds) --------------------------------------- + a.opt_int( + "max_event_client_delay_ms", + record.max_event_client_delay_ms, + ); + a.opt_int( + "time_to_first_action_execution_ms", + record.time_to_first_action_execution_ms, + ); + a.opt_int("time_to_command_start_ms", record.time_to_command_start_ms); + a.opt_int( + "time_to_command_critical_section_ms", + record.time_to_command_critical_section_ms, + ); + a.opt_int( + "time_to_first_analysis_ms", + record.time_to_first_analysis_ms, + ); + a.opt_int( + "time_to_load_first_build_file_ms", + record.time_to_load_first_build_file_ms, + ); + a.opt_int( + "time_to_first_command_execution_start_ms", + record.time_to_first_command_execution_start_ms, + ); + a.opt_int( + "time_to_last_action_execution_end_ms", + record.time_to_last_action_execution_end_ms, + ); + a.opt_int( + "time_to_first_test_discovery_ms", + record.time_to_first_test_discovery_ms, + ); + a.opt_int( + "time_to_first_test_run_ms", + record.time_to_first_test_run_ms, + ); + a.opt_int( + "time_to_first_pass_test_result_ms", + record.time_to_first_pass_test_result_ms, + ); + a.opt_int( + "time_to_first_fail_test_result_ms", + record.time_to_first_fail_test_result_ms, + ); + a.opt_int( + "time_to_first_fatal_test_result_ms", + record.time_to_first_fatal_test_result_ms, + ); + a.opt_int( + "time_to_first_timeout_test_result_ms", + record.time_to_first_timeout_test_result_ms, + ); + a.opt_int( + "time_to_first_skip_test_result_ms", + record.time_to_first_skip_test_result_ms, + ); + a.opt_int( + "time_to_first_unknown_test_result_ms", + record.time_to_first_unknown_test_result_ms, + ); + a.opt_int( + "time_to_first_infra_failure_test_result_ms", + record.time_to_first_infra_failure_test_result_ms, + ); + a.int("exec_time_ms", record.exec_time_ms); + + // -- Memory / system --------------------------------------------------------------------- + a.opt_int("max_malloc_bytes_active", record.max_malloc_bytes_active); + a.opt_int( + "max_malloc_bytes_allocated", + record.max_malloc_bytes_allocated, + ); + a.opt_int( + "system_total_memory_bytes", + record.system_total_memory_bytes, + ); + a.opt_int( + "peak_process_memory_bytes", + record.peak_process_memory_bytes, + ); + a.opt_int( + "peak_used_disk_space_bytes", + record.peak_used_disk_space_bytes, + ); + a.opt_int("total_disk_space_bytes", record.total_disk_space_bytes); + a.opt_int("memory_max_anon_allprocs", record.memory_max_anon_allprocs); + a.opt_int( + "memory_max_anon_forkserver_actions", + record.memory_max_anon_forkserver_actions, + ); + a.opt_int( + "memory_max_total_allprocs", + record.memory_max_total_allprocs, + ); + a.opt_int( + "memory_max_total_forkserver_actions", + record.memory_max_total_forkserver_actions, + ); + + // -- DICE / in-progress peaks ------------------------------------------------------------ + a.opt_int( + "max_dice_in_progress_keys", + record.max_dice_in_progress_keys, + ); + a.opt_int("max_dice_compute_keys", record.max_dice_compute_keys); + a.opt_int("max_in_progress_actions", record.max_in_progress_actions); + a.opt_int( + "max_in_progress_local_actions", + record.max_in_progress_local_actions, + ); + a.opt_int( + "max_in_progress_remote_actions", + record.max_in_progress_remote_actions, + ); + a.opt_int( + "max_in_progress_remote_uploads", + record.max_in_progress_remote_uploads, + ); + + // -- Remote execution bytes / speeds ----------------------------------------------------- + a.opt_int("re_upload_bytes", record.re_upload_bytes); + a.opt_int("re_download_bytes", record.re_download_bytes); + a.opt_int("re_max_download_speed", record.re_max_download_speed); + a.opt_int("re_max_upload_speed", record.re_max_upload_speed); + a.opt_int("re_avg_download_speed", record.re_avg_download_speed); + a.opt_int("re_avg_upload_speed", record.re_avg_upload_speed); + a.opt_float( + "re_average_local_cache_lookup_microseconds", + record.re_average_local_cache_lookup_microseconds, + ); + + // -- Local cache ------------------------------------------------------------------------- + a.opt_int("local_cache_hits_files", record.local_cache_hits_files); + a.opt_int("local_cache_hits_bytes", record.local_cache_hits_bytes); + a.opt_int("local_cache_misses_files", record.local_cache_misses_files); + a.opt_int("local_cache_misses_bytes", record.local_cache_misses_bytes); + a.opt_int( + "local_cache_hits_files_from_memory_cache", + record.local_cache_hits_files_from_memory_cache, + ); + a.opt_int( + "local_cache_hits_files_from_filesystem_cache", + record.local_cache_hits_files_from_filesystem_cache, + ); + a.opt_int("local_cache_lookups", record.local_cache_lookups); + + // -- Storage backends (zdb / zgateway / manifold / hedwig) ------------------------------- + a.opt_int("zdb_download_queries", record.zdb_download_queries); + a.opt_int("zdb_download_bytes", record.zdb_download_bytes); + a.opt_int("zdb_upload_queries", record.zdb_upload_queries); + a.opt_int("zdb_upload_bytes", record.zdb_upload_bytes); + a.opt_int( + "zgateway_download_queries", + record.zgateway_download_queries, + ); + a.opt_int("zgateway_download_bytes", record.zgateway_download_bytes); + a.opt_int("zgateway_upload_queries", record.zgateway_upload_queries); + a.opt_int("zgateway_upload_bytes", record.zgateway_upload_bytes); + a.opt_int( + "manifold_download_queries", + record.manifold_download_queries, + ); + a.opt_int("manifold_download_bytes", record.manifold_download_bytes); + a.opt_int("manifold_upload_queries", record.manifold_upload_queries); + a.opt_int("manifold_upload_bytes", record.manifold_upload_bytes); + a.opt_int("hedwig_download_queries", record.hedwig_download_queries); + a.opt_int("hedwig_download_bytes", record.hedwig_download_bytes); + a.opt_int("hedwig_upload_queries", record.hedwig_upload_queries); + a.opt_int("hedwig_upload_bytes", record.hedwig_upload_bytes); + + // -- Event sink -------------------------------------------------------------------------- + a.opt_int("sink_success_count", record.sink_success_count); + a.opt_int("sink_failure_count", record.sink_failure_count); + a.opt_int("sink_dropped_count", record.sink_dropped_count); + a.opt_int("sink_bytes_written", record.sink_bytes_written); + a.opt_int("sink_max_buffer_depth", record.sink_max_buffer_depth); + + // -- IO syscall counts ------------------------------------------------------------------- + a.opt_int("io_copy_count", record.io_copy_count); + a.opt_int("io_symlink_count", record.io_symlink_count); + a.opt_int("io_hardlink_count", record.io_hardlink_count); + a.opt_int("io_mkdir_count", record.io_mkdir_count); + a.opt_int("io_readdir_count", record.io_readdir_count); + a.opt_int("io_readdir_eden_count", record.io_readdir_eden_count); + a.opt_int("io_rmdir_count", record.io_rmdir_count); + a.opt_int("io_rmdir_all_count", record.io_rmdir_all_count); + a.opt_int("io_stat_count", record.io_stat_count); + a.opt_int("io_stat_eden_count", record.io_stat_eden_count); + a.opt_int("io_chmod_count", record.io_chmod_count); + a.opt_int("io_readlink_count", record.io_readlink_count); + a.opt_int("io_remove_count", record.io_remove_count); + a.opt_int("io_rename_count", record.io_rename_count); + a.opt_int("io_read_count", record.io_read_count); + a.opt_int("io_write_count", record.io_write_count); + a.opt_int("io_canonicalize_count", record.io_canonicalize_count); + a.opt_int("io_eden_settle_count", record.io_eden_settle_count); + + // -- Versions / environment -------------------------------------------------------------- + a.opt_string("watchman_version", record.watchman_version.clone()); + a.opt_string("eden_version", record.eden_version.clone()); + a.opt_string("file_watcher", record.file_watcher.clone()); + a.opt_string( + "persistent_cache_mode", + record.persistent_cache_mode.clone(), + ); + a.opt_int("file_watcher_duration_ms", record.file_watcher_duration_ms); + a.opt_int( + "initial_materializer_entries_from_sqlite", + record.initial_materializer_entries_from_sqlite, + ); + + // -- Source control ---------------------------------------------------------------------- + // A repo is either hg or git, so at most one of these is set; both map to the same OTel + // `vcs.ref.head.revision` (the current checkout's revision). + a.opt_string_std(VCS_REF_HEAD_REVISION, record.hg_revision.clone()); + a.opt_string_std(VCS_REF_HEAD_REVISION, record.git_revision.clone()); + a.opt_bool("has_local_changes", record.has_local_changes); + a.strings( + "version_control_errors", + record.version_control_errors.iter().cloned(), + ); + a.strings( + "representative_config_flags", + record.representative_config_flags.iter().cloned(), + ); + + // -- Outcome / status -------------------------------------------------------------------- + // The process exit status -- OTel `process.exit.code`. + a.opt_int_std(PROCESS_EXIT_CODE, record.exit_code); + a.opt_string("exit_result_name", record.exit_result_name.clone()); + a.opt_bool("has_command_result", record.has_command_result); + a.opt_bool("has_end_of_stream", record.has_end_of_stream); + a.opt_bool( + "instant_command_is_success", + record.instant_command_is_success, + ); + a.opt_bool( + "daemon_connection_failure", + record.daemon_connection_failure, + ); + a.opt_bool("should_restart", record.should_restart); + a.opt_bool("eligible_for_full_hybrid", record.eligible_for_full_hybrid); + a.opt_bool("new_configs_used", record.new_configs_used); + a.opt_string( + "critical_path_backend", + record.critical_path_backend.clone(), + ); + a.opt_int( + "compressed_event_log_size_bytes", + record.compressed_event_log_size_bytes, + ); + a.opt_int("event_log_manifold_ttl_s", record.event_log_manifold_ttl_s); + a.opt_int("wrapper_start_time", record.wrapper_start_time); + a.opt_string("installer_log_url", record.installer_log_url.clone()); + + // -- Enums (stored as i32; emit the proto enum name) ------------------------------------- + if let Some(outcome) = record + .outcome + .and_then(|v| buck2_data::InvocationOutcome::try_from(v).ok()) + { + a.string("outcome", outcome.as_str_name()); + } + if let Some(reason) = record + .daemon_was_started + .and_then(|v| buck2_data::DaemonWasStartedReason::try_from(v).ok()) + { + a.string("daemon_was_started", reason.as_str_name()); + } + a.strings( + "active_networks_kinds", + record + .active_networks_kinds + .iter() + .filter_map(|&v| buck2_data::NetworkKind::try_from(v).ok()) + .map(|k| k.as_str_name()), + ); + + // -- Resolved target patterns (unwrap the single-field `TargetPattern` wrapper) ---------- + if let Some(patterns) = &record.parsed_target_patterns { + a.strings( + "parsed_target_patterns", + patterns.target_patterns.iter().map(|p| p.value.clone()), + ); + } + + // -- Dynamic key-value maps: each entry's key becomes its own attribute key --------------- + // `metadata` is the explicit Scuba passthrough; `client_metadata` and `install_device_metadata` + // are likewise lists of (key, value) pairs, so we emit `.` rather than two + // parallel `.key`/`.value` arrays (which can't be correlated by a backend). These keys are + // dynamic and not guaranteed unique across entries, so the final dedup pass (last-wins) resolves + // any collisions. + if let Some(metadata) = &record.metadata { + for (k, v) in &metadata.strings { + a.push(format!("metadata.{k}"), v.clone()); + } + for (k, v) in &metadata.ints { + a.push(format!("metadata.{k}"), *v); + } + } + for m in &record.client_metadata { + a.push(format!("client_metadata.{}", m.key), m.value.clone()); + } + for entry in record + .install_device_metadata + .iter() + .flat_map(|d| d.entry.iter()) + { + a.push( + format!("install_device_metadata.{}", entry.key), + entry.value.clone(), + ); + } + + // -- Repeated messages projected field-wise into parallel arrays ------------------------- + a.strings( + "soft_error_categories.category", + record + .soft_error_categories + .iter() + .map(|e| e.category.clone()), + ); + a.bools( + "soft_error_categories.is_quiet", + record.soft_error_categories.iter().map(|e| e.is_quiet), + ); + + a.strings( + "errors.message", + record.errors.iter().map(|e| e.message.clone()), + ); + a.strings( + "errors.category", + record.errors.iter().filter_map(|e| e.category.clone()), + ); + a.strings( + "errors.category_key", + record.errors.iter().filter_map(|e| e.category_key.clone()), + ); + a.strings( + "errors.best_tag", + record.errors.iter().filter_map(|e| e.best_tag.clone()), + ); + a.strings( + "errors.source_area", + record.errors.iter().filter_map(|e| e.source_area.clone()), + ); + a.strings( + "errors.source_location", + record + .errors + .iter() + .filter_map(|e| e.source_location.clone()), + ); + a.strings( + "errors.telemetry_message", + record + .errors + .iter() + .filter_map(|e| e.telemetry_message.clone()), + ); + a.strings( + "errors.tags", + record.errors.iter().flat_map(|e| e.tags.iter().cloned()), + ); + a.strings( + "errors.sub_error_categories", + record + .errors + .iter() + .flat_map(|e| e.sub_error_categories.iter().cloned()), + ); + + // -- Nested sub-messages ----------------------------------------------------------------- + if let Some(snapshot) = &record.first_snapshot { + push_snapshot(&mut a, "first_snapshot", snapshot); + } + if let Some(snapshot) = &record.last_snapshot { + push_snapshot(&mut a, "last_snapshot", snapshot); + } + if let Some(command_end) = &record.command_end { + push_command_end(&mut a, command_end); + } + if let Some(stats) = &record.file_watcher_stats { + push_file_watcher_stats(&mut a, stats); + } + if let Some(options) = &record.command_options { + push_command_options(&mut a, options); + } + if let Some(target_cfg) = &record.target_cfg { + push_target_cfg(&mut a, target_cfg); + } + if let Some(revision) = &record.version_control_revision { + push_version_control_revision(&mut a, revision); + } + + a.into_attrs() +} + +fn push_command_options(a: &mut Attrs, o: &buck2_data::CommandOptions) { + a.int( + "command_options.configured_parallelism", + o.configured_parallelism, + ); + a.int( + "command_options.available_parallelism", + o.available_parallelism, + ); +} + +fn push_target_cfg(a: &mut Attrs, t: &buck2_data::TargetCfg) { + a.strings( + "target_cfg.target_platforms", + t.target_platforms.iter().cloned(), + ); + a.strings("target_cfg.cli_modifiers", t.cli_modifiers.iter().cloned()); +} + +fn push_version_control_revision(a: &mut Attrs, r: &buck2_data::VersionControlRevision) { + a.opt_string( + "version_control_revision.hg_revision", + r.hg_revision.clone(), + ); + a.opt_string( + "version_control_revision.git_revision", + r.git_revision.clone(), + ); + a.opt_bool( + "version_control_revision.has_local_changes", + r.has_local_changes, + ); + a.opt_string( + "version_control_revision.command_error", + r.command_error.clone(), + ); +} + +fn push_file_watcher_stats(a: &mut Attrs, s: &buck2_data::FileWatcherStats) { + a.bool("file_watcher_stats.fresh_instance", s.fresh_instance); + a.int("file_watcher_stats.events_total", s.events_total); + a.int("file_watcher_stats.events_processed", s.events_processed); + // The mergebase the working copy branched from -- OTel `vcs.ref.base.revision`. + a.opt_string_std(VCS_REF_BASE_REVISION, s.branched_from_revision.clone()); + a.opt_int( + "file_watcher_stats.branched_from_global_rev", + s.branched_from_global_rev, + ); + a.opt_string( + "file_watcher_stats.incomplete_events_reason", + s.incomplete_events_reason.clone(), + ); + a.opt_string( + "file_watcher_stats.watchman_version", + s.watchman_version.clone(), + ); + a.opt_int( + "file_watcher_stats.branched_from_revision_timestamp", + s.branched_from_revision_timestamp, + ); + a.opt_string("file_watcher_stats.eden_version", s.eden_version.clone()); + + // `events` is a repeated message: project field-wise into parallel arrays. + a.strings( + "file_watcher_stats.events.path", + s.events.iter().map(|e| e.path.clone()), + ); + a.strings( + "file_watcher_stats.events.event", + s.events + .iter() + .filter_map(|e| buck2_data::FileWatcherEventType::try_from(e.event).ok()) + .map(|e| e.as_str_name()), + ); + a.strings( + "file_watcher_stats.events.kind", + s.events + .iter() + .filter_map(|e| buck2_data::FileWatcherKind::try_from(e.kind).ok()) + .map(|e| e.as_str_name()), + ); + + if let Some(fresh) = &s.fresh_instance_data { + a.bool( + "file_watcher_stats.fresh_instance_data.new_mergebase", + fresh.new_mergebase, + ); + a.bool( + "file_watcher_stats.fresh_instance_data.cleared_dice", + fresh.cleared_dice, + ); + a.bool( + "file_watcher_stats.fresh_instance_data.cleared_dep_files", + fresh.cleared_dep_files, + ); + } +} + +fn push_snapshot(a: &mut Attrs, prefix: &str, s: &buck2_data::Snapshot) { + let k = |n: &str| format!("{prefix}.{n}"); + + a.opt_int(k("buck2_rss"), s.buck2_rss); + a.int(k("buck2_max_rss"), s.buck2_max_rss); + a.int(k("buck2_user_cpu_us"), s.buck2_user_cpu_us); + a.int(k("buck2_system_cpu_us"), s.buck2_system_cpu_us); + a.int( + k("blocking_executor_io_queue_size"), + s.blocking_executor_io_queue_size, + ); + a.int( + k("tokio_blocking_queue_depth"), + s.tokio_blocking_queue_depth, + ); + a.int( + k("tokio_num_idle_blocking_threads"), + s.tokio_num_idle_blocking_threads, + ); + a.int( + k("tokio_num_blocking_threads"), + s.tokio_num_blocking_threads, + ); + + a.int(k("re_download_bytes"), s.re_download_bytes); + a.int(k("re_upload_bytes"), s.re_upload_bytes); + a.int(k("re_uploads_started"), s.re_uploads_started); + a.int( + k("re_uploads_finished_successfully"), + s.re_uploads_finished_successfully, + ); + a.int( + k("re_uploads_finished_with_error"), + s.re_uploads_finished_with_error, + ); + a.int(k("re_downloads_started"), s.re_downloads_started); + a.int( + k("re_downloads_finished_successfully"), + s.re_downloads_finished_successfully, + ); + a.int( + k("re_downloads_finished_with_error"), + s.re_downloads_finished_with_error, + ); + a.int(k("re_action_cache_started"), s.re_action_cache_started); + a.int( + k("re_action_cache_finished_successfully"), + s.re_action_cache_finished_successfully, + ); + a.int( + k("re_action_cache_finished_with_error"), + s.re_action_cache_finished_with_error, + ); + a.int(k("re_executes_started"), s.re_executes_started); + a.int( + k("re_executes_finished_successfully"), + s.re_executes_finished_successfully, + ); + a.int( + k("re_executes_finished_with_error"), + s.re_executes_finished_with_error, + ); + a.int(k("re_materializes_started"), s.re_materializes_started); + a.int( + k("re_materializes_finished_successfully"), + s.re_materializes_finished_successfully, + ); + a.int( + k("re_materializes_finished_with_error"), + s.re_materializes_finished_with_error, + ); + a.int( + k("re_write_action_results_started"), + s.re_write_action_results_started, + ); + a.int( + k("re_write_action_results_finished_successfully"), + s.re_write_action_results_finished_successfully, + ); + a.int( + k("re_write_action_results_finished_with_error"), + s.re_write_action_results_finished_with_error, + ); + a.int( + k("re_get_digest_expirations_started"), + s.re_get_digest_expirations_started, + ); + a.int( + k("re_get_digest_expirations_finished_successfully"), + s.re_get_digest_expirations_finished_successfully, + ); + a.int( + k("re_get_digest_expirations_finished_with_error"), + s.re_get_digest_expirations_finished_with_error, + ); + + a.int(k("io_in_flight_copy"), s.io_in_flight_copy); + a.int(k("io_in_flight_symlink"), s.io_in_flight_symlink); + a.int(k("io_in_flight_hardlink"), s.io_in_flight_hardlink); + a.int(k("io_in_flight_mk_dir"), s.io_in_flight_mk_dir); + a.int(k("io_in_flight_read_dir"), s.io_in_flight_read_dir); + a.int( + k("io_in_flight_read_dir_eden"), + s.io_in_flight_read_dir_eden, + ); + a.int(k("io_in_flight_rm_dir"), s.io_in_flight_rm_dir); + a.int(k("io_in_flight_rm_dir_all"), s.io_in_flight_rm_dir_all); + a.int(k("io_in_flight_stat"), s.io_in_flight_stat); + a.int(k("io_in_flight_stat_eden"), s.io_in_flight_stat_eden); + a.int(k("io_in_flight_chmod"), s.io_in_flight_chmod); + a.int(k("io_in_flight_read_link"), s.io_in_flight_read_link); + a.int(k("io_in_flight_remove"), s.io_in_flight_remove); + a.int(k("io_in_flight_rename"), s.io_in_flight_rename); + a.int(k("io_in_flight_read"), s.io_in_flight_read); + a.int(k("io_in_flight_write"), s.io_in_flight_write); + a.int(k("io_in_flight_canonicalize"), s.io_in_flight_canonicalize); + a.int(k("io_in_flight_eden_settle"), s.io_in_flight_eden_settle); + + a.int(k("daemon_uptime_s"), s.daemon_uptime_s); + + a.opt_int(k("malloc_bytes_active"), s.malloc_bytes_active); + a.opt_int(k("malloc_bytes_allocated"), s.malloc_bytes_allocated); + a.opt_int(k("used_disk_space_bytes"), s.used_disk_space_bytes); + + a.opt_int(k("host_cpu_usage_system_ms"), s.host_cpu_usage_system_ms); + a.opt_int(k("host_cpu_usage_user_ms"), s.host_cpu_usage_user_ms); + + a.int(k("dice_key_count"), s.dice_key_count); + a.int( + k("dice_currently_active_key_count"), + s.dice_currently_active_key_count, + ); + a.int( + k("dice_active_transaction_count"), + s.dice_active_transaction_count, + ); + + a.int( + k("deferred_materializer_queue_size"), + s.deferred_materializer_queue_size, + ); + + a.opt_int(k("sink_successes"), s.sink_successes); + a.opt_int(k("sink_failures"), s.sink_failures); + a.opt_int( + k("sink_failures_invalid_request"), + s.sink_failures_invalid_request, + ); + a.opt_int( + k("sink_failures_unauthorized"), + s.sink_failures_unauthorized, + ); + a.opt_int( + k("sink_failures_rate_limited"), + s.sink_failures_rate_limited, + ); + a.opt_int(k("sink_failures_pushed_back"), s.sink_failures_pushed_back); + a.opt_int( + k("sink_failures_enqueue_failed"), + s.sink_failures_enqueue_failed, + ); + a.opt_int( + k("sink_failures_internal_error"), + s.sink_failures_internal_error, + ); + a.opt_int(k("sink_failures_timed_out"), s.sink_failures_timed_out); + a.opt_int(k("sink_failures_unknown"), s.sink_failures_unknown); + a.opt_int(k("sink_buffer_depth"), s.sink_buffer_depth); + a.opt_int(k("sink_dropped"), s.sink_dropped); + a.opt_int(k("sink_bytes_written"), s.sink_bytes_written); + + // `network_interface_stats` is a map keyed by interface name; emit each interface's counters + // under its own dynamic key (like `metadata`). + for (interface, stats) in &s.network_interface_stats { + a.int( + format!("{prefix}.network_interface_stats.{interface}.tx_bytes"), + stats.tx_bytes, + ); + a.int( + format!("{prefix}.network_interface_stats.{interface}.rx_bytes"), + stats.rx_bytes, + ); + if let Ok(network_kind) = buck2_data::NetworkKind::try_from(stats.network_kind) { + a.string( + format!("{prefix}.network_interface_stats.{interface}.network_kind"), + network_kind.as_str_name(), + ); + } + } + + a.int(k("http_download_bytes"), s.http_download_bytes); + + a.int( + k("deferred_materializer_declares"), + s.deferred_materializer_declares, + ); + a.int( + k("deferred_materializer_declares_reused"), + s.deferred_materializer_declares_reused, + ); + + if let Some(unix) = &s.unix_system_stats { + a.float(k("unix_system_stats.load1"), unix.load1); + a.float(k("unix_system_stats.load5"), unix.load5); + a.float(k("unix_system_stats.load15"), unix.load15); + } + + a.int(k("zdb_download_queries"), s.zdb_download_queries); + a.int(k("zdb_download_bytes"), s.zdb_download_bytes); + a.int(k("zdb_upload_queries"), s.zdb_upload_queries); + a.int(k("zdb_upload_bytes"), s.zdb_upload_bytes); + + a.int(k("zgateway_download_queries"), s.zgateway_download_queries); + a.int(k("zgateway_download_bytes"), s.zgateway_download_bytes); + a.int(k("zgateway_upload_queries"), s.zgateway_upload_queries); + a.int(k("zgateway_upload_bytes"), s.zgateway_upload_bytes); + + a.int(k("manifold_download_queries"), s.manifold_download_queries); + a.int(k("manifold_download_bytes"), s.manifold_download_bytes); + a.int(k("manifold_upload_queries"), s.manifold_upload_queries); + a.int(k("manifold_upload_bytes"), s.manifold_upload_bytes); + + a.int(k("hedwig_download_queries"), s.hedwig_download_queries); + a.int(k("hedwig_download_bytes"), s.hedwig_download_bytes); + a.int(k("hedwig_upload_queries"), s.hedwig_upload_queries); + a.int(k("hedwig_upload_bytes"), s.hedwig_upload_bytes); + + a.int(k("local_cache_hits_files"), s.local_cache_hits_files); + a.int(k("local_cache_hits_bytes"), s.local_cache_hits_bytes); + a.int(k("local_cache_misses_files"), s.local_cache_misses_files); + a.int(k("local_cache_misses_bytes"), s.local_cache_misses_bytes); + + a.int( + k("local_cache_hits_files_from_memory_cache"), + s.local_cache_hits_files_from_memory_cache, + ); + a.int( + k("local_cache_hits_files_from_filesystem_cache"), + s.local_cache_hits_files_from_filesystem_cache, + ); + + a.int(k("local_cache_lookups"), s.local_cache_lookups); + a.int( + k("local_cache_lookup_latency_microseconds"), + s.local_cache_lookup_latency_microseconds, + ); + + a.opt_int( + k("this_event_client_delay_ms"), + s.this_event_client_delay_ms, + ); + a.opt_int(k("client_cpu_percents"), s.client_cpu_percents); + + if let Some(cgroup) = &s.allprocs_cgroup { + push_cgroup_memory_stats(a, &k("allprocs_cgroup"), cgroup); + } + if let Some(cgroup) = &s.forkserver_actions_cgroup { + push_cgroup_memory_stats(a, &k("forkserver_actions_cgroup"), cgroup); + } +} + +fn push_cgroup_memory_stats(a: &mut Attrs, prefix: &str, s: &buck2_data::UnixCgroupMemoryStats) { + a.int(format!("{prefix}.anon"), s.anon); + a.int(format!("{prefix}.file"), s.file); + a.int(format!("{prefix}.kernel"), s.kernel); +} + +fn push_command_end(a: &mut Attrs, c: &buck2_data::CommandEnd) { + a.bool("command_end.is_success", c.is_success); + if let Some(build_result) = &c.build_result { + a.bool( + "command_end.build_result.build_completed", + build_result.build_completed, + ); + } + match &c.data { + Some(buck2_data::command_end::Data::Build(e)) => { + a.string("command_end.kind", "build"); + push_build_command_end(a, e); + } + Some(buck2_data::command_end::Data::Targets(e)) => { + a.string("command_end.kind", "targets"); + push_targets_command_end(a, e); + } + Some(buck2_data::command_end::Data::Query(e)) => { + a.string("command_end.kind", "query"); + push_query_command_end(a, e); + } + Some(buck2_data::command_end::Data::Cquery(e)) => { + a.string("command_end.kind", "cquery"); + push_cquery_command_end(a, e); + } + Some(buck2_data::command_end::Data::Test(e)) => { + a.string("command_end.kind", "test"); + push_test_command_end(a, e); + } + Some(buck2_data::command_end::Data::Audit(e)) => { + a.string("command_end.kind", "audit"); + push_audit_command_end(a, e); + } + Some(buck2_data::command_end::Data::Docs(e)) => { + a.string("command_end.kind", "docs"); + push_docs_command_end(a, e); + } + Some(buck2_data::command_end::Data::Clean(e)) => { + a.string("command_end.kind", "clean"); + push_clean_command_end(a, e); + } + Some(buck2_data::command_end::Data::Aquery(e)) => { + a.string("command_end.kind", "aquery"); + push_aquery_command_end(a, e); + } + Some(buck2_data::command_end::Data::Install(e)) => { + a.string("command_end.kind", "install"); + push_install_command_end(a, e); + } + Some(buck2_data::command_end::Data::Materialize(e)) => { + a.string("command_end.kind", "materialize"); + push_materialize_command_end(a, e); + } + Some(buck2_data::command_end::Data::Profile(e)) => { + a.string("command_end.kind", "profile"); + push_profile_command_end(a, e); + } + Some(buck2_data::command_end::Data::Bxl(e)) => { + a.string("command_end.kind", "bxl"); + push_bxl_command_end(a, e); + } + Some(buck2_data::command_end::Data::Lsp(e)) => { + a.string("command_end.kind", "lsp"); + push_lsp_command_end(a, e); + } + Some(buck2_data::command_end::Data::FileStatus(e)) => { + a.string("command_end.kind", "file_status"); + push_file_status_command_end(a, e); + } + Some(buck2_data::command_end::Data::Starlark(e)) => { + a.string("command_end.kind", "starlark"); + push_starlark_command_end(a, e); + } + Some(buck2_data::command_end::Data::Subscribe(e)) => { + a.string("command_end.kind", "subscribe"); + push_subscription_command_end(a, e); + } + Some(buck2_data::command_end::Data::Trace(e)) => { + a.string("command_end.kind", "trace"); + push_trace_io_command_end(a, e); + } + Some(buck2_data::command_end::Data::Ctargets(e)) => { + a.string("command_end.kind", "ctargets"); + push_configured_targets_command_end(a, e); + } + Some(buck2_data::command_end::Data::StarlarkDebugAttach(e)) => { + a.string("command_end.kind", "starlark_debug_attach"); + push_starlark_debug_attach_command_end(a, e); + } + Some(buck2_data::command_end::Data::Explain(e)) => { + a.string("command_end.kind", "explain"); + push_explain_command_end(a, e); + } + Some(buck2_data::command_end::Data::ExpandExternalCell(e)) => { + a.string("command_end.kind", "expand_external_cell"); + push_expand_external_cells_command_end(a, e); + } + Some(buck2_data::command_end::Data::Complete(e)) => { + a.string("command_end.kind", "complete"); + push_complete_command_end(a, e); + } + Some(buck2_data::command_end::Data::HydrationPageOut(e)) => { + a.string("command_end.kind", "hydration_page_out"); + push_hydration_page_out_command_end(a, e); + } + None => {} + } +} + +fn push_build_command_end(a: &mut Attrs, e: &buck2_data::BuildCommandEnd) { + a.strings( + "command_end.build.unresolved_target_patterns", + e.unresolved_target_patterns.iter().map(|p| p.value.clone()), + ); +} + +fn push_targets_command_end(a: &mut Attrs, e: &buck2_data::TargetsCommandEnd) { + a.strings( + "command_end.targets.unresolved_target_patterns", + e.unresolved_target_patterns.iter().map(|p| p.value.clone()), + ); +} + +fn push_query_command_end(_a: &mut Attrs, _e: &buck2_data::QueryCommandEnd) {} + +fn push_cquery_command_end(_a: &mut Attrs, _e: &buck2_data::CQueryCommandEnd) {} + +fn push_test_command_end(a: &mut Attrs, e: &buck2_data::TestCommandEnd) { + a.strings( + "command_end.test.unresolved_target_patterns", + e.unresolved_target_patterns.iter().map(|p| p.value.clone()), + ); +} + +fn push_audit_command_end(_a: &mut Attrs, _e: &buck2_data::AuditCommandEnd) {} + +fn push_docs_command_end(_a: &mut Attrs, _e: &buck2_data::DocsCommandEnd) {} + +fn push_clean_command_end(a: &mut Attrs, e: &buck2_data::CleanCommandEnd) { + if let Some(s) = &e.clean_stale_stats { + a.int( + "command_end.clean.clean_stale_stats.stale_artifact_count", + s.stale_artifact_count, + ); + a.int( + "command_end.clean.clean_stale_stats.stale_bytes", + s.stale_bytes, + ); + a.int( + "command_end.clean.clean_stale_stats.retained_artifact_count", + s.retained_artifact_count, + ); + a.int( + "command_end.clean.clean_stale_stats.retained_bytes", + s.retained_bytes, + ); + a.int( + "command_end.clean.clean_stale_stats.untracked_artifact_count", + s.untracked_artifact_count, + ); + a.int( + "command_end.clean.clean_stale_stats.untracked_bytes", + s.untracked_bytes, + ); + a.int( + "command_end.clean.clean_stale_stats.cleaned_artifact_count", + s.cleaned_artifact_count, + ); + a.int( + "command_end.clean.clean_stale_stats.cleaned_bytes", + s.cleaned_bytes, + ); + a.int( + "command_end.clean.clean_stale_stats.total_duration_s", + s.total_duration_s, + ); + a.int( + "command_end.clean.clean_stale_stats.scan_duration_s", + s.scan_duration_s, + ); + a.int( + "command_end.clean.clean_stale_stats.clean_duration_s", + s.clean_duration_s, + ); + } +} + +fn push_aquery_command_end(_a: &mut Attrs, _e: &buck2_data::AqueryCommandEnd) {} + +fn push_install_command_end(a: &mut Attrs, e: &buck2_data::InstallCommandEnd) { + a.strings( + "command_end.install.unresolved_target_patterns", + e.unresolved_target_patterns.iter().map(|p| p.value.clone()), + ); +} + +fn push_materialize_command_end(_a: &mut Attrs, _e: &buck2_data::MaterializeCommandEnd) {} + +fn push_profile_command_end(_a: &mut Attrs, _e: &buck2_data::ProfileCommandEnd) {} + +fn push_bxl_command_end(a: &mut Attrs, e: &buck2_data::BxlCommandEnd) { + a.string("command_end.bxl.bxl_label", e.bxl_label.clone()); +} + +fn push_lsp_command_end(_a: &mut Attrs, _e: &buck2_data::LspCommandEnd) {} + +fn push_file_status_command_end(_a: &mut Attrs, _e: &buck2_data::FileStatusCommandEnd) {} + +fn push_starlark_command_end(_a: &mut Attrs, _e: &buck2_data::StarlarkCommandEnd) {} + +fn push_subscription_command_end(_a: &mut Attrs, _e: &buck2_data::SubscriptionCommandEnd) {} + +fn push_trace_io_command_end(_a: &mut Attrs, _e: &buck2_data::TraceIoCommandEnd) {} + +fn push_configured_targets_command_end( + _a: &mut Attrs, + _e: &buck2_data::ConfiguredTargetsCommandEnd, +) { +} + +fn push_starlark_debug_attach_command_end( + _a: &mut Attrs, + _e: &buck2_data::StarlarkDebugAttachCommandEnd, +) { +} + +fn push_explain_command_end(_a: &mut Attrs, _e: &buck2_data::ExplainCommandEnd) {} + +fn push_expand_external_cells_command_end( + _a: &mut Attrs, + _e: &buck2_data::ExpandExternalCellsCommandEnd, +) { +} + +fn push_complete_command_end(_a: &mut Attrs, _e: &buck2_data::CompleteCommandEnd) {} + +fn push_hydration_page_out_command_end( + _a: &mut Attrs, + _e: &buck2_data::HydrationPageOutCommandEnd, +) { +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn find<'a>(attrs: &'a [KeyValue], key: &str) -> Option<&'a Value> { + attrs + .iter() + .find(|kv| kv.key.as_str() == key) + .map(|kv| &kv.value) + } + + fn string(attrs: &[KeyValue], key: &str) -> Option { + match find(attrs, key)? { + Value::String(s) => Some(s.as_str().to_owned()), + _ => None, + } + } + + fn int(attrs: &[KeyValue], key: &str) -> Option { + match find(attrs, key)? { + Value::I64(i) => Some(*i), + _ => None, + } + } + + fn boolean(attrs: &[KeyValue], key: &str) -> Option { + match find(attrs, key)? { + Value::Bool(b) => Some(*b), + _ => None, + } + } + + fn strings(attrs: &[KeyValue], key: &str) -> Option> { + match find(attrs, key)? { + Value::Array(Array::String(v)) => { + Some(v.iter().map(|s| s.as_str().to_owned()).collect()) + } + _ => None, + } + } + + /// Exercises one representative field of each mapping shape: scalars, optionals (set and unset), + /// the `_us` duration conversion, repeated scalars, an enum projected to its proto name, a nested + /// sub-message prefix, the dynamic `metadata` passthrough, and a repeated message projected + /// field-wise into parallel arrays. If the mapping for any of these shapes regresses, one of + /// these assertions fails. + #[test] + fn maps_representative_fields() { + let record = buck2_data::InvocationRecord { + // Scalar string. + re_session_id: "session-123".to_owned(), + // Repeated scalar -> single string array. + cli_args: vec!["build".to_owned(), "//foo:bar".to_owned()], + // Required int. + run_local_count: 7, + // Optional int, present. + run_fallback_count: Some(3), + // Optional int, absent -- `run_local_only_count` is left None below. + // Duration -> integer microseconds. + command_duration: Some(::prost_types::Duration { + seconds: 2, + nanos: 500_000, + }), + // Enum stored as i32 -> proto enum name. + outcome: Some(buck2_data::InvocationOutcome::Success as i32), + // Fields mapped onto OpenTelemetry semantic conventions. + exit_code: Some(0), + git_revision: Some("deadbeef".to_owned()), + // Nested sub-message, emitted under a prefix. + first_snapshot: Some(buck2_data::Snapshot { + buck2_max_rss: 42, + ..Default::default() + }), + // Nested sub-message carrying a semantic-convention field (`branched_from_revision`). + file_watcher_stats: Some(buck2_data::FileWatcherStats { + branched_from_revision: Some("cafef00d".to_owned()), + ..Default::default() + }), + // Dynamic passthrough map. + metadata: Some(buck2_data::TypedMetadata { + strings: HashMap::from([("flavor".to_owned(), "vanilla".to_owned())]), + ints: HashMap::from([("answer".to_owned(), 42)]), + }), + // Repeated message -> parallel field-wise arrays. The second error has no category, so + // `errors.category` is shorter than `errors.message` (it filters out the None). + errors: vec![ + buck2_data::ProcessedErrorReport { + message: "boom".to_owned(), + category: Some("INFRA".to_owned()), + ..Default::default() + }, + buck2_data::ProcessedErrorReport { + message: "bad".to_owned(), + category: None, + ..Default::default() + }, + ], + ..Default::default() + }; + + let attrs = invocation_record_attributes(&record); + + // Static marker so backends can segment this wide event. + assert_eq!( + string(&attrs, "buck2.event_type").as_deref(), + Some("invocation_record") + ); + + // buck2-specific fields are namespaced under `buck2.`. + assert_eq!( + string(&attrs, "buck2.re_session_id").as_deref(), + Some("session-123") + ); + assert_eq!(int(&attrs, "buck2.run_local_count"), Some(7)); + assert_eq!(int(&attrs, "buck2.run_fallback_count"), Some(3)); + + // Fields that map onto OpenTelemetry semantic conventions use the standardized name and + // are *not* `buck2.`-prefixed. + assert_eq!( + strings(&attrs, PROCESS_COMMAND_ARGS), + Some(vec!["build".to_owned(), "//foo:bar".to_owned()]) + ); + assert!(find(&attrs, "buck2.cli_args").is_none()); + assert_eq!(int(&attrs, PROCESS_EXIT_CODE), Some(0)); + assert_eq!( + string(&attrs, VCS_REF_HEAD_REVISION).as_deref(), + Some("deadbeef") + ); + assert_eq!( + string(&attrs, VCS_REF_BASE_REVISION).as_deref(), + Some("cafef00d") + ); + + // Unset optional fields produce no attribute at all (rather than a zero/empty column). + assert!(find(&attrs, "buck2.run_local_only_count").is_none()); + + // 2s + 500_000ns = 2_000_000us + 500us. + assert_eq!(int(&attrs, "buck2.command_duration_us"), Some(2_000_500)); + + assert_eq!(string(&attrs, "buck2.outcome").as_deref(), Some("Success")); + assert_eq!(int(&attrs, "buck2.first_snapshot.buck2_max_rss"), Some(42)); + + assert_eq!( + string(&attrs, "buck2.metadata.flavor").as_deref(), + Some("vanilla") + ); + assert_eq!(int(&attrs, "buck2.metadata.answer"), Some(42)); + + assert_eq!( + strings(&attrs, "buck2.errors.message"), + Some(vec!["boom".to_owned(), "bad".to_owned()]) + ); + assert_eq!( + strings(&attrs, "buck2.errors.category"), + Some(vec!["INFRA".to_owned()]) + ); + } + + /// Empty repeated fields and absent optionals must not produce attributes -- an empty column + /// carries no information and just inflates every event. The default record has nothing set. + #[test] + fn skips_empty_and_absent_fields() { + let attrs = invocation_record_attributes(&buck2_data::InvocationRecord::default()); + + // Empty repeated scalar: no array attribute. + assert!(find(&attrs, PROCESS_COMMAND_ARGS).is_none()); + assert!(find(&attrs, "buck2.tags").is_none()); + // Absent optional duration / enum / nested message. + assert!(find(&attrs, "buck2.command_duration_us").is_none()); + assert!(find(&attrs, "buck2.outcome").is_none()); + assert!(find(&attrs, "buck2.first_snapshot.buck2_max_rss").is_none()); + + // The static marker is unconditional, so it is always present even for an empty record. + assert_eq!( + string(&attrs, "buck2.event_type").as_deref(), + Some("invocation_record") + ); + } + + /// `bool` optionals: present when set, absent when not. (The default-record test covers absence + /// of most shapes; bools get their own check because `false` is a meaningful set value that must + /// not be confused with "unset".) + #[test] + fn maps_optional_bools() { + let record = buck2_data::InvocationRecord { + has_local_changes: Some(false), + ..Default::default() + }; + let attrs = invocation_record_attributes(&record); + // Explicitly set to `false` -- must be emitted, not skipped. + assert_eq!(boolean(&attrs, "buck2.has_local_changes"), Some(false)); + // Never set -- must be absent. + assert!(find(&attrs, "buck2.new_configs_used").is_none()); + } + + /// `client_metadata` / `install_device_metadata` are key-value maps: each entry's key becomes + /// its own attribute (`.`), not parallel `.key`/`.value` arrays. Duplicate keys + /// (here, two install devices sharing `os`) are resolved last-wins by the dedup pass. + #[test] + fn maps_key_value_metadata() { + let record = buck2_data::InvocationRecord { + client_metadata: vec![buck2_data::ClientMetadata { + key: "user".to_owned(), + value: "alice".to_owned(), + }], + install_device_metadata: vec![ + buck2_data::DeviceMetadata { + entry: vec![buck2_data::device_metadata::Entry { + key: "os".to_owned(), + value: "ios".to_owned(), + }], + }, + buck2_data::DeviceMetadata { + entry: vec![buck2_data::device_metadata::Entry { + key: "os".to_owned(), + value: "android".to_owned(), + }], + }, + ], + ..Default::default() + }; + let attrs = invocation_record_attributes(&record); + + // Entry key becomes the attribute key; no parallel `.key`/`.value` arrays. + assert_eq!( + string(&attrs, "buck2.client_metadata.user").as_deref(), + Some("alice") + ); + assert!(find(&attrs, "buck2.client_metadata.key").is_none()); + assert!(find(&attrs, "buck2.client_metadata.value").is_none()); + + // Duplicate key across devices: last write wins, and only one attribute is emitted. + assert_eq!( + string(&attrs, "buck2.install_device_metadata.os").as_deref(), + Some("android") + ); + assert_eq!( + attrs + .iter() + .filter(|kv| kv.key.as_str() == "buck2.install_device_metadata.os") + .count(), + 1 + ); + } +} From a5cf021281f08c0d00ea700032a0f6001e5389b2 Mon Sep 17 00:00:00 2001 From: Cameron Samak Date: Mon, 6 Jul 2026 07:45:06 -0700 Subject: [PATCH 20/29] Stop doing a tset projection walk in write_json validation (#16) Unnecessary given validation in TransitiveSet:new. Already upstreamed. --- app/buck2_build_api/src/actions/impls/json.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/buck2_build_api/src/actions/impls/json.rs b/app/buck2_build_api/src/actions/impls/json.rs index 6552be24f2a8a..5064524e8575f 100644 --- a/app/buck2_build_api/src/actions/impls/json.rs +++ b/app/buck2_build_api/src/actions/impls/json.rs @@ -192,7 +192,14 @@ impl<'a, 'v> Serialize for SerializeValue<'a, 'v> { } JsonUnpack::Enum(x) => x.serialize(serializer), JsonUnpack::TransitiveSetJsonProjection(x) => { - serializer.collect_seq(err(x.iter_values())?.map(|v| self.with_value(v))) + match self.fs { + // Skip validation when fs == None because it can be expensive. + // TransitiveSet::new already did validate_json for projected values. + None => serializer.collect_seq(std::iter::empty::()), + Some(_) => { + serializer.collect_seq(err(x.iter_values())?.map(|v| self.with_value(v))) + } + } } JsonUnpack::TargetLabel(x) => { // Users could do this with `str(ctx.label.raw_target())`, but in some benchmarks that causes From df688907637a7fa5543c2983afeadcdece6619b6 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 6 Jul 2026 15:25:33 -0400 Subject: [PATCH 21/29] DUX-5483 Cargo.toml: opentelemetry-otlp: Add reqwest-rustls (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenTelemetry exporter we're using was added with `default-features = false`, which adds `reqwest` without a TLS stack, which ends up breaking OpenTelemetry export in our current CI setup. Follow-up to https://github.com/MercuryTechnologies/buck2/pull/14 `cargo tree` confirms that we're using the system store: ``` $ cargo tree --edges=normal --invert=rustls-platform-verifier --package=buck2_core rustls-platform-verifier v0.7.0 └── reqwest v0.13.4 ├── opentelemetry-http v0.32.0 │ └── opentelemetry-otlp v0.32.0 │ └── buck2_core v0.1.0 (/Users/wiggles/buck2-2/app/buck2_core) └── opentelemetry-otlp v0.32.0 (*) ``` See: - https://github.com/rustls/rustls-platform-verifier - "A certificate verification library for rustls that uses the operating system's verifier"
Cargo.lock diff ```diff --- /Users/wiggles/mwb/nix/packages/buck2-source/Cargo.lock 2026-07-06 10:44:58.618882289 -0700 +++ Cargo.lock 2026-07-06 11:50:41.817123538 -0700 @@ -3270,6 +3270,17 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] name = "chrono" version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3376,6 +3387,16 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] name = "common-path" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4642,8 +4663,10 @@ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -4665,11 +4688,13 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -5379,6 +5404,55 @@ ] [[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] name = "jobserver" version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5677,6 +5751,12 @@ ] [[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] name = "lsp-server" version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6935,6 +7015,63 @@ ] [[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases 0.2.1", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand 0.10.1", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases 0.2.1", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] name = "quote" version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6992,6 +7129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ + "chacha20", "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -7061,6 +7199,15 @@ ] [[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] name = "rand_xorshift" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -7280,13 +7427,19 @@ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "sync_wrapper 1.0.2", "tokio", + "tokio-rustls", "tower 0.5.3", "tower-http", "tower-service", @@ -7475,10 +7628,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] [[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs 0.8.3", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework 3.7.0", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] name = "rustls-webpki" version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -7834,6 +8015,22 @@ checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] name = "siphasher" version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -8551,6 +8748,21 @@ ] [[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] name = "tokio" version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -9207,6 +9419,25 @@ ] [[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] name = "webpki-roots" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" ```
--- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9722fefba14c0..c583bbe87a02a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -260,7 +260,7 @@ num_enum = "0.7.4" object = "0.36.7" once_cell = "1.21" opentelemetry = "0.32.0" -opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] } +opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "reqwest-rustls"] } opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } opentelemetry_sdk = "0.32.1" os_str_bytes = { version = "6.6.0", features = ["conversions"] } From 5ada3fb942aa601fb9ab0feac258eeadf9ca6611 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 6 Jul 2026 19:18:12 -0400 Subject: [PATCH 22/29] DUX-5497 OpenTelemetry: Respect `$TRACEPARENT` and similar (#18) This adds support for more OpenTelemetry environment variables: - `$OTEL_EXPORTER_OTLP_*` - `$OTEL_SDK_DISABLED` - `$OTEL_SERVICE_NAME` - `$OTEL_RESOURCE_ATTRIBUTES` - `$TRACEPARENT` - `TRACESTATE` --- app/buck2_core/src/logging/otel.rs | 144 +++++++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 7 deletions(-) diff --git a/app/buck2_core/src/logging/otel.rs b/app/buck2_core/src/logging/otel.rs index 0f838f4b7a546..1be9b9d76d1af 100644 --- a/app/buck2_core/src/logging/otel.rs +++ b/app/buck2_core/src/logging/otel.rs @@ -20,6 +20,22 @@ //! to callers: anything that reaches the global provider stored here -- a `tracing` layer bridged on //! top, or an out-of-band "wide event" span -- rides the same exporter. //! +//! ## Environment variables +//! +//! We honor the standard OpenTelemetry environment variables so buck2 slots into an existing +//! telemetry setup without buck2-specific configuration: +//! +//! * `OTEL_EXPORTER_OTLP_*` -- endpoint, headers, timeout, compression, TLS. Read by the exporter +//! builder itself; we only check the two endpoint variables to decide *whether* to export. +//! * `OTEL_SDK_DISABLED=true` -- the spec's global kill-switch. `opentelemetry-rust` does not honor +//! it on its own, so we check it here and skip building the exporter entirely. +//! * `OTEL_SERVICE_NAME` / `OTEL_RESOURCE_ATTRIBUTES` -- extra/overriding resource attributes, +//! merged in by [`resource`]. Our own authoritative attributes (pid, arch, os, host) win on +//! conflict; `service.name` defaults to `buck2` but yields to a user-provided value. +//! * `TRACEPARENT` / `TRACESTATE` -- W3C Trace Context of the launching process (CI job, wrapper +//! script, ...). When present, our wide-event span is parented under that trace instead of being +//! a disconnected root. See [`export_span`]. +//! //! ## Deferred activation (important) //! //! The OTLP batch exporter spawns background threads (a batch-processor thread plus an HTTP client @@ -41,19 +57,25 @@ //! we drain it explicitly. [`shutdown`] does that drain and must be called on every exit path. See //! . +use std::collections::HashMap; use std::sync::OnceLock; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::time::SystemTime; use buck2_error::conversion::from_any_with_tag; +use opentelemetry::Context; use opentelemetry::KeyValue; +use opentelemetry::propagation::TextMapPropagator as _; use opentelemetry::trace::Span as _; use opentelemetry::trace::Tracer as _; use opentelemetry::trace::TracerProvider as _; use opentelemetry_otlp::Protocol; use opentelemetry_otlp::WithExportConfig; use opentelemetry_sdk::Resource; +use opentelemetry_sdk::propagation::TraceContextPropagator; +use opentelemetry_sdk::resource::EnvResourceDetector; +use opentelemetry_sdk::resource::TelemetryResourceDetector; use opentelemetry_sdk::trace::BatchConfigBuilder; use opentelemetry_sdk::trace::BatchSpanProcessor; use opentelemetry_sdk::trace::SdkTracerProvider; @@ -63,7 +85,6 @@ use opentelemetry_semantic_conventions::resource::HOST_NAME; use opentelemetry_semantic_conventions::resource::OS_TYPE; use opentelemetry_semantic_conventions::resource::PROCESS_PID; use opentelemetry_semantic_conventions::resource::SERVICE_INSTANCE_ID; -use opentelemetry_semantic_conventions::resource::SERVICE_NAME; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use uuid::Uuid; @@ -95,6 +116,13 @@ fn otlp_endpoint_configured() -> bool { .any(|var| std::env::var_os(var).is_some_and(|v| !v.is_empty())) } +/// The OpenTelemetry SDK kill-switch. Per the spec, `OTEL_SDK_DISABLED=true` (case-insensitive) +/// disables the SDK; any other value, or an unset variable, leaves it enabled. `opentelemetry-rust` +/// does not read this variable itself, so we check it and skip building the exporter. +fn otel_sdk_disabled() -> bool { + std::env::var_os("OTEL_SDK_DISABLED").is_some_and(|v| v.eq_ignore_ascii_case("true")) +} + /// Map Rust's [`std::env::consts::OS`] to the OpenTelemetry `os.type` value set, passing through any /// value without a standardized equivalent (e.g. `ios`, `android`) as-is. Most names already match; /// only a couple are spelled differently. @@ -123,9 +151,15 @@ fn otel_host_arch(arch: &str) -> &str { /// Resource attributes identifying this build invocation, following OpenTelemetry semantic /// conventions (). +/// +/// Attributes are layered in ascending priority -- each step below overrides earlier ones on +/// conflict -- so `service.name` falls back to `buck2` but yields to whatever the user configured, +/// while the values buck2 measures itself always win. Expressing precedence by ordering lets the +/// user's `service.name` simply override our default instead of us having to detect whether they set +/// one. (We can't use the all-in-one `Resource::builder()`: it runs its detectors *first*, i.e. at +/// the lowest priority, leaving no way to slip our `buck2` default underneath them.) fn resource(version: &'static str) -> Resource { - let mut attributes = vec![ - KeyValue::new(SERVICE_NAME, "buck2"), + let mut authoritative = vec![ // buck2 is not a deployed service, so `service.version` is just this binary's build version. // The caller passes it in (`BuckVersion::get_version()`, the same string `buck2 --version` // prints) because the richer version -- the source revision stamped at build time via @@ -141,15 +175,36 @@ fn resource(version: &'static str) -> Resource { KeyValue::new(PROCESS_PID, i64::from(std::process::id())), ]; if let Ok(Some(hostname)) = hostname::get().map(|h| h.into_string().ok()) { - attributes.push(KeyValue::new(HOST_NAME, hostname)); + authoritative.push(KeyValue::new(HOST_NAME, hostname)); + } + + let mut builder = Resource::builder_empty() + // Lowest priority: our default `service.name`, overridden by anything the user sets below. + .with_service_name("buck2") + // `telemetry.sdk.*`, plus any attributes from `OTEL_RESOURCE_ATTRIBUTES` (which may carry a + // `service.name` that then wins over our default, and other attributes like + // `deployment.environment` that flow through untouched). + .with_detectors(&[ + Box::new(TelemetryResourceDetector), + Box::new(EnvResourceDetector::new()), + ]); + // `OTEL_SERVICE_NAME` is the dedicated variable and outranks a `service.name` in + // `OTEL_RESOURCE_ATTRIBUTES`; `EnvResourceDetector` only reads the latter, so apply it ourselves. + if let Some(name) = std::env::var_os("OTEL_SERVICE_NAME") + .and_then(|v| v.into_string().ok()) + .filter(|v| !v.is_empty()) + { + builder = builder.with_service_name(name); } - Resource::builder().with_attributes(attributes).build() + // Highest priority: the attributes buck2 measures itself (pid, arch, os, host) win over anything + // the environment supplies. + builder.with_attributes(authoritative).build() } /// Build the OTLP exporter and tracer provider (spawning the exporter's background threads) and store /// the provider in [`PROVIDER`]. No-op when no endpoint is configured. fn build_provider(version: &'static str) -> buck2_error::Result<()> { - if !otlp_endpoint_configured() { + if otel_sdk_disabled() || !otlp_endpoint_configured() { return Ok(()); } @@ -213,10 +268,41 @@ pub fn export_span( .span_builder(name) .with_start_time(start) .with_attributes(attributes) - .start(&tracer); + .start_with_context(&tracer, &parent_context_from_env()); span.end_with_timestamp(end); } +/// Extract a parent trace context from the environment, following the W3C Trace Context convention +/// that CI systems and wrappers (`otel-cli`, the Jenkins / GitHub Actions OpenTelemetry plugins, +/// Buildkite, ...) use to hand an in-progress trace to child processes via `$TRACEPARENT` (and its +/// companion `$TRACESTATE`). When buck2 is launched under such a trace, this makes our wide-event +/// span a child of the launching trace instead of a disconnected root. +/// +/// `opentelemetry-rust` only ever extracts trace context from HTTP-style carriers, never from the +/// process environment, so we assemble a carrier from the env ourselves and run the standard +/// `TraceContextPropagator` over it. Returns an empty (root) context when `$TRACEPARENT` is absent +/// or malformed -- i.e. the previous always-root behavior. +fn parent_context_from_env() -> Context { + let env = |var| std::env::var_os(var).and_then(|v| v.into_string().ok()); + parent_context_from_carrier( + env("TRACEPARENT").as_deref(), + env("TRACESTATE").as_deref(), + ) +} + +/// The environment-independent core of [`parent_context_from_env`], split out so it can be tested +/// without mutating process-global environment state. +fn parent_context_from_carrier(traceparent: Option<&str>, tracestate: Option<&str>) -> Context { + let mut carrier: HashMap = HashMap::new(); + // The propagator keys off the lowercased header names, matching how these would arrive over HTTP. + for (header, value) in [("traceparent", traceparent), ("tracestate", tracestate)] { + if let Some(value) = value.filter(|v| !v.is_empty()) { + carrier.insert(header.to_owned(), value.to_owned()); + } + } + TraceContextPropagator::new().extract(&carrier) +} + /// Build the OTLP exporter and start exporting, if telemetry is configured. /// /// This spawns the exporter's background threads, so it MUST be called only after the process has @@ -261,8 +347,52 @@ pub fn shutdown() { #[cfg(test)] mod tests { + use opentelemetry::trace::TraceContextExt as _; + use super::*; + #[test] + fn traceparent_becomes_remote_parent() { + // A well-formed W3C traceparent yields a valid, remote parent span context whose trace id + // our span will inherit. + let cx = parent_context_from_carrier( + Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"), + None, + ); + let span_context = cx.span().span_context().clone(); + assert!(span_context.is_valid()); + assert!(span_context.is_remote()); + assert_eq!( + span_context.trace_id().to_string(), + "0af7651916cd43dd8448eb211c80319c" + ); + } + + #[test] + fn no_or_malformed_traceparent_is_root() { + // No traceparent -> no parent (a fresh root span, as before). + assert!( + !parent_context_from_carrier(None, None) + .span() + .span_context() + .is_valid() + ); + // An empty value is treated as absent, not as a malformed header. + assert!( + !parent_context_from_carrier(Some(""), Some("foo=bar")) + .span() + .span_context() + .is_valid() + ); + // Garbage is ignored rather than propagated. + assert!( + !parent_context_from_carrier(Some("not-a-traceparent"), None) + .span() + .span_context() + .is_valid() + ); + } + #[test] fn os_type_remaps_and_passes_through() { // Rust spellings that differ from the OpenTelemetry value set. From fe47b62740eabac22aac5b15686161b4cf819d80 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Tue, 7 Jul 2026 17:33:24 -0400 Subject: [PATCH 23/29] GHA: Fix CI (#21) --- .github/workflows/build-and-examples.yml | 4 ++-- .github/workflows/build-and-test.yml | 6 +++--- app/buck2_build_signals_impl/src/enhancement.rs | 2 +- app/buck2_core/src/bzl.rs | 12 ++++++++---- app/buck2_core/src/logging/otel.rs | 6 ++++-- app/buck2_events/src/sink/otel.rs | 2 +- app/buck2_events/src/sink/otel_record.rs | 7 +++++++ app/buck2_interpreter/src/package_imports.rs | 2 +- .../src/interpreter.rs | 4 ++-- app/buck2_server/src/daemon/state.rs | 1 - app/buck2_test/src/command.rs | 6 +++--- remote_execution/oss/re_grpc/src/client.rs | 1 + 12 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-and-examples.yml b/.github/workflows/build-and-examples.yml index 707a90b0beefc..14d7da62db95e 100644 --- a/.github/workflows/build-and-examples.yml +++ b/.github/workflows/build-and-examples.yml @@ -7,7 +7,7 @@ jobs: steps: - uses: maxim-lobanov/setup-xcode@v1.6.0 with: - xcode-version: 16.4.0 + xcode-version: latest-stable - uses: actions/checkout@v6 - uses: ./.github/actions/setup_macos_env - uses: ./.github/actions/init_opam @@ -73,7 +73,7 @@ jobs: - uses: ./.github/actions/setup_reindeer - uses: ./.github/actions/build_bootstrap windows-build-examples: - runs-on: windows-8-core + runs-on: windows-latest steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup_windows_env diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 39a85e7b2dcde..fc11c37acfaa9 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: # allows manual triggering jobs: linux-build-and-test: - runs-on: 4-core-ubuntu + runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup_linux_env @@ -16,13 +16,13 @@ jobs: steps: - uses: maxim-lobanov/setup-xcode@v1.6.0 with: - xcode-version: 16.4.0 + xcode-version: latest-stable - uses: actions/checkout@v6 - uses: ./.github/actions/setup_macos_env - uses: ./.github/actions/build_debug - uses: ./.github/actions/run_test_py windows-build-and-test: - runs-on: windows-8-core + runs-on: windows-latest steps: - uses: actions/checkout@v6 - uses: ./.github/actions/setup_windows_env diff --git a/app/buck2_build_signals_impl/src/enhancement.rs b/app/buck2_build_signals_impl/src/enhancement.rs index 93a65a5d27ef7..ef6528718d770 100644 --- a/app/buck2_build_signals_impl/src/enhancement.rs +++ b/app/buck2_build_signals_impl/src/enhancement.rs @@ -240,7 +240,7 @@ impl CriticalPathProtoEnhancer { let duration_proto = duration_to_proto_saturating(duration); buck2_data::CriticalPathEntry2 { span_ids: Vec::new(), - duration: Some(duration_proto), + duration: Some(duration_proto.clone()), user_duration: Some(prost_types::Duration::default()), queue_duration: None, total_duration: Some(duration_proto), diff --git a/app/buck2_core/src/bzl.rs b/app/buck2_core/src/bzl.rs index 2ca5d8e3659e4..e0731660ca42f 100644 --- a/app/buck2_core/src/bzl.rs +++ b/app/buck2_core/src/bzl.rs @@ -53,7 +53,7 @@ impl ImportPath { pub fn new_with_build_file_cells( path: CellPath, - mut build_file_cell: BuildFileCell, + _build_file_cell: BuildFileCell, ) -> buck2_error::Result { if path.parent().is_none() { return Err(ImportPathError::Invalid(path).into()); @@ -67,7 +67,9 @@ impl ImportPath { return Err(ImportPathError::Suffix(path).into()); } - build_file_cell = BuildFileCell::new(path.cell()); + // Deliberately ignore the user-provided `build_file_cell`. + // See: https://github.com/MercuryTechnologies/buck2/commit/f74d3594e8cf90db0e3cfaaeb748e96ce5b13a62 + let build_file_cell = BuildFileCell::new(path.cell()); Ok(Self { path, @@ -78,7 +80,7 @@ impl ImportPath { /// LSP creates imports for non-bzl files. pub fn new_hack_for_lsp( path: CellPath, - mut build_file_cell: BuildFileCell, + _build_file_cell: BuildFileCell, ) -> buck2_error::Result { if path.parent().is_none() { return Err(ImportPathError::Invalid(path).into()); @@ -88,7 +90,9 @@ impl ImportPath { return Err(ImportPathError::Invalid(path).into()); } - build_file_cell = BuildFileCell::new(path.cell()); + // Deliberately ignore the user-provided `build_file_cell`. + // See: https://github.com/MercuryTechnologies/buck2/commit/f74d3594e8cf90db0e3cfaaeb748e96ce5b13a62 + let build_file_cell = BuildFileCell::new(path.cell()); Ok(Self { path, diff --git a/app/buck2_core/src/logging/otel.rs b/app/buck2_core/src/logging/otel.rs index 1be9b9d76d1af..4d6ba36998647 100644 --- a/app/buck2_core/src/logging/otel.rs +++ b/app/buck2_core/src/logging/otel.rs @@ -30,7 +30,7 @@ //! * `OTEL_SDK_DISABLED=true` -- the spec's global kill-switch. `opentelemetry-rust` does not honor //! it on its own, so we check it here and skip building the exporter entirely. //! * `OTEL_SERVICE_NAME` / `OTEL_RESOURCE_ATTRIBUTES` -- extra/overriding resource attributes, -//! merged in by [`resource`]. Our own authoritative attributes (pid, arch, os, host) win on +//! merged in by `resource`. Our own authoritative attributes (pid, arch, os, host) win on //! conflict; `service.name` defaults to `buck2` but yields to a user-provided value. //! * `TRACEPARENT` / `TRACESTATE` -- W3C Trace Context of the launching process (CI job, wrapper //! script, ...). When present, our wide-event span is parented under that trace instead of being @@ -234,7 +234,9 @@ fn build_provider(version: &'static str) -> buck2_error::Result<()> { .build(); // `activate` runs once, so ignore an already-set slot. - let _ = PROVIDER.set(provider); + // Note: If the `set` call fails, the `provider` we pass in is returned, and our `provider` has + // a `Drop` impl which shuts it down, so we need an actual binding here to avoid a Clippy lint. + let _result = PROVIDER.set(provider); Ok(()) } diff --git a/app/buck2_events/src/sink/otel.rs b/app/buck2_events/src/sink/otel.rs index 6d12c6814b12b..e67ce7e33b457 100644 --- a/app/buck2_events/src/sink/otel.rs +++ b/app/buck2_events/src/sink/otel.rs @@ -16,7 +16,7 @@ //! record to it exactly as they route to Scribe, with no special-casing. Unlike Scribe it is not //! Meta-specific, so it is the wide-event export path in open-source builds too. //! -//! The actual proto -> attribute mapping lives in [`crate::sink::otel_record`]; spans are emitted +//! The actual proto -> attribute mapping lives in `crate::sink::otel_record`; spans are emitted //! through [`buck2_core::logging::otel`], which owns the exporter lifecycle (activation after the //! daemon fork, flushing on exit). diff --git a/app/buck2_events/src/sink/otel_record.rs b/app/buck2_events/src/sink/otel_record.rs index 7dd612b6c59c4..c065d6fb676e2 100644 --- a/app/buck2_events/src/sink/otel_record.rs +++ b/app/buck2_events/src/sink/otel_record.rs @@ -39,6 +39,13 @@ //! //! [semantic convention]: https://opentelemetry.io/docs/specs/semconv/ +// The `push_*_command_end` helpers all take their proto message by reference so they share a +// uniform signature with their non-empty siblings and can be dispatched identically from the +// `command_end` match. Several of those messages are empty (zero-sized), which trips +// `trivially_copy_pass_by_ref`; passing them by value would only force clones/derefs at the call +// sites for no benefit, so allow it file-wide. +#![expect(clippy::trivially_copy_pass_by_ref)] + use std::collections::HashMap; use opentelemetry::Array; diff --git a/app/buck2_interpreter/src/package_imports.rs b/app/buck2_interpreter/src/package_imports.rs index 4723efc36d09b..c6a40e017e527 100644 --- a/app/buck2_interpreter/src/package_imports.rs +++ b/app/buck2_interpreter/src/package_imports.rs @@ -193,7 +193,7 @@ mod tests { assert_eq!("root//bin.bzl", import.import().to_string()); let import = expect_import("root", "other/bin"); - assert_eq!("cell1//other.bzl@root", import.import().to_string()); + assert_eq!("cell1//other.bzl", import.import().to_string()); assert_eq!("symbol1", import.lookup_alias("alias1")); assert_eq!("symbol2", import.lookup_alias("alias2")); assert_eq!("symbol3", import.lookup_alias("symbol3")); diff --git a/app/buck2_interpreter_for_build_tests/src/interpreter.rs b/app/buck2_interpreter_for_build_tests/src/interpreter.rs index c2a453d891c4e..781dec1dfd7ea 100644 --- a/app/buck2_interpreter_for_build_tests/src/interpreter.rs +++ b/app/buck2_interpreter_for_build_tests/src/interpreter.rs @@ -242,9 +242,9 @@ fn test_find_imports() { assert_eq!( &[ - "root//imports/one.bzl@cell1", + "root//imports/one.bzl", "cell1//one.bzl", - "cell2//two.bzl@cell1", + "cell2//two.bzl", "cell1//config/foo/other.bzl", "cell1//config/bar/three.bzl", ], diff --git a/app/buck2_server/src/daemon/state.rs b/app/buck2_server/src/daemon/state.rs index d5c41fca34131..3366841ff14c9 100644 --- a/app/buck2_server/src/daemon/state.rs +++ b/app/buck2_server/src/daemon/state.rs @@ -51,7 +51,6 @@ use buck2_events::source::ChannelEventSource; use buck2_execute::digest_config::DigestConfig; use buck2_execute::execute::blocking::BlockingExecutor; use buck2_execute::execute::blocking::BuckBlockingExecutor; -use buck2_execute::execute::blocking::DirectIoExecutor; use buck2_execute::materialize::materializer::MaterializationMethod; use buck2_execute::materialize::materializer::Materializer; use buck2_execute::re::manager::ReConnectionManager; diff --git a/app/buck2_test/src/command.rs b/app/buck2_test/src/command.rs index 90b620ca84d00..4a39c5255cd46 100644 --- a/app/buck2_test/src/command.rs +++ b/app/buck2_test/src/command.rs @@ -598,13 +598,13 @@ async fn test( // We don't have per-test details. Rather than showing all output // (which would include passing tests), show nothing. // This ensures we never show passing test output with --test-output=errors - let has_failures = test_statuses.failed.as_ref().map_or(false, |f| f.count > 0) - || test_statuses.fatals.as_ref().map_or(false, |f| f.count > 0); + let has_failures = test_statuses.failed.as_ref().is_some_and(|f| f.count > 0) + || test_statuses.fatals.as_ref().is_some_and(|f| f.count > 0); if has_failures { // Add a note explaining why output is suppressed let msg = "\nNote: Test output suppressed with --test-output=errors because the test executor does not provide per-test output details.\n"; - (msg.to_string(), String::new()) + (msg.to_owned(), String::new()) } else { (String::new(), String::new()) } diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index b79105698845f..02f0f555749ae 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -256,6 +256,7 @@ pub struct RERuntimeOpts { /// Maximum retries for network requests. max_retries: usize, /// Timeout for RPC requests. + #[expect(unused)] rpc_timeout: Duration, } From 4c819e9325eb9c7613c321a305285566887be2a1 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 8 Jul 2026 12:17:46 -0400 Subject: [PATCH 24/29] buck2_core: Use patched opentelemetry-sdk for percent-decoding (#20) This integrates the Mercury `opentelemetry-rust` fork with fixes from a couple PRs: See: - https://github.com/open-telemetry/opentelemetry-rust/pull/3586 - https://github.com/open-telemetry/opentelemetry-rust/pull/3587 Follow-up to Arian's review: https://github.com/MercuryTechnologies/buck2/pull/18#pullrequestreview-4646273415 --- Cargo.toml | 11 ++-- app/buck2_core/src/logging/otel.rs | 84 ++++++++++++++++-------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c583bbe87a02a..0fae8e735b2f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -259,10 +259,13 @@ num_cpus = "1.16" num_enum = "0.7.4" object = "0.36.7" once_cell = "1.21" -opentelemetry = "0.32.0" -opentelemetry-otlp = { version = "0.32.0", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "reqwest-rustls"] } -opentelemetry-semantic-conventions = { version = "0.32.0", features = ["semconv_experimental"] } -opentelemetry_sdk = "0.32.1" +# See: +# - https://github.com/open-telemetry/opentelemetry-rust/pull/3586 +# - https://github.com/open-telemetry/opentelemetry-rust/pull/3587 +opentelemetry = { version = "0.32.0", git = "https://github.com/MercuryTechnologies/opentelemetry-rust.git", branch = "mercury-head", features = ["experimental_propagation_env_carrier"] } +opentelemetry-otlp = { version = "0.32.0", git = "https://github.com/MercuryTechnologies/opentelemetry-rust.git", branch = "mercury-head", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "reqwest-rustls"] } +opentelemetry-semantic-conventions = { version = "0.32.0", git = "https://github.com/MercuryTechnologies/opentelemetry-rust.git", branch = "mercury-head", features = ["semconv_experimental"] } +opentelemetry_sdk = { version = "0.32.1", git = "https://github.com/MercuryTechnologies/opentelemetry-rust.git", branch = "mercury-head" } os_str_bytes = { version = "6.6.0", features = ["conversions"] } parking_lot = { version = "0.12.1", features = ["send_guard"] } paste = "1.0.15" diff --git a/app/buck2_core/src/logging/otel.rs b/app/buck2_core/src/logging/otel.rs index 4d6ba36998647..f11b94edea767 100644 --- a/app/buck2_core/src/logging/otel.rs +++ b/app/buck2_core/src/logging/otel.rs @@ -57,7 +57,6 @@ //! we drain it explicitly. [`shutdown`] does that drain and must be called on every exit path. See //! . -use std::collections::HashMap; use std::sync::OnceLock; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; @@ -67,6 +66,7 @@ use buck2_error::conversion::from_any_with_tag; use opentelemetry::Context; use opentelemetry::KeyValue; use opentelemetry::propagation::TextMapPropagator as _; +use opentelemetry::propagation::environment::EnvironmentExtractor; use opentelemetry::trace::Span as _; use opentelemetry::trace::Tracer as _; use opentelemetry::trace::TracerProvider as _; @@ -116,9 +116,14 @@ fn otlp_endpoint_configured() -> bool { .any(|var| std::env::var_os(var).is_some_and(|v| !v.is_empty())) } -/// The OpenTelemetry SDK kill-switch. Per the spec, `OTEL_SDK_DISABLED=true` (case-insensitive) -/// disables the SDK; any other value, or an unset variable, leaves it enabled. `opentelemetry-rust` -/// does not read this variable itself, so we check it and skip building the exporter. +/// Detect `$OTEL_SDK_DISABLED`. +/// +/// Per the spec, `OTEL_SDK_DISABLED=true` (case-insensitive) disables the SDK; any other value, or +/// an unset variable, leaves it enabled. `opentelemetry-rust` does not read this variable itself, +/// so we check it and skip building the exporter. +/// +/// This can be deleted when the upstream issue is resolved: +/// - fn otel_sdk_disabled() -> bool { std::env::var_os("OTEL_SDK_DISABLED").is_some_and(|v| v.eq_ignore_ascii_case("true")) } @@ -183,7 +188,10 @@ fn resource(version: &'static str) -> Resource { .with_service_name("buck2") // `telemetry.sdk.*`, plus any attributes from `OTEL_RESOURCE_ATTRIBUTES` (which may carry a // `service.name` that then wins over our default, and other attributes like - // `deployment.environment` that flow through untouched). + // `deployment.environment` that flow through untouched). `EnvResourceDetector` percent-decodes + // the values per the Resource SDK spec -- but only in our Mercury fork of `opentelemetry-rust` + // (see the workspace `Cargo.toml`); upstream does not yet. See + // . .with_detectors(&[ Box::new(TelemetryResourceDetector), Box::new(EnvResourceDetector::new()), @@ -274,35 +282,20 @@ pub fn export_span( span.end_with_timestamp(end); } -/// Extract a parent trace context from the environment, following the W3C Trace Context convention -/// that CI systems and wrappers (`otel-cli`, the Jenkins / GitHub Actions OpenTelemetry plugins, -/// Buildkite, ...) use to hand an in-progress trace to child processes via `$TRACEPARENT` (and its -/// companion `$TRACESTATE`). When buck2 is launched under such a trace, this makes our wide-event -/// span a child of the launching trace instead of a disconnected root. +/// Extract a parent trace context from the environment. +/// +/// Follows the W3C Trace Context convention that CI systems and wrappers (`otel-cli`, the Jenkins / +/// GitHub Actions OpenTelemetry plugins, Buildkite, ...) use to hand an in-progress trace to child +/// processes via `$TRACEPARENT` (and its companion `$TRACESTATE`). When buck2 is launched under +/// such a trace, this makes our wide-event span a child of the launching trace instead of a +/// disconnected root. /// -/// `opentelemetry-rust` only ever extracts trace context from HTTP-style carriers, never from the -/// process environment, so we assemble a carrier from the env ourselves and run the standard -/// `TraceContextPropagator` over it. Returns an empty (root) context when `$TRACEPARENT` is absent -/// or malformed -- i.e. the previous always-root behavior. +/// `EnvironmentExtractor` snapshots the process environment as a propagation carrier, normalizing +/// lookup keys so the `TraceContextPropagator`'s `traceparent`/`tracestate` reads resolve to +/// `$TRACEPARENT`/`$TRACESTATE`. Returns an empty (root) context when `$TRACEPARENT` is absent or +/// malformed -- i.e. the previous always-root behavior. fn parent_context_from_env() -> Context { - let env = |var| std::env::var_os(var).and_then(|v| v.into_string().ok()); - parent_context_from_carrier( - env("TRACEPARENT").as_deref(), - env("TRACESTATE").as_deref(), - ) -} - -/// The environment-independent core of [`parent_context_from_env`], split out so it can be tested -/// without mutating process-global environment state. -fn parent_context_from_carrier(traceparent: Option<&str>, tracestate: Option<&str>) -> Context { - let mut carrier: HashMap = HashMap::new(); - // The propagator keys off the lowercased header names, matching how these would arrive over HTTP. - for (header, value) in [("traceparent", traceparent), ("tracestate", tracestate)] { - if let Some(value) = value.filter(|v| !v.is_empty()) { - carrier.insert(header.to_owned(), value.to_owned()); - } - } - TraceContextPropagator::new().extract(&carrier) + TraceContextPropagator::new().extract(&EnvironmentExtractor::new()) } /// Build the OTLP exporter and start exporting, if telemetry is configured. @@ -353,14 +346,25 @@ mod tests { use super::*; + /// Build a parent context the way [`parent_context_from_env`] does, but from an explicit + /// environment snapshot (via `EnvironmentExtractor`'s `FromIterator`) so we needn't mutate the + /// process environment. Keys must be given already-normalized (uppercase), matching how a real + /// `$TRACEPARENT`/`$TRACESTATE` reaches the extractor. + fn parent_context_from_vars(vars: &[(&str, &str)]) -> Context { + let extractor = EnvironmentExtractor::from_iter( + vars.iter().map(|(k, v)| (k.to_string(), v.to_string())), + ); + TraceContextPropagator::new().extract(&extractor) + } + #[test] fn traceparent_becomes_remote_parent() { // A well-formed W3C traceparent yields a valid, remote parent span context whose trace id // our span will inherit. - let cx = parent_context_from_carrier( - Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"), - None, - ); + let cx = parent_context_from_vars(&[( + "TRACEPARENT", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + )]); let span_context = cx.span().span_context().clone(); assert!(span_context.is_valid()); assert!(span_context.is_remote()); @@ -374,21 +378,21 @@ mod tests { fn no_or_malformed_traceparent_is_root() { // No traceparent -> no parent (a fresh root span, as before). assert!( - !parent_context_from_carrier(None, None) + !parent_context_from_vars(&[]) .span() .span_context() .is_valid() ); - // An empty value is treated as absent, not as a malformed header. + // An empty value parses as no valid parent, even alongside a tracestate. assert!( - !parent_context_from_carrier(Some(""), Some("foo=bar")) + !parent_context_from_vars(&[("TRACEPARENT", ""), ("TRACESTATE", "foo=bar")]) .span() .span_context() .is_valid() ); // Garbage is ignored rather than propagated. assert!( - !parent_context_from_carrier(Some("not-a-traceparent"), None) + !parent_context_from_vars(&[("TRACEPARENT", "not-a-traceparent")]) .span() .span_context() .is_valid() From 47423b26a6deb4300021dddb1bee7c8cd79d7b1c Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 8 Jul 2026 18:57:59 -0400 Subject: [PATCH 25/29] GHA: build-and-test: Don't run on `mercury-head` (#23) Right now we run CI on `pull_request` and `push`, which means it runs twice on PRs. Let's only run it once. --- .github/workflows/build-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index fc11c37acfaa9..a8e389adfbc9b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,6 +1,8 @@ name: Build and test on: push: + branches: + - mercury-head pull_request: workflow_dispatch: # allows manual triggering jobs: From 06a7233f2e385ea67dd8e8cbcd5e8a8e98080b6c Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 8 Jul 2026 20:06:26 -0400 Subject: [PATCH 26/29] DUX-5509 re_grpc: request zstd-compressed `BatchReadBlobs` responses (#22) There are many small `BatchReadBlobs` requests, and they're large enough that compression would help. When the server advertises zstd support in its capabilities, advertise zstd as an acceptable response compressor and decompress any zstd-compressed blobs the server returns. NB: It would be nice if we could only compress blobs above a certain size, but we don't know the size when we make the request. We continue to advertise IDENTITY an acceptable response compressor, so we may need to patch our cache server to choose an appropriate size threshold above which to compress data (Bazel chooses 100 bytes, AIUI). In a moderately sized build with ~81,000 BatchReadBlobs requests, this patch shows significant benefits: - Request latency dropped: - p100: 16,653ms to 3,468ms (4.8x speedup) - p90: 953ms to 593ms - p75: 529ms to 312ms - p50: 235ms to 161ms - p25: 110ms to 96ms - p0: 67ms to 68ms - Cumulative blob size over the network dropped from 5,807MiB to 1,626MiB (3.57x compression ratio) - Blob size distribution over the wire improved: - p100: 3.7M to 741.8K - p90: 188.4K to 58.0K - p75: 103.2K to 33.9K - p50: 12.9K to 3.7K - p25: 32B to 45B (regression) - p0: 15B to 2B (regression) --- remote_execution/oss/re_grpc/src/client.rs | 107 ++++++++++++++++++++- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index 02f0f555749ae..a00d4448c8a73 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -240,7 +240,8 @@ pub struct RECapabilities { /// Largest size of a message before being uploaded using bytestream service. /// 0 indicates no limit beyond constraint of underlying transport (which is unknown). max_total_batch_size: usize, - /// Compressors supported by the "compressed-blobs" bytestream resources. + /// Compressors supported by the "compressed-blobs" bytestream resources. Also used to decide + /// whether we can request zstd-compressed `BatchReadBlobs` responses. supported_compressors: Vec, } @@ -258,6 +259,8 @@ pub struct RERuntimeOpts { /// Timeout for RPC requests. #[expect(unused)] rpc_timeout: Duration, + /// Whether to request zstd-compressed `BatchReadBlobs` responses (server supports zstd). + batch_read_zstd: bool, } #[derive(Clone)] @@ -309,6 +312,15 @@ impl Compressor { } } +/// Decompress a zstd-compressed blob returned in a `BatchReadBlobs` response. +async fn zstd_decompress_blob(data: &[u8]) -> anyhow::Result> { + let mut decoder = ZstdDecoder::new(Cursor::new(data)); + decoder.multiple_members(true); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).await?; + Ok(out) +} + pub struct REClientBuilder; impl REClientBuilder { @@ -465,6 +477,9 @@ impl REClientBuilder { cas_ttl_secs: opts.cas_ttl_secs.unwrap_or(3 * 60 * 60), max_retries: opts.max_retries, rpc_timeout: Duration::from_secs(opts.grpc_timeout), + batch_read_zstd: capabilities + .supported_compressors + .contains(&Compressor::Zstd), }, grpc_clients, capabilities, @@ -1500,6 +1515,17 @@ where let inlined_digests = request.inlined_digests.unwrap_or_default(); let file_digests = request.file_digests.unwrap_or_default(); + // Advertise zstd for BatchReadBlobs responses when the server supports it; Identity is always + // acceptable as a fallback and the server decides per-blob what to actually return. + let acceptable_compressors = if opts.batch_read_zstd { + vec![ + compressor::Value::Zstd as i32, + compressor::Value::Identity as i32, + ] + } else { + vec![compressor::Value::Identity as i32] + }; + let mut curr_size = 0; let mut requests = vec![]; let mut curr_digests = vec![]; @@ -1520,7 +1546,7 @@ where let read_blob_req = BatchReadBlobsRequest { instance_name: instance_name.as_str().to_owned(), digests: std::mem::take(&mut curr_digests), - acceptable_compressors: vec![compressor::Value::Identity as i32], + acceptable_compressors: acceptable_compressors.clone(), ..Default::default() }; requests.push(read_blob_req); @@ -1533,7 +1559,7 @@ where let read_blob_req = BatchReadBlobsRequest { instance_name: instance_name.as_str().to_owned(), digests: std::mem::take(&mut curr_digests), - acceptable_compressors: vec![compressor::Value::Identity as i32], + acceptable_compressors: acceptable_compressors.clone(), ..Default::default() }; requests.push(read_blob_req); @@ -1548,7 +1574,20 @@ where for r in resp.responses.into_iter() { let digest = tdigest_from(r.digest.context("Response digest not found.")?); check_status(r.status.unwrap_or_default())?; - batched_blobs_response.insert(digest, r.data); + let data = if r.compressor == compressor::Value::Identity as i32 { + r.data + } else if r.compressor == compressor::Value::Zstd as i32 { + zstd_decompress_blob(&r.data) + .await + .context("Failed to decompress zstd BatchReadBlobs response")? + } else { + return Err(anyhow::anyhow!( + "BatchReadBlobs response for `{}` used an unsupported compressor: {}", + digest, + r.compressor + )); + }; + batched_blobs_response.insert(digest, data); } } @@ -2005,6 +2044,7 @@ mod tests { cas_ttl_secs: 0, max_retries: 0, rpc_timeout: Duration::from_secs(60), + batch_read_zstd: false, } } @@ -2613,6 +2653,65 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_batch_download_zstd() -> anyhow::Result<()> { + // The server returns a zstd-compressed blob; we advertise zstd and decompress it. + let blob = vec![9u8; 500]; + let digest = TDigest { + hash: "dd".to_owned(), + size_in_bytes: blob.len() as i64, + ..Default::default() + }; + + let mut compressed = vec![]; + ZstdEncoder::new(Cursor::new(blob.clone())) + .read_to_end(&mut compressed) + .await + .unwrap(); + + let req = DownloadRequest { + inlined_digests: Some(vec![digest.clone()]), + ..Default::default() + }; + + let res = BatchReadBlobsResponse { + responses: vec![batch_read_blobs_response::Response { + digest: Some(tdigest_to(digest.clone())), + data: compressed.clone(), + status: Some(Status::default()), + compressor: compressor::Value::Zstd as i32, + }], + }; + + let mut opts = test_re_runtime_opts(); + opts.batch_read_zstd = true; + + let expected = blob.clone(); + let d_resp = download_impl( + &opts, + &InstanceName(None), + req, + None, + 10000, + |req| { + let res = res.clone(); + async move { + // We advertised zstd as an acceptable response compressor. + assert!( + req.acceptable_compressors + .contains(&(compressor::Value::Zstd as i32)) + ); + Ok(res) + } + }, + |_digest| async move { anyhow::Ok(Box::pin(futures::stream::iter(vec![]))) }, + ) + .await?; + + assert_eq!(d_resp.inlined_blobs.unwrap()[0].blob, expected); + Ok(()) + } + #[tokio::test] async fn test_upload_large_named() -> anyhow::Result<()> { let blob_data = vec![ From 02bfd31b66d6df2ea7e2b690039b4fb82b0f4312 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 9 Jul 2026 15:43:40 -0400 Subject: [PATCH 27/29] DUX-5510 re_grpc: zstd-compress large `BatchUpdateBlobs` blobs (#26) Compress batched blob uploads whose `data` exceeds a configurable threshold (default 100 bytes) with zstd, when the server advertises zstd in its `supported_batch_update_compressors` capability. --- app/buck2_re_configuration/src/lib.rs | 7 + remote_execution/oss/re_grpc/src/client.rs | 213 +++++++++++++++++++-- remote_execution/oss/re_grpc/src/retry.rs | 15 +- 3 files changed, 213 insertions(+), 22 deletions(-) diff --git a/app/buck2_re_configuration/src/lib.rs b/app/buck2_re_configuration/src/lib.rs index a75b202802c7b..9bdcb59fbe1da 100644 --- a/app/buck2_re_configuration/src/lib.rs +++ b/app/buck2_re_configuration/src/lib.rs @@ -458,6 +458,9 @@ pub struct Buck2OssReConfiguration { pub max_decoding_message_size: Option, /// The max cumulative blob size for `Read` and `BatchReadBlobs` methods. pub max_total_batch_size: Option, + /// Minimum size in bytes of a blob's `data` before it is zstd-compressed in `BatchUpdateBlobs` + /// uploads. + pub batch_compression_threshold_bytes: Option, /// Maximum number of concurrent upload requests for each action. pub max_concurrent_uploads_per_action: Option, /// Time that digests are assumed to live in CAS after being touched. @@ -571,6 +574,10 @@ impl Buck2OssReConfiguration { section: BUCK2_RE_CLIENT_CFG_SECTION, property: "max_total_batch_size", })?, + batch_compression_threshold_bytes: legacy_config.parse(BuckconfigKeyRef { + section: BUCK2_RE_CLIENT_CFG_SECTION, + property: "batch_compression_threshold_bytes", + })?, max_concurrent_uploads_per_action: legacy_config.parse(BuckconfigKeyRef { section: BUCK2_RE_CLIENT_CFG_SECTION, property: "max_concurrent_uploads_per_action", diff --git a/remote_execution/oss/re_grpc/src/client.rs b/remote_execution/oss/re_grpc/src/client.rs index a00d4448c8a73..7b727731e85d7 100644 --- a/remote_execution/oss/re_grpc/src/client.rs +++ b/remote_execution/oss/re_grpc/src/client.rs @@ -107,6 +107,8 @@ use crate::retry::retrying_stream; use crate::stats::CountingConnector; const DEFAULT_MAX_TOTAL_BATCH_SIZE: usize = 4 * 1000 * 1000; +/// Default minimum `data` size before a batched blob is zstd-compressed. +const DEFAULT_BATCH_COMPRESSION_THRESHOLD: usize = 100; const INITIAL_DELAY: Duration = Duration::from_millis(100); const MAX_DELAY: Duration = Duration::from_secs(10); @@ -243,6 +245,8 @@ pub struct RECapabilities { /// Compressors supported by the "compressed-blobs" bytestream resources. Also used to decide /// whether we can request zstd-compressed `BatchReadBlobs` responses. supported_compressors: Vec, + /// Compressors the server accepts for `BatchUpdateBlobs` request `data`. + supported_batch_update_compressors: Vec, } /// Contains runtime options for the remote execution client as set under `buck2_re_client` @@ -259,6 +263,8 @@ pub struct RERuntimeOpts { /// Timeout for RPC requests. #[expect(unused)] rpc_timeout: Duration, + /// Minimum `data` size before a batched blob is zstd-compressed in `BatchUpdateBlobs` uploads. + batch_compression_threshold: usize, /// Whether to request zstd-compressed `BatchReadBlobs` responses (server supports zstd). batch_read_zstd: bool, } @@ -312,6 +318,14 @@ impl Compressor { } } +/// zstd-compress an in-memory blob for a `BatchUpdateBlobs` request. +async fn zstd_compress_blob(data: &[u8]) -> anyhow::Result> { + let mut encoder = ZstdEncoder::new(Cursor::new(data)); + let mut out = Vec::new(); + encoder.read_to_end(&mut out).await?; + Ok(out) +} + /// Decompress a zstd-compressed blob returned in a `BatchReadBlobs` response. async fn zstd_decompress_blob(data: &[u8]) -> anyhow::Result> { let mut decoder = ZstdDecoder::new(Cursor::new(data)); @@ -414,6 +428,7 @@ impl REClientBuilder { RECapabilities { max_total_batch_size: DEFAULT_MAX_TOTAL_BATCH_SIZE, supported_compressors: Vec::new(), + supported_batch_update_compressors: Vec::new(), } }; @@ -477,6 +492,9 @@ impl REClientBuilder { cas_ttl_secs: opts.cas_ttl_secs.unwrap_or(3 * 60 * 60), max_retries: opts.max_retries, rpc_timeout: Duration::from_secs(opts.grpc_timeout), + batch_compression_threshold: opts + .batch_compression_threshold_bytes + .unwrap_or(DEFAULT_BATCH_COMPRESSION_THRESHOLD), batch_read_zstd: capabilities .supported_compressors .contains(&Compressor::Zstd), @@ -514,6 +532,17 @@ impl REClientBuilder { Vec::new() }; + let supported_batch_update_compressors = if let Some(cache_cap) = &resp.cache_capabilities { + cache_cap + .supported_batch_update_compressors + .iter() + .cloned() + .filter_map(Compressor::from_grpc) + .collect() + } else { + Vec::new() + }; + let max_total_batch_size_from_capabilities: Option = if let Some(cache_cap) = resp.cache_capabilities { let size = cache_cap.max_batch_total_size_bytes as usize; @@ -534,6 +563,7 @@ impl REClientBuilder { Ok(RECapabilities { max_total_batch_size, supported_compressors, + supported_batch_update_compressors, }) } } @@ -949,12 +979,20 @@ impl REClient { metadata: RemoteExecutionMetadata, request: UploadRequest, ) -> anyhow::Result { + // Only compress batched blobs if the server advertised zstd support for BatchUpdateBlobs. + let batch_zstd_threshold = self + .capabilities + .supported_batch_update_compressors + .contains(&Compressor::Zstd) + .then_some(self.runtime_opts.batch_compression_threshold); + upload_impl( &self.instance_name, request, self.bystream_compressor, self.capabilities.max_total_batch_size, self.runtime_opts.max_concurrent_uploads_per_action, + batch_zstd_threshold, |re_request| async { let metadata = metadata.clone(); let cas_client = self.grpc_clients.cas_client.clone(); @@ -1698,6 +1736,9 @@ async fn upload_impl( bystream_compressor: Option, max_total_batch_size: usize, max_concurrent_uploads: Option, + // Minimum blob `data` size before zstd-compressing it in a `BatchUpdateBlobs` request. + // `None` disables batch compression (e.g. server doesn't advertise zstd support). + batch_zstd_threshold: Option, cas_f: impl Fn(BatchUpdateBlobsRequest) -> Cas + Sync + Send + Copy, bystream_fut: impl Fn(Vec) -> Byt + Sync + Send + Copy, ) -> anyhow::Result @@ -1847,14 +1888,8 @@ where ..Default::default() }; for blob in batch { - match blob { - BatchUploadRequest::Blob(blob) => { - re_request.requests.push(Request { - digest: Some(tdigest_to(blob.digest.clone())), - data: blob.blob.clone(), - compressor: compressor::Value::Identity as i32, - }); - } + let (digest, data) = match blob { + BatchUploadRequest::Blob(blob) => (blob.digest.clone(), blob.blob.clone()), BatchUploadRequest::File(file) => { // These should be small files, so no need to use a buffered reader. let mut fin = tokio::fs::File::open(&file.name) @@ -1862,14 +1897,24 @@ where .with_context(|| format!("Opening {} for reading failed", file.name))?; let mut data = vec![]; fin.read_to_end(&mut data).await?; + (file.digest.clone(), data) + } + }; - re_request.requests.push(Request { - digest: Some(tdigest_to(file.digest.clone())), - data, - compressor: compressor::Value::Identity as i32, - }); + // The digest always refers to the *uncompressed* content; only the transmitted + // `data` is compressed. Skip small blobs where compression isn't worth it. + let (data, compressor) = match batch_zstd_threshold { + Some(threshold) if data.len() > threshold => { + (zstd_compress_blob(&data).await?, compressor::Value::Zstd) } - } + _ => (data, compressor::Value::Identity), + }; + + re_request.requests.push(Request { + digest: Some(tdigest_to(digest)), + data, + compressor: compressor as i32, + }); } let blob_hashes = re_request .requests @@ -2044,10 +2089,62 @@ mod tests { cas_ttl_secs: 0, max_retries: 0, rpc_timeout: Duration::from_secs(60), + batch_compression_threshold: DEFAULT_BATCH_COMPRESSION_THRESHOLD, batch_read_zstd: false, } } + /// The all-OK response a server would send for a `BatchUpdateBlobs` of these digests. + fn batch_update_ok(digests: &[&TDigest]) -> BatchUpdateBlobsResponse { + BatchUpdateBlobsResponse { + responses: digests + .iter() + .map(|digest| batch_update_blobs_response::Response { + digest: Some(tdigest_to((*digest).clone())), + status: Some(Status::default()), + }) + .collect(), + } + } + + /// An upload request inlining the given (hash, blob) pairs, plus the all-OK + /// `BatchUpdateBlobs` response the server would send for it. + fn inlined_upload(blobs: &[(&str, &[u8])]) -> (UploadRequest, BatchUpdateBlobsResponse) { + let digests: Vec = blobs + .iter() + .map(|&(hash, blob)| TDigest { + hash: hash.to_owned(), + size_in_bytes: blob.len() as i64, + ..Default::default() + }) + .collect(); + let req = UploadRequest { + inlined_blobs_with_digest: Some( + digests + .iter() + .zip(blobs) + .map(|(digest, &(_, blob))| InlinedBlobWithDigest { + digest: digest.clone(), + blob: blob.to_vec(), + ..Default::default() + }) + .collect(), + ), + ..Default::default() + }; + let res = batch_update_ok(&digests.iter().collect::>()); + (req, res) + } + + async fn zstd_decompress(data: &[u8]) -> Vec { + let mut out = vec![]; + ZstdDecoder::new(Cursor::new(data.to_vec())) + .read_to_end(&mut out) + .await + .unwrap(); + out + } + #[tokio::test] async fn test_download_named() -> anyhow::Result<()> { let work = tempfile::tempdir()?; @@ -2633,6 +2730,7 @@ mod tests { None, 10000, None, + None, |req| { let res = res.clone(); let digest1 = digest1.clone(); @@ -2653,6 +2751,84 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_batch_upload_zstd_threshold() -> anyhow::Result<()> { + // A blob larger than the threshold is zstd-compressed; a small one is left uncompressed. + let big = vec![7u8; 1000]; + let (req, res) = inlined_upload(&[("big", &big), ("small", b"hi")]); + + let expected_big = big.clone(); + upload_impl( + &InstanceName(None), + req, + None, + 10000, + None, + Some(100), // batch zstd threshold + |req| { + let res = res.clone(); + let expected_big = expected_big.clone(); + async move { + let by_hash = |h: &str| { + req.requests + .iter() + .find(|r| r.digest.as_ref().unwrap().hash == h) + .unwrap() + .clone() + }; + + let big_req = by_hash("big"); + assert_eq!(big_req.compressor, compressor::Value::Zstd as i32); + // The digest still refers to the uncompressed content... + assert_eq!(big_req.digest.as_ref().unwrap().size_bytes, 1000); + // ...but the transmitted data is the zstd-compressed form. + assert_eq!(zstd_decompress(&big_req.data).await, expected_big); + + let small_req = by_hash("small"); + assert_eq!(small_req.compressor, compressor::Value::Identity as i32); + assert_eq!(small_req.data, b"hi"); + + Ok(res) + } + }, + |_req| async { panic!("A Bytestream upload should not be triggered") }, + ) + .await?; + + Ok(()) + } + + #[tokio::test] + async fn test_batch_upload_zstd_disabled() -> anyhow::Result<()> { + // With no threshold (batch compression disabled), everything stays Identity. + let big = vec![7u8; 1000]; + let (req, res) = inlined_upload(&[("big", &big)]); + + upload_impl( + &InstanceName(None), + req, + None, + 10000, + None, + None, // batch compression disabled + |req| { + let res = res.clone(); + async move { + assert_eq!( + req.requests[0].compressor, + compressor::Value::Identity as i32 + ); + assert_eq!(req.requests[0].data.len(), 1000); + Ok(res) + } + }, + |_req| async { panic!("A Bytestream upload should not be triggered") }, + ) + .await?; + + Ok(()) + } + #[tokio::test] async fn test_batch_download_zstd() -> anyhow::Result<()> { // The server returns a zstd-compressed blob; we advertise zstd and decompress it. @@ -2776,6 +2952,7 @@ mod tests { None, 10, // kept small to simulate a large file upload None, + None, |req| { let res = res.clone(); let digest1 = digest1.clone(); @@ -2851,6 +3028,7 @@ mod tests { None, 10, // kept small to simulate a large inlined upload None, + None, |req| { let res = res.clone(); let digest1 = digest1.clone(); @@ -2913,6 +3091,7 @@ mod tests { None, 10, None, + None, |_req| async move { panic!("This should not be called as there are no blobs to upload in batch"); }, @@ -2975,6 +3154,7 @@ mod tests { None, 3, None, + None, |_req| async move { panic!("Not called"); }, @@ -3022,6 +3202,7 @@ mod tests { compressor, 0, // max_total_batch_size=0 forces bytestream API None, + None, |_req| async move { panic!("Not called"); }, @@ -3047,6 +3228,7 @@ mod tests { compressor, 1024, // forces the batch API None, + None, |_req| async move { panic!("Not called"); }, @@ -3094,6 +3276,7 @@ mod tests { None, 1, None, + None, |_req| async move { panic!("Not called"); }, @@ -3141,6 +3324,7 @@ mod tests { Some(Compressor::Zstd), 1, None, + None, |_req| async move { panic!("Not called"); }, @@ -3210,6 +3394,7 @@ mod tests { Some(Compressor::Zstd), 1, None, + None, |_req| async move { panic!("Not called"); }, diff --git a/remote_execution/oss/re_grpc/src/retry.rs b/remote_execution/oss/re_grpc/src/retry.rs index 14f3845b560d0..197afce4d1cfe 100644 --- a/remote_execution/oss/re_grpc/src/retry.rs +++ b/remote_execution/oss/re_grpc/src/retry.rs @@ -7,14 +7,17 @@ * of this source tree. */ +use std::future::Future; use std::sync::Arc; use std::time::Duration; -use std::future::Future; + use futures::Stream; use futures::TryStreamExt; -use crate::error::{REClientError, TCode}; use tracing::warn; +use crate::error::REClientError; +use crate::error::TCode; + pub async fn retry( method: &str, mut f: F, @@ -173,9 +176,7 @@ where { warn!( "Retrying {} stream open after error: {}. Attempts remaining: {}", - method, - err, - state.retries_left, + method, err, state.retries_left, ); tokio::time::sleep(state.delay).await; state.delay *= 2; @@ -201,9 +202,7 @@ where { warn!( "Retrying {} stream read after error: {}. Attempts remaining: {}", - method, - err, - state.retries_left, + method, err, state.retries_left, ); tokio::time::sleep(state.delay).await; state.delay *= 2; From 780ef1343928ea055b45ef8b03db9d834b84f60f Mon Sep 17 00:00:00 2001 From: Harry Garrood Date: Tue, 14 Jul 2026 13:31:11 +0100 Subject: [PATCH 28/29] Fix clang-specific cc/cxx invocations in cargo_buildscript --- prelude/rust/cargo_buildscript.bzl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prelude/rust/cargo_buildscript.bzl b/prelude/rust/cargo_buildscript.bzl index f5ca6e37aebe2..7674fad37d2fb 100644 --- a/prelude/rust/cargo_buildscript.bzl +++ b/prelude/rust/cargo_buildscript.bzl @@ -301,6 +301,8 @@ def _cargo_buildscript_impl(ctx: AnalysisContext) -> list[Provider]: ) deps_link = deps_tset.project_as_args("default") sanitizer_flags = ["-fno-sanitize=all"] + cc_is_clang = cxx_toolchain_info.c_compiler_info.compiler_type.startswith("clang") + cxx_is_clang = cxx_toolchain_info.cxx_compiler_info.compiler_type.startswith("clang") env["LD"] = _make_cc_shim( ctx = ctx, name = "__ld_shim", @@ -316,7 +318,7 @@ def _cargo_buildscript_impl(ctx: AnalysisContext) -> list[Provider]: name = "__cc_shim", cmd = cmd_args( cxx_toolchain_info.c_compiler_info.compiler, - cmd_args(env["LD"], format = "--ld-path={}"), + cmd_args(env["LD"], format = "--ld-path={}") if cc_is_clang else cmd_args(), cxx_toolchain_info.c_compiler_info.preprocessor_flags, cxx_toolchain_info.c_compiler_info.compiler_flags, deps_preprocessor_flags, @@ -331,7 +333,7 @@ def _cargo_buildscript_impl(ctx: AnalysisContext) -> list[Provider]: name = "__cxx_shim", cmd = cmd_args( cxx_toolchain_info.cxx_compiler_info.compiler, - cmd_args(env["LD"], format = "--ld-path={}"), + cmd_args(env["LD"], format = "--ld-path={}") if cxx_is_clang else cmd_args(), cxx_toolchain_info.cxx_compiler_info.preprocessor_flags, cxx_toolchain_info.cxx_compiler_info.compiler_flags, deps_preprocessor_flags, From d58a73486bb01b3647eede35601b8d5a8cdbe0ef Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 24 Jul 2026 13:53:09 -0700 Subject: [PATCH 29/29] buck2_test: add an allowlist for env-vars in buckconfig This allows consumers to allowlist environment variables without patching buck itself. Concretely, my motivation on this is propagating things like OpenTelemetry endpoints into test harnesses: at Mercury we have a fancy test harness which starts postgres and stuff around tests which we would like to have observability for. We would like to *not* make that Facebook's problem, thus the buckconfig addition. --- .../src/execute/environment_inheritance.rs | 203 +++++++++++++++--- app/buck2_test/src/orchestrator.rs | 31 ++- tests/core/test/test_env_allowlist.py | 101 +++++++++ .../test/test_env_allowlist_data/.buckconfig | 15 ++ .../test/test_env_allowlist_data/.buckroot | 0 .../test_env_allowlist_data/TARGETS.fixture | 20 ++ .../platforms/TARGETS.fixture | 18 ++ .../test/test_env_allowlist_data/rules.bzl | 64 ++++++ 8 files changed, 427 insertions(+), 25 deletions(-) create mode 100644 tests/core/test/test_env_allowlist.py create mode 100644 tests/core/test/test_env_allowlist_data/.buckconfig create mode 100644 tests/core/test/test_env_allowlist_data/.buckroot create mode 100644 tests/core/test/test_env_allowlist_data/TARGETS.fixture create mode 100644 tests/core/test/test_env_allowlist_data/platforms/TARGETS.fixture create mode 100644 tests/core/test/test_env_allowlist_data/rules.bzl diff --git a/app/buck2_execute/src/execute/environment_inheritance.rs b/app/buck2_execute/src/execute/environment_inheritance.rs index e7df138fe8224..24ec8ab607d3a 100644 --- a/app/buck2_execute/src/execute/environment_inheritance.rs +++ b/app/buck2_execute/src/execute/environment_inheritance.rs @@ -9,6 +9,7 @@ */ use std::ffi::OsString; +use std::sync::Arc; use std::sync::OnceLock; use dupe::Dupe; @@ -66,34 +67,78 @@ const ENV_ALLOW_LIST: &[&str] = &[ "WINDIR", ]; -#[derive(Copy, Clone, Dupe, Debug)] +/// N.B. this forces [`std::env::var_os()`] to be instantiated with concrete +/// type parameters. +fn real_getenv(key: &str) -> Option { + std::env::var_os(key) +} + +/// The values captured from the built-in allowlist alone. We compute this *once* since getenv is +/// actually not cheap (being O(n) of the environment size). +fn builtin_test_allowlist_values() -> &'static Arc<[(String, OsString)]> { + static TEST_CELL: OnceLock> = OnceLock::new(); + + TEST_CELL.get_or_init(|| { + EnvironmentInheritance::test_allowlist_from(ENV_ALLOW_LIST, &[], &real_getenv).values + }) +} + +fn no_values() -> Arc<[(String, OsString)]> { + static EMPTY: OnceLock> = OnceLock::new(); + EMPTY.get_or_init(|| Vec::new().into()).dupe() +} + +#[derive(Clone, Dupe, Debug)] pub struct EnvironmentInheritance { clear: bool, - values: &'static [(&'static str, OsString)], + values: Arc<[(String, OsString)]>, exclusions: &'static [&'static str], } impl EnvironmentInheritance { pub fn test_allowlist() -> Self { - // This is made to be a list of lists in case we want to include lists from different - // provenances, like the test_env_allowlist::ENV_LIST_HACKY. - let allowlists = &[ENV_ALLOW_LIST]; - - // We create this *once* since getenv is actually not cheap (being O(n) of the environment - // size). - static TEST_CELL: OnceLock> = OnceLock::new(); - - let values = TEST_CELL.get_or_init(|| { - let mut ret = Vec::new(); - for list in allowlists.iter() { - for key in list.iter() { - if let Some(value) = std::env::var_os(key) { - ret.push((*key, value)); - } + Self::test_allowlist_with_extra(&[]) + } + + /// The built-in test allowlist, plus any extra variable names supplied by + /// the caller (in practice, the `[test] env_allowlist` buckconfig). + /// + /// Note: The environment of tests derives from the *daemon's* environment, + /// not the client's. + pub fn test_allowlist_with_extra(extra: &[String]) -> Self { + if extra.is_empty() { + return Self { + clear: true, + values: builtin_test_allowlist_values().dupe(), + exclusions: &[], + }; + } + Self::test_allowlist_from(ENV_ALLOW_LIST, extra, &real_getenv) + } + + /// Capture the values of `builtin` and `extra` according to `getenv`. + fn test_allowlist_from( + builtin: &[&str], + extra: &[String], + getenv: &dyn Fn(&str) -> Option, + ) -> Self { + let values = builtin + .iter() + .copied() + .chain(extra.iter().map(String::as_str)) + .filter_map(|key| { + // Trim and skip empties, as buck2's other comma-separated list configs do — see + // `IgnoreSet::from_ignore_spec` and `CellPackageBoundaryExceptions::new`. + // Otherwise `FOO, BAR` would silently do nothing for `BAR`. + let key = key.trim(); + if key.is_empty() { + return None; } - } - ret - }); + // A name appearing in both lists is harmless: both copies get the same value from + // `getenv`, and every consumer of `values()` is last-wins. + Some((key.to_owned(), getenv(key)?)) + }) + .collect(); Self { clear: true, @@ -107,7 +152,7 @@ impl EnvironmentInheritance { pub fn local_command_exclusions() -> Self { Self { clear: false, - values: &[], + values: no_values(), exclusions: &[ "PYTHONPATH", "PYTHONHOME", @@ -120,14 +165,14 @@ impl EnvironmentInheritance { pub fn empty() -> Self { Self { - values: &[], + values: no_values(), exclusions: &[], clear: true, } } - pub fn values(&self) -> impl Iterator + use<> { - self.values.iter().map(|(k, v)| (*k, v)) + pub fn values(&self) -> impl Iterator + use<'_> { + self.values.iter().map(|(k, v)| (k.as_str(), v)) } pub fn exclusions(&self) -> impl Iterator + use<> { @@ -138,3 +183,113 @@ impl EnvironmentInheritance { self.clear } } + +#[cfg(test)] +mod tests { + use super::*; + + const BUILTIN: &[&str] = &["SET_BUILTIN", "UNSET_BUILTIN"]; + + /// The only variables that exist as far as these tests are concerned. + fn getenv(key: &str) -> Option { + match key { + "SET_BUILTIN" => Some(OsString::from("builtin-value")), + "SET_EXTRA" => Some(OsString::from("extra-value")), + "SET_EXTRA_2" => Some(OsString::from("extra-value-2")), + _ => None, + } + } + + fn captured(extra: &[&str]) -> Vec<(String, OsString)> { + let extra: Vec = extra.iter().map(|k| (*k).to_owned()).collect(); + EnvironmentInheritance::test_allowlist_from(BUILTIN, &extra, &getenv) + .values() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect() + } + + fn pairs(expected: &[(&str, &str)]) -> Vec<(String, OsString)> { + expected + .iter() + .map(|(k, v)| ((*k).to_owned(), OsString::from(*v))) + .collect() + } + + #[test] + fn test_builtin_only_drops_unset_names() { + assert_eq!(captured(&[]), pairs(&[("SET_BUILTIN", "builtin-value")])); + } + + #[test] + fn test_extra_names_are_appended() { + assert_eq!( + captured(&["SET_EXTRA", "SET_EXTRA_2"]), + pairs(&[ + ("SET_BUILTIN", "builtin-value"), + ("SET_EXTRA", "extra-value"), + ("SET_EXTRA_2", "extra-value-2"), + ]), + ); + } + + #[test] + fn test_extra_names_are_trimmed() { + // As the names arrive from `env_allowlist = SET_EXTRA, SET_EXTRA_2`. + assert_eq!( + captured(&["SET_EXTRA", " SET_EXTRA_2"]), + captured(&["SET_EXTRA", "SET_EXTRA_2"]), + ); + } + + #[test] + fn test_extra_ignores_unset_names() { + assert_eq!(captured(&["NEVER_SET"]), captured(&[])); + } + + #[test] + fn test_extra_ignores_empty_names() { + // What `env_allowlist =` and `env_allowlist = SET_EXTRA,` parse to. + assert_eq!(captured(&["", " "]), captured(&[])); + assert_eq!(captured(&["SET_EXTRA", ""]), captured(&["SET_EXTRA"])); + } + + #[test] + fn test_repeating_a_builtin_name_keeps_the_same_value() { + // Duplicates are tolerated because every consumer of `values()` is last-wins, which is + // only safe as long as both copies carry the same value. + assert_eq!( + captured(&["SET_BUILTIN"]), + pairs(&[ + ("SET_BUILTIN", "builtin-value"), + ("SET_BUILTIN", "builtin-value"), + ]), + ); + } + + #[test] + fn test_allowlist_clears_the_environment_and_excludes_nothing() { + let inheritance = EnvironmentInheritance::test_allowlist_from( + BUILTIN, + &["SET_EXTRA".to_owned()], + &getenv, + ); + assert!(inheritance.clear()); + assert_eq!(inheritance.exclusions().count(), 0); + } + + #[test] + fn test_empty_extra_takes_the_memoized_path() { + // The `extra.is_empty()` fast path in `test_allowlist_with_extra` reads from a different + // (memoized) source than `test_allowlist_from`, so check the two agree. + let memoized: Vec<_> = EnvironmentInheritance::test_allowlist() + .values() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); + let direct: Vec<_> = + EnvironmentInheritance::test_allowlist_from(ENV_ALLOW_LIST, &[], &real_getenv) + .values() + .map(|(k, v)| (k.to_owned(), v.clone())) + .collect(); + assert_eq!(memoized, direct); + } +} diff --git a/app/buck2_test/src/orchestrator.rs b/app/buck2_test/src/orchestrator.rs index cc288c6527198..f97aceb80e290 100644 --- a/app/buck2_test/src/orchestrator.rs +++ b/app/buck2_test/src/orchestrator.rs @@ -54,6 +54,8 @@ use buck2_build_signals::env::WaitingData; use buck2_cli_proto::test_request::TestOutputMode; use buck2_common::dice::cells::HasCellResolver; use buck2_common::events::HasEvents; +use buck2_common::legacy_configs::dice::HasLegacyConfigs; +use buck2_common::legacy_configs::key::BuckconfigKeyRef; use buck2_common::liveliness_observer::LivelinessObserver; use buck2_common::local_resource_state::LocalResourceState; use buck2_core::cells::cell_root_path::CellRootPathBuf; @@ -1541,6 +1543,30 @@ impl BuckTestOrchestrator<'_> { }) } + /// Environment variables to inherit into test processes on top of the built-in allowlist, + /// specified in the root cell's `[test] env_allowlist` buckconfig. The built-in list is + /// deliberately minimal, and this is the escape hatch for passing variables which shouldn't + /// invaliate test executions (tracing context, proxy settings, and so on). + /// + /// Note that inherited variables are deliberately not part of the action digest, so changing + /// the *value* of one of these will not invalidate test executions. Changing the config itself + /// will, since reading it here records a dependency edge. + async fn extra_test_env_allowlist( + dice: &mut DiceComputations<'_>, + ) -> buck2_error::Result> { + let root_cell = dice.get_cell_resolver().await?.root_cell(); + Ok(dice + .parse_legacy_config_list_property::( + root_cell, + BuckconfigKeyRef { + section: "test", + property: "env_allowlist", + }, + ) + .await? + .unwrap_or_default()) + } + async fn create_command_execution_request( dice: &mut DiceComputations<'_>, cwd: ProjectRelativePathBuf, @@ -1591,9 +1617,12 @@ impl BuckTestOrchestrator<'_> { .get::() .unwrap() .0; + let extra_env_allowlist = Self::extra_test_env_allowlist(dice).await?; request = request .with_working_directory(cwd) - .with_local_environment_inheritance(EnvironmentInheritance::test_allowlist()) + .with_local_environment_inheritance(EnvironmentInheritance::test_allowlist_with_extra( + &extra_env_allowlist, + )) .with_disable_miniperf(!has_resource_control) .with_worker(worker) .with_remote_execution_custom_image(re_dynamic_image) diff --git a/tests/core/test/test_env_allowlist.py b/tests/core/test/test_env_allowlist.py new file mode 100644 index 0000000000000..1ef0d17af1711 --- /dev/null +++ b/tests/core/test/test_env_allowlist.py @@ -0,0 +1,101 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# pyre-strict + +from buck2.tests.e2e_util.api.buck import Buck +from buck2.tests.e2e_util.asserts import expect_failure +from buck2.tests.e2e_util.buck_workspace import buck_test, env + +# `@env` sets these on the buck2 client, so the daemon it spawns inherits them. +# The `//:expect_*` targets then assert on what actually reaches the test +# process, which is what `[test] env_allowlist` controls. +PROBE = "BUCK2_E2E_ENV_PROBE" +PROBE_2 = "BUCK2_E2E_ENV_PROBE_2" + +# The allowlist only applies to local execution, so pin every test to it. +LOCAL_ONLY = [ + "-c", + "test.local_enabled=true", + "-c", + "test.remote_enabled=false", +] + + +@buck_test() +@env(PROBE, "probe-value") +async def test_env_not_inherited_by_default(buck: Buck) -> None: + # The var is in the daemon's environment but not in the allowlist, so the + # test process must not see it. + await buck.test(*LOCAL_ONLY, "//:expect_unset") + + +@buck_test() +@env(PROBE, "probe-value") +async def test_env_inherited_when_allowlisted(buck: Buck) -> None: + await buck.test( + *LOCAL_ONLY, + "-c", + f"test.env_allowlist={PROBE}", + "//:expect_set", + ) + + +@buck_test() +@env(PROBE, "probe-value") +@env(PROBE_2, "probe-value-2") +async def test_env_allowlist_accepts_a_list(buck: Buck) -> None: + await buck.test( + *LOCAL_ONLY, + "-c", + f"test.env_allowlist={PROBE},{PROBE_2}", + "//:expect_both_set", + ) + + +@buck_test() +@env(PROBE, "probe-value") +@env(PROBE_2, "probe-value-2") +async def test_env_allowlist_only_covers_what_it_names(buck: Buck) -> None: + # Allowlisting one var must not drag its neighbour along. + await expect_failure( + buck.test( + *LOCAL_ONLY, + "-c", + f"test.env_allowlist={PROBE}", + "//:expect_both_set", + ), + ) + + +@buck_test() +@env(PROBE, "probe-value") +async def test_env_allowlist_ignores_unset_vars(buck: Buck) -> None: + # Naming a var that isn't in the daemon's environment is not an error, and + # must not show up in the test process as an empty string. + await buck.test( + *LOCAL_ONLY, + "-c", + "test.env_allowlist=BUCK2_E2E_ENV_PROBE_NEVER_SET", + "//:expect_unset", + ) + + +@buck_test() +@env(PROBE, "probe-value") +async def test_changing_env_allowlist_reruns_the_test(buck: Buck) -> None: + # Reading the config records a dep edge on it, so dropping the var from the + # allowlist has to invalidate the successful run above rather than let the + # daemon reuse it. + await buck.test( + *LOCAL_ONLY, + "-c", + f"test.env_allowlist={PROBE}", + "//:expect_set", + ) + await expect_failure(buck.test(*LOCAL_ONLY, "//:expect_set")) diff --git a/tests/core/test/test_env_allowlist_data/.buckconfig b/tests/core/test/test_env_allowlist_data/.buckconfig new file mode 100644 index 0000000000000..65dec17e1f663 --- /dev/null +++ b/tests/core/test/test_env_allowlist_data/.buckconfig @@ -0,0 +1,15 @@ +[buildfile] +name=TARGETS.fixture + +[repositories] +root = . +nano_prelude = nano_prelude + +[cell_aliases] +prelude = nano_prelude + +[external_cells] +nano_prelude = bundled + +[build] +execution_platforms = root//platforms:platforms diff --git a/tests/core/test/test_env_allowlist_data/.buckroot b/tests/core/test/test_env_allowlist_data/.buckroot new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/core/test/test_env_allowlist_data/TARGETS.fixture b/tests/core/test/test_env_allowlist_data/TARGETS.fixture new file mode 100644 index 0000000000000..66e58fd591fc9 --- /dev/null +++ b/tests/core/test/test_env_allowlist_data/TARGETS.fixture @@ -0,0 +1,20 @@ +load(":rules.bzl", "env_probe_test") + +# An empty expected value means "must not be set at all". +env_probe_test( + name = "expect_unset", + expect_env = {"BUCK2_E2E_ENV_PROBE": ""}, +) + +env_probe_test( + name = "expect_set", + expect_env = {"BUCK2_E2E_ENV_PROBE": "probe-value"}, +) + +env_probe_test( + name = "expect_both_set", + expect_env = { + "BUCK2_E2E_ENV_PROBE": "probe-value", + "BUCK2_E2E_ENV_PROBE_2": "probe-value-2", + }, +) diff --git a/tests/core/test/test_env_allowlist_data/platforms/TARGETS.fixture b/tests/core/test/test_env_allowlist_data/platforms/TARGETS.fixture new file mode 100644 index 0000000000000..e7a5179f98653 --- /dev/null +++ b/tests/core/test/test_env_allowlist_data/platforms/TARGETS.fixture @@ -0,0 +1,18 @@ +local_enabled = read_config("test", "local_enabled", "true") +remote_enabled = read_config("test", "remote_enabled", "false") + +platform( + name = "platform", +) + +execution_platform( + name = "exec_platform", + platform = ":platform", + local_enabled = local_enabled == "true", + remote_enabled = remote_enabled == "true", +) + +execution_platforms( + name = "platforms", + platforms = [":exec_platform"], +) diff --git a/tests/core/test/test_env_allowlist_data/rules.bzl b/tests/core/test/test_env_allowlist_data/rules.bzl new file mode 100644 index 0000000000000..437def757ef9e --- /dev/null +++ b/tests/core/test/test_env_allowlist_data/rules.bzl @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# A test whose command asserts that specific environment variables were +# inherited from the daemon with specific values, and fails otherwise. Used to +# observe what the local execution env allowlist actually lets through. + +_SCRIPT_PREFIX = """ +import os +import sys + +if "--list" in sys.argv: + print("test1\\n") + sys.exit(0) + +failures = [] + +def check(name, want): + got = os.environ.get(name) + if got != want: + failures.append("expected " + name + "=" + repr(want) + ", got " + repr(got)) + +""" + +_SCRIPT_SUFFIX = """ +if failures: + sys.stderr.write("\\n".join(failures) + "\\n") + sys.exit(1) +sys.exit(0) +""" + +def _env_probe_test_impl(ctx): + # An empty expected value means "must not be set at all". + checks = [ + 'check("{}", {})'.format(name, '"{}"'.format(want) if want else "None") + for name, want in ctx.attrs.expect_env.items() + ] + script = _SCRIPT_PREFIX + "\n".join(checks) + _SCRIPT_SUFFIX + + out = ctx.actions.declare_output("file", has_content_based_path = False) + ctx.actions.run( + ["touch", out.as_output()], + category = "touch", + ) + return [ + DefaultInfo(out), + ExternalRunnerTestInfo( + command = ["fbpython", "-c", script], + use_project_relative_paths = True, + type = "lionhead", + ), + ] + +env_probe_test = rule( + attrs = { + "expect_env": attrs.dict(attrs.string(), attrs.string(), default = {}), + }, + impl = _env_probe_test_impl, +)