From 2ce88821329571fd4e138757abb3ec5262957829 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 31 Aug 2026 11:12:17 -0400 Subject: [PATCH] refactor(clp-tdl-package): Move `clp_binary_path` and `s3_credential_env` into a shared `task::utils` module. --- .../src/task/compression/compress.rs | 106 +--------------- components/clp-tdl-package/src/task/mod.rs | 1 + components/clp-tdl-package/src/task/utils.rs | 116 ++++++++++++++++++ 3 files changed, 119 insertions(+), 104 deletions(-) create mode 100644 components/clp-tdl-package/src/task/utils.rs diff --git a/components/clp-tdl-package/src/task/compression/compress.rs b/components/clp-tdl-package/src/task/compression/compress.rs index a1ccc35b72..0cdf035603 100644 --- a/components/clp-tdl-package/src/task/compression/compress.rs +++ b/components/clp-tdl-package/src/task/compression/compress.rs @@ -10,10 +10,7 @@ use std::process::Command; use std::process::Stdio; use anyhow::Context; -use aws_config::BehaviorVersion; -use aws_sdk_s3::config::ProvideCredentials; use clp_rust_utils::aws::AWS_DEFAULT_REGION; -use clp_rust_utils::clp_config::AwsAuthentication; use clp_rust_utils::clp_config::S3Config; use clp_rust_utils::clp_config::package::config::ArchiveOutput; use clp_rust_utils::clp_config::package::config::ArchiveOutputStorage; @@ -30,6 +27,8 @@ use non_empty_string::NonEmptyString; use crate::common::clp_home; use crate::common::runtime; +use crate::task::utils::clp_binary_path; +use crate::task::utils::s3_credential_env; /// Compresses the given S3 objects into archives, uploads them to S3, and returns their metadata /// for the commit task. @@ -345,74 +344,6 @@ fn build_s3_logs_list(input_source: &S3InputSource) -> anyhow::Result { Ok(list) } -/// Resolves the AWS credential env vars clp-s needs to access the S3 objects. -/// -/// # Returns -/// -/// The env-var name-value pairs with the following environment variables set: -/// -/// * `AWS_ACCESS_KEY_ID` -/// * `AWS_SECRET_ACCESS_KEY` -/// * `AWS_SESSION_TOKEN` (if any) -/// -/// # Errors -/// -/// Returns an error if: -/// -/// * The default AWS SDK credential provider chain has no provider. -/// * Forwards [`ProvideCredentials::provide_credentials`]'s return values on failure. -fn s3_credential_env( - runtime: &tokio::runtime::Handle, - region: &str, - auth: &AwsAuthentication, -) -> anyhow::Result> { - /// The env var holding the AWS access key ID. - const AWS_ACCESS_KEY_ID_ENV_VAR: &str = "AWS_ACCESS_KEY_ID"; - - /// The env var holding the AWS secret access key. - const AWS_SECRET_ACCESS_KEY_ENV_VAR: &str = "AWS_SECRET_ACCESS_KEY"; - - /// The env var holding the AWS session token. - const AWS_SESSION_TOKEN_ENV_VAR: &str = "AWS_SESSION_TOKEN"; - - let (access_key_id, secret_access_key, session_token) = match auth { - AwsAuthentication::Credentials { credentials } => ( - credentials.access_key_id.clone(), - credentials.secret_access_key.clone(), - credentials.session_token.clone(), - ), - AwsAuthentication::Default => { - let sdk_config = runtime.block_on( - aws_config::defaults(BehaviorVersion::latest()) - .region(aws_sdk_s3::config::Region::new(region.to_string())) - .load(), - ); - let provider = sdk_config - .credentials_provider() - .context("default AWS SDK credential provider is unavailable")?; - let credentials = runtime - .block_on(provider.provide_credentials()) - .context("failed to resolve credentials from the default AWS SDK provider chain")?; - ( - credentials.access_key_id().to_string(), - credentials.secret_access_key().to_string(), - credentials - .session_token() - .map(std::string::ToString::to_string), - ) - } - }; - - let mut env = vec![ - (AWS_ACCESS_KEY_ID_ENV_VAR, access_key_id), - (AWS_SECRET_ACCESS_KEY_ENV_VAR, secret_access_key), - ]; - if let Some(session_token) = session_token { - env.push((AWS_SESSION_TOKEN_ENV_VAR, session_token)); - } - Ok(env) -} - /// Parses a single clp-s `--print-archive-stats` stdout line into an [`ArchiveMetadata`]. /// /// NOTE: clp-s emits a superset of [`ArchiveMetadata`]'s fields per line; unknown fields are @@ -585,15 +516,6 @@ fn build_log_converter_args(output_dir: &Path, inputs_from_path: &Path) -> Vec PathBuf { - clp_home.join("bin").join(binary) -} - /// Resolves the S3 config the archives are uploaded to from `config`. /// /// # Returns @@ -905,7 +827,6 @@ mod tests { use std::path::PathBuf; use clp_rust_utils::clp_config::AwsAuthentication; - use clp_rust_utils::clp_config::AwsCredentials; use clp_rust_utils::clp_config::S3Config; use clp_rust_utils::clp_config::package::config::ArchiveOutput; use clp_rust_utils::clp_config::package::config::ArchiveOutputStorage; @@ -923,7 +844,6 @@ mod tests { use super::build_s3_logs_list; use super::create_archive_s3_key; use super::parse_archive_stats; - use super::s3_credential_env; #[test] fn build_s3_logs_list_default_endpoint() -> anyhow::Result<()> { @@ -947,28 +867,6 @@ mod tests { Ok(()) } - #[test] - fn s3_credential_env_credentials() { - let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime"); - let auth = AwsAuthentication::Credentials { - credentials: AwsCredentials { - access_key_id: "the-access-key".to_string(), - secret_access_key: "the-secret-key".to_string(), - session_token: Some("the-session-token".to_string()), - }, - }; - - assert_eq!( - s3_credential_env(runtime.handle(), "us-east-1", &auth) - .expect("failed to resolve credentials"), - vec![ - ("AWS_ACCESS_KEY_ID", "the-access-key".to_string()), - ("AWS_SECRET_ACCESS_KEY", "the-secret-key".to_string()), - ("AWS_SESSION_TOKEN", "the-session-token".to_string()), - ] - ); - } - #[test] fn parse_archive_stats_ignores_extra_keys() { let line = concat!( diff --git a/components/clp-tdl-package/src/task/mod.rs b/components/clp-tdl-package/src/task/mod.rs index f672b3ee36..32fbf2c96f 100644 --- a/components/clp-tdl-package/src/task/mod.rs +++ b/components/clp-tdl-package/src/task/mod.rs @@ -1,3 +1,4 @@ //! The task implementations this package registers with Spider. pub mod compression; +pub mod utils; diff --git a/components/clp-tdl-package/src/task/utils.rs b/components/clp-tdl-package/src/task/utils.rs new file mode 100644 index 0000000000..197eed0c2e --- /dev/null +++ b/components/clp-tdl-package/src/task/utils.rs @@ -0,0 +1,116 @@ +//! Helpers shared by the tasks that invoke CLP's core binaries. + +use std::path::Path; +use std::path::PathBuf; + +use anyhow::Context; +use aws_config::BehaviorVersion; +use aws_sdk_s3::config::ProvideCredentials; +use clp_rust_utils::clp_config::AwsAuthentication; + +/// Resolves the path of a CLP binary under `clp_home`, joining `bin/{binary}`. +/// +/// # Returns +/// +/// The path to the named binary under the CLP installation. +pub(super) fn clp_binary_path(clp_home: &Path, binary: &str) -> PathBuf { + clp_home.join("bin").join(binary) +} + +/// Resolves the AWS credential env vars clp-s needs to access the S3 objects. +/// +/// # Returns +/// +/// The env-var name-value pairs with the following environment variables set: +/// +/// * `AWS_ACCESS_KEY_ID` +/// * `AWS_SECRET_ACCESS_KEY` +/// * `AWS_SESSION_TOKEN` (if any) +/// +/// # Errors +/// +/// Returns an error if: +/// +/// * The default AWS SDK credential provider chain has no provider. +/// * Forwards [`ProvideCredentials::provide_credentials`]'s return values on failure. +pub(super) fn s3_credential_env( + runtime: &tokio::runtime::Handle, + region: &str, + auth: &AwsAuthentication, +) -> anyhow::Result> { + /// The env var holding the AWS access key ID. + const AWS_ACCESS_KEY_ID_ENV_VAR: &str = "AWS_ACCESS_KEY_ID"; + + /// The env var holding the AWS secret access key. + const AWS_SECRET_ACCESS_KEY_ENV_VAR: &str = "AWS_SECRET_ACCESS_KEY"; + + /// The env var holding the AWS session token. + const AWS_SESSION_TOKEN_ENV_VAR: &str = "AWS_SESSION_TOKEN"; + + let (access_key_id, secret_access_key, session_token) = match auth { + AwsAuthentication::Credentials { credentials } => ( + credentials.access_key_id.clone(), + credentials.secret_access_key.clone(), + credentials.session_token.clone(), + ), + AwsAuthentication::Default => { + let sdk_config = runtime.block_on( + aws_config::defaults(BehaviorVersion::latest()) + .region(aws_sdk_s3::config::Region::new(region.to_string())) + .load(), + ); + let provider = sdk_config + .credentials_provider() + .context("default AWS SDK credential provider is unavailable")?; + let credentials = runtime + .block_on(provider.provide_credentials()) + .context("failed to resolve credentials from the default AWS SDK provider chain")?; + ( + credentials.access_key_id().to_string(), + credentials.secret_access_key().to_string(), + credentials + .session_token() + .map(std::string::ToString::to_string), + ) + } + }; + + let mut env = vec![ + (AWS_ACCESS_KEY_ID_ENV_VAR, access_key_id), + (AWS_SECRET_ACCESS_KEY_ENV_VAR, secret_access_key), + ]; + if let Some(session_token) = session_token { + env.push((AWS_SESSION_TOKEN_ENV_VAR, session_token)); + } + Ok(env) +} + +#[cfg(test)] +mod tests { + use clp_rust_utils::clp_config::AwsAuthentication; + use clp_rust_utils::clp_config::AwsCredentials; + + use super::s3_credential_env; + + #[test] + fn s3_credential_env_credentials() { + let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime"); + let auth = AwsAuthentication::Credentials { + credentials: AwsCredentials { + access_key_id: "the-access-key".to_string(), + secret_access_key: "the-secret-key".to_string(), + session_token: Some("the-session-token".to_string()), + }, + }; + + assert_eq!( + s3_credential_env(runtime.handle(), "us-east-1", &auth) + .expect("failed to resolve credentials"), + vec![ + ("AWS_ACCESS_KEY_ID", "the-access-key".to_string()), + ("AWS_SECRET_ACCESS_KEY", "the-secret-key".to_string()), + ("AWS_SESSION_TOKEN", "the-session-token".to_string()), + ] + ); + } +}