Skip to content

refactor(clp-tdl-package): Move clp_binary_path and s3_credential_env into a shared task::utils module. #2508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LinZhihao-723 wants to merge 3 commits into
base: main
Choose a base branch
from
+119 −104
Open

refactor(clp-tdl-package): Move clp_binary_path and s3_credential_env into a shared task::utils module. #2508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2ce8882
refactor(clp-tdl-package): Move `clp_binary_path` and `s3_credential_…
LinZhihao-723 Aug 31, 2026
3b3b9c4
Merge branch 'main' into tdl-task-utils-mod
LinZhihao-723 Aug 31, 2026
3f36b5a
Merge branch 'main' into tdl-task-utils-mod
LinZhihao-723 Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 2 additions & 104 deletions components/clp-tdl-package/src/task/compression/compress.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@ use std::process::Command;
use std::process::Stdio;

use anyhow::Context;
use aws_config::BehaviorVersion;
use aws_sdk_s3::config::ProvideCredentials;
use clp_rust_utils::aws::AWS_DEFAULT_REGION;
use clp_rust_utils::clp_config::AwsAuthentication;
use clp_rust_utils::clp_config::S3Config;
use clp_rust_utils::clp_config::package::config::ArchiveOutput;
use clp_rust_utils::clp_config::package::config::ArchiveOutputStorage;
Expand All @@ -30,6 +27,8 @@ use non_empty_string::NonEmptyString;

use crate::common::clp_home;
use crate::common::runtime;
use crate::task::utils::clp_binary_path;
use crate::task::utils::s3_credential_env;

/// Compresses the given S3 objects into archives, uploads them to S3, and returns their metadata
/// for the commit task.
Expand Down Expand Up @@ -345,74 +344,6 @@ fn build_s3_logs_list(input_source: &S3InputSource) -> anyhow::Result<String> {
Ok(list)
}

/// Resolves the AWS credential env vars clp-s needs to access the S3 objects.
///
/// # Returns
///
/// The env-var name-value pairs with the following environment variables set:
///
/// * `AWS_ACCESS_KEY_ID`
/// * `AWS_SECRET_ACCESS_KEY`
/// * `AWS_SESSION_TOKEN` (if any)
///
/// # Errors
///
/// Returns an error if:
///
/// * The default AWS SDK credential provider chain has no provider.
/// * Forwards [`ProvideCredentials::provide_credentials`]'s return values on failure.
fn s3_credential_env(
runtime: &tokio::runtime::Handle,
region: &str,
auth: &AwsAuthentication,
) -> anyhow::Result<Vec<(&'static str, String)>> {
/// The env var holding the AWS access key ID.
const AWS_ACCESS_KEY_ID_ENV_VAR: &str = "AWS_ACCESS_KEY_ID";

/// The env var holding the AWS secret access key.
const AWS_SECRET_ACCESS_KEY_ENV_VAR: &str = "AWS_SECRET_ACCESS_KEY";

/// The env var holding the AWS session token.
const AWS_SESSION_TOKEN_ENV_VAR: &str = "AWS_SESSION_TOKEN";

let (access_key_id, secret_access_key, session_token) = match auth {
AwsAuthentication::Credentials { credentials } => (
credentials.access_key_id.clone(),
credentials.secret_access_key.clone(),
credentials.session_token.clone(),
),
AwsAuthentication::Default => {
let sdk_config = runtime.block_on(
aws_config::defaults(BehaviorVersion::latest())
.region(aws_sdk_s3::config::Region::new(region.to_string()))
.load(),
);
let provider = sdk_config
.credentials_provider()
.context("default AWS SDK credential provider is unavailable")?;
let credentials = runtime
.block_on(provider.provide_credentials())
.context("failed to resolve credentials from the default AWS SDK provider chain")?;
(
credentials.access_key_id().to_string(),
credentials.secret_access_key().to_string(),
credentials
.session_token()
.map(std::string::ToString::to_string),
)
}
};

let mut env = vec![
(AWS_ACCESS_KEY_ID_ENV_VAR, access_key_id),
(AWS_SECRET_ACCESS_KEY_ENV_VAR, secret_access_key),
];
if let Some(session_token) = session_token {
env.push((AWS_SESSION_TOKEN_ENV_VAR, session_token));
}
Ok(env)
}

/// Parses a single clp-s `--print-archive-stats` stdout line into an [`ArchiveMetadata`].
///
/// NOTE: clp-s emits a superset of [`ArchiveMetadata`]'s fields per line; unknown fields are
Expand Down Expand Up @@ -585,15 +516,6 @@ fn build_log_converter_args(output_dir: &Path, inputs_from_path: &Path) -> Vec<O
]
}

/// Resolves the path of a CLP binary under `clp_home`, joining `bin/{binary}`.
///
/// # Returns
///
/// The path to the named binary under the CLP installation.
fn clp_binary_path(clp_home: &Path, binary: &str) -> PathBuf {
clp_home.join("bin").join(binary)
}

/// Resolves the S3 config the archives are uploaded to from `config`.
///
/// # Returns
Expand Down Expand Up @@ -905,7 +827,6 @@ mod tests {
use std::path::PathBuf;

use clp_rust_utils::clp_config::AwsAuthentication;
use clp_rust_utils::clp_config::AwsCredentials;
use clp_rust_utils::clp_config::S3Config;
use clp_rust_utils::clp_config::package::config::ArchiveOutput;
use clp_rust_utils::clp_config::package::config::ArchiveOutputStorage;
Expand All @@ -923,7 +844,6 @@ mod tests {
use super::build_s3_logs_list;
use super::create_archive_s3_key;
use super::parse_archive_stats;
use super::s3_credential_env;

#[test]
fn build_s3_logs_list_default_endpoint() -> anyhow::Result<()> {
Expand All @@ -947,28 +867,6 @@ mod tests {
Ok(())
}

#[test]
fn s3_credential_env_credentials() {
let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime");
let auth = AwsAuthentication::Credentials {
credentials: AwsCredentials {
access_key_id: "the-access-key".to_string(),
secret_access_key: "the-secret-key".to_string(),
session_token: Some("the-session-token".to_string()),
},
};

assert_eq!(
s3_credential_env(runtime.handle(), "us-east-1", &auth)
.expect("failed to resolve credentials"),
vec![
("AWS_ACCESS_KEY_ID", "the-access-key".to_string()),
("AWS_SECRET_ACCESS_KEY", "the-secret-key".to_string()),
("AWS_SESSION_TOKEN", "the-session-token".to_string()),
]
);
}

#[test]
fn parse_archive_stats_ignores_extra_keys() {
let line = concat!(
Expand Down
1 change: 1 addition & 0 deletions components/clp-tdl-package/src/task/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//! The task implementations this package registers with Spider.

pub mod compression;
pub mod utils;
116 changes: 116 additions & 0 deletions components/clp-tdl-package/src/task/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
//! Helpers shared by the tasks that invoke CLP's core binaries.

use std::path::Path;
use std::path::PathBuf;

use anyhow::Context;
use aws_config::BehaviorVersion;
use aws_sdk_s3::config::ProvideCredentials;
use clp_rust_utils::clp_config::AwsAuthentication;

/// Resolves the path of a CLP binary under `clp_home`, joining `bin/{binary}`.
///
/// # Returns
///
/// The path to the named binary under the CLP installation.
pub(super) fn clp_binary_path(clp_home: &Path, binary: &str) -> PathBuf {
clp_home.join("bin").join(binary)
}

/// Resolves the AWS credential env vars clp-s needs to access the S3 objects.
///
/// # Returns
///
/// The env-var name-value pairs with the following environment variables set:
///
/// * `AWS_ACCESS_KEY_ID`
/// * `AWS_SECRET_ACCESS_KEY`
/// * `AWS_SESSION_TOKEN` (if any)
///
/// # Errors
///
/// Returns an error if:
///
/// * The default AWS SDK credential provider chain has no provider.
/// * Forwards [`ProvideCredentials::provide_credentials`]'s return values on failure.
pub(super) fn s3_credential_env(
runtime: &tokio::runtime::Handle,
region: &str,
auth: &AwsAuthentication,
) -> anyhow::Result<Vec<(&'static str, String)>> {
/// The env var holding the AWS access key ID.
const AWS_ACCESS_KEY_ID_ENV_VAR: &str = "AWS_ACCESS_KEY_ID";

/// The env var holding the AWS secret access key.
const AWS_SECRET_ACCESS_KEY_ENV_VAR: &str = "AWS_SECRET_ACCESS_KEY";

/// The env var holding the AWS session token.
const AWS_SESSION_TOKEN_ENV_VAR: &str = "AWS_SESSION_TOKEN";

let (access_key_id, secret_access_key, session_token) = match auth {
AwsAuthentication::Credentials { credentials } => (
credentials.access_key_id.clone(),
credentials.secret_access_key.clone(),
credentials.session_token.clone(),
),
AwsAuthentication::Default => {
let sdk_config = runtime.block_on(
aws_config::defaults(BehaviorVersion::latest())
.region(aws_sdk_s3::config::Region::new(region.to_string()))
.load(),
);
let provider = sdk_config
.credentials_provider()
.context("default AWS SDK credential provider is unavailable")?;
let credentials = runtime
.block_on(provider.provide_credentials())
.context("failed to resolve credentials from the default AWS SDK provider chain")?;
(
credentials.access_key_id().to_string(),
credentials.secret_access_key().to_string(),
credentials
.session_token()
.map(std::string::ToString::to_string),
)
}
};

let mut env = vec![
(AWS_ACCESS_KEY_ID_ENV_VAR, access_key_id),
(AWS_SECRET_ACCESS_KEY_ENV_VAR, secret_access_key),
];
if let Some(session_token) = session_token {
env.push((AWS_SESSION_TOKEN_ENV_VAR, session_token));
}
Ok(env)
}

#[cfg(test)]
mod tests {
use clp_rust_utils::clp_config::AwsAuthentication;
use clp_rust_utils::clp_config::AwsCredentials;

use super::s3_credential_env;

#[test]
fn s3_credential_env_credentials() {
let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime");
let auth = AwsAuthentication::Credentials {
credentials: AwsCredentials {
access_key_id: "the-access-key".to_string(),
secret_access_key: "the-secret-key".to_string(),
session_token: Some("the-session-token".to_string()),
},
};

assert_eq!(
s3_credential_env(runtime.handle(), "us-east-1", &auth)
.expect("failed to resolve credentials"),
vec![
("AWS_ACCESS_KEY_ID", "the-access-key".to_string()),
("AWS_SECRET_ACCESS_KEY", "the-secret-key".to_string()),
("AWS_SESSION_TOKEN", "the-session-token".to_string()),
]
);
}
}
Loading