Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 83 additions & 20 deletions src/storages/fs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,16 @@ pub struct FileSystem {
root: PathBuf,
}

/// Per-bucket dir for the server's own bookkeeping (object metadata, multipart
/// parts). Kept on the writable volume, not the maybe-read-only fs-root, and
/// reserved: user keys under it are refused and it is hidden from listings.
const RESERVED_DIR: &str = ".wasmer-s3";

/// Does a key fall under the reserved dir at the bucket root?
fn key_under_reserved(key: &str) -> bool {
key.trim_start_matches('/').split('/').next() == Some(RESERVED_DIR)
}

impl FileSystem {
/// Constructs a file system storage located at `root`
/// # Errors
Expand Down Expand Up @@ -82,6 +92,12 @@ impl FileSystem {

/// resolve object path under the virtual root
fn get_object_path(&self, bucket: &str, key: &str) -> io::Result<PathBuf> {
if key_under_reserved(key) {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
format!("keys under {RESERVED_DIR}/ are reserved"),
));
}
let dir = Path::new(&bucket);
let file_path = Path::new(&key);
let ans = dir.join(file_path).absolutize_virtually(&self.root)?.into();
Expand All @@ -95,18 +111,24 @@ impl FileSystem {
Ok(ans)
}

/// resolve metadata path under the virtual root (custom format)
fn get_metadata_path(&self, bucket: &str, key: &str) -> io::Result<PathBuf> {
let encode = |s: &str| base64_simd::URL_SAFE_NO_PAD.encode_to_string(s);
/// Resolve `<bucket>/RESERVED_DIR/<sub>/<name>` on the writable volume.
fn reserved_path(&self, bucket: &str, sub: &str, name: &str) -> io::Result<PathBuf> {
let rel = Path::new(bucket).join(RESERVED_DIR).join(sub).join(name);
Ok(rel.absolutize_virtually(&self.root)?.into())
}

let file_path_str = format!(
".bucket-{}.object-{}.metadata.json",
encode(bucket),
encode(key),
/// Object metadata sidecar path (in the bucket's reserved dir).
fn get_metadata_path(&self, bucket: &str, key: &str) -> io::Result<PathBuf> {
let name = format!(
"{}.json",
base64_simd::URL_SAFE_NO_PAD.encode_to_string(key)
);
let file_path = Path::new(&file_path_str);
let ans = file_path.absolutize_virtually(&self.root)?.into();
Ok(ans)
self.reserved_path(bucket, "meta", &name)
}

/// Multipart part path (in the bucket's reserved dir).
fn get_part_path(&self, bucket: &str, upload_id: &str, part: i64) -> io::Result<PathBuf> {
self.reserved_path(bucket, "mpu", &format!("{upload_id}.part-{part}"))
}

/// load metadata from fs
Expand Down Expand Up @@ -134,6 +156,9 @@ impl FileSystem {
metadata: &HashMap<String, String>,
) -> io::Result<()> {
let path = self.get_metadata_path(bucket, key)?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let content = serde_json::to_vec(metadata)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
fs::write(&path, &content).await
Expand Down Expand Up @@ -258,6 +283,9 @@ impl S3Storage for FileSystem {
let src_metadata_path = trace_try!(self.get_metadata_path(bucket, key));
if src_metadata_path.exists() {
let dst_metadata_path = trace_try!(self.get_metadata_path(&input.bucket, &input.key));
if let Some(parent) = dst_metadata_path.parent() {
trace_try!(fs::create_dir_all(parent).await);
}
let _ = trace_try!(fs::copy(src_metadata_path, dst_metadata_path).await);
}

Expand Down Expand Up @@ -290,6 +318,10 @@ impl S3Storage for FileSystem {
&self,
input: DeleteObjectRequest,
) -> S3StorageResult<DeleteObjectOutput, DeleteObjectError> {
if key_under_reserved(&input.key) {
// reserved keys read as absent; delete is idempotent
return Ok(DeleteObjectOutput::default());
}
let path = trace_try!(self.get_object_path(&input.bucket, &input.key));
if input.key.ends_with('/') {
let mut dir = trace_try!(fs::read_dir(&path).await);
Expand All @@ -311,6 +343,9 @@ impl S3Storage for FileSystem {
) -> S3StorageResult<DeleteObjectsOutput, DeleteObjectsError> {
let mut objects: Vec<(PathBuf, String)> = Vec::new();
for object in input.delete.objects {
if key_under_reserved(&object.key) {
continue;
}
let path = trace_try!(self.get_object_path(&input.bucket, &object.key));
if path.exists() {
objects.push((path, object.key));
Expand Down Expand Up @@ -356,6 +391,10 @@ impl S3Storage for FileSystem {
&self,
input: GetObjectRequest,
) -> S3StorageResult<GetObjectOutput, GetObjectError> {
if key_under_reserved(&input.key) {
let err = code_error!(NoSuchKey, "The specified key does not exist.");
return Err(err.into());
}
let object_path = trace_try!(self.get_object_path(&input.bucket, &input.key));

let parse_range = |s: &str| {
Expand Down Expand Up @@ -465,6 +504,10 @@ impl S3Storage for FileSystem {
&self,
input: HeadObjectRequest,
) -> S3StorageResult<HeadObjectOutput, HeadObjectError> {
if key_under_reserved(&input.key) {
let err = code_error!(NoSuchKey, "The specified key does not exist.");
return Err(err.into());
}
let path = trace_try!(self.get_object_path(&input.bucket, &input.key));

if !path.exists() {
Expand Down Expand Up @@ -539,10 +582,14 @@ impl S3Storage for FileSystem {
let delimiter_is_dir = input.delimiter.as_deref() == Some("/");

while let Some(dir) = dir_queue.pop_front() {
let mut entries = trace_try!(fs::read_dir(dir).await);
let mut entries = trace_try!(fs::read_dir(&dir).await);
loop {
let entry = trace_try!(entries.next_entry().await);
if let Some(entry) = entry {
// hide the reserved bookkeeping dir at the bucket root
if dir == path && entry.file_name().to_str() == Some(RESERVED_DIR) {
continue;
}
let file_type = trace_try!(entry.file_type().await);
if file_type.is_dir() {
if delimiter_is_dir {
Expand Down Expand Up @@ -631,10 +678,14 @@ impl S3Storage for FileSystem {
dir_queue.push_back(path.clone());

while let Some(dir) = dir_queue.pop_front() {
let mut entries = trace_try!(fs::read_dir(dir).await);
let mut entries = trace_try!(fs::read_dir(&dir).await);
loop {
let entry = trace_try!(entries.next_entry().await);
if let Some(entry) = entry {
// hide the reserved bookkeeping dir at the bucket root
if dir == path && entry.file_name().to_str() == Some(RESERVED_DIR) {
continue;
}
let file_type = trace_try!(entry.file_type().await);
if file_type.is_dir() {
dir_queue.push_back(entry.path());
Expand Down Expand Up @@ -720,6 +771,14 @@ impl S3Storage for FileSystem {
code_error!(IncompleteBody,"You did not provide the number of bytes specified by the Content-Length HTTP header.")
})?;

if key_under_reserved(&key) {
let err = code_error!(
AccessDenied,
format!("the \"{RESERVED_DIR}/\" key prefix is reserved for internal use")
);
return Err(err.into());
}

if key.ends_with('/') {
if content_length == Some(0) {
let object_path = trace_try!(self.get_object_path(&bucket, &key));
Expand All @@ -739,6 +798,12 @@ impl S3Storage for FileSystem {
trace_try!(fs::create_dir_all(&dir_path).await);
}

// write metadata before the object, so a metadata failure leaves nothing
// stored (a clean error) rather than a stored object plus a 500
if let Some(ref metadata) = metadata {
trace_try!(self.save_metadata(&bucket, &key, metadata).await);
}

let mut md5_hash = Md5::new();
let stream = body.inspect_ok(|bytes| md5_hash.update(bytes.as_ref()));

Expand All @@ -757,10 +822,6 @@ impl S3Storage for FileSystem {
"PutObject: write file",
);

if let Some(ref metadata) = metadata {
trace_try!(self.save_metadata(&bucket, &key, metadata).await);
}

let output = PutObjectOutput {
e_tag: Some(format!("\"{}\"", md5_sum)),
..PutObjectOutput::default()
Expand Down Expand Up @@ -793,6 +854,7 @@ impl S3Storage for FileSystem {
) -> S3StorageResult<UploadPartOutput, UploadPartError> {
let UploadPartRequest {
body,
bucket,
upload_id,
part_number,
..
Expand All @@ -802,8 +864,10 @@ impl S3Storage for FileSystem {
code_error!(IncompleteBody, "You did not provide the number of bytes specified by the Content-Length HTTP header.")
})?;

let file_path_str = format!(".upload_id-{}.part-{}", upload_id, part_number);
let file_path = trace_try!(Path::new(&file_path_str).absolutize_virtually(&self.root));
let file_path = trace_try!(self.get_part_path(&bucket, &upload_id, part_number));
if let Some(parent) = file_path.parent() {
trace_try!(fs::create_dir_all(parent).await);
}

let mut md5_hash = Md5::new();
let stream = body.inspect_ok(|bytes| md5_hash.update(bytes.as_ref()));
Expand Down Expand Up @@ -875,8 +939,7 @@ impl S3Storage for FileSystem {
"InvalidPartOrder"
)));
}
let part_path_str = format!(".upload_id-{}.part-{}", upload_id, part_number);
let part_path = trace_try!(Path::new(&part_path_str).absolutize_virtually(&self.root));
let part_path = trace_try!(self.get_part_path(&bucket, &upload_id, part_number));

let mut reader = trace_try!(File::open(&part_path).await);
let (ret, duration) =
Expand Down
135 changes: 134 additions & 1 deletion tests/service/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use std::io;
use std::path::PathBuf;

use anyhow::{anyhow, Result};
use hyper::header::HeaderValue;
use hyper::header::{HeaderName, HeaderValue};
use hyper::{Body, Method, StatusCode};
use tracing::{debug_span, error};
use uuid::Uuid;
Expand Down Expand Up @@ -322,6 +322,91 @@ mod success {

Ok(())
}

#[tokio::test]
async fn put_object_metadata_in_reserved_dir() -> Result<()> {
let (root, service) = setup_service().unwrap();
let bucket = "asd";
let key = "dir/obj";
let content = "hi";
fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap();

let mut req = Request::new(Body::from(content));
*req.method_mut() = Method::PUT;
*req.uri_mut() = format!("http://localhost/{}/{}", bucket, key)
.parse()
.unwrap();
req.headers_mut().insert(
X_AMZ_CONTENT_SHA256,
HeaderValue::from_static("UNSIGNED-PAYLOAD"),
);
req.headers_mut().insert(
HeaderName::from_static("x-amz-meta-foo"),
HeaderValue::from_static("bar"),
);

assert_eq!(
service.hyper_call(req).await.unwrap().status(),
StatusCode::OK
);

// object stored at its real path
let obj = generate_path(&root, S3Path::Object { bucket, key });
assert_eq!(fs::read_to_string(obj).unwrap(), content);

// metadata sidecar lives in the bucket's reserved dir, nothing at the fs-root
let meta_dir = root.join(bucket).join(".wasmer-s3").join("meta");
assert_eq!(fs::read_dir(&meta_dir).unwrap().count(), 1);
let root_has_sidecar = fs::read_dir(&root)
.unwrap()
.filter_map(|e| e.ok())
.any(|e| e.file_name().to_string_lossy().contains(".metadata.json"));
assert!(!root_has_sidecar);

Ok(())
}

#[tokio::test]
async fn list_omits_reserved_dir() -> Result<()> {
let (root, service) = setup_service().unwrap();
let bucket = "asd";
fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap();

let mut put = Request::new(Body::from("x"));
*put.method_mut() = Method::PUT;
*put.uri_mut() = format!("http://localhost/{}/obj", bucket).parse().unwrap();
put.headers_mut().insert(
X_AMZ_CONTENT_SHA256,
HeaderValue::from_static("UNSIGNED-PAYLOAD"),
);
put.headers_mut().insert(
HeaderName::from_static("x-amz-meta-foo"),
HeaderValue::from_static("bar"),
);
assert_eq!(
service.hyper_call(put).await.unwrap().status(),
StatusCode::OK
);
assert!(root.join(bucket).join(".wasmer-s3").exists());

let mut req = Request::new(Body::empty());
*req.method_mut() = Method::GET;
*req.uri_mut() = format!("http://localhost/{}?list-type=2", bucket)
.parse()
.unwrap();
req.headers_mut().insert(
X_AMZ_CONTENT_SHA256,
HeaderValue::from_static("UNSIGNED-PAYLOAD"),
);

let mut res = service.hyper_call(req).await.unwrap();
let body = recv_body_string(&mut res).await.unwrap();
assert_eq!(res.status(), StatusCode::OK);
assert!(body.contains("obj"));
assert!(!body.contains(".wasmer-s3"));

Ok(())
}
}

mod error {
Expand Down Expand Up @@ -435,4 +520,52 @@ mod error {

Ok(())
}

#[tokio::test]
async fn put_reserved_key() -> Result<()> {
let (root, service) = setup_service().unwrap();
let bucket = "asd";
fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap();

let mut req = Request::new(Body::from("x"));
*req.method_mut() = Method::PUT;
*req.uri_mut() = format!("http://localhost/{}/.wasmer-s3/evil", bucket)
.parse()
.unwrap();
req.headers_mut().insert(
X_AMZ_CONTENT_SHA256,
HeaderValue::from_static("UNSIGNED-PAYLOAD"),
);

let mut res = service.hyper_call(req).await.unwrap();
let body = recv_body_string(&mut res).await.unwrap();

assert_eq!(res.status(), StatusCode::FORBIDDEN);
assert!(body.contains("AccessDenied"));
assert!(body.contains("reserved for internal use"));
assert!(!root.join(bucket).join(".wasmer-s3").join("evil").exists());

Ok(())
}

#[tokio::test]
async fn get_reserved_key() {
let (_, service) = setup_service().unwrap();

let mut req = Request::new(Body::empty());
*req.method_mut() = Method::GET;
*req.uri_mut() = "http://localhost/asd/.wasmer-s3/meta/x.json"
.parse()
.unwrap();
req.headers_mut().insert(
X_AMZ_CONTENT_SHA256,
HeaderValue::from_static("UNSIGNED-PAYLOAD"),
);

let mut res = service.hyper_call(req).await.unwrap();
let body = recv_body_string(&mut res).await.unwrap();

assert_eq!(res.status(), StatusCode::NOT_FOUND);
assert!(body.contains("NoSuchKey"));
}
}
Loading