diff --git a/src/storages/fs.rs b/src/storages/fs.rs index a3e4886..28cd844 100644 --- a/src/storages/fs.rs +++ b/src/storages/fs.rs @@ -47,6 +47,16 @@ pub struct FileSystem { root: PathBuf, } +/// Per-bucket dir for the server's own bookkeeping (object metadata, multipart +/// parts). Kept on the writable volume, not the maybe-read-only fs-root, and +/// reserved: user keys under it are refused and it is hidden from listings. +const RESERVED_DIR: &str = ".wasmer-s3"; + +/// Does a key fall under the reserved dir at the bucket root? +fn key_under_reserved(key: &str) -> bool { + key.trim_start_matches('/').split('/').next() == Some(RESERVED_DIR) +} + impl FileSystem { /// Constructs a file system storage located at `root` /// # Errors @@ -82,6 +92,12 @@ impl FileSystem { /// resolve object path under the virtual root fn get_object_path(&self, bucket: &str, key: &str) -> io::Result { + if key_under_reserved(key) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("keys under {RESERVED_DIR}/ are reserved"), + )); + } let dir = Path::new(&bucket); let file_path = Path::new(&key); let ans = dir.join(file_path).absolutize_virtually(&self.root)?.into(); @@ -95,18 +111,24 @@ impl FileSystem { Ok(ans) } - /// resolve metadata path under the virtual root (custom format) - fn get_metadata_path(&self, bucket: &str, key: &str) -> io::Result { - let encode = |s: &str| base64_simd::URL_SAFE_NO_PAD.encode_to_string(s); + /// Resolve `/RESERVED_DIR//` on the writable volume. + fn reserved_path(&self, bucket: &str, sub: &str, name: &str) -> io::Result { + let rel = Path::new(bucket).join(RESERVED_DIR).join(sub).join(name); + Ok(rel.absolutize_virtually(&self.root)?.into()) + } - let file_path_str = format!( - ".bucket-{}.object-{}.metadata.json", - encode(bucket), - encode(key), + /// Object metadata sidecar path (in the bucket's reserved dir). + fn get_metadata_path(&self, bucket: &str, key: &str) -> io::Result { + let name = format!( + "{}.json", + base64_simd::URL_SAFE_NO_PAD.encode_to_string(key) ); - let file_path = Path::new(&file_path_str); - let ans = file_path.absolutize_virtually(&self.root)?.into(); - Ok(ans) + self.reserved_path(bucket, "meta", &name) + } + + /// Multipart part path (in the bucket's reserved dir). + fn get_part_path(&self, bucket: &str, upload_id: &str, part: i64) -> io::Result { + self.reserved_path(bucket, "mpu", &format!("{upload_id}.part-{part}")) } /// load metadata from fs @@ -134,6 +156,9 @@ impl FileSystem { metadata: &HashMap, ) -> io::Result<()> { let path = self.get_metadata_path(bucket, key)?; + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).await?; + } let content = serde_json::to_vec(metadata) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; fs::write(&path, &content).await @@ -258,6 +283,9 @@ impl S3Storage for FileSystem { let src_metadata_path = trace_try!(self.get_metadata_path(bucket, key)); if src_metadata_path.exists() { let dst_metadata_path = trace_try!(self.get_metadata_path(&input.bucket, &input.key)); + if let Some(parent) = dst_metadata_path.parent() { + trace_try!(fs::create_dir_all(parent).await); + } let _ = trace_try!(fs::copy(src_metadata_path, dst_metadata_path).await); } @@ -290,6 +318,10 @@ impl S3Storage for FileSystem { &self, input: DeleteObjectRequest, ) -> S3StorageResult { + if key_under_reserved(&input.key) { + // reserved keys read as absent; delete is idempotent + return Ok(DeleteObjectOutput::default()); + } let path = trace_try!(self.get_object_path(&input.bucket, &input.key)); if input.key.ends_with('/') { let mut dir = trace_try!(fs::read_dir(&path).await); @@ -311,6 +343,9 @@ impl S3Storage for FileSystem { ) -> S3StorageResult { let mut objects: Vec<(PathBuf, String)> = Vec::new(); for object in input.delete.objects { + if key_under_reserved(&object.key) { + continue; + } let path = trace_try!(self.get_object_path(&input.bucket, &object.key)); if path.exists() { objects.push((path, object.key)); @@ -356,6 +391,10 @@ impl S3Storage for FileSystem { &self, input: GetObjectRequest, ) -> S3StorageResult { + if key_under_reserved(&input.key) { + let err = code_error!(NoSuchKey, "The specified key does not exist."); + return Err(err.into()); + } let object_path = trace_try!(self.get_object_path(&input.bucket, &input.key)); let parse_range = |s: &str| { @@ -465,6 +504,10 @@ impl S3Storage for FileSystem { &self, input: HeadObjectRequest, ) -> S3StorageResult { + if key_under_reserved(&input.key) { + let err = code_error!(NoSuchKey, "The specified key does not exist."); + return Err(err.into()); + } let path = trace_try!(self.get_object_path(&input.bucket, &input.key)); if !path.exists() { @@ -539,10 +582,14 @@ impl S3Storage for FileSystem { let delimiter_is_dir = input.delimiter.as_deref() == Some("/"); while let Some(dir) = dir_queue.pop_front() { - let mut entries = trace_try!(fs::read_dir(dir).await); + let mut entries = trace_try!(fs::read_dir(&dir).await); loop { let entry = trace_try!(entries.next_entry().await); if let Some(entry) = entry { + // hide the reserved bookkeeping dir at the bucket root + if dir == path && entry.file_name().to_str() == Some(RESERVED_DIR) { + continue; + } let file_type = trace_try!(entry.file_type().await); if file_type.is_dir() { if delimiter_is_dir { @@ -631,10 +678,14 @@ impl S3Storage for FileSystem { dir_queue.push_back(path.clone()); while let Some(dir) = dir_queue.pop_front() { - let mut entries = trace_try!(fs::read_dir(dir).await); + let mut entries = trace_try!(fs::read_dir(&dir).await); loop { let entry = trace_try!(entries.next_entry().await); if let Some(entry) = entry { + // hide the reserved bookkeeping dir at the bucket root + if dir == path && entry.file_name().to_str() == Some(RESERVED_DIR) { + continue; + } let file_type = trace_try!(entry.file_type().await); if file_type.is_dir() { dir_queue.push_back(entry.path()); @@ -720,6 +771,14 @@ impl S3Storage for FileSystem { code_error!(IncompleteBody,"You did not provide the number of bytes specified by the Content-Length HTTP header.") })?; + if key_under_reserved(&key) { + let err = code_error!( + AccessDenied, + format!("the \"{RESERVED_DIR}/\" key prefix is reserved for internal use") + ); + return Err(err.into()); + } + if key.ends_with('/') { if content_length == Some(0) { let object_path = trace_try!(self.get_object_path(&bucket, &key)); @@ -739,6 +798,12 @@ impl S3Storage for FileSystem { trace_try!(fs::create_dir_all(&dir_path).await); } + // write metadata before the object, so a metadata failure leaves nothing + // stored (a clean error) rather than a stored object plus a 500 + if let Some(ref metadata) = metadata { + trace_try!(self.save_metadata(&bucket, &key, metadata).await); + } + let mut md5_hash = Md5::new(); let stream = body.inspect_ok(|bytes| md5_hash.update(bytes.as_ref())); @@ -757,10 +822,6 @@ impl S3Storage for FileSystem { "PutObject: write file", ); - if let Some(ref metadata) = metadata { - trace_try!(self.save_metadata(&bucket, &key, metadata).await); - } - let output = PutObjectOutput { e_tag: Some(format!("\"{}\"", md5_sum)), ..PutObjectOutput::default() @@ -793,6 +854,7 @@ impl S3Storage for FileSystem { ) -> S3StorageResult { let UploadPartRequest { body, + bucket, upload_id, part_number, .. @@ -802,8 +864,10 @@ impl S3Storage for FileSystem { code_error!(IncompleteBody, "You did not provide the number of bytes specified by the Content-Length HTTP header.") })?; - let file_path_str = format!(".upload_id-{}.part-{}", upload_id, part_number); - let file_path = trace_try!(Path::new(&file_path_str).absolutize_virtually(&self.root)); + let file_path = trace_try!(self.get_part_path(&bucket, &upload_id, part_number)); + if let Some(parent) = file_path.parent() { + trace_try!(fs::create_dir_all(parent).await); + } let mut md5_hash = Md5::new(); let stream = body.inspect_ok(|bytes| md5_hash.update(bytes.as_ref())); @@ -875,8 +939,7 @@ impl S3Storage for FileSystem { "InvalidPartOrder" ))); } - let part_path_str = format!(".upload_id-{}.part-{}", upload_id, part_number); - let part_path = trace_try!(Path::new(&part_path_str).absolutize_virtually(&self.root)); + let part_path = trace_try!(self.get_part_path(&bucket, &upload_id, part_number)); let mut reader = trace_try!(File::open(&part_path).await); let (ret, duration) = diff --git a/tests/service/main.rs b/tests/service/main.rs index ef43a0e..9aa4fe4 100644 --- a/tests/service/main.rs +++ b/tests/service/main.rs @@ -15,7 +15,7 @@ use std::io; use std::path::PathBuf; use anyhow::{anyhow, Result}; -use hyper::header::HeaderValue; +use hyper::header::{HeaderName, HeaderValue}; use hyper::{Body, Method, StatusCode}; use tracing::{debug_span, error}; use uuid::Uuid; @@ -322,6 +322,91 @@ mod success { Ok(()) } + + #[tokio::test] + async fn put_object_metadata_in_reserved_dir() -> Result<()> { + let (root, service) = setup_service().unwrap(); + let bucket = "asd"; + let key = "dir/obj"; + let content = "hi"; + fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap(); + + let mut req = Request::new(Body::from(content)); + *req.method_mut() = Method::PUT; + *req.uri_mut() = format!("http://localhost/{}/{}", bucket, key) + .parse() + .unwrap(); + req.headers_mut().insert( + X_AMZ_CONTENT_SHA256, + HeaderValue::from_static("UNSIGNED-PAYLOAD"), + ); + req.headers_mut().insert( + HeaderName::from_static("x-amz-meta-foo"), + HeaderValue::from_static("bar"), + ); + + assert_eq!( + service.hyper_call(req).await.unwrap().status(), + StatusCode::OK + ); + + // object stored at its real path + let obj = generate_path(&root, S3Path::Object { bucket, key }); + assert_eq!(fs::read_to_string(obj).unwrap(), content); + + // metadata sidecar lives in the bucket's reserved dir, nothing at the fs-root + let meta_dir = root.join(bucket).join(".wasmer-s3").join("meta"); + assert_eq!(fs::read_dir(&meta_dir).unwrap().count(), 1); + let root_has_sidecar = fs::read_dir(&root) + .unwrap() + .filter_map(|e| e.ok()) + .any(|e| e.file_name().to_string_lossy().contains(".metadata.json")); + assert!(!root_has_sidecar); + + Ok(()) + } + + #[tokio::test] + async fn list_omits_reserved_dir() -> Result<()> { + let (root, service) = setup_service().unwrap(); + let bucket = "asd"; + fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap(); + + let mut put = Request::new(Body::from("x")); + *put.method_mut() = Method::PUT; + *put.uri_mut() = format!("http://localhost/{}/obj", bucket).parse().unwrap(); + put.headers_mut().insert( + X_AMZ_CONTENT_SHA256, + HeaderValue::from_static("UNSIGNED-PAYLOAD"), + ); + put.headers_mut().insert( + HeaderName::from_static("x-amz-meta-foo"), + HeaderValue::from_static("bar"), + ); + assert_eq!( + service.hyper_call(put).await.unwrap().status(), + StatusCode::OK + ); + assert!(root.join(bucket).join(".wasmer-s3").exists()); + + let mut req = Request::new(Body::empty()); + *req.method_mut() = Method::GET; + *req.uri_mut() = format!("http://localhost/{}?list-type=2", bucket) + .parse() + .unwrap(); + req.headers_mut().insert( + X_AMZ_CONTENT_SHA256, + HeaderValue::from_static("UNSIGNED-PAYLOAD"), + ); + + let mut res = service.hyper_call(req).await.unwrap(); + let body = recv_body_string(&mut res).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); + assert!(body.contains("obj")); + assert!(!body.contains(".wasmer-s3")); + + Ok(()) + } } mod error { @@ -435,4 +520,52 @@ mod error { Ok(()) } + + #[tokio::test] + async fn put_reserved_key() -> Result<()> { + let (root, service) = setup_service().unwrap(); + let bucket = "asd"; + fs::create_dir(generate_path(&root, S3Path::Bucket { bucket })).unwrap(); + + let mut req = Request::new(Body::from("x")); + *req.method_mut() = Method::PUT; + *req.uri_mut() = format!("http://localhost/{}/.wasmer-s3/evil", bucket) + .parse() + .unwrap(); + req.headers_mut().insert( + X_AMZ_CONTENT_SHA256, + HeaderValue::from_static("UNSIGNED-PAYLOAD"), + ); + + let mut res = service.hyper_call(req).await.unwrap(); + let body = recv_body_string(&mut res).await.unwrap(); + + assert_eq!(res.status(), StatusCode::FORBIDDEN); + assert!(body.contains("AccessDenied")); + assert!(body.contains("reserved for internal use")); + assert!(!root.join(bucket).join(".wasmer-s3").join("evil").exists()); + + Ok(()) + } + + #[tokio::test] + async fn get_reserved_key() { + let (_, service) = setup_service().unwrap(); + + let mut req = Request::new(Body::empty()); + *req.method_mut() = Method::GET; + *req.uri_mut() = "http://localhost/asd/.wasmer-s3/meta/x.json" + .parse() + .unwrap(); + req.headers_mut().insert( + X_AMZ_CONTENT_SHA256, + HeaderValue::from_static("UNSIGNED-PAYLOAD"), + ); + + let mut res = service.hyper_call(req).await.unwrap(); + let body = recv_body_string(&mut res).await.unwrap(); + + assert_eq!(res.status(), StatusCode::NOT_FOUND); + assert!(body.contains("NoSuchKey")); + } }