From e03272975354f5c01abc476af6d1863735ce5024 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:32:42 +0100 Subject: [PATCH 1/9] Remove feature from `resource` --- Cargo.lock | 1 + resource/Cargo.toml | 1 + resource/src/lib.rs | 1 - resource/src/uri.rs | 73 +++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63f233d..82dbac0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4002,6 +4002,7 @@ dependencies = [ "directories", "flate2", "futures", + "http-body", "indicatif", "log", "pin-project", diff --git a/resource/Cargo.toml b/resource/Cargo.toml index ef218ad..a1a2ea6 100644 --- a/resource/Cargo.toml +++ b/resource/Cargo.toml @@ -28,6 +28,7 @@ tokio-util = { version = "0.7", features = ["io", "compat"] } url = { version = "2", features = ["serde"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +http-body = "1" suppaftp = { version = "6", features = ["async"] } tokio = { version = "1", features = ["fs", "process"] } diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 197d6e8..8181d70 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(impl_trait_in_assoc_type)] #![expect(async_fn_in_trait)] // TODO pub mod buffered; diff --git a/resource/src/uri.rs b/resource/src/uri.rs index 7aed6e0..14e41e2 100644 --- a/resource/src/uri.rs +++ b/resource/src/uri.rs @@ -1,7 +1,12 @@ -use std::{fmt, sync::LazyLock}; +use std::{ + fmt, + pin::Pin, + sync::LazyLock, + task::{Context, Poll}, +}; use bytes::Bytes; -use futures::{Stream, TryStreamExt}; +use futures::Stream; use reqwest::IntoUrl; use url::Url; @@ -142,8 +147,7 @@ impl RawResource for UrlResource { panic!("UrlResource::read is not supported on wasm32, use the non-blocking version."); } - type AsyncReader = - tokio_util::io::StreamReader>, Bytes>; + type AsyncReader = tokio_util::io::StreamReader; async fn size_async(&self) -> std::io::Result { static CLIENT: LazyLock = LazyLock::new(reqwest::Client::new); let response = CLIENT @@ -177,8 +181,65 @@ impl RawResource for UrlResource { ))); } - let stream = response.bytes_stream().map_err(std::io::Error::other); - Ok(tokio_util::io::StreamReader::new(stream)) + Ok(tokio_util::io::StreamReader::new(UrlByteStream::new( + response, + ))) + } +} + +/// Mirrors `reqwest::Response::bytes_stream`, which returns an opaque unnameable type. +pub struct UrlByteStream { + #[cfg(not(target_arch = "wasm32"))] + body: reqwest::Body, + #[cfg(target_arch = "wasm32")] + stream: Pin>>>, +} +impl UrlByteStream { + fn new(response: reqwest::Response) -> Self { + #[cfg(not(target_arch = "wasm32"))] + { + Self { + body: response.into(), + } + } + #[cfg(target_arch = "wasm32")] + { + use futures::{StreamExt, TryStreamExt}; + Self { + stream: response.bytes_stream().map_err(reqwest_error).boxed_local(), + } + } + } +} +impl Stream for UrlByteStream { + type Item = std::io::Result; + + #[cfg(not(target_arch = "wasm32"))] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + use http_body::Body; + use std::task::ready; + + // Impl taken from `reqwest::async_impl::body::DataStream`. + + loop { + return match ready!(Pin::new(&mut self.body).poll_frame(cx)) { + Some(Ok(frame)) => { + // skip non-data frames + if let Ok(buf) = frame.into_data() { + Poll::Ready(Some(Ok(buf))) + } else { + continue; + } + } + Some(Err(err)) => Poll::Ready(Some(Err(reqwest_error(err)))), + None => Poll::Ready(None), + }; + } + } + + #[cfg(target_arch = "wasm32")] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_next(cx) } } From a596315ec092ae0f13a664efcf23bc9ee415ec88 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:35:59 +0100 Subject: [PATCH 2/9] `RawResource` -> `Resource` --- ensembl/src/contig/grch37_meta.rs | 2 +- ensembl/src/contig/grch38_meta.rs | 2 +- ensembl/src/resource.rs | 8 ++++---- genomes1000/src/contig/grch37_meta.rs | 2 +- genomes1000/src/contig/grch38_meta.rs | 2 +- genomes1000/src/lib.rs | 16 ++++++++-------- genomes1000/src/slow.rs | 2 +- genomes1000/src/source.rs | 8 ++++---- gwas_catalog/src/lib.rs | 8 ++++---- hail/src/contig/grch37_meta.rs | 2 +- hail/src/contig/grch38_meta.rs | 2 +- hail/src/lib.rs | 2 +- hail/src/source.rs | 8 ++++---- liftover/src/parse.rs | 6 +++--- liftover/src/sources.rs | 14 +++++++------- liftover/tests/fuzz.rs | 2 +- liftover/tests/fuzz/internal.rs | 2 +- liftover/tests/fuzz/testpoints.rs | 2 +- liftover/tests/fuzz/ucsc.rs | 2 +- pan_ukbb/src/lib.rs | 14 +++++++------- pgs_catalog/src/lib.rs | 16 ++++++++-------- pgs_catalog/src/metadata.rs | 2 +- resource/src/buffered.rs | 6 +++--- resource/src/cached.rs | 18 +++++++++--------- resource/src/compression.rs | 10 +++++----- resource/src/fs.rs | 4 ++-- resource/src/iter.rs | 4 ++-- resource/src/lib.rs | 10 +++++----- resource/src/progress.rs | 6 +++--- resource/src/uri.rs | 4 ++-- 30 files changed, 93 insertions(+), 93 deletions(-) diff --git a/ensembl/src/contig/grch37_meta.rs b/ensembl/src/contig/grch37_meta.rs index f025c7f..6f56e8e 100755 --- a/ensembl/src/contig/grch37_meta.rs +++ b/ensembl/src/contig/grch37_meta.rs @@ -94,7 +94,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{RawResourceExt, Resource}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/contig/grch38_meta.rs b/ensembl/src/contig/grch38_meta.rs index b45a8c8..df23e8e 100755 --- a/ensembl/src/contig/grch38_meta.rs +++ b/ensembl/src/contig/grch38_meta.rs @@ -715,7 +715,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{Resource, RawResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/resource.rs b/ensembl/src/resource.rs index fa4db5d..8cc4fff 100644 --- a/ensembl/src/resource.rs +++ b/ensembl/src/resource.rs @@ -1,5 +1,5 @@ +use resource::{Resource, UrlResource}; use url::Url; -use resource::{RawResource, UrlResource}; const GRCH38_REFERENCE_GENOME_INDEXED: &str = "fasta/homo_sapiens/dna_index/Homo_sapiens.GRCh38.dna.toplevel.fa.gz"; @@ -106,7 +106,7 @@ impl EnsemblResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for EnsemblResource { +impl Resource for EnsemblResource { const NAMESPACE: &'static str = "ensembl"; fn key(&self) -> String { @@ -117,7 +117,7 @@ impl RawResource for EnsemblResource { resource::Compression::infer(&self.key) } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -125,7 +125,7 @@ impl RawResource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/genomes1000/src/contig/grch37_meta.rs b/genomes1000/src/contig/grch37_meta.rs index b779559..7c5c536 100755 --- a/genomes1000/src/contig/grch37_meta.rs +++ b/genomes1000/src/contig/grch37_meta.rs @@ -101,7 +101,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{Resource, RawResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/contig/grch38_meta.rs b/genomes1000/src/contig/grch38_meta.rs index 5b20ba6..0a2f8b5 100755 --- a/genomes1000/src/contig/grch38_meta.rs +++ b/genomes1000/src/contig/grch38_meta.rs @@ -3388,7 +3388,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{RawResourceExt, Resource}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/lib.rs b/genomes1000/src/lib.rs index a619946..0ed1c25 100755 --- a/genomes1000/src/lib.rs +++ b/genomes1000/src/lib.rs @@ -19,7 +19,7 @@ use biocore::{ location::{ContigPosition, ContigRange}, vcf::IndexedVcfReader, }; -use resource::{RawResource, RawResourceExt, fs::FsCache}; +use resource::{RawResourceExt, Resource, fs::FsCache}; use utile::{io::FromUtf8Bytes, iter::IteratorExt}; use self::{pedigree::Pedigree, simplified::SimplifiedRecord, source::Genomes1000Resource}; @@ -435,7 +435,7 @@ pub async fn load_contig( parse::parse(resource.read()?, sample_reading_function(c)) } -pub async fn load_pedigree(resource: impl RawResource) -> io::Result> { +pub async fn load_pedigree(resource: impl Resource) -> io::Result> { Ok(csv::ReaderBuilder::new() .delimiter(b' ') .from_reader(resource.read()?) @@ -458,11 +458,11 @@ pub async fn load_pedigree(resource: impl RawResource) -> io::Result( fasta: F, - index: impl RawResource, + index: impl Resource, ) -> io::Result> where - F: RawResource, - ::Reader: std::io::BufRead, + F: Resource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } @@ -471,11 +471,11 @@ where /// It should also implement [Seek](std::io::Seek) if random access is needed. pub async fn load_grch37_reference_genome( fasta: F, - index: impl RawResource, + index: impl Resource, ) -> io::Result> where - F: RawResource, - ::Reader: std::io::BufRead, + F: Resource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } diff --git a/genomes1000/src/slow.rs b/genomes1000/src/slow.rs index 8ca4d67..e54df57 100755 --- a/genomes1000/src/slow.rs +++ b/genomes1000/src/slow.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, RawResourceExt}; +use resource::{RawResourceExt, Resource}; use std::{collections::HashMap, io::Read}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/source.rs b/genomes1000/src/source.rs index a588573..b0ee4ed 100755 --- a/genomes1000/src/source.rs +++ b/genomes1000/src/source.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, UrlResource}; +use resource::{Resource, UrlResource}; use url::Url; use crate::contig::GRCh38Contig; @@ -158,7 +158,7 @@ impl Genomes1000Resource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for Genomes1000Resource { +impl Resource for Genomes1000Resource { const NAMESPACE: &'static str = "1000genomes"; fn key(&self) -> String { @@ -173,7 +173,7 @@ impl RawResource for Genomes1000Resource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -181,7 +181,7 @@ impl RawResource for Genomes1000Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/gwas_catalog/src/lib.rs b/gwas_catalog/src/lib.rs index 963477b..b772064 100644 --- a/gwas_catalog/src/lib.rs +++ b/gwas_catalog/src/lib.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use biocore::location::ContigPosition; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{Resource, RawResourceExt, UrlResource}; use utile::io::reqwest_error; #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -39,7 +39,7 @@ impl GwasCatalogResource { Self::ANCESTRY_URL.parse().unwrap() } } -impl RawResource for GwasCatalogResource { +impl Resource for GwasCatalogResource { const NAMESPACE: &'static str = "gwas_catalog"; fn key(&self) -> String { @@ -50,7 +50,7 @@ impl RawResource for GwasCatalogResource { None } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { Ok(self.size) } @@ -59,7 +59,7 @@ impl RawResource for GwasCatalogResource { UrlResource::new(self.url).unwrap().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { Ok(self.size) } diff --git a/hail/src/contig/grch37_meta.rs b/hail/src/contig/grch37_meta.rs index 9ab7232..7795f65 100755 --- a/hail/src/contig/grch37_meta.rs +++ b/hail/src/contig/grch37_meta.rs @@ -93,7 +93,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{RawResourceExt, Resource}; use crate::source::HailCommonResource; diff --git a/hail/src/contig/grch38_meta.rs b/hail/src/contig/grch38_meta.rs index 35cfc2a..b5a958b 100755 --- a/hail/src/contig/grch38_meta.rs +++ b/hail/src/contig/grch38_meta.rs @@ -3375,7 +3375,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{RawResourceExt, Resource}; use crate::source::HailCommonResource; diff --git a/hail/src/lib.rs b/hail/src/lib.rs index 42dd9d6..18a6801 100644 --- a/hail/src/lib.rs +++ b/hail/src/lib.rs @@ -3,7 +3,7 @@ pub mod contig; pub mod source; -use resource::{RawResource, RawResourceExt}; +use resource::{RawResourceExt, Resource}; use source::HailCommonResource; pub async fn load_grch38_reference_genome() diff --git a/hail/src/source.rs b/hail/src/source.rs index b92074b..5432681 100644 --- a/hail/src/source.rs +++ b/hail/src/source.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, UrlResource}; +use resource::{Resource, UrlResource}; use url::Url; const HAIL_COMMON_BUCKET: &str = "hail-common"; @@ -44,7 +44,7 @@ impl HailCommonResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for HailCommonResource { +impl Resource for HailCommonResource { const NAMESPACE: &'static str = "hail_common"; fn key(&self) -> String { @@ -64,7 +64,7 @@ impl RawResource for HailCommonResource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -72,7 +72,7 @@ impl RawResource for HailCommonResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/src/parse.rs b/liftover/src/parse.rs index 86a3b0e..f8e2ff2 100755 --- a/liftover/src/parse.rs +++ b/liftover/src/parse.rs @@ -8,7 +8,7 @@ use std::{ }; use biocore::{genome::ArcContig, location::orientation::Stranded}; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, RawResourceExt}; use utile::io::read_ext::AsyncReadInto; use super::{ @@ -16,10 +16,10 @@ use super::{ }; impl Liftover { - pub fn load(resource: impl RawResource) -> anyhow::Result { + pub fn load(resource: impl Resource) -> anyhow::Result { Ok(Self::read(resource.decompressed().buffered().read()?)?) } - pub async fn load_async(resource: impl RawResource) -> anyhow::Result { + pub async fn load_async(resource: impl Resource) -> anyhow::Result { Ok(Self::read( &*resource .decompressed() diff --git a/liftover/src/sources.rs b/liftover/src/sources.rs index 4d38e73..93f7a64 100755 --- a/liftover/src/sources.rs +++ b/liftover/src/sources.rs @@ -1,6 +1,6 @@ use url::Url; -use resource::{Compression, RawResource, UrlResource}; +use resource::{Compression, Resource, UrlResource}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct EnsemblResource { @@ -42,7 +42,7 @@ impl EnsemblResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for EnsemblResource { +impl Resource for EnsemblResource { const NAMESPACE: &'static str = "ensembl"; fn key(&self) -> String { self.key.clone() @@ -56,7 +56,7 @@ impl RawResource for EnsemblResource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -64,7 +64,7 @@ impl RawResource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -131,7 +131,7 @@ impl UcscResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for UcscResource { +impl Resource for UcscResource { const NAMESPACE: &'static str = "ucsc"; fn key(&self) -> String { self.key.clone() @@ -145,7 +145,7 @@ impl RawResource for UcscResource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -153,7 +153,7 @@ impl RawResource for UcscResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/tests/fuzz.rs b/liftover/tests/fuzz.rs index 4e1effa..df4880b 100755 --- a/liftover/tests/fuzz.rs +++ b/liftover/tests/fuzz.rs @@ -10,7 +10,7 @@ mod ucsc; use std::path::PathBuf; use biocore::location::ContigRange; -use resource::{RawResource, RawResourceExt, fs::FsCache}; +use resource::{RawResourceExt, Resource, fs::FsCache}; use liftover::{ Liftover, diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index 942210c..5ccb259 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -4,7 +4,7 @@ use liftover::{ Liftover, LiftoverIndexed, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, RawResourceExt}; #[ignore] #[test] diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index 27f03b3..6b4e206 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -5,7 +5,7 @@ use biocore::{ genome::Contig, location::{ContigPosition, ContigRange}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{RawResourceExt, Resource}; use liftover::{ Chain, Liftover, diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index 515057b..f2d0650 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -7,7 +7,7 @@ use liftover::{ bindings::{self, ucsc::UcscLiftoverSettings}, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, RawResourceExt}; #[ignore] #[tokio::test] diff --git a/pan_ukbb/src/lib.rs b/pan_ukbb/src/lib.rs index 2cdf855..9df80e3 100755 --- a/pan_ukbb/src/lib.rs +++ b/pan_ukbb/src/lib.rs @@ -11,7 +11,7 @@ use ordered_float::NotNan; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{RawResourceExt, Resource, UrlResource}; const URL_BASE: &str = "https://pan-ukb-us-east-1.s3.amazonaws.com"; const PHENOTYPE_MANIFEST_KEY: &str = "sumstats_release/phenotype_manifest.tsv.bgz"; @@ -36,7 +36,7 @@ impl PanUKBBS3Resource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for PanUKBBS3Resource { +impl Resource for PanUKBBS3Resource { const NAMESPACE: &'static str = "pan_ukbb"; fn key(&self) -> String { @@ -51,7 +51,7 @@ impl RawResource for PanUKBBS3Resource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -59,7 +59,7 @@ impl RawResource for PanUKBBS3Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -407,7 +407,7 @@ impl PhenotypeManifestEntry { Self::load(resource) } - pub fn load(resource: impl RawResource) -> csv::Result> { + pub fn load(resource: impl Resource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -415,7 +415,7 @@ impl PhenotypeManifestEntry { .into_deserialize() .try_collect() } - pub async fn load_async(resource: impl RawResource) -> csv::Result> { + pub async fn load_async(resource: impl Resource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -607,7 +607,7 @@ pub struct SummaryStats { pub low_confidence_MID: Option, } impl SummaryStats { - pub fn load(resource: impl RawResource) -> io::Result>> + pub fn load(resource: impl Resource) -> io::Result>> where Contig: DeserializeOwned, { diff --git a/pgs_catalog/src/lib.rs b/pgs_catalog/src/lib.rs index 228738b..1fbf8a9 100755 --- a/pgs_catalog/src/lib.rs +++ b/pgs_catalog/src/lib.rs @@ -11,7 +11,7 @@ use simplified::SimplificationError; use url::Url; use biocore::dna::DnaSequence; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{Resource, RawResourceExt, UrlResource}; pub use ids::{pgs::PgsId, rs::RsId}; @@ -46,7 +46,7 @@ impl PgsCatalogResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for PgsCatalogResource { +impl Resource for PgsCatalogResource { const NAMESPACE: &'static str = "pgs_catalog"; fn key(&self) -> String { @@ -62,7 +62,7 @@ impl RawResource for PgsCatalogResource { } } - type Reader = ::Reader; + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -70,7 +70,7 @@ impl RawResource for PgsCatalogResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -566,8 +566,8 @@ impl Study { resource: R, ) -> io::Result>> where - R: RawResource, - ::Reader: io::BufRead, + R: Resource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; @@ -594,8 +594,8 @@ impl HarmonizedStudy { resource: R, ) -> io::Result>> where - R: RawResource, - ::Reader: io::BufRead, + R: Resource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; diff --git a/pgs_catalog/src/metadata.rs b/pgs_catalog/src/metadata.rs index 3aadc33..00f9a13 100644 --- a/pgs_catalog/src/metadata.rs +++ b/pgs_catalog/src/metadata.rs @@ -11,7 +11,7 @@ use ids::{ pgs::{PgsId, pgp::PgpId, ppm::PpmId, pss::PssId}, pubmed::PubmedId, }; -use resource::{RawResource, RawResourceExt}; +use resource::{RawResourceExt, Resource}; use url::Url; use crate::{PgsCatalogResource, WeightType}; diff --git a/resource/src/buffered.rs b/resource/src/buffered.rs index f10ce44..b8014a3 100644 --- a/resource/src/buffered.rs +++ b/resource/src/buffered.rs @@ -1,15 +1,15 @@ -use super::{Compression, RawResource}; +use super::{Compression, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BufferedResource { resource: R, } -impl BufferedResource { +impl BufferedResource { pub fn new(resource: R) -> Self { Self { resource } } } -impl RawResource for BufferedResource { +impl Resource for BufferedResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) diff --git a/resource/src/cached.rs b/resource/src/cached.rs index 9265c9b..bf9951b 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -1,7 +1,7 @@ use std::{fmt, path::PathBuf}; use crate::{ - Compression, RawResource, RawResourceExt, ResourceRef, + Compression, Resource, RawResourceExt, ResourceRef, fs::{FsCache, FsCacheEntry}, }; @@ -18,7 +18,7 @@ impl fmt::Display for FsCacheResource { impl FsCacheResource { pub fn new(cache: &FsCache, resource: R) -> Self where - R: RawResource, + R: Resource, { Self { entry: FsCacheEntry::new(cache, PathBuf::from(R::NAMESPACE).join(resource.key())), @@ -40,7 +40,7 @@ impl FsCacheResource { pub fn ensure_cached(self) -> std::io::Result where - R: RawResource, + R: Resource, { self.cache()?; Ok(self) @@ -48,7 +48,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: RawResource, + R: Resource, { self.cache_async().await?; Ok(self) @@ -56,14 +56,14 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: RawResource, + R: Resource, { panic!("FsCacheResource is not supported on wasm32"); } pub fn cache(&self) -> std::io::Result where - R: RawResource, + R: Resource, { if !self.try_exists()? { self.read()?; @@ -73,7 +73,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: RawResource, + R: Resource, { if !self.try_exists_async().await? { self.read_async().await?; @@ -83,7 +83,7 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: RawResource, + R: Resource, { panic!("FsCacheResource is not supported on wasm32"); } @@ -100,7 +100,7 @@ impl FsCacheResource { panic!("FsCacheResource is not supported on wasm32"); } } -impl RawResource for FsCacheResource { +impl Resource for FsCacheResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) diff --git a/resource/src/compression.rs b/resource/src/compression.rs index 5c96324..aa062cc 100644 --- a/resource/src/compression.rs +++ b/resource/src/compression.rs @@ -7,14 +7,14 @@ use std::{ use pin_project::pin_project; -use super::{Compression, RawResource, RawResourceExt, ResourceRef}; +use super::{Compression, RawResourceExt, Resource, ResourceRef}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DecompressedResource { resource: R, compression: Option, } -impl DecompressedResource { +impl DecompressedResource { pub fn new(resource: R) -> Self { Self { resource, @@ -28,7 +28,7 @@ impl DecompressedResource { } } } -impl RawResource for DecompressedResource { +impl Resource for DecompressedResource { const NAMESPACE: &'static str = "decompressed"; fn key(&self) -> String { let key = self.resource.key(); @@ -149,7 +149,7 @@ pub struct CompressedResource { resource: R, compression: Compression, } -impl CompressedResource { +impl CompressedResource { pub fn new(resource: R, compression: Compression) -> Self { Self { resource, @@ -157,7 +157,7 @@ impl CompressedResource { } } } -impl RawResource for CompressedResource { +impl Resource for CompressedResource { const NAMESPACE: &'static str = "compressed"; fn key(&self) -> String { format!( diff --git a/resource/src/fs.rs b/resource/src/fs.rs index 1beb850..7fa4a14 100644 --- a/resource/src/fs.rs +++ b/resource/src/fs.rs @@ -8,7 +8,7 @@ use directories::ProjectDirs; use utile::io::not_found_error; -use crate::RawResource; +use crate::Resource; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct FsCache { @@ -143,7 +143,7 @@ impl FsCacheEntry { tokio::fs::remove_file(&self).await } } -impl RawResource for FsCacheEntry { +impl Resource for FsCacheEntry { const NAMESPACE: &'static str = "fs_cache"; fn key(&self) -> String { self.path.to_string_lossy().as_ref().to_owned() diff --git a/resource/src/iter.rs b/resource/src/iter.rs index 5b1d22d..40c5797 100644 --- a/resource/src/iter.rs +++ b/resource/src/iter.rs @@ -1,6 +1,6 @@ use utile::jsonl::JsonLinesReader; -use super::RawResource; +use super::Resource; pub struct IterToJsonLinesResource { key: String, @@ -18,7 +18,7 @@ where } } } -impl RawResource for IterToJsonLinesResource +impl Resource for IterToJsonLinesResource where I: Iterator + Clone, T: serde::Serialize, diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 8181d70..5283ad5 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -33,7 +33,7 @@ pub use self::cached::FsCacheResource; type JsonStreamDeserializer = StreamDeserializer<'static, serde_json::de::IoRead>, T>; -pub trait RawResource { +pub trait Resource { const NAMESPACE: &'static str; fn key(&self) -> String; @@ -47,7 +47,7 @@ pub trait RawResource { async fn size_async(&self) -> io::Result; async fn read_async(&self) -> io::Result; } -pub trait RawResourceExt: RawResource + Sized { +pub trait RawResourceExt: Resource + Sized { fn buffered(self) -> BufferedResource { BufferedResource::new(self) } @@ -121,7 +121,7 @@ pub trait RawResourceExt: RawResource + Sized { Ok(stream::try_unfold((), |()| async move { todo!() })) } } -impl RawResourceExt for T {} +impl RawResourceExt for T {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Compression { @@ -177,12 +177,12 @@ impl Compression { struct ResourceRef<'a, R> { resource: &'a R, } -impl<'a, R: RawResource> ResourceRef<'a, R> { +impl<'a, R: Resource> ResourceRef<'a, R> { pub fn new(resource: &'a R) -> Self { Self { resource } } } -impl<'a, R: RawResource> RawResource for ResourceRef<'a, R> { +impl<'a, R: Resource> Resource for ResourceRef<'a, R> { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(self.resource) diff --git a/resource/src/progress.rs b/resource/src/progress.rs index 22470e6..1d10285 100644 --- a/resource/src/progress.rs +++ b/resource/src/progress.rs @@ -2,7 +2,7 @@ use std::pin::Pin; use indicatif::ProgressStyle; -use super::{Compression, RawResource}; +use super::{Compression, Resource}; const PROGRESS_BAR_STYLE: &str = "{spinner} {bytes} ({percent}%) of {total_bytes} | {bytes_per_sec} {wide_bar} {eta}"; @@ -11,12 +11,12 @@ const PROGRESS_BAR_STYLE: &str = pub struct ProgressResource { resource: R, } -impl ProgressResource { +impl ProgressResource { pub fn new(resource: R) -> Self { Self { resource } } } -impl RawResource for ProgressResource { +impl Resource for ProgressResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) diff --git a/resource/src/uri.rs b/resource/src/uri.rs index 14e41e2..b92cf3e 100644 --- a/resource/src/uri.rs +++ b/resource/src/uri.rs @@ -12,7 +12,7 @@ use url::Url; use utile::io::{get_filesize_from_headers, reqwest_error}; -use super::{Compression, RawResource}; +use super::{Compression, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct UrlResource(Url); @@ -83,7 +83,7 @@ impl UrlResource { unreachable!() } } -impl RawResource for UrlResource { +impl Resource for UrlResource { const NAMESPACE: &'static str = "url"; fn key(&self) -> String { self.0.to_string() From 5dea85709c808519ac75c930336bf79304558c39 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:36:28 +0100 Subject: [PATCH 3/9] `RawResourceExt` -> `ResourceExt` --- ensembl/src/contig/grch37_meta.rs | 2 +- ensembl/src/contig/grch38_meta.rs | 2 +- genomes1000/examples/load_all.rs | 2 +- genomes1000/src/contig/grch37_meta.rs | 2 +- genomes1000/src/contig/grch38_meta.rs | 2 +- genomes1000/src/lib.rs | 2 +- genomes1000/src/slow.rs | 2 +- gwas_catalog/src/lib.rs | 2 +- hail/src/contig/grch37_meta.rs | 2 +- hail/src/contig/grch38_meta.rs | 2 +- hail/src/lib.rs | 2 +- liftover/examples/load_all.rs | 2 +- liftover/src/parse.rs | 2 +- liftover/tests/fuzz.rs | 2 +- liftover/tests/fuzz/internal.rs | 4 ++-- liftover/tests/fuzz/testpoints.rs | 4 ++-- liftover/tests/fuzz/ucsc.rs | 4 ++-- pan_ukbb/src/lib.rs | 2 +- pgs_catalog/src/lib.rs | 2 +- pgs_catalog/src/metadata.rs | 2 +- resource/src/cached.rs | 2 +- resource/src/compression.rs | 2 +- resource/src/lib.rs | 4 ++-- 23 files changed, 27 insertions(+), 27 deletions(-) diff --git a/ensembl/src/contig/grch37_meta.rs b/ensembl/src/contig/grch37_meta.rs index 6f56e8e..cbfd7fb 100755 --- a/ensembl/src/contig/grch37_meta.rs +++ b/ensembl/src/contig/grch37_meta.rs @@ -94,7 +94,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResourceExt, Resource}; + use resource::{Resource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/contig/grch38_meta.rs b/ensembl/src/contig/grch38_meta.rs index df23e8e..f6bb334 100755 --- a/ensembl/src/contig/grch38_meta.rs +++ b/ensembl/src/contig/grch38_meta.rs @@ -715,7 +715,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, RawResourceExt}; + use resource::{Resource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/genomes1000/examples/load_all.rs b/genomes1000/examples/load_all.rs index 0f150c1..067c5ff 100755 --- a/genomes1000/examples/load_all.rs +++ b/genomes1000/examples/load_all.rs @@ -1,4 +1,4 @@ -use resource::RawResourceExt; +use resource::ResourceExt; use genomes1000::{GRCh38Contig, load_grch38_reference_genome, source::Genomes1000Resource}; diff --git a/genomes1000/src/contig/grch37_meta.rs b/genomes1000/src/contig/grch37_meta.rs index 7c5c536..59c504b 100755 --- a/genomes1000/src/contig/grch37_meta.rs +++ b/genomes1000/src/contig/grch37_meta.rs @@ -101,7 +101,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, RawResourceExt}; + use resource::{Resource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/contig/grch38_meta.rs b/genomes1000/src/contig/grch38_meta.rs index 0a2f8b5..321ea1f 100755 --- a/genomes1000/src/contig/grch38_meta.rs +++ b/genomes1000/src/contig/grch38_meta.rs @@ -3388,7 +3388,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResourceExt, Resource}; + use resource::{Resource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/lib.rs b/genomes1000/src/lib.rs index 0ed1c25..c568074 100755 --- a/genomes1000/src/lib.rs +++ b/genomes1000/src/lib.rs @@ -19,7 +19,7 @@ use biocore::{ location::{ContigPosition, ContigRange}, vcf::IndexedVcfReader, }; -use resource::{RawResourceExt, Resource, fs::FsCache}; +use resource::{ResourceExt, Resource, fs::FsCache}; use utile::{io::FromUtf8Bytes, iter::IteratorExt}; use self::{pedigree::Pedigree, simplified::SimplifiedRecord, source::Genomes1000Resource}; diff --git a/genomes1000/src/slow.rs b/genomes1000/src/slow.rs index e54df57..3b6cf62 100755 --- a/genomes1000/src/slow.rs +++ b/genomes1000/src/slow.rs @@ -1,4 +1,4 @@ -use resource::{RawResourceExt, Resource}; +use resource::{ResourceExt, Resource}; use std::{collections::HashMap, io::Read}; use crate::source::Genomes1000Resource; diff --git a/gwas_catalog/src/lib.rs b/gwas_catalog/src/lib.rs index b772064..70fa8f1 100644 --- a/gwas_catalog/src/lib.rs +++ b/gwas_catalog/src/lib.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use biocore::location::ContigPosition; -use resource::{Resource, RawResourceExt, UrlResource}; +use resource::{Resource, ResourceExt, UrlResource}; use utile::io::reqwest_error; #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/hail/src/contig/grch37_meta.rs b/hail/src/contig/grch37_meta.rs index 7795f65..2f3b2d6 100755 --- a/hail/src/contig/grch37_meta.rs +++ b/hail/src/contig/grch37_meta.rs @@ -93,7 +93,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResourceExt, Resource}; + use resource::{ResourceExt, Resource}; use crate::source::HailCommonResource; diff --git a/hail/src/contig/grch38_meta.rs b/hail/src/contig/grch38_meta.rs index b5a958b..09412f3 100755 --- a/hail/src/contig/grch38_meta.rs +++ b/hail/src/contig/grch38_meta.rs @@ -3375,7 +3375,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResourceExt, Resource}; + use resource::{Resource, ResourceExt}; use crate::source::HailCommonResource; diff --git a/hail/src/lib.rs b/hail/src/lib.rs index 18a6801..6872f9b 100644 --- a/hail/src/lib.rs +++ b/hail/src/lib.rs @@ -3,7 +3,7 @@ pub mod contig; pub mod source; -use resource::{RawResourceExt, Resource}; +use resource::{ResourceExt, Resource}; use source::HailCommonResource; pub async fn load_grch38_reference_genome() diff --git a/liftover/examples/load_all.rs b/liftover/examples/load_all.rs index c428618..5d84d21 100755 --- a/liftover/examples/load_all.rs +++ b/liftover/examples/load_all.rs @@ -2,7 +2,7 @@ use liftover::{ Liftover, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::RawResourceExt; +use resource::ResourceExt; #[tokio::main] async fn main() -> anyhow::Result<()> { diff --git a/liftover/src/parse.rs b/liftover/src/parse.rs index f8e2ff2..ce1784a 100755 --- a/liftover/src/parse.rs +++ b/liftover/src/parse.rs @@ -8,7 +8,7 @@ use std::{ }; use biocore::{genome::ArcContig, location::orientation::Stranded}; -use resource::{Resource, RawResourceExt}; +use resource::{Resource, ResourceExt}; use utile::io::read_ext::AsyncReadInto; use super::{ diff --git a/liftover/tests/fuzz.rs b/liftover/tests/fuzz.rs index df4880b..85e69b9 100755 --- a/liftover/tests/fuzz.rs +++ b/liftover/tests/fuzz.rs @@ -10,7 +10,7 @@ mod ucsc; use std::path::PathBuf; use biocore::location::ContigRange; -use resource::{RawResourceExt, Resource, fs::FsCache}; +use resource::{Resource, ResourceExt, fs::FsCache}; use liftover::{ Liftover, diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index 5ccb259..c16743f 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -4,7 +4,7 @@ use liftover::{ Liftover, LiftoverIndexed, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{Resource, RawResourceExt}; +use resource::{Resource, ResourceExt}; #[ignore] #[test] @@ -170,7 +170,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ResourceExt, fs::FsCacheEntry}; pub fn store( snps_internal: Vec>, diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index 6b4e206..bc22704 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -5,7 +5,7 @@ use biocore::{ genome::Contig, location::{ContigPosition, ContigRange}, }; -use resource::{RawResourceExt, Resource}; +use resource::{ResourceExt, Resource}; use liftover::{ Chain, Liftover, @@ -231,7 +231,7 @@ pub mod cache { use std::path::PathBuf; use liftover::Liftover; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ResourceExt, fs::FsCacheEntry}; pub fn store(liftover: &Liftover, prefix: &str, key: &str) { let (snps, ranges) = super::get(liftover); diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index f2d0650..23a184d 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -7,7 +7,7 @@ use liftover::{ bindings::{self, ucsc::UcscLiftoverSettings}, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{Resource, RawResourceExt}; +use resource::{Resource, ResourceExt}; #[ignore] #[tokio::test] @@ -171,7 +171,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ResourceExt, fs::FsCacheEntry}; pub fn get(prefix: &str, key: &str) -> (Vec>, Vec>) { ( diff --git a/pan_ukbb/src/lib.rs b/pan_ukbb/src/lib.rs index 9df80e3..af49705 100755 --- a/pan_ukbb/src/lib.rs +++ b/pan_ukbb/src/lib.rs @@ -11,7 +11,7 @@ use ordered_float::NotNan; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; -use resource::{RawResourceExt, Resource, UrlResource}; +use resource::{ResourceExt, Resource, UrlResource}; const URL_BASE: &str = "https://pan-ukb-us-east-1.s3.amazonaws.com"; const PHENOTYPE_MANIFEST_KEY: &str = "sumstats_release/phenotype_manifest.tsv.bgz"; diff --git a/pgs_catalog/src/lib.rs b/pgs_catalog/src/lib.rs index 1fbf8a9..6abb250 100755 --- a/pgs_catalog/src/lib.rs +++ b/pgs_catalog/src/lib.rs @@ -11,7 +11,7 @@ use simplified::SimplificationError; use url::Url; use biocore::dna::DnaSequence; -use resource::{Resource, RawResourceExt, UrlResource}; +use resource::{Resource, ResourceExt, UrlResource}; pub use ids::{pgs::PgsId, rs::RsId}; diff --git a/pgs_catalog/src/metadata.rs b/pgs_catalog/src/metadata.rs index 00f9a13..c3349da 100644 --- a/pgs_catalog/src/metadata.rs +++ b/pgs_catalog/src/metadata.rs @@ -11,7 +11,7 @@ use ids::{ pgs::{PgsId, pgp::PgpId, ppm::PpmId, pss::PssId}, pubmed::PubmedId, }; -use resource::{RawResourceExt, Resource}; +use resource::{Resource, ResourceExt}; use url::Url; use crate::{PgsCatalogResource, WeightType}; diff --git a/resource/src/cached.rs b/resource/src/cached.rs index bf9951b..0210ef6 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -1,7 +1,7 @@ use std::{fmt, path::PathBuf}; use crate::{ - Compression, Resource, RawResourceExt, ResourceRef, + Compression, Resource, ResourceExt, ResourceRef, fs::{FsCache, FsCacheEntry}, }; diff --git a/resource/src/compression.rs b/resource/src/compression.rs index aa062cc..835e48a 100644 --- a/resource/src/compression.rs +++ b/resource/src/compression.rs @@ -7,7 +7,7 @@ use std::{ use pin_project::pin_project; -use super::{Compression, RawResourceExt, Resource, ResourceRef}; +use super::{Compression, ResourceExt, Resource, ResourceRef}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DecompressedResource { diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 5283ad5..1c8c3eb 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -47,7 +47,7 @@ pub trait Resource { async fn size_async(&self) -> io::Result; async fn read_async(&self) -> io::Result; } -pub trait RawResourceExt: Resource + Sized { +pub trait ResourceExt: Resource + Sized { fn buffered(self) -> BufferedResource { BufferedResource::new(self) } @@ -121,7 +121,7 @@ pub trait RawResourceExt: Resource + Sized { Ok(stream::try_unfold((), |()| async move { todo!() })) } } -impl RawResourceExt for T {} +impl ResourceExt for T {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Compression { From 304be4cec38988ec4f492afdfe4b92de95a16488 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sun, 16 Aug 2026 22:43:11 +0100 Subject: [PATCH 4/9] `ReadResource` --- ensembl/src/contig/grch37_meta.rs | 2 +- ensembl/src/contig/grch38_meta.rs | 2 +- ensembl/src/resource.rs | 9 ++-- genomes1000/src/contig/grch37_meta.rs | 2 +- genomes1000/src/contig/grch38_meta.rs | 2 +- genomes1000/src/lib.rs | 16 +++--- genomes1000/src/slow.rs | 2 +- genomes1000/src/source.rs | 9 ++-- gwas_catalog/src/lib.rs | 9 ++-- hail/src/contig/grch37_meta.rs | 2 +- hail/src/contig/grch38_meta.rs | 2 +- hail/src/lib.rs | 2 +- hail/src/source.rs | 9 ++-- liftover/src/parse.rs | 6 +-- liftover/src/sources.rs | 16 +++--- liftover/tests/fuzz/internal.rs | 2 +- liftover/tests/fuzz/testpoints.rs | 4 +- liftover/tests/fuzz/ucsc.rs | 2 +- pan_ukbb/src/lib.rs | 15 +++--- pgs_catalog/src/lib.rs | 17 ++++--- pgs_catalog/src/metadata.rs | 2 +- resource/src/buffered.rs | 5 +- resource/src/cached.rs | 36 ++++++++------ resource/src/compression.rs | 8 +-- resource/src/fs.rs | 5 +- resource/src/iter.rs | 9 +++- resource/src/lib.rs | 72 ++++++++++++++------------- resource/src/progress.rs | 5 +- resource/src/uri.rs | 5 +- 29 files changed, 153 insertions(+), 124 deletions(-) diff --git a/ensembl/src/contig/grch37_meta.rs b/ensembl/src/contig/grch37_meta.rs index cbfd7fb..c7058b0 100755 --- a/ensembl/src/contig/grch37_meta.rs +++ b/ensembl/src/contig/grch37_meta.rs @@ -94,7 +94,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, ResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/contig/grch38_meta.rs b/ensembl/src/contig/grch38_meta.rs index f6bb334..416e6fd 100755 --- a/ensembl/src/contig/grch38_meta.rs +++ b/ensembl/src/contig/grch38_meta.rs @@ -715,7 +715,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, ResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/resource.rs b/ensembl/src/resource.rs index 8cc4fff..0611397 100644 --- a/ensembl/src/resource.rs +++ b/ensembl/src/resource.rs @@ -1,4 +1,4 @@ -use resource::{Resource, UrlResource}; +use resource::{ReadResource, Resource, UrlResource}; use url::Url; const GRCH38_REFERENCE_GENOME_INDEXED: &str = @@ -116,8 +116,9 @@ impl Resource for EnsemblResource { fn compression(&self) -> Option { resource::Compression::infer(&self.key) } - - type Reader = ::Reader; +} +impl ReadResource for EnsemblResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -125,7 +126,7 @@ impl Resource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/genomes1000/src/contig/grch37_meta.rs b/genomes1000/src/contig/grch37_meta.rs index 59c504b..bf8e099 100755 --- a/genomes1000/src/contig/grch37_meta.rs +++ b/genomes1000/src/contig/grch37_meta.rs @@ -101,7 +101,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, ResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/contig/grch38_meta.rs b/genomes1000/src/contig/grch38_meta.rs index 321ea1f..a03ed47 100755 --- a/genomes1000/src/contig/grch38_meta.rs +++ b/genomes1000/src/contig/grch38_meta.rs @@ -3388,7 +3388,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, ResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/lib.rs b/genomes1000/src/lib.rs index c568074..14a51ce 100755 --- a/genomes1000/src/lib.rs +++ b/genomes1000/src/lib.rs @@ -19,7 +19,7 @@ use biocore::{ location::{ContigPosition, ContigRange}, vcf::IndexedVcfReader, }; -use resource::{ResourceExt, Resource, fs::FsCache}; +use resource::{ReadResource, ResourceExt, fs::FsCache}; use utile::{io::FromUtf8Bytes, iter::IteratorExt}; use self::{pedigree::Pedigree, simplified::SimplifiedRecord, source::Genomes1000Resource}; @@ -435,7 +435,7 @@ pub async fn load_contig( parse::parse(resource.read()?, sample_reading_function(c)) } -pub async fn load_pedigree(resource: impl Resource) -> io::Result> { +pub async fn load_pedigree(resource: impl ReadResource) -> io::Result> { Ok(csv::ReaderBuilder::new() .delimiter(b' ') .from_reader(resource.read()?) @@ -458,11 +458,11 @@ pub async fn load_pedigree(resource: impl Resource) -> io::Result> /// It should also implement [Seek](std::io::Seek) if random access is needed. pub async fn load_grch38_reference_genome( fasta: F, - index: impl Resource, + index: impl ReadResource, ) -> io::Result> where - F: Resource, - ::Reader: std::io::BufRead, + F: ReadResource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } @@ -471,11 +471,11 @@ where /// It should also implement [Seek](std::io::Seek) if random access is needed. pub async fn load_grch37_reference_genome( fasta: F, - index: impl Resource, + index: impl ReadResource, ) -> io::Result> where - F: Resource, - ::Reader: std::io::BufRead, + F: ReadResource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } diff --git a/genomes1000/src/slow.rs b/genomes1000/src/slow.rs index 3b6cf62..2b854ce 100755 --- a/genomes1000/src/slow.rs +++ b/genomes1000/src/slow.rs @@ -1,4 +1,4 @@ -use resource::{ResourceExt, Resource}; +use resource::{ReadResource, ResourceExt}; use std::{collections::HashMap, io::Read}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/source.rs b/genomes1000/src/source.rs index b0ee4ed..b29c00c 100755 --- a/genomes1000/src/source.rs +++ b/genomes1000/src/source.rs @@ -1,4 +1,4 @@ -use resource::{Resource, UrlResource}; +use resource::{ReadResource, Resource, UrlResource}; use url::Url; use crate::contig::GRCh38Contig; @@ -172,8 +172,9 @@ impl Resource for Genomes1000Resource { resource::Compression::infer(&self.key) } } - - type Reader = ::Reader; +} +impl ReadResource for Genomes1000Resource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -181,7 +182,7 @@ impl Resource for Genomes1000Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/gwas_catalog/src/lib.rs b/gwas_catalog/src/lib.rs index 70fa8f1..c880acc 100644 --- a/gwas_catalog/src/lib.rs +++ b/gwas_catalog/src/lib.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use biocore::location::ContigPosition; -use resource::{Resource, ResourceExt, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; use utile::io::reqwest_error; #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -49,8 +49,9 @@ impl Resource for GwasCatalogResource { fn compression(&self) -> Option { None } - - type Reader = ::Reader; +} +impl ReadResource for GwasCatalogResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { Ok(self.size) } @@ -59,7 +60,7 @@ impl Resource for GwasCatalogResource { UrlResource::new(self.url).unwrap().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { Ok(self.size) } diff --git a/hail/src/contig/grch37_meta.rs b/hail/src/contig/grch37_meta.rs index 2f3b2d6..da230ab 100755 --- a/hail/src/contig/grch37_meta.rs +++ b/hail/src/contig/grch37_meta.rs @@ -93,7 +93,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{ResourceExt, Resource}; + use resource::{ReadResource, ResourceExt}; use crate::source::HailCommonResource; diff --git a/hail/src/contig/grch38_meta.rs b/hail/src/contig/grch38_meta.rs index 09412f3..911999d 100755 --- a/hail/src/contig/grch38_meta.rs +++ b/hail/src/contig/grch38_meta.rs @@ -3375,7 +3375,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{Resource, ResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::HailCommonResource; diff --git a/hail/src/lib.rs b/hail/src/lib.rs index 6872f9b..ca3a959 100644 --- a/hail/src/lib.rs +++ b/hail/src/lib.rs @@ -3,7 +3,7 @@ pub mod contig; pub mod source; -use resource::{ResourceExt, Resource}; +use resource::{ReadResource, ResourceExt}; use source::HailCommonResource; pub async fn load_grch38_reference_genome() diff --git a/hail/src/source.rs b/hail/src/source.rs index 5432681..b3cdd60 100644 --- a/hail/src/source.rs +++ b/hail/src/source.rs @@ -1,4 +1,4 @@ -use resource::{Resource, UrlResource}; +use resource::{ReadResource, Resource, UrlResource}; use url::Url; const HAIL_COMMON_BUCKET: &str = "hail-common"; @@ -63,8 +63,9 @@ impl Resource for HailCommonResource { _ => resource::Compression::infer_strict(&self.key), } } - - type Reader = ::Reader; +} +impl ReadResource for HailCommonResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -72,7 +73,7 @@ impl Resource for HailCommonResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/src/parse.rs b/liftover/src/parse.rs index ce1784a..4e019e7 100755 --- a/liftover/src/parse.rs +++ b/liftover/src/parse.rs @@ -8,7 +8,7 @@ use std::{ }; use biocore::{genome::ArcContig, location::orientation::Stranded}; -use resource::{Resource, ResourceExt}; +use resource::{ReadResource, ResourceExt}; use utile::io::read_ext::AsyncReadInto; use super::{ @@ -16,10 +16,10 @@ use super::{ }; impl Liftover { - pub fn load(resource: impl Resource) -> anyhow::Result { + pub fn load(resource: impl ReadResource) -> anyhow::Result { Ok(Self::read(resource.decompressed().buffered().read()?)?) } - pub async fn load_async(resource: impl Resource) -> anyhow::Result { + pub async fn load_async(resource: impl ReadResource) -> anyhow::Result { Ok(Self::read( &*resource .decompressed() diff --git a/liftover/src/sources.rs b/liftover/src/sources.rs index 93f7a64..56c4b29 100755 --- a/liftover/src/sources.rs +++ b/liftover/src/sources.rs @@ -1,6 +1,6 @@ use url::Url; -use resource::{Compression, Resource, UrlResource}; +use resource::{Compression, ReadResource, Resource, UrlResource}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct EnsemblResource { @@ -55,8 +55,9 @@ impl Resource for EnsemblResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for EnsemblResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -64,7 +65,7 @@ impl Resource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -144,8 +145,9 @@ impl Resource for UcscResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for UcscResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -153,7 +155,7 @@ impl Resource for UcscResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index c16743f..1a86281 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -170,7 +170,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, fs::FsCacheEntry}; pub fn store( snps_internal: Vec>, diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index bc22704..cccef68 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -5,7 +5,7 @@ use biocore::{ genome::Contig, location::{ContigPosition, ContigRange}, }; -use resource::{ResourceExt, Resource}; +use resource::{Resource, ResourceExt}; use liftover::{ Chain, Liftover, @@ -231,7 +231,7 @@ pub mod cache { use std::path::PathBuf; use liftover::Liftover; - use resource::{ResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, fs::FsCacheEntry}; pub fn store(liftover: &Liftover, prefix: &str, key: &str) { let (snps, ranges) = super::get(liftover); diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index 23a184d..871e67c 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -171,7 +171,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, fs::FsCacheEntry}; pub fn get(prefix: &str, key: &str) -> (Vec>, Vec>) { ( diff --git a/pan_ukbb/src/lib.rs b/pan_ukbb/src/lib.rs index af49705..7e9582b 100755 --- a/pan_ukbb/src/lib.rs +++ b/pan_ukbb/src/lib.rs @@ -11,7 +11,7 @@ use ordered_float::NotNan; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; -use resource::{ResourceExt, Resource, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; const URL_BASE: &str = "https://pan-ukb-us-east-1.s3.amazonaws.com"; const PHENOTYPE_MANIFEST_KEY: &str = "sumstats_release/phenotype_manifest.tsv.bgz"; @@ -50,8 +50,9 @@ impl Resource for PanUKBBS3Resource { None } } - - type Reader = ::Reader; +} +impl ReadResource for PanUKBBS3Resource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -59,7 +60,7 @@ impl Resource for PanUKBBS3Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -407,7 +408,7 @@ impl PhenotypeManifestEntry { Self::load(resource) } - pub fn load(resource: impl Resource) -> csv::Result> { + pub fn load(resource: impl ReadResource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -415,7 +416,7 @@ impl PhenotypeManifestEntry { .into_deserialize() .try_collect() } - pub async fn load_async(resource: impl Resource) -> csv::Result> { + pub async fn load_async(resource: impl ReadResource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -607,7 +608,7 @@ pub struct SummaryStats { pub low_confidence_MID: Option, } impl SummaryStats { - pub fn load(resource: impl Resource) -> io::Result>> + pub fn load(resource: impl ReadResource) -> io::Result>> where Contig: DeserializeOwned, { diff --git a/pgs_catalog/src/lib.rs b/pgs_catalog/src/lib.rs index 6abb250..c000a63 100755 --- a/pgs_catalog/src/lib.rs +++ b/pgs_catalog/src/lib.rs @@ -11,7 +11,7 @@ use simplified::SimplificationError; use url::Url; use biocore::dna::DnaSequence; -use resource::{Resource, ResourceExt, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; pub use ids::{pgs::PgsId, rs::RsId}; @@ -61,8 +61,9 @@ impl Resource for PgsCatalogResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for PgsCatalogResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -70,7 +71,7 @@ impl Resource for PgsCatalogResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -566,8 +567,8 @@ impl Study { resource: R, ) -> io::Result>> where - R: Resource, - ::Reader: io::BufRead, + R: ReadResource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; @@ -594,8 +595,8 @@ impl HarmonizedStudy { resource: R, ) -> io::Result>> where - R: Resource, - ::Reader: io::BufRead, + R: ReadResource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; diff --git a/pgs_catalog/src/metadata.rs b/pgs_catalog/src/metadata.rs index c3349da..ad20e2b 100644 --- a/pgs_catalog/src/metadata.rs +++ b/pgs_catalog/src/metadata.rs @@ -11,7 +11,7 @@ use ids::{ pgs::{PgsId, pgp::PgpId, ppm::PpmId, pss::PssId}, pubmed::PubmedId, }; -use resource::{Resource, ResourceExt}; +use resource::{ReadResource, ResourceExt}; use url::Url; use crate::{PgsCatalogResource, WeightType}; diff --git a/resource/src/buffered.rs b/resource/src/buffered.rs index b8014a3..d248eb6 100644 --- a/resource/src/buffered.rs +++ b/resource/src/buffered.rs @@ -1,4 +1,4 @@ -use super::{Compression, Resource}; +use super::{Compression, ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BufferedResource { @@ -17,7 +17,8 @@ impl Resource for BufferedResource { fn compression(&self) -> Option { self.resource.compression() } - +} +impl ReadResource for BufferedResource { type Reader = std::io::BufReader; fn size(&self) -> std::io::Result { self.resource.size() diff --git a/resource/src/cached.rs b/resource/src/cached.rs index 0210ef6..50a6cca 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -1,7 +1,7 @@ use std::{fmt, path::PathBuf}; use crate::{ - Compression, Resource, ResourceExt, ResourceRef, + Compression, ReadResource, Resource, ResourceExt, ResourceRef, fs::{FsCache, FsCacheEntry}, }; @@ -40,7 +40,7 @@ impl FsCacheResource { pub fn ensure_cached(self) -> std::io::Result where - R: Resource, + R: ReadResource, { self.cache()?; Ok(self) @@ -48,7 +48,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: Resource, + R: ReadResource, { self.cache_async().await?; Ok(self) @@ -56,14 +56,14 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: Resource, + R: ReadResource, { panic!("FsCacheResource is not supported on wasm32"); } pub fn cache(&self) -> std::io::Result where - R: Resource, + R: ReadResource, { if !self.try_exists()? { self.read()?; @@ -73,7 +73,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: Resource, + R: ReadResource, { if !self.try_exists_async().await? { self.read_async().await?; @@ -83,7 +83,7 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: Resource, + R: ReadResource, { panic!("FsCacheResource is not supported on wasm32"); } @@ -109,7 +109,9 @@ impl Resource for FsCacheResource { fn compression(&self) -> Option { self.resource.compression() } - +} +#[cfg(not(target_arch = "wasm32"))] // TODO +impl ReadResource for FsCacheResource { type Reader = std::fs::File; fn size(&self) -> std::io::Result { if let Ok(size) = self.entry.size() { @@ -134,9 +136,7 @@ impl Resource for FsCacheResource { self.entry.read() } - #[cfg(not(target_arch = "wasm32"))] // TODO type AsyncReader = tokio::fs::File; - #[cfg(not(target_arch = "wasm32"))] // TODO async fn size_async(&self) -> std::io::Result { if let Ok(size) = self.entry.size_async().await { Ok(size) @@ -144,7 +144,7 @@ impl Resource for FsCacheResource { self.resource.size_async().await } } - #[cfg(not(target_arch = "wasm32"))] // TODO + async fn read_async(&self) -> std::io::Result { if self.try_exists_async().await? { log::info!("Cache hit at {self}"); @@ -166,13 +166,21 @@ impl Resource for FsCacheResource { self.entry.read_async().await } - #[cfg(target_arch = "wasm32")] // TODO +} +#[cfg(target_arch = "wasm32")] // TODO +impl ReadResource for FsCacheResource { + type Reader = std::fs::File; + fn size(&self) -> std::io::Result { + panic!("FsCacheResource is not supported on wasm32"); + } + fn read(&self) -> std::io::Result { + panic!("FsCacheResource is not supported on wasm32"); + } + type AsyncReader = std::io::Cursor<&'static [u8]>; - #[cfg(target_arch = "wasm32")] // TODO async fn size_async(&self) -> std::io::Result { panic!("FsCacheResource is not supported on wasm32"); } - #[cfg(target_arch = "wasm32")] // TODO async fn read_async(&self) -> std::io::Result { panic!("FsCacheResource is not supported on wasm32"); } diff --git a/resource/src/compression.rs b/resource/src/compression.rs index 835e48a..5207c03 100644 --- a/resource/src/compression.rs +++ b/resource/src/compression.rs @@ -7,7 +7,7 @@ use std::{ use pin_project::pin_project; -use super::{Compression, ResourceExt, Resource, ResourceRef}; +use super::{Compression, ReadResource, Resource, ResourceExt, ResourceRef}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DecompressedResource { @@ -41,7 +41,8 @@ impl Resource for DecompressedResource { fn compression(&self) -> Option { None } - +} +impl ReadResource for DecompressedResource { type Reader = DecompressedReader; fn size(&self) -> std::io::Result { Err(std::io::Error::new( @@ -170,7 +171,8 @@ impl Resource for CompressedResource { fn compression(&self) -> Option { Some(self.compression) } - +} +impl ReadResource for CompressedResource { type Reader = CompressedReader; fn size(&self) -> std::io::Result { diff --git a/resource/src/fs.rs b/resource/src/fs.rs index 7fa4a14..5f6bb3b 100644 --- a/resource/src/fs.rs +++ b/resource/src/fs.rs @@ -8,7 +8,7 @@ use directories::ProjectDirs; use utile::io::not_found_error; -use crate::Resource; +use crate::{ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct FsCache { @@ -152,7 +152,8 @@ impl Resource for FsCacheEntry { fn compression(&self) -> Option { None } - +} +impl ReadResource for FsCacheEntry { type Reader = std::fs::File; fn size(&self) -> std::io::Result { std::fs::metadata(self).map(|m| m.len()) diff --git a/resource/src/iter.rs b/resource/src/iter.rs index 40c5797..3f389a3 100644 --- a/resource/src/iter.rs +++ b/resource/src/iter.rs @@ -1,6 +1,6 @@ use utile::jsonl::JsonLinesReader; -use super::Resource; +use super::{ReadResource, Resource}; pub struct IterToJsonLinesResource { key: String, @@ -31,7 +31,12 @@ where fn compression(&self) -> Option { None } - +} +impl ReadResource for IterToJsonLinesResource +where + I: Iterator + Clone, + T: serde::Serialize, +{ type Reader = JsonLinesReader; fn size(&self) -> std::io::Result { diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 1c8c3eb..26960e3 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -38,7 +38,8 @@ pub trait Resource { fn key(&self) -> String; fn compression(&self) -> Option; - +} +pub trait ReadResource: Resource { type Reader: io::Read; fn size(&self) -> io::Result; fn read(&self) -> io::Result; @@ -46,36 +47,6 @@ pub trait Resource { type AsyncReader: tokio::io::AsyncRead; async fn size_async(&self) -> io::Result; async fn read_async(&self) -> io::Result; -} -pub trait ResourceExt: Resource + Sized { - fn buffered(self) -> BufferedResource { - BufferedResource::new(self) - } - - fn with_fs_cache(self, cache: &crate::fs::FsCache) -> FsCacheResource { - FsCacheResource::new(cache, self) - } - fn with_global_fs_cache(self) -> FsCacheResource { - FsCacheResource::new(&crate::fs::FsCache::global(), self) - } - - fn log_progress(self) -> ProgressResource { - ProgressResource::new(self) - } - - fn decompressed(self) -> DecompressedResource { - DecompressedResource::new(self) - } - fn decompressed_with(self, compression: Compression) -> DecompressedResource { - DecompressedResource::new_with(self, compression) - } - - fn compressed(self) -> CompressedResource { - CompressedResource::new(self, Compression::Gzip) - } - fn compressed_with(self, compression: Compression) -> CompressedResource { - CompressedResource::new(self, compression) - } fn read_vec(&self) -> io::Result> { let mut reader = ResourceRef::new(self).read()?; @@ -121,6 +92,36 @@ pub trait ResourceExt: Resource + Sized { Ok(stream::try_unfold((), |()| async move { todo!() })) } } +pub trait ResourceExt: Resource + Sized { + fn buffered(self) -> BufferedResource { + BufferedResource::new(self) + } + + fn with_fs_cache(self, cache: &crate::fs::FsCache) -> FsCacheResource { + FsCacheResource::new(cache, self) + } + fn with_global_fs_cache(self) -> FsCacheResource { + FsCacheResource::new(&crate::fs::FsCache::global(), self) + } + + fn log_progress(self) -> ProgressResource { + ProgressResource::new(self) + } + + fn decompressed(self) -> DecompressedResource { + DecompressedResource::new(self) + } + fn decompressed_with(self, compression: Compression) -> DecompressedResource { + DecompressedResource::new_with(self, compression) + } + + fn compressed(self) -> CompressedResource { + CompressedResource::new(self, Compression::Gzip) + } + fn compressed_with(self, compression: Compression) -> CompressedResource { + CompressedResource::new(self, compression) + } +} impl ResourceExt for T {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -174,15 +175,15 @@ impl Compression { /// or requiring a `Clone` bound in some places. /// (The blanket impl would allow the builder api to take a reference /// which in practice can cause annoying lifetime issues.) -struct ResourceRef<'a, R> { +struct ResourceRef<'a, R: ?Sized> { resource: &'a R, } -impl<'a, R: Resource> ResourceRef<'a, R> { +impl<'a, R: Resource + ?Sized> ResourceRef<'a, R> { pub fn new(resource: &'a R) -> Self { Self { resource } } } -impl<'a, R: Resource> Resource for ResourceRef<'a, R> { +impl<'a, R: Resource + ?Sized> Resource for ResourceRef<'a, R> { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(self.resource) @@ -190,7 +191,8 @@ impl<'a, R: Resource> Resource for ResourceRef<'a, R> { fn compression(&self) -> Option { R::compression(self.resource) } - +} +impl<'a, R: ReadResource + ?Sized> ReadResource for ResourceRef<'a, R> { type Reader = R::Reader; fn size(&self) -> io::Result { R::size(self.resource) diff --git a/resource/src/progress.rs b/resource/src/progress.rs index 1d10285..aa8a123 100644 --- a/resource/src/progress.rs +++ b/resource/src/progress.rs @@ -2,7 +2,7 @@ use std::pin::Pin; use indicatif::ProgressStyle; -use super::{Compression, Resource}; +use super::{Compression, ReadResource, Resource}; const PROGRESS_BAR_STYLE: &str = "{spinner} {bytes} ({percent}%) of {total_bytes} | {bytes_per_sec} {wide_bar} {eta}"; @@ -25,7 +25,8 @@ impl Resource for ProgressResource { fn compression(&self) -> Option { self.resource.compression() } - +} +impl ReadResource for ProgressResource { type Reader = indicatif::ProgressBarIter; fn size(&self) -> std::io::Result { self.resource.size() diff --git a/resource/src/uri.rs b/resource/src/uri.rs index b92cf3e..2206cbf 100644 --- a/resource/src/uri.rs +++ b/resource/src/uri.rs @@ -12,7 +12,7 @@ use url::Url; use utile::io::{get_filesize_from_headers, reqwest_error}; -use super::{Compression, Resource}; +use super::{Compression, ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct UrlResource(Url); @@ -92,7 +92,8 @@ impl Resource for UrlResource { fn compression(&self) -> Option { None } - +} +impl ReadResource for UrlResource { #[cfg(not(target_arch = "wasm32"))] type Reader = reqwest::blocking::Response; #[cfg(not(target_arch = "wasm32"))] From 9a200dd6fed1d7188d4c75a90ed6c2c45d76ab12 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:39:21 +0100 Subject: [PATCH 5/9] `WriteResource` --- liftover/tests/fuzz/internal.rs | 2 +- liftover/tests/fuzz/testpoints.rs | 2 +- liftover/tests/fuzz/ucsc.rs | 2 +- resource/Cargo.toml | 2 +- resource/src/cached.rs | 13 ++-- resource/src/fs.rs | 111 +++++++++++++----------------- resource/src/lib.rs | 79 +++++++++++++++++++-- 7 files changed, 131 insertions(+), 80 deletions(-) diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index 1a86281..b96169a 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -170,7 +170,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ReadResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; pub fn store( snps_internal: Vec>, diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index cccef68..69ab52a 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -231,7 +231,7 @@ pub mod cache { use std::path::PathBuf; use liftover::Liftover; - use resource::{ReadResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; pub fn store(liftover: &Liftover, prefix: &str, key: &str) { let (snps, ranges) = super::get(liftover); diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index 871e67c..5b30758 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -171,7 +171,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ReadResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; pub fn get(prefix: &str, key: &str) -> (Vec>, Vec>) { ( diff --git a/resource/Cargo.toml b/resource/Cargo.toml index a1a2ea6..038cdc0 100644 --- a/resource/Cargo.toml +++ b/resource/Cargo.toml @@ -33,4 +33,4 @@ suppaftp = { version = "6", features = ["async"] } tokio = { version = "1", features = ["fs", "process"] } [dev-dependencies] -tokio = { version = "1", features = ["macros"] } +tokio = { version = "1", features = ["macros", "rt"] } diff --git a/resource/src/cached.rs b/resource/src/cached.rs index 50a6cca..9484ed5 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -1,9 +1,11 @@ use std::{fmt, path::PathBuf}; use crate::{ - Compression, ReadResource, Resource, ResourceExt, ResourceRef, + Compression, ReadResource, Resource, fs::{FsCache, FsCacheEntry}, }; +#[cfg(not(target_arch = "wasm32"))] +use crate::{ResourceExt, ResourceRef, WriteResource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct FsCacheResource { @@ -129,7 +131,7 @@ impl ReadResource for FsCacheResource { log::info!("Cache miss at {self} from {self}"); self.entry - .write_file(ResourceRef::new(&self.resource).buffered().read()?)?; + .write_resource(&ResourceRef::new(&self.resource).buffered())?; log::info!("Retrieved {self}"); @@ -154,12 +156,7 @@ impl ReadResource for FsCacheResource { log::info!("Cache miss at {self} from {self}"); self.entry - .write_file_async( - ResourceRef::new(&self.resource) - .buffered() - .read_async() - .await?, - ) + .write_resource_async(&ResourceRef::new(&self.resource).buffered()) .await?; log::info!("Retrieved {self}"); diff --git a/resource/src/fs.rs b/resource/src/fs.rs index 5f6bb3b..55f4e2f 100644 --- a/resource/src/fs.rs +++ b/resource/src/fs.rs @@ -1,13 +1,17 @@ use std::{ - fmt, io, + fmt, path::{Path, PathBuf}, sync::LazyLock, }; +#[cfg(not(target_arch = "wasm32"))] +use std::{io, pin::Pin}; use directories::ProjectDirs; use utile::io::not_found_error; +#[cfg(not(target_arch = "wasm32"))] +use crate::WriteResource; use crate::{ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -77,62 +81,12 @@ impl FsCacheEntry { pub fn try_exists(&self) -> std::io::Result { self.as_ref().try_exists() } + #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn try_exists_async(&self) -> std::io::Result { tokio::fs::try_exists(&self).await } - pub fn write_file(&self, mut data: impl std::io::BufRead) -> std::io::Result<()> { - self.write_file_with(|tmp_file| std::io::copy(&mut data, tmp_file).map(drop)) - } - pub fn write_file_with( - &self, - f: impl FnOnce(&mut tempfile::NamedTempFile) -> std::io::Result<()>, - ) -> std::io::Result<()> { - std::fs::create_dir_all(self.path.parent().unwrap())?; - - let mut tmp_file = tempfile::Builder::new() - .prefix("tempfile_") - .suffix("_utile") - .tempfile_in(self.path.parent().unwrap())?; - f(&mut tmp_file)?; - - rename_or_copy(tmp_file, self)?; - - Ok(()) - } - #[cfg(not(target_arch = "wasm32"))] // TODO - pub async fn write_file_async( - &self, - data: impl tokio::io::AsyncBufRead, - ) -> std::io::Result<()> { - tokio::fs::create_dir_all(self.path.parent().unwrap()).await?; - - let tmp_file = tempfile::Builder::new() - .prefix("tempfile_") - .suffix("_utile") - .tempfile_in(self.path.parent().unwrap())?; - tokio::io::copy( - &mut std::pin::pin!(data), - &mut tokio::fs::File::create(tmp_file.path()).await?, - ) - .await?; - - rename_or_copy_async(tmp_file, &self).await?; - - Ok(()) - } - - pub fn write_json(&self, data: &T) -> std::io::Result<()> { - self.write_file_with(|file| Ok(serde_json::to_writer(file, data)?)) - } - pub fn write_json_lines( - &self, - data: impl IntoIterator, - ) -> std::io::Result<()> { - self.write_file(utile::jsonl::JsonLinesReader::new(data.into_iter())) - } - /// Unfortunately some sources aren't pure. pub fn invalidate(&self) -> std::io::Result<()> { std::fs::remove_file(self) @@ -185,29 +139,58 @@ impl ReadResource for FsCacheEntry { panic!("FsCacheEntry is not supported on wasm32"); } } +#[cfg(not(target_arch = "wasm32"))] +impl WriteResource for FsCacheEntry { + type Writer = tempfile::NamedTempFile; + fn write_with(&self, f: impl FnOnce(&mut Self::Writer) -> io::Result<()>) -> io::Result<()> { + std::fs::create_dir_all(self.path.parent().unwrap())?; + + let mut tmp_file = tempfile::Builder::new() + .prefix("tempfile_") + .tempfile_in(self.path.parent().unwrap())?; + f(&mut tmp_file)?; + + rename_or_copy(tmp_file, self) + } + + type AsyncWriter = tokio::fs::File; + async fn write_async_with( + &self, + f: impl AsyncFnOnce(Pin<&mut Self::AsyncWriter>) -> io::Result<()>, + ) -> io::Result<()> { + tokio::fs::create_dir_all(self.path.parent().unwrap()).await?; + + let tmp_file = tempfile::Builder::new() + .prefix("tempfile_") + .tempfile_in(self.path.parent().unwrap())?; + let mut writer = tokio::fs::File::create(tmp_file.path()).await?; + f(Pin::new(&mut writer)).await?; + drop(writer); + + rename_or_copy_async(tmp_file, self).await + } +} -// Add these new helper functions -fn rename_or_copy(from: impl AsRef, to: impl AsRef) -> std::io::Result<()> { +#[cfg(not(target_arch = "wasm32"))] +fn rename_or_copy(from: impl AsRef, to: impl AsRef) -> io::Result<()> { match std::fs::rename(from.as_ref(), to.as_ref()) { Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { + Err(error) if error.kind() == io::ErrorKind::CrossesDevices => { std::fs::copy(from.as_ref(), to.as_ref())?; - std::fs::remove_file(from.as_ref())?; - Ok(()) + std::fs::remove_file(from.as_ref()) } - Err(e) => Err(e), + Err(error) => Err(error), } } -#[cfg(not(target_arch = "wasm32"))] // TODO -async fn rename_or_copy_async(from: impl AsRef, to: impl AsRef) -> std::io::Result<()> { +#[cfg(not(target_arch = "wasm32"))] +async fn rename_or_copy_async(from: impl AsRef, to: impl AsRef) -> io::Result<()> { match tokio::fs::rename(from.as_ref(), to.as_ref()).await { Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { + Err(error) if error.kind() == io::ErrorKind::CrossesDevices => { tokio::fs::copy(from.as_ref(), to.as_ref()).await?; - tokio::fs::remove_file(from.as_ref()).await?; - Ok(()) + tokio::fs::remove_file(from.as_ref()).await } - Err(e) => Err(e), + Err(error) => Err(error), } } diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 26960e3..3dd0311 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -10,14 +10,14 @@ pub mod uri; use std::{ fmt::Debug, - io::{self, Read}, - pin::pin, + io::{self, Read, Write as _}, + pin::{Pin, pin}, }; -use futures::{Stream, stream}; +use futures::{Stream, StreamExt as _, stream}; use serde::de::DeserializeOwned; use serde_json::StreamDeserializer; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt as _}; use utile::io::read_ext::AsyncReadInto; @@ -92,6 +92,77 @@ pub trait ReadResource: Resource { Ok(stream::try_unfold((), |()| async move { todo!() })) } } +pub trait WriteResource: Resource { + type Writer: io::Write; + fn write_with(&self, f: impl FnOnce(&mut Self::Writer) -> io::Result<()>) -> io::Result<()>; + + type AsyncWriter: tokio::io::AsyncWrite; + async fn write_async_with( + &self, + f: impl AsyncFnOnce(Pin<&mut Self::AsyncWriter>) -> io::Result<()>, + ) -> io::Result<()>; + + fn write_resource(&self, resource: &impl ReadResource) -> io::Result<()> { + self.write_with(|writer| std::io::copy(&mut resource.read()?, writer).map(drop)) + } + async fn write_resource_async(&self, resource: &impl ReadResource) -> io::Result<()> { + self.write_async_with(async |mut writer| { + tokio::io::copy(&mut pin!(resource.read_async().await?), &mut writer) + .await + .map(drop) + }) + .await + } + + fn write_slice(&self, data: &[u8]) -> io::Result<()> { + self.write_with(|writer| writer.write_all(data)) + } + async fn write_slice_async(&self, data: &[u8]) -> io::Result<()> { + self.write_async_with(async |mut writer| writer.write_all(data).await) + .await + } + + fn write_json(&self, data: &T) -> std::io::Result<()> { + self.write_with(|writer| Ok(serde_json::to_writer(writer, data)?)) + } + async fn write_json_async(&self, data: &T) -> std::io::Result<()> { + // TODO: avoid buffering in memory + let data = serde_json::to_vec(data)?; + self.write_slice_async(&data).await + } + + fn write_json_lines( + &self, + data: impl IntoIterator, + ) -> std::io::Result<()> { + self.write_with(|writer| { + std::io::copy( + &mut utile::jsonl::JsonLinesReader::new(data.into_iter()), + writer, + ) + .map(drop) + }) + } + async fn write_json_lines_async( + &self, + data: impl IntoIterator, + ) -> io::Result<()> { + self.write_async_with(async |mut writer| { + let items = stream::iter(data); + let mut items = pin!(items); + + let mut vec = Vec::new(); + while let Some(item) = items.next().await { + vec.clear(); + serde_json::to_writer(&mut vec, &item)?; + vec.push(b'\n'); + writer.write_all(&vec).await?; + } + Ok(()) + }) + .await + } +} pub trait ResourceExt: Resource + Sized { fn buffered(self) -> BufferedResource { BufferedResource::new(self) From 82c3d8a677ac24a424ecbb33bbe745d5159c10ac Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:08:34 +0100 Subject: [PATCH 6/9] Move file --- genomes1000/src/lib.rs | 2 +- liftover/tests/fuzz.rs | 2 +- liftover/tests/fuzz/internal.rs | 2 +- liftover/tests/fuzz/testpoints.rs | 2 +- liftover/tests/fuzz/ucsc.rs | 2 +- resource/src/{ => cache}/fs.rs | 0 resource/src/cache/mod.rs | 1 + resource/src/cached.rs | 2 +- resource/src/lib.rs | 6 +++--- 9 files changed, 10 insertions(+), 9 deletions(-) rename resource/src/{ => cache}/fs.rs (100%) create mode 100644 resource/src/cache/mod.rs diff --git a/genomes1000/src/lib.rs b/genomes1000/src/lib.rs index 14a51ce..e7be9ad 100755 --- a/genomes1000/src/lib.rs +++ b/genomes1000/src/lib.rs @@ -19,7 +19,7 @@ use biocore::{ location::{ContigPosition, ContigRange}, vcf::IndexedVcfReader, }; -use resource::{ReadResource, ResourceExt, fs::FsCache}; +use resource::{ReadResource, ResourceExt, cache::fs::FsCache}; use utile::{io::FromUtf8Bytes, iter::IteratorExt}; use self::{pedigree::Pedigree, simplified::SimplifiedRecord, source::Genomes1000Resource}; diff --git a/liftover/tests/fuzz.rs b/liftover/tests/fuzz.rs index 85e69b9..9bb3491 100755 --- a/liftover/tests/fuzz.rs +++ b/liftover/tests/fuzz.rs @@ -10,7 +10,7 @@ mod ucsc; use std::path::PathBuf; use biocore::location::ContigRange; -use resource::{Resource, ResourceExt, fs::FsCache}; +use resource::{Resource, ResourceExt, cache::fs::FsCache}; use liftover::{ Liftover, diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index b96169a..06b491d 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -170,7 +170,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn store( snps_internal: Vec>, diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index 69ab52a..bd8c57e 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -231,7 +231,7 @@ pub mod cache { use std::path::PathBuf; use liftover::Liftover; - use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn store(liftover: &Liftover, prefix: &str, key: &str) { let (snps, ranges) = super::get(liftover); diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index 5b30758..3b08a8d 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -171,7 +171,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{ReadResource, WriteResource, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn get(prefix: &str, key: &str) -> (Vec>, Vec>) { ( diff --git a/resource/src/fs.rs b/resource/src/cache/fs.rs similarity index 100% rename from resource/src/fs.rs rename to resource/src/cache/fs.rs diff --git a/resource/src/cache/mod.rs b/resource/src/cache/mod.rs new file mode 100644 index 0000000..d521fbd --- /dev/null +++ b/resource/src/cache/mod.rs @@ -0,0 +1 @@ +pub mod fs; diff --git a/resource/src/cached.rs b/resource/src/cached.rs index 9484ed5..aa67b6e 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -2,7 +2,7 @@ use std::{fmt, path::PathBuf}; use crate::{ Compression, ReadResource, Resource, - fs::{FsCache, FsCacheEntry}, + cache::fs::{FsCache, FsCacheEntry}, }; #[cfg(not(target_arch = "wasm32"))] use crate::{ResourceExt, ResourceRef, WriteResource}; diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 3dd0311..5bcac04 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -1,9 +1,9 @@ #![expect(async_fn_in_trait)] // TODO pub mod buffered; +pub mod cache; pub mod cached; pub mod compression; -pub mod fs; pub mod iter; pub mod progress; pub mod uri; @@ -168,11 +168,11 @@ pub trait ResourceExt: Resource + Sized { BufferedResource::new(self) } - fn with_fs_cache(self, cache: &crate::fs::FsCache) -> FsCacheResource { + fn with_fs_cache(self, cache: &crate::cache::fs::FsCache) -> FsCacheResource { FsCacheResource::new(cache, self) } fn with_global_fs_cache(self) -> FsCacheResource { - FsCacheResource::new(&crate::fs::FsCache::global(), self) + FsCacheResource::new(&crate::cache::fs::FsCache::global(), self) } fn log_progress(self) -> ProgressResource { From 6ae901b8802fd6ae109045bef2f753be77f485d0 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:01:19 +0100 Subject: [PATCH 7/9] Fix lint --- utile/src/hash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utile/src/hash.rs b/utile/src/hash.rs index cb80f29..266032b 100644 --- a/utile/src/hash.rs +++ b/utile/src/hash.rs @@ -52,7 +52,7 @@ impl FromStr for Sha256Hash { )); } let mut bytes = [0u8; 32]; - for (i, chunk) in s.as_bytes().chunks_exact(2).enumerate() { + for (i, chunk) in s.as_bytes().as_chunks::<2>().0.iter().enumerate() { bytes[i] = u8::from_str_radix( std::str::from_utf8(chunk).map_err(crate::io::invalid_data)?, 16, From 27cee02277bc9efd70b8208bf4b7e06b9a3a7389 Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:02:56 +0100 Subject: [PATCH 8/9] Pin stable toolchain in CI --- .github/workflows/check.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b972869..cfc6f3c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -43,6 +43,8 @@ jobs: clippy-stable: runs-on: ubuntu-latest + env: + RUST_VERSION: "1.98.0" steps: - name: Checkout uses: actions/checkout@v4 @@ -59,20 +61,20 @@ jobs: ~/.cargo/git/db/ ~/.rustup/toolchains/ target/ - key: ${{ runner.os }}-cargo-check-stable-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-cargo-check-${{ env.RUST_VERSION }}-${{ hashFiles('**/Cargo.lock') }} - name: Install stable toolchain - run: rustup toolchain install stable + run: rustup toolchain install ${{ env.RUST_VERSION }} --component clippy - name: Install WASM toolchain - run: rustup target add wasm32-unknown-unknown --toolchain stable + run: rustup target add wasm32-unknown-unknown --toolchain ${{ env.RUST_VERSION }} - name: Check on stable - run: cargo +stable clippy --all-targets -p utile -p puv -- -D warnings + run: cargo +${{ env.RUST_VERSION }} clippy --all-targets -p utile -p puv -- -D warnings - name: Check project (WASM) on stable # Note lack of --all-targets here because tests and examples are not wasm compatible - run: cargo +stable clippy --target wasm32-unknown-unknown -p utile -p puv -- -D warnings + run: cargo +${{ env.RUST_VERSION }} clippy --target wasm32-unknown-unknown -p utile -p puv -- -D warnings test: runs-on: ubuntu-latest From c7d1bb865b73ace5707f1ef6fb94a04f1c4637bc Mon Sep 17 00:00:00 2001 From: QuartzLibrary <81446760+QuartzLibrary@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:19:49 +0100 Subject: [PATCH 9/9] Simplify renaming The fallback is not needed if we put the temporary file in the same directory. --- resource/src/cache/fs.rs | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/resource/src/cache/fs.rs b/resource/src/cache/fs.rs index 55f4e2f..c40ba9b 100644 --- a/resource/src/cache/fs.rs +++ b/resource/src/cache/fs.rs @@ -150,7 +150,7 @@ impl WriteResource for FsCacheEntry { .tempfile_in(self.path.parent().unwrap())?; f(&mut tmp_file)?; - rename_or_copy(tmp_file, self) + std::fs::rename(tmp_file.path(), self) } type AsyncWriter = tokio::fs::File; @@ -167,30 +167,6 @@ impl WriteResource for FsCacheEntry { f(Pin::new(&mut writer)).await?; drop(writer); - rename_or_copy_async(tmp_file, self).await - } -} - -#[cfg(not(target_arch = "wasm32"))] -fn rename_or_copy(from: impl AsRef, to: impl AsRef) -> io::Result<()> { - match std::fs::rename(from.as_ref(), to.as_ref()) { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::CrossesDevices => { - std::fs::copy(from.as_ref(), to.as_ref())?; - std::fs::remove_file(from.as_ref()) - } - Err(error) => Err(error), - } -} - -#[cfg(not(target_arch = "wasm32"))] -async fn rename_or_copy_async(from: impl AsRef, to: impl AsRef) -> io::Result<()> { - match tokio::fs::rename(from.as_ref(), to.as_ref()).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::CrossesDevices => { - tokio::fs::copy(from.as_ref(), to.as_ref()).await?; - tokio::fs::remove_file(from.as_ref()).await - } - Err(error) => Err(error), + tokio::fs::rename(tmp_file.path(), self).await } }