diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b972869..cfc6f3c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -43,6 +43,8 @@ jobs: clippy-stable: runs-on: ubuntu-latest + env: + RUST_VERSION: "1.98.0" steps: - name: Checkout uses: actions/checkout@v4 @@ -59,20 +61,20 @@ jobs: ~/.cargo/git/db/ ~/.rustup/toolchains/ target/ - key: ${{ runner.os }}-cargo-check-stable-${{ hashFiles('**/Cargo.lock') }} + key: ${{ runner.os }}-cargo-check-${{ env.RUST_VERSION }}-${{ hashFiles('**/Cargo.lock') }} - name: Install stable toolchain - run: rustup toolchain install stable + run: rustup toolchain install ${{ env.RUST_VERSION }} --component clippy - name: Install WASM toolchain - run: rustup target add wasm32-unknown-unknown --toolchain stable + run: rustup target add wasm32-unknown-unknown --toolchain ${{ env.RUST_VERSION }} - name: Check on stable - run: cargo +stable clippy --all-targets -p utile -p puv -- -D warnings + run: cargo +${{ env.RUST_VERSION }} clippy --all-targets -p utile -p puv -- -D warnings - name: Check project (WASM) on stable # Note lack of --all-targets here because tests and examples are not wasm compatible - run: cargo +stable clippy --target wasm32-unknown-unknown -p utile -p puv -- -D warnings + run: cargo +${{ env.RUST_VERSION }} clippy --target wasm32-unknown-unknown -p utile -p puv -- -D warnings test: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 63f233d..82dbac0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4002,6 +4002,7 @@ dependencies = [ "directories", "flate2", "futures", + "http-body", "indicatif", "log", "pin-project", diff --git a/ensembl/src/contig/grch37_meta.rs b/ensembl/src/contig/grch37_meta.rs index f025c7f..c7058b0 100755 --- a/ensembl/src/contig/grch37_meta.rs +++ b/ensembl/src/contig/grch37_meta.rs @@ -94,7 +94,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/contig/grch38_meta.rs b/ensembl/src/contig/grch38_meta.rs index b45a8c8..416e6fd 100755 --- a/ensembl/src/contig/grch38_meta.rs +++ b/ensembl/src/contig/grch38_meta.rs @@ -715,7 +715,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::resource::EnsemblResource; diff --git a/ensembl/src/resource.rs b/ensembl/src/resource.rs index fa4db5d..0611397 100644 --- a/ensembl/src/resource.rs +++ b/ensembl/src/resource.rs @@ -1,5 +1,5 @@ +use resource::{ReadResource, Resource, UrlResource}; use url::Url; -use resource::{RawResource, UrlResource}; const GRCH38_REFERENCE_GENOME_INDEXED: &str = "fasta/homo_sapiens/dna_index/Homo_sapiens.GRCh38.dna.toplevel.fa.gz"; @@ -106,7 +106,7 @@ impl EnsemblResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for EnsemblResource { +impl Resource for EnsemblResource { const NAMESPACE: &'static str = "ensembl"; fn key(&self) -> String { @@ -116,8 +116,9 @@ impl RawResource for EnsemblResource { fn compression(&self) -> Option { resource::Compression::infer(&self.key) } - - type Reader = ::Reader; +} +impl ReadResource for EnsemblResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -125,7 +126,7 @@ impl RawResource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/genomes1000/examples/load_all.rs b/genomes1000/examples/load_all.rs index 0f150c1..067c5ff 100755 --- a/genomes1000/examples/load_all.rs +++ b/genomes1000/examples/load_all.rs @@ -1,4 +1,4 @@ -use resource::RawResourceExt; +use resource::ResourceExt; use genomes1000::{GRCh38Contig, load_grch38_reference_genome, source::Genomes1000Resource}; diff --git a/genomes1000/src/contig/grch37_meta.rs b/genomes1000/src/contig/grch37_meta.rs index b779559..bf8e099 100755 --- a/genomes1000/src/contig/grch37_meta.rs +++ b/genomes1000/src/contig/grch37_meta.rs @@ -101,7 +101,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/contig/grch38_meta.rs b/genomes1000/src/contig/grch38_meta.rs index 5b20ba6..a03ed47 100755 --- a/genomes1000/src/contig/grch38_meta.rs +++ b/genomes1000/src/contig/grch38_meta.rs @@ -3388,7 +3388,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/lib.rs b/genomes1000/src/lib.rs index a619946..e7be9ad 100755 --- a/genomes1000/src/lib.rs +++ b/genomes1000/src/lib.rs @@ -19,7 +19,7 @@ use biocore::{ location::{ContigPosition, ContigRange}, vcf::IndexedVcfReader, }; -use resource::{RawResource, RawResourceExt, fs::FsCache}; +use resource::{ReadResource, ResourceExt, cache::fs::FsCache}; use utile::{io::FromUtf8Bytes, iter::IteratorExt}; use self::{pedigree::Pedigree, simplified::SimplifiedRecord, source::Genomes1000Resource}; @@ -435,7 +435,7 @@ pub async fn load_contig( parse::parse(resource.read()?, sample_reading_function(c)) } -pub async fn load_pedigree(resource: impl RawResource) -> io::Result> { +pub async fn load_pedigree(resource: impl ReadResource) -> io::Result> { Ok(csv::ReaderBuilder::new() .delimiter(b' ') .from_reader(resource.read()?) @@ -458,11 +458,11 @@ pub async fn load_pedigree(resource: impl RawResource) -> io::Result( fasta: F, - index: impl RawResource, + index: impl ReadResource, ) -> io::Result> where - F: RawResource, - ::Reader: std::io::BufRead, + F: ReadResource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } @@ -471,11 +471,11 @@ where /// It should also implement [Seek](std::io::Seek) if random access is needed. pub async fn load_grch37_reference_genome( fasta: F, - index: impl RawResource, + index: impl ReadResource, ) -> io::Result> where - F: RawResource, - ::Reader: std::io::BufRead, + F: ReadResource, + ::Reader: std::io::BufRead, { biocore::fasta::IndexedFastaReader::new(fasta.read()?, index.decompressed().buffered().read()?) } diff --git a/genomes1000/src/slow.rs b/genomes1000/src/slow.rs index 8ca4d67..2b854ce 100755 --- a/genomes1000/src/slow.rs +++ b/genomes1000/src/slow.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, RawResourceExt}; +use resource::{ReadResource, ResourceExt}; use std::{collections::HashMap, io::Read}; use crate::source::Genomes1000Resource; diff --git a/genomes1000/src/source.rs b/genomes1000/src/source.rs index a588573..b29c00c 100755 --- a/genomes1000/src/source.rs +++ b/genomes1000/src/source.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, UrlResource}; +use resource::{ReadResource, Resource, UrlResource}; use url::Url; use crate::contig::GRCh38Contig; @@ -158,7 +158,7 @@ impl Genomes1000Resource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for Genomes1000Resource { +impl Resource for Genomes1000Resource { const NAMESPACE: &'static str = "1000genomes"; fn key(&self) -> String { @@ -172,8 +172,9 @@ impl RawResource for Genomes1000Resource { resource::Compression::infer(&self.key) } } - - type Reader = ::Reader; +} +impl ReadResource for Genomes1000Resource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -181,7 +182,7 @@ impl RawResource for Genomes1000Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/gwas_catalog/src/lib.rs b/gwas_catalog/src/lib.rs index 963477b..c880acc 100644 --- a/gwas_catalog/src/lib.rs +++ b/gwas_catalog/src/lib.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use biocore::location::ContigPosition; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; use utile::io::reqwest_error; #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -39,7 +39,7 @@ impl GwasCatalogResource { Self::ANCESTRY_URL.parse().unwrap() } } -impl RawResource for GwasCatalogResource { +impl Resource for GwasCatalogResource { const NAMESPACE: &'static str = "gwas_catalog"; fn key(&self) -> String { @@ -49,8 +49,9 @@ impl RawResource for GwasCatalogResource { fn compression(&self) -> Option { None } - - type Reader = ::Reader; +} +impl ReadResource for GwasCatalogResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { Ok(self.size) } @@ -59,7 +60,7 @@ impl RawResource for GwasCatalogResource { UrlResource::new(self.url).unwrap().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { Ok(self.size) } diff --git a/hail/src/contig/grch37_meta.rs b/hail/src/contig/grch37_meta.rs index 9ab7232..da230ab 100755 --- a/hail/src/contig/grch37_meta.rs +++ b/hail/src/contig/grch37_meta.rs @@ -93,7 +93,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::HailCommonResource; diff --git a/hail/src/contig/grch38_meta.rs b/hail/src/contig/grch38_meta.rs index 35cfc2a..911999d 100755 --- a/hail/src/contig/grch38_meta.rs +++ b/hail/src/contig/grch38_meta.rs @@ -3375,7 +3375,7 @@ pub static META: phf::Map<&'static str, ContigMeta> = phf::phf_map! { #[cfg(test)] mod tests { - use resource::{RawResource, RawResourceExt}; + use resource::{ReadResource, ResourceExt}; use crate::source::HailCommonResource; diff --git a/hail/src/lib.rs b/hail/src/lib.rs index 42dd9d6..ca3a959 100644 --- a/hail/src/lib.rs +++ b/hail/src/lib.rs @@ -3,7 +3,7 @@ pub mod contig; pub mod source; -use resource::{RawResource, RawResourceExt}; +use resource::{ReadResource, ResourceExt}; use source::HailCommonResource; pub async fn load_grch38_reference_genome() diff --git a/hail/src/source.rs b/hail/src/source.rs index b92074b..b3cdd60 100644 --- a/hail/src/source.rs +++ b/hail/src/source.rs @@ -1,4 +1,4 @@ -use resource::{RawResource, UrlResource}; +use resource::{ReadResource, Resource, UrlResource}; use url::Url; const HAIL_COMMON_BUCKET: &str = "hail-common"; @@ -44,7 +44,7 @@ impl HailCommonResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for HailCommonResource { +impl Resource for HailCommonResource { const NAMESPACE: &'static str = "hail_common"; fn key(&self) -> String { @@ -63,8 +63,9 @@ impl RawResource for HailCommonResource { _ => resource::Compression::infer_strict(&self.key), } } - - type Reader = ::Reader; +} +impl ReadResource for HailCommonResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -72,7 +73,7 @@ impl RawResource for HailCommonResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/examples/load_all.rs b/liftover/examples/load_all.rs index c428618..5d84d21 100755 --- a/liftover/examples/load_all.rs +++ b/liftover/examples/load_all.rs @@ -2,7 +2,7 @@ use liftover::{ Liftover, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::RawResourceExt; +use resource::ResourceExt; #[tokio::main] async fn main() -> anyhow::Result<()> { diff --git a/liftover/src/parse.rs b/liftover/src/parse.rs index 86a3b0e..4e019e7 100755 --- a/liftover/src/parse.rs +++ b/liftover/src/parse.rs @@ -8,7 +8,7 @@ use std::{ }; use biocore::{genome::ArcContig, location::orientation::Stranded}; -use resource::{RawResource, RawResourceExt}; +use resource::{ReadResource, ResourceExt}; use utile::io::read_ext::AsyncReadInto; use super::{ @@ -16,10 +16,10 @@ use super::{ }; impl Liftover { - pub fn load(resource: impl RawResource) -> anyhow::Result { + pub fn load(resource: impl ReadResource) -> anyhow::Result { Ok(Self::read(resource.decompressed().buffered().read()?)?) } - pub async fn load_async(resource: impl RawResource) -> anyhow::Result { + pub async fn load_async(resource: impl ReadResource) -> anyhow::Result { Ok(Self::read( &*resource .decompressed() diff --git a/liftover/src/sources.rs b/liftover/src/sources.rs index 4d38e73..56c4b29 100755 --- a/liftover/src/sources.rs +++ b/liftover/src/sources.rs @@ -1,6 +1,6 @@ use url::Url; -use resource::{Compression, RawResource, UrlResource}; +use resource::{Compression, ReadResource, Resource, UrlResource}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct EnsemblResource { @@ -42,7 +42,7 @@ impl EnsemblResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for EnsemblResource { +impl Resource for EnsemblResource { const NAMESPACE: &'static str = "ensembl"; fn key(&self) -> String { self.key.clone() @@ -55,8 +55,9 @@ impl RawResource for EnsemblResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for EnsemblResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -64,7 +65,7 @@ impl RawResource for EnsemblResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -131,7 +132,7 @@ impl UcscResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for UcscResource { +impl Resource for UcscResource { const NAMESPACE: &'static str = "ucsc"; fn key(&self) -> String { self.key.clone() @@ -144,8 +145,9 @@ impl RawResource for UcscResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for UcscResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -153,7 +155,7 @@ impl RawResource for UcscResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } diff --git a/liftover/tests/fuzz.rs b/liftover/tests/fuzz.rs index 4e1effa..9bb3491 100755 --- a/liftover/tests/fuzz.rs +++ b/liftover/tests/fuzz.rs @@ -10,7 +10,7 @@ mod ucsc; use std::path::PathBuf; use biocore::location::ContigRange; -use resource::{RawResource, RawResourceExt, fs::FsCache}; +use resource::{Resource, ResourceExt, cache::fs::FsCache}; use liftover::{ Liftover, diff --git a/liftover/tests/fuzz/internal.rs b/liftover/tests/fuzz/internal.rs index 942210c..06b491d 100755 --- a/liftover/tests/fuzz/internal.rs +++ b/liftover/tests/fuzz/internal.rs @@ -4,7 +4,7 @@ use liftover::{ Liftover, LiftoverIndexed, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, ResourceExt}; #[ignore] #[test] @@ -170,7 +170,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn store( snps_internal: Vec>, diff --git a/liftover/tests/fuzz/testpoints.rs b/liftover/tests/fuzz/testpoints.rs index 27f03b3..bd8c57e 100755 --- a/liftover/tests/fuzz/testpoints.rs +++ b/liftover/tests/fuzz/testpoints.rs @@ -5,7 +5,7 @@ use biocore::{ genome::Contig, location::{ContigPosition, ContigRange}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, ResourceExt}; use liftover::{ Chain, Liftover, @@ -231,7 +231,7 @@ pub mod cache { use std::path::PathBuf; use liftover::Liftover; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn store(liftover: &Liftover, prefix: &str, key: &str) { let (snps, ranges) = super::get(liftover); diff --git a/liftover/tests/fuzz/ucsc.rs b/liftover/tests/fuzz/ucsc.rs index 515057b..3b08a8d 100755 --- a/liftover/tests/fuzz/ucsc.rs +++ b/liftover/tests/fuzz/ucsc.rs @@ -7,7 +7,7 @@ use liftover::{ bindings::{self, ucsc::UcscLiftoverSettings}, sources::{EnsemblHG, EnsemblResource, UcscHG, UcscResource}, }; -use resource::{RawResource, RawResourceExt}; +use resource::{Resource, ResourceExt}; #[ignore] #[tokio::test] @@ -171,7 +171,7 @@ pub mod cache { use std::path::PathBuf; use biocore::location::{ContigPosition, ContigRange}; - use resource::{RawResourceExt, fs::FsCacheEntry}; + use resource::{ReadResource, WriteResource, cache::fs::FsCacheEntry}; pub fn get(prefix: &str, key: &str) -> (Vec>, Vec>) { ( diff --git a/pan_ukbb/src/lib.rs b/pan_ukbb/src/lib.rs index 2cdf855..7e9582b 100755 --- a/pan_ukbb/src/lib.rs +++ b/pan_ukbb/src/lib.rs @@ -11,7 +11,7 @@ use ordered_float::NotNan; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use url::Url; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; const URL_BASE: &str = "https://pan-ukb-us-east-1.s3.amazonaws.com"; const PHENOTYPE_MANIFEST_KEY: &str = "sumstats_release/phenotype_manifest.tsv.bgz"; @@ -36,7 +36,7 @@ impl PanUKBBS3Resource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for PanUKBBS3Resource { +impl Resource for PanUKBBS3Resource { const NAMESPACE: &'static str = "pan_ukbb"; fn key(&self) -> String { @@ -50,8 +50,9 @@ impl RawResource for PanUKBBS3Resource { None } } - - type Reader = ::Reader; +} +impl ReadResource for PanUKBBS3Resource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -59,7 +60,7 @@ impl RawResource for PanUKBBS3Resource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -407,7 +408,7 @@ impl PhenotypeManifestEntry { Self::load(resource) } - pub fn load(resource: impl RawResource) -> csv::Result> { + pub fn load(resource: impl ReadResource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -415,7 +416,7 @@ impl PhenotypeManifestEntry { .into_deserialize() .try_collect() } - pub async fn load_async(resource: impl RawResource) -> csv::Result> { + pub async fn load_async(resource: impl ReadResource) -> csv::Result> { csv::ReaderBuilder::new() .delimiter(b'\t') .has_headers(true) @@ -607,7 +608,7 @@ pub struct SummaryStats { pub low_confidence_MID: Option, } impl SummaryStats { - pub fn load(resource: impl RawResource) -> io::Result>> + pub fn load(resource: impl ReadResource) -> io::Result>> where Contig: DeserializeOwned, { diff --git a/pgs_catalog/src/lib.rs b/pgs_catalog/src/lib.rs index 228738b..c000a63 100755 --- a/pgs_catalog/src/lib.rs +++ b/pgs_catalog/src/lib.rs @@ -11,7 +11,7 @@ use simplified::SimplificationError; use url::Url; use biocore::dna::DnaSequence; -use resource::{RawResource, RawResourceExt, UrlResource}; +use resource::{ReadResource, Resource, ResourceExt, UrlResource}; pub use ids::{pgs::PgsId, rs::RsId}; @@ -46,7 +46,7 @@ impl PgsCatalogResource { UrlResource::new(self.url()).unwrap() } } -impl RawResource for PgsCatalogResource { +impl Resource for PgsCatalogResource { const NAMESPACE: &'static str = "pgs_catalog"; fn key(&self) -> String { @@ -61,8 +61,9 @@ impl RawResource for PgsCatalogResource { None } } - - type Reader = ::Reader; +} +impl ReadResource for PgsCatalogResource { + type Reader = ::Reader; fn size(&self) -> std::io::Result { self.url_resource().size() } @@ -70,7 +71,7 @@ impl RawResource for PgsCatalogResource { self.url_resource().read() } - type AsyncReader = ::AsyncReader; + type AsyncReader = ::AsyncReader; async fn size_async(&self) -> std::io::Result { self.url_resource().size_async().await } @@ -566,8 +567,8 @@ impl Study { resource: R, ) -> io::Result>> where - R: RawResource, - ::Reader: io::BufRead, + R: ReadResource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; @@ -594,8 +595,8 @@ impl HarmonizedStudy { resource: R, ) -> io::Result>> where - R: RawResource, - ::Reader: io::BufRead, + R: ReadResource, + ::Reader: io::BufRead, { let mut file = resource.read()?; let _header = comments::read(&mut file)?; diff --git a/pgs_catalog/src/metadata.rs b/pgs_catalog/src/metadata.rs index 3aadc33..ad20e2b 100644 --- a/pgs_catalog/src/metadata.rs +++ b/pgs_catalog/src/metadata.rs @@ -11,7 +11,7 @@ use ids::{ pgs::{PgsId, pgp::PgpId, ppm::PpmId, pss::PssId}, pubmed::PubmedId, }; -use resource::{RawResource, RawResourceExt}; +use resource::{ReadResource, ResourceExt}; use url::Url; use crate::{PgsCatalogResource, WeightType}; diff --git a/resource/Cargo.toml b/resource/Cargo.toml index ef218ad..038cdc0 100644 --- a/resource/Cargo.toml +++ b/resource/Cargo.toml @@ -28,8 +28,9 @@ tokio-util = { version = "0.7", features = ["io", "compat"] } url = { version = "2", features = ["serde"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +http-body = "1" suppaftp = { version = "6", features = ["async"] } tokio = { version = "1", features = ["fs", "process"] } [dev-dependencies] -tokio = { version = "1", features = ["macros"] } +tokio = { version = "1", features = ["macros", "rt"] } diff --git a/resource/src/buffered.rs b/resource/src/buffered.rs index f10ce44..d248eb6 100644 --- a/resource/src/buffered.rs +++ b/resource/src/buffered.rs @@ -1,15 +1,15 @@ -use super::{Compression, RawResource}; +use super::{Compression, ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct BufferedResource { resource: R, } -impl BufferedResource { +impl BufferedResource { pub fn new(resource: R) -> Self { Self { resource } } } -impl RawResource for BufferedResource { +impl Resource for BufferedResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) @@ -17,7 +17,8 @@ impl RawResource for BufferedResource { fn compression(&self) -> Option { self.resource.compression() } - +} +impl ReadResource for BufferedResource { type Reader = std::io::BufReader; fn size(&self) -> std::io::Result { self.resource.size() diff --git a/resource/src/fs.rs b/resource/src/cache/fs.rs similarity index 66% rename from resource/src/fs.rs rename to resource/src/cache/fs.rs index 1beb850..c40ba9b 100644 --- a/resource/src/fs.rs +++ b/resource/src/cache/fs.rs @@ -1,14 +1,18 @@ use std::{ - fmt, io, + fmt, path::{Path, PathBuf}, sync::LazyLock, }; +#[cfg(not(target_arch = "wasm32"))] +use std::{io, pin::Pin}; use directories::ProjectDirs; use utile::io::not_found_error; -use crate::RawResource; +#[cfg(not(target_arch = "wasm32"))] +use crate::WriteResource; +use crate::{ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct FsCache { @@ -77,62 +81,12 @@ impl FsCacheEntry { pub fn try_exists(&self) -> std::io::Result { self.as_ref().try_exists() } + #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn try_exists_async(&self) -> std::io::Result { tokio::fs::try_exists(&self).await } - pub fn write_file(&self, mut data: impl std::io::BufRead) -> std::io::Result<()> { - self.write_file_with(|tmp_file| std::io::copy(&mut data, tmp_file).map(drop)) - } - pub fn write_file_with( - &self, - f: impl FnOnce(&mut tempfile::NamedTempFile) -> std::io::Result<()>, - ) -> std::io::Result<()> { - std::fs::create_dir_all(self.path.parent().unwrap())?; - - let mut tmp_file = tempfile::Builder::new() - .prefix("tempfile_") - .suffix("_utile") - .tempfile_in(self.path.parent().unwrap())?; - f(&mut tmp_file)?; - - rename_or_copy(tmp_file, self)?; - - Ok(()) - } - #[cfg(not(target_arch = "wasm32"))] // TODO - pub async fn write_file_async( - &self, - data: impl tokio::io::AsyncBufRead, - ) -> std::io::Result<()> { - tokio::fs::create_dir_all(self.path.parent().unwrap()).await?; - - let tmp_file = tempfile::Builder::new() - .prefix("tempfile_") - .suffix("_utile") - .tempfile_in(self.path.parent().unwrap())?; - tokio::io::copy( - &mut std::pin::pin!(data), - &mut tokio::fs::File::create(tmp_file.path()).await?, - ) - .await?; - - rename_or_copy_async(tmp_file, &self).await?; - - Ok(()) - } - - pub fn write_json(&self, data: &T) -> std::io::Result<()> { - self.write_file_with(|file| Ok(serde_json::to_writer(file, data)?)) - } - pub fn write_json_lines( - &self, - data: impl IntoIterator, - ) -> std::io::Result<()> { - self.write_file(utile::jsonl::JsonLinesReader::new(data.into_iter())) - } - /// Unfortunately some sources aren't pure. pub fn invalidate(&self) -> std::io::Result<()> { std::fs::remove_file(self) @@ -143,7 +97,7 @@ impl FsCacheEntry { tokio::fs::remove_file(&self).await } } -impl RawResource for FsCacheEntry { +impl Resource for FsCacheEntry { const NAMESPACE: &'static str = "fs_cache"; fn key(&self) -> String { self.path.to_string_lossy().as_ref().to_owned() @@ -152,7 +106,8 @@ impl RawResource for FsCacheEntry { fn compression(&self) -> Option { None } - +} +impl ReadResource for FsCacheEntry { type Reader = std::fs::File; fn size(&self) -> std::io::Result { std::fs::metadata(self).map(|m| m.len()) @@ -184,29 +139,34 @@ impl RawResource for FsCacheEntry { panic!("FsCacheEntry is not supported on wasm32"); } } +#[cfg(not(target_arch = "wasm32"))] +impl WriteResource for FsCacheEntry { + type Writer = tempfile::NamedTempFile; + fn write_with(&self, f: impl FnOnce(&mut Self::Writer) -> io::Result<()>) -> io::Result<()> { + std::fs::create_dir_all(self.path.parent().unwrap())?; -// Add these new helper functions -fn rename_or_copy(from: impl AsRef, to: impl AsRef) -> std::io::Result<()> { - match std::fs::rename(from.as_ref(), to.as_ref()) { - Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { - std::fs::copy(from.as_ref(), to.as_ref())?; - std::fs::remove_file(from.as_ref())?; - Ok(()) - } - Err(e) => Err(e), + let mut tmp_file = tempfile::Builder::new() + .prefix("tempfile_") + .tempfile_in(self.path.parent().unwrap())?; + f(&mut tmp_file)?; + + std::fs::rename(tmp_file.path(), self) } -} -#[cfg(not(target_arch = "wasm32"))] // TODO -async fn rename_or_copy_async(from: impl AsRef, to: impl AsRef) -> std::io::Result<()> { - match tokio::fs::rename(from.as_ref(), to.as_ref()).await { - Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::CrossesDevices => { - tokio::fs::copy(from.as_ref(), to.as_ref()).await?; - tokio::fs::remove_file(from.as_ref()).await?; - Ok(()) - } - Err(e) => Err(e), + type AsyncWriter = tokio::fs::File; + async fn write_async_with( + &self, + f: impl AsyncFnOnce(Pin<&mut Self::AsyncWriter>) -> io::Result<()>, + ) -> io::Result<()> { + tokio::fs::create_dir_all(self.path.parent().unwrap()).await?; + + let tmp_file = tempfile::Builder::new() + .prefix("tempfile_") + .tempfile_in(self.path.parent().unwrap())?; + let mut writer = tokio::fs::File::create(tmp_file.path()).await?; + f(Pin::new(&mut writer)).await?; + drop(writer); + + tokio::fs::rename(tmp_file.path(), self).await } } diff --git a/resource/src/cache/mod.rs b/resource/src/cache/mod.rs new file mode 100644 index 0000000..d521fbd --- /dev/null +++ b/resource/src/cache/mod.rs @@ -0,0 +1 @@ +pub mod fs; diff --git a/resource/src/cached.rs b/resource/src/cached.rs index 9265c9b..aa67b6e 100644 --- a/resource/src/cached.rs +++ b/resource/src/cached.rs @@ -1,9 +1,11 @@ use std::{fmt, path::PathBuf}; use crate::{ - Compression, RawResource, RawResourceExt, ResourceRef, - fs::{FsCache, FsCacheEntry}, + Compression, ReadResource, Resource, + cache::fs::{FsCache, FsCacheEntry}, }; +#[cfg(not(target_arch = "wasm32"))] +use crate::{ResourceExt, ResourceRef, WriteResource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct FsCacheResource { @@ -18,7 +20,7 @@ impl fmt::Display for FsCacheResource { impl FsCacheResource { pub fn new(cache: &FsCache, resource: R) -> Self where - R: RawResource, + R: Resource, { Self { entry: FsCacheEntry::new(cache, PathBuf::from(R::NAMESPACE).join(resource.key())), @@ -40,7 +42,7 @@ impl FsCacheResource { pub fn ensure_cached(self) -> std::io::Result where - R: RawResource, + R: ReadResource, { self.cache()?; Ok(self) @@ -48,7 +50,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: RawResource, + R: ReadResource, { self.cache_async().await?; Ok(self) @@ -56,14 +58,14 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn ensure_cached_async(self) -> std::io::Result where - R: RawResource, + R: ReadResource, { panic!("FsCacheResource is not supported on wasm32"); } pub fn cache(&self) -> std::io::Result where - R: RawResource, + R: ReadResource, { if !self.try_exists()? { self.read()?; @@ -73,7 +75,7 @@ impl FsCacheResource { #[cfg(not(target_arch = "wasm32"))] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: RawResource, + R: ReadResource, { if !self.try_exists_async().await? { self.read_async().await?; @@ -83,7 +85,7 @@ impl FsCacheResource { #[cfg(target_arch = "wasm32")] // TODO pub async fn cache_async(&self) -> std::io::Result where - R: RawResource, + R: ReadResource, { panic!("FsCacheResource is not supported on wasm32"); } @@ -100,7 +102,7 @@ impl FsCacheResource { panic!("FsCacheResource is not supported on wasm32"); } } -impl RawResource for FsCacheResource { +impl Resource for FsCacheResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) @@ -109,7 +111,9 @@ impl RawResource for FsCacheResource { fn compression(&self) -> Option { self.resource.compression() } - +} +#[cfg(not(target_arch = "wasm32"))] // TODO +impl ReadResource for FsCacheResource { type Reader = std::fs::File; fn size(&self) -> std::io::Result { if let Ok(size) = self.entry.size() { @@ -127,16 +131,14 @@ impl RawResource for FsCacheResource { log::info!("Cache miss at {self} from {self}"); self.entry - .write_file(ResourceRef::new(&self.resource).buffered().read()?)?; + .write_resource(&ResourceRef::new(&self.resource).buffered())?; log::info!("Retrieved {self}"); self.entry.read() } - #[cfg(not(target_arch = "wasm32"))] // TODO type AsyncReader = tokio::fs::File; - #[cfg(not(target_arch = "wasm32"))] // TODO async fn size_async(&self) -> std::io::Result { if let Ok(size) = self.entry.size_async().await { Ok(size) @@ -144,7 +146,7 @@ impl RawResource for FsCacheResource { self.resource.size_async().await } } - #[cfg(not(target_arch = "wasm32"))] // TODO + async fn read_async(&self) -> std::io::Result { if self.try_exists_async().await? { log::info!("Cache hit at {self}"); @@ -154,25 +156,28 @@ impl RawResource for FsCacheResource { log::info!("Cache miss at {self} from {self}"); self.entry - .write_file_async( - ResourceRef::new(&self.resource) - .buffered() - .read_async() - .await?, - ) + .write_resource_async(&ResourceRef::new(&self.resource).buffered()) .await?; log::info!("Retrieved {self}"); self.entry.read_async().await } - #[cfg(target_arch = "wasm32")] // TODO +} +#[cfg(target_arch = "wasm32")] // TODO +impl ReadResource for FsCacheResource { + type Reader = std::fs::File; + fn size(&self) -> std::io::Result { + panic!("FsCacheResource is not supported on wasm32"); + } + fn read(&self) -> std::io::Result { + panic!("FsCacheResource is not supported on wasm32"); + } + type AsyncReader = std::io::Cursor<&'static [u8]>; - #[cfg(target_arch = "wasm32")] // TODO async fn size_async(&self) -> std::io::Result { panic!("FsCacheResource is not supported on wasm32"); } - #[cfg(target_arch = "wasm32")] // TODO async fn read_async(&self) -> std::io::Result { panic!("FsCacheResource is not supported on wasm32"); } diff --git a/resource/src/compression.rs b/resource/src/compression.rs index 5c96324..5207c03 100644 --- a/resource/src/compression.rs +++ b/resource/src/compression.rs @@ -7,14 +7,14 @@ use std::{ use pin_project::pin_project; -use super::{Compression, RawResource, RawResourceExt, ResourceRef}; +use super::{Compression, ReadResource, Resource, ResourceExt, ResourceRef}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DecompressedResource { resource: R, compression: Option, } -impl DecompressedResource { +impl DecompressedResource { pub fn new(resource: R) -> Self { Self { resource, @@ -28,7 +28,7 @@ impl DecompressedResource { } } } -impl RawResource for DecompressedResource { +impl Resource for DecompressedResource { const NAMESPACE: &'static str = "decompressed"; fn key(&self) -> String { let key = self.resource.key(); @@ -41,7 +41,8 @@ impl RawResource for DecompressedResource { fn compression(&self) -> Option { None } - +} +impl ReadResource for DecompressedResource { type Reader = DecompressedReader; fn size(&self) -> std::io::Result { Err(std::io::Error::new( @@ -149,7 +150,7 @@ pub struct CompressedResource { resource: R, compression: Compression, } -impl CompressedResource { +impl CompressedResource { pub fn new(resource: R, compression: Compression) -> Self { Self { resource, @@ -157,7 +158,7 @@ impl CompressedResource { } } } -impl RawResource for CompressedResource { +impl Resource for CompressedResource { const NAMESPACE: &'static str = "compressed"; fn key(&self) -> String { format!( @@ -170,7 +171,8 @@ impl RawResource for CompressedResource { fn compression(&self) -> Option { Some(self.compression) } - +} +impl ReadResource for CompressedResource { type Reader = CompressedReader; fn size(&self) -> std::io::Result { diff --git a/resource/src/iter.rs b/resource/src/iter.rs index 5b1d22d..3f389a3 100644 --- a/resource/src/iter.rs +++ b/resource/src/iter.rs @@ -1,6 +1,6 @@ use utile::jsonl::JsonLinesReader; -use super::RawResource; +use super::{ReadResource, Resource}; pub struct IterToJsonLinesResource { key: String, @@ -18,7 +18,7 @@ where } } } -impl RawResource for IterToJsonLinesResource +impl Resource for IterToJsonLinesResource where I: Iterator + Clone, T: serde::Serialize, @@ -31,7 +31,12 @@ where fn compression(&self) -> Option { None } - +} +impl ReadResource for IterToJsonLinesResource +where + I: Iterator + Clone, + T: serde::Serialize, +{ type Reader = JsonLinesReader; fn size(&self) -> std::io::Result { diff --git a/resource/src/lib.rs b/resource/src/lib.rs index 197d6e8..5bcac04 100755 --- a/resource/src/lib.rs +++ b/resource/src/lib.rs @@ -1,24 +1,23 @@ -#![feature(impl_trait_in_assoc_type)] #![expect(async_fn_in_trait)] // TODO pub mod buffered; +pub mod cache; pub mod cached; pub mod compression; -pub mod fs; pub mod iter; pub mod progress; pub mod uri; use std::{ fmt::Debug, - io::{self, Read}, - pin::pin, + io::{self, Read, Write as _}, + pin::{Pin, pin}, }; -use futures::{Stream, stream}; +use futures::{Stream, StreamExt as _, stream}; use serde::de::DeserializeOwned; use serde_json::StreamDeserializer; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt as _}; use utile::io::read_ext::AsyncReadInto; @@ -34,12 +33,13 @@ pub use self::cached::FsCacheResource; type JsonStreamDeserializer = StreamDeserializer<'static, serde_json::de::IoRead>, T>; -pub trait RawResource { +pub trait Resource { const NAMESPACE: &'static str; fn key(&self) -> String; fn compression(&self) -> Option; - +} +pub trait ReadResource: Resource { type Reader: io::Read; fn size(&self) -> io::Result; fn read(&self) -> io::Result; @@ -47,36 +47,6 @@ pub trait RawResource { type AsyncReader: tokio::io::AsyncRead; async fn size_async(&self) -> io::Result; async fn read_async(&self) -> io::Result; -} -pub trait RawResourceExt: RawResource + Sized { - fn buffered(self) -> BufferedResource { - BufferedResource::new(self) - } - - fn with_fs_cache(self, cache: &crate::fs::FsCache) -> FsCacheResource { - FsCacheResource::new(cache, self) - } - fn with_global_fs_cache(self) -> FsCacheResource { - FsCacheResource::new(&crate::fs::FsCache::global(), self) - } - - fn log_progress(self) -> ProgressResource { - ProgressResource::new(self) - } - - fn decompressed(self) -> DecompressedResource { - DecompressedResource::new(self) - } - fn decompressed_with(self, compression: Compression) -> DecompressedResource { - DecompressedResource::new_with(self, compression) - } - - fn compressed(self) -> CompressedResource { - CompressedResource::new(self, Compression::Gzip) - } - fn compressed_with(self, compression: Compression) -> CompressedResource { - CompressedResource::new(self, compression) - } fn read_vec(&self) -> io::Result> { let mut reader = ResourceRef::new(self).read()?; @@ -122,7 +92,108 @@ pub trait RawResourceExt: RawResource + Sized { Ok(stream::try_unfold((), |()| async move { todo!() })) } } -impl RawResourceExt for T {} +pub trait WriteResource: Resource { + type Writer: io::Write; + fn write_with(&self, f: impl FnOnce(&mut Self::Writer) -> io::Result<()>) -> io::Result<()>; + + type AsyncWriter: tokio::io::AsyncWrite; + async fn write_async_with( + &self, + f: impl AsyncFnOnce(Pin<&mut Self::AsyncWriter>) -> io::Result<()>, + ) -> io::Result<()>; + + fn write_resource(&self, resource: &impl ReadResource) -> io::Result<()> { + self.write_with(|writer| std::io::copy(&mut resource.read()?, writer).map(drop)) + } + async fn write_resource_async(&self, resource: &impl ReadResource) -> io::Result<()> { + self.write_async_with(async |mut writer| { + tokio::io::copy(&mut pin!(resource.read_async().await?), &mut writer) + .await + .map(drop) + }) + .await + } + + fn write_slice(&self, data: &[u8]) -> io::Result<()> { + self.write_with(|writer| writer.write_all(data)) + } + async fn write_slice_async(&self, data: &[u8]) -> io::Result<()> { + self.write_async_with(async |mut writer| writer.write_all(data).await) + .await + } + + fn write_json(&self, data: &T) -> std::io::Result<()> { + self.write_with(|writer| Ok(serde_json::to_writer(writer, data)?)) + } + async fn write_json_async(&self, data: &T) -> std::io::Result<()> { + // TODO: avoid buffering in memory + let data = serde_json::to_vec(data)?; + self.write_slice_async(&data).await + } + + fn write_json_lines( + &self, + data: impl IntoIterator, + ) -> std::io::Result<()> { + self.write_with(|writer| { + std::io::copy( + &mut utile::jsonl::JsonLinesReader::new(data.into_iter()), + writer, + ) + .map(drop) + }) + } + async fn write_json_lines_async( + &self, + data: impl IntoIterator, + ) -> io::Result<()> { + self.write_async_with(async |mut writer| { + let items = stream::iter(data); + let mut items = pin!(items); + + let mut vec = Vec::new(); + while let Some(item) = items.next().await { + vec.clear(); + serde_json::to_writer(&mut vec, &item)?; + vec.push(b'\n'); + writer.write_all(&vec).await?; + } + Ok(()) + }) + .await + } +} +pub trait ResourceExt: Resource + Sized { + fn buffered(self) -> BufferedResource { + BufferedResource::new(self) + } + + fn with_fs_cache(self, cache: &crate::cache::fs::FsCache) -> FsCacheResource { + FsCacheResource::new(cache, self) + } + fn with_global_fs_cache(self) -> FsCacheResource { + FsCacheResource::new(&crate::cache::fs::FsCache::global(), self) + } + + fn log_progress(self) -> ProgressResource { + ProgressResource::new(self) + } + + fn decompressed(self) -> DecompressedResource { + DecompressedResource::new(self) + } + fn decompressed_with(self, compression: Compression) -> DecompressedResource { + DecompressedResource::new_with(self, compression) + } + + fn compressed(self) -> CompressedResource { + CompressedResource::new(self, Compression::Gzip) + } + fn compressed_with(self, compression: Compression) -> CompressedResource { + CompressedResource::new(self, compression) + } +} +impl ResourceExt for T {} #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Compression { @@ -175,15 +246,15 @@ impl Compression { /// or requiring a `Clone` bound in some places. /// (The blanket impl would allow the builder api to take a reference /// which in practice can cause annoying lifetime issues.) -struct ResourceRef<'a, R> { +struct ResourceRef<'a, R: ?Sized> { resource: &'a R, } -impl<'a, R: RawResource> ResourceRef<'a, R> { +impl<'a, R: Resource + ?Sized> ResourceRef<'a, R> { pub fn new(resource: &'a R) -> Self { Self { resource } } } -impl<'a, R: RawResource> RawResource for ResourceRef<'a, R> { +impl<'a, R: Resource + ?Sized> Resource for ResourceRef<'a, R> { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(self.resource) @@ -191,7 +262,8 @@ impl<'a, R: RawResource> RawResource for ResourceRef<'a, R> { fn compression(&self) -> Option { R::compression(self.resource) } - +} +impl<'a, R: ReadResource + ?Sized> ReadResource for ResourceRef<'a, R> { type Reader = R::Reader; fn size(&self) -> io::Result { R::size(self.resource) diff --git a/resource/src/progress.rs b/resource/src/progress.rs index 22470e6..aa8a123 100644 --- a/resource/src/progress.rs +++ b/resource/src/progress.rs @@ -2,7 +2,7 @@ use std::pin::Pin; use indicatif::ProgressStyle; -use super::{Compression, RawResource}; +use super::{Compression, ReadResource, Resource}; const PROGRESS_BAR_STYLE: &str = "{spinner} {bytes} ({percent}%) of {total_bytes} | {bytes_per_sec} {wide_bar} {eta}"; @@ -11,12 +11,12 @@ const PROGRESS_BAR_STYLE: &str = pub struct ProgressResource { resource: R, } -impl ProgressResource { +impl ProgressResource { pub fn new(resource: R) -> Self { Self { resource } } } -impl RawResource for ProgressResource { +impl Resource for ProgressResource { const NAMESPACE: &'static str = R::NAMESPACE; fn key(&self) -> String { R::key(&self.resource) @@ -25,7 +25,8 @@ impl RawResource for ProgressResource { fn compression(&self) -> Option { self.resource.compression() } - +} +impl ReadResource for ProgressResource { type Reader = indicatif::ProgressBarIter; fn size(&self) -> std::io::Result { self.resource.size() diff --git a/resource/src/uri.rs b/resource/src/uri.rs index 7aed6e0..2206cbf 100644 --- a/resource/src/uri.rs +++ b/resource/src/uri.rs @@ -1,13 +1,18 @@ -use std::{fmt, sync::LazyLock}; +use std::{ + fmt, + pin::Pin, + sync::LazyLock, + task::{Context, Poll}, +}; use bytes::Bytes; -use futures::{Stream, TryStreamExt}; +use futures::Stream; use reqwest::IntoUrl; use url::Url; use utile::io::{get_filesize_from_headers, reqwest_error}; -use super::{Compression, RawResource}; +use super::{Compression, ReadResource, Resource}; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct UrlResource(Url); @@ -78,7 +83,7 @@ impl UrlResource { unreachable!() } } -impl RawResource for UrlResource { +impl Resource for UrlResource { const NAMESPACE: &'static str = "url"; fn key(&self) -> String { self.0.to_string() @@ -87,7 +92,8 @@ impl RawResource for UrlResource { fn compression(&self) -> Option { None } - +} +impl ReadResource for UrlResource { #[cfg(not(target_arch = "wasm32"))] type Reader = reqwest::blocking::Response; #[cfg(not(target_arch = "wasm32"))] @@ -142,8 +148,7 @@ impl RawResource for UrlResource { panic!("UrlResource::read is not supported on wasm32, use the non-blocking version."); } - type AsyncReader = - tokio_util::io::StreamReader>, Bytes>; + type AsyncReader = tokio_util::io::StreamReader; async fn size_async(&self) -> std::io::Result { static CLIENT: LazyLock = LazyLock::new(reqwest::Client::new); let response = CLIENT @@ -177,8 +182,65 @@ impl RawResource for UrlResource { ))); } - let stream = response.bytes_stream().map_err(std::io::Error::other); - Ok(tokio_util::io::StreamReader::new(stream)) + Ok(tokio_util::io::StreamReader::new(UrlByteStream::new( + response, + ))) + } +} + +/// Mirrors `reqwest::Response::bytes_stream`, which returns an opaque unnameable type. +pub struct UrlByteStream { + #[cfg(not(target_arch = "wasm32"))] + body: reqwest::Body, + #[cfg(target_arch = "wasm32")] + stream: Pin>>>, +} +impl UrlByteStream { + fn new(response: reqwest::Response) -> Self { + #[cfg(not(target_arch = "wasm32"))] + { + Self { + body: response.into(), + } + } + #[cfg(target_arch = "wasm32")] + { + use futures::{StreamExt, TryStreamExt}; + Self { + stream: response.bytes_stream().map_err(reqwest_error).boxed_local(), + } + } + } +} +impl Stream for UrlByteStream { + type Item = std::io::Result; + + #[cfg(not(target_arch = "wasm32"))] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + use http_body::Body; + use std::task::ready; + + // Impl taken from `reqwest::async_impl::body::DataStream`. + + loop { + return match ready!(Pin::new(&mut self.body).poll_frame(cx)) { + Some(Ok(frame)) => { + // skip non-data frames + if let Ok(buf) = frame.into_data() { + Poll::Ready(Some(Ok(buf))) + } else { + continue; + } + } + Some(Err(err)) => Poll::Ready(Some(Err(reqwest_error(err)))), + None => Poll::Ready(None), + }; + } + } + + #[cfg(target_arch = "wasm32")] + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_next(cx) } } diff --git a/utile/src/hash.rs b/utile/src/hash.rs index cb80f29..266032b 100644 --- a/utile/src/hash.rs +++ b/utile/src/hash.rs @@ -52,7 +52,7 @@ impl FromStr for Sha256Hash { )); } let mut bytes = [0u8; 32]; - for (i, chunk) in s.as_bytes().chunks_exact(2).enumerate() { + for (i, chunk) in s.as_bytes().as_chunks::<2>().0.iter().enumerate() { bytes[i] = u8::from_str_radix( std::str::from_utf8(chunk).map_err(crate::io::invalid_data)?, 16,