@@ -8,15 +8,14 @@ use rand::prelude::*;
88use tokio_stream:: wrappers:: ReceiverStream ;
99
1010use callisto:: lfs_locks;
11- use common:: config:: PackConfig ;
1211use common:: errors:: { GitLFSError , MegaError } ;
1312use jupiter:: storage:: Storage ;
1413use jupiter:: storage:: lfs_db_storage:: LfsDbStorage ;
1514
1615use crate :: lfs:: lfs_structs:: {
17- BatchRequest , BatchResponse , ChunkDownloadObject , Link , Lock , LockList , LockListQuery ,
18- LockRequest , MetaObject , ObjectError , Operation , RequestObject , ResCondition , ResponseObject ,
19- TransferMode , UnlockRequest , VerifiableLockList , VerifiableLockRequest ,
16+ BatchRequest , BatchResponse , Lock , LockList , LockListQuery , LockRequest , MetaObject ,
17+ ObjectError , Operation , RequestObject , ResCondition , ResponseObject , TransferMode ,
18+ UnlockRequest , VerifiableLockList , VerifiableLockRequest ,
2019} ;
2120
2221pub async fn lfs_retrieve_lock (
@@ -210,8 +209,7 @@ pub async fn lfs_process_batch(
210209 }
211210 }
212211 } ;
213- let enable_split = meta. splited && config. local . enable_split ;
214- let file_exist = file_storage. exist_object ( & meta. oid , enable_split) . await ;
212+ let file_exist = file_storage. exist_object ( & meta. oid ) . await ;
215213 let download_url = file_storage
216214 . download_url ( & meta. oid , listen_addr)
217215 . await
@@ -240,70 +238,13 @@ pub async fn lfs_process_batch(
240238 } )
241239}
242240
243- /// if server enable split, then return a list of chunk ids.
244- /// else return an error.
245- pub async fn lfs_fetch_chunk_ids (
246- storage : & Storage ,
247- oid : & str ,
248- listen_addr : & str ,
249- ) -> Result < Vec < ChunkDownloadObject > , GitLFSError > {
250- let config = storage. config ( ) . lfs . clone ( ) ;
251-
252- if !config. local . enable_split {
253- return Err ( GitLFSError :: GeneralError (
254- "Server didn't run in `split` mode, didn't support chunk ids" . to_string ( ) ,
255- ) ) ;
256- }
257- let db_storage = storage. lfs_db_storage ( ) ;
258-
259- let meta = lfs_get_meta ( db_storage. clone ( ) , oid)
260- . await
261- . map_err ( |_| GitLFSError :: GeneralError ( "" . to_string ( ) ) ) ?
262- . unwrap ( ) ;
263- assert ! ( meta. splited, "database didn't match the split mode" ) ;
264-
265- let relations = db_storage
266- . get_lfs_relations ( oid)
267- . await
268- . map_err ( |_| GitLFSError :: GeneralError ( "" . to_string ( ) ) ) ?;
269-
270- if relations. is_empty ( ) {
271- return Err ( GitLFSError :: GeneralError (
272- "oid didn't have chunks" . to_string ( ) ,
273- ) ) ;
274- }
275- let mut response_objects = Vec :: < ChunkDownloadObject > :: new ( ) ;
276-
277- for relation in relations {
278- // Reuse RequestArgs to create a link
279- let req_obj = RequestObject {
280- oid : relation. sub_oid . clone ( ) ,
281- size : relation. size ,
282- ..Default :: default ( )
283- } ;
284- let download_url = storage
285- . lfs_file_storage ( )
286- . download_url ( & req_obj. oid , listen_addr)
287- . await
288- . unwrap ( ) ;
289- response_objects. push ( ChunkDownloadObject {
290- sub_oid : relation. sub_oid ,
291- size : relation. size ,
292- offset : relation. offset ,
293- link : Link :: new ( & download_url) ,
294- } ) ;
295- }
296- Ok ( response_objects)
297- }
298-
299241/// Upload object to storage.
300242/// if server enable split, split the object and upload each part to storage, save the relationship to database.
301243pub async fn lfs_upload_object (
302244 storage : & Storage ,
303245 req_obj : & RequestObject ,
304246 body_bytes : Vec < u8 > ,
305247) -> Result < ( ) , GitLFSError > {
306- let config = storage. config ( ) . lfs . clone ( ) ;
307248 let db_storage = storage. lfs_db_storage ( ) ;
308249 let file_storage = storage. lfs_file_storage ( ) ;
309250
@@ -313,36 +254,14 @@ pub async fn lfs_upload_object(
313254 } else {
314255 return Err ( GitLFSError :: GeneralError ( String :: from ( "Not found " ) ) ) ;
315256 } ;
316- let split_size = match PackConfig :: get_size_from_str ( & config. local . split_size , || Ok ( 0 ) ) {
317- Ok ( split_size) => split_size,
318- Err ( err) => return Err ( GitLFSError :: GeneralError ( err) ) ,
319- } ;
320257
321- if config. local . enable_split && meta. splited {
322- // assert!(request_vars.size == body_bytes.len() as i64, "size didn't match: {} != {}", request_vars.size, body_bytes.len()); // TODO: git client, request_vars.size is `0`!!
323- // split object to blocks
324- match file_storage
325- . put_object_with_chunk ( & meta. oid , & body_bytes, split_size)
326- . await
327- {
328- Ok ( _) => { }
329- Err ( _) => {
330- lfs_delete_meta ( & db_storage, req_obj) . await ?;
331- // TODO: whether/how to delete the uploaded blocks.
332- return Err ( GitLFSError :: GeneralError ( String :: from (
333- "Header not acceptable!" ,
334- ) ) ) ;
335- }
336- } ;
337- } else {
338- // normal mode
339- let res = file_storage. put_object ( & meta. oid , body_bytes) . await ;
340- if res. is_err ( ) {
341- lfs_delete_meta ( & db_storage, req_obj) . await . unwrap ( ) ;
342- return Err ( GitLFSError :: GeneralError ( String :: from (
343- "Header not acceptable!" ,
344- ) ) ) ;
345- }
258+ // normal mode
259+ let res = file_storage. put_object ( & meta. oid , body_bytes) . await ;
260+ if res. is_err ( ) {
261+ lfs_delete_meta ( & db_storage, req_obj) . await . unwrap ( ) ;
262+ return Err ( GitLFSError :: GeneralError ( String :: from (
263+ "Header not acceptable!" ,
264+ ) ) ) ;
346265 }
347266 Ok ( ( ) )
348267}
@@ -358,53 +277,14 @@ pub async fn lfs_download_object(
358277
359278 let meta = lfs_get_meta ( db_storage. clone ( ) , & oid) . await ?;
360279 match meta {
361- Some ( meta) => {
362- if meta. splited {
363- // client didn't support split, splice the object and return it.
364- let relations = db_storage. get_lfs_relations ( & meta. oid ) . await . unwrap ( ) ;
365- if relations. is_empty ( ) {
366- return Err ( GitLFSError :: GeneralError (
367- "oid didn't have chunks" . to_string ( ) ,
368- ) ) ;
369- }
370- let ( tx, rx) = tokio:: sync:: mpsc:: channel ( 1 ) ;
371- let oid = oid. clone ( ) ;
372- tokio:: spawn ( async move {
373- let chunks_len = relations. len ( ) ;
374- for relation in relations {
375- let sub_bytes = file_storage. get_object ( & relation. sub_oid ) . await . unwrap ( ) ;
376- if let Err ( err) = tx. send ( Ok ( sub_bytes) ) . await {
377- tracing:: error!(
378- "lfs object download failed, failed to send chunk [{}], error: {}" ,
379- relation. offset,
380- err
381- ) ;
382- break ;
383- }
384- }
385- tracing:: debug!(
386- "lfs object download completed for oid: {}, {} chunks" ,
387- oid,
388- chunks_len
389- ) ;
390- } ) ;
391- Ok ( ReceiverStream :: new ( rx) )
392- } else {
393- let meta = lfs_get_meta ( db_storage, & oid) . await ?. unwrap ( ) ;
394- let bytes = file_storage. get_object ( & meta. oid ) . await . unwrap ( ) ;
395- let ( tx, rx) = tokio:: sync:: mpsc:: channel ( 1 ) ;
396- tx. send ( Ok ( bytes) ) . await . unwrap ( ) ;
397- Ok ( ReceiverStream :: new ( rx) )
398- }
280+ Some ( _) => {
281+ let meta = lfs_get_meta ( db_storage, & oid) . await ?. unwrap ( ) ;
282+ let bytes = file_storage. get_object ( & meta. oid ) . await . unwrap ( ) ;
283+ let ( tx, rx) = tokio:: sync:: mpsc:: channel ( 1 ) ;
284+ tx. send ( Ok ( bytes) ) . await . unwrap ( ) ;
285+ Ok ( ReceiverStream :: new ( rx) )
399286 }
400287 None => {
401- // check if the oid is a part of a split object, if so, return the part.
402- if !lfs_check_sub_oid_exist ( db_storage, & oid) . await . unwrap ( ) {
403- return Err ( GitLFSError :: GeneralError (
404- "oid didn't belong to any object" . to_string ( ) ,
405- ) ) ;
406- }
407-
408288 let bytes = file_storage. get_object ( & oid) . await . unwrap ( ) ;
409289 // because return type must be `ReceiverStream`, so we need to wrap the bytes into a stream.
410290 let ( tx, rx) = tokio:: sync:: mpsc:: channel ( 1 ) ;
@@ -414,46 +294,6 @@ pub async fn lfs_download_object(
414294 }
415295}
416296
417- /// Download a chunk from a large object.
418- /// It's used when server didn't have splited chunk, but client request a chunk.
419- /// If the server enable split, then the chunk must be a splited chunk, rather than a random part of the object.
420- pub async fn lfs_download_chunk (
421- storage : Storage ,
422- origin_oid : & str ,
423- chunk_oid : & String ,
424- offset : u64 ,
425- size : u64 ,
426- ) -> Result < Bytes , GitLFSError > {
427- let config = & storage. config ( ) . lfs ;
428- let db_storage = storage. lfs_db_storage ( ) ;
429- let file_storage = storage. lfs_file_storage ( ) ;
430-
431- // check if the chunk is already exist.
432- if config. local . enable_split {
433- let relations = db_storage. get_lfs_relations ( origin_oid) . await . unwrap ( ) ;
434- let chunk_relation = relations. iter ( ) . find ( |r| & r. sub_oid == chunk_oid) ;
435- if chunk_relation. is_none ( ) {
436- return Err ( GitLFSError :: GeneralError (
437- "Chunk not found in split object" . to_string ( ) ,
438- ) ) ;
439- }
440- let chunk = file_storage. get_object ( chunk_oid) . await . unwrap ( ) ;
441- Ok ( chunk)
442- } else {
443- // return part of the original object.
444- let bytes = file_storage. get_object ( origin_oid) . await . unwrap ( ) ;
445- let chunk_bytes = bytes[ offset as usize ..( offset + size) as usize ] . to_vec ( ) ;
446- // check hash
447- let chunk_hash = hex:: encode ( ring:: digest:: digest ( & ring:: digest:: SHA256 , & chunk_bytes) ) ;
448- if chunk_hash != * chunk_oid {
449- return Err ( GitLFSError :: GeneralError (
450- "Chunk hash not match" . to_string ( ) ,
451- ) ) ;
452- }
453- Ok ( Bytes :: from ( chunk_bytes) )
454- }
455- }
456-
457297async fn lfs_get_filtered_locks (
458298 storage : LfsDbStorage ,
459299 refspec : & str ,
@@ -591,7 +431,6 @@ async fn lfs_delete_meta(
591431 req_obj : & RequestObject ,
592432) -> Result < ( ) , GitLFSError > {
593433 let res = storage. delete_lfs_object ( req_obj. oid . to_owned ( ) ) . await ;
594- storage. delete_lfs_relations ( & req_obj. oid ) . await . unwrap ( ) ;
595434 match res {
596435 Ok ( _) => Ok ( ( ) ) ,
597436 Err ( _) => Err ( GitLFSError :: GeneralError ( "" . to_string ( ) ) ) ,
@@ -669,14 +508,6 @@ async fn delete_lock(
669508 }
670509}
671510
672- async fn lfs_check_sub_oid_exist (
673- storage : LfsDbStorage ,
674- sub_oid : & str ,
675- ) -> Result < bool , GitLFSError > {
676- let result = storage. get_lfs_relations_ori_oid ( sub_oid) . await . unwrap ( ) ;
677- Ok ( !result. is_empty ( ) )
678- }
679-
680511#[ cfg( test) ]
681512mod tests {
682513 use super :: * ;
0 commit comments