Skip to content
This repository was archived by the owner on Oct 19, 2026. It is now read-only.

Commit 57ceddb

Browse files
authored
chore: remove Relay / MQ / LFS / raw_blob table (#1766)
* feat: remove Relay / MQ / LFS / raw_blob table * fix: fix some problems
1 parent e8aa84a commit 57ceddb

24 files changed

Lines changed: 180 additions & 1116 deletions

‎ceres/src/lfs/handler.rs‎

Lines changed: 17 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,14 @@ use rand::prelude::*;
88
use tokio_stream::wrappers::ReceiverStream;
99

1010
use callisto::lfs_locks;
11-
use common::config::PackConfig;
1211
use common::errors::{GitLFSError, MegaError};
1312
use jupiter::storage::Storage;
1413
use jupiter::storage::lfs_db_storage::LfsDbStorage;
1514

1615
use crate::lfs::lfs_structs::{
17-
BatchRequest, BatchResponse, ChunkDownloadObject, Link, Lock, LockList, LockListQuery,
18-
LockRequest, MetaObject, ObjectError, Operation, RequestObject, ResCondition, ResponseObject,
19-
TransferMode, UnlockRequest, VerifiableLockList, VerifiableLockRequest,
16+
BatchRequest, BatchResponse, Lock, LockList, LockListQuery, LockRequest, MetaObject,
17+
ObjectError, Operation, RequestObject, ResCondition, ResponseObject, TransferMode,
18+
UnlockRequest, VerifiableLockList, VerifiableLockRequest,
2019
};
2120

2221
pub async fn lfs_retrieve_lock(
@@ -210,8 +209,7 @@ pub async fn lfs_process_batch(
210209
}
211210
}
212211
};
213-
let enable_split = meta.splited && config.local.enable_split;
214-
let file_exist = file_storage.exist_object(&meta.oid, enable_split).await;
212+
let file_exist = file_storage.exist_object(&meta.oid).await;
215213
let download_url = file_storage
216214
.download_url(&meta.oid, listen_addr)
217215
.await
@@ -240,70 +238,13 @@ pub async fn lfs_process_batch(
240238
})
241239
}
242240

243-
/// if server enable split, then return a list of chunk ids.
244-
/// else return an error.
245-
pub async fn lfs_fetch_chunk_ids(
246-
storage: &Storage,
247-
oid: &str,
248-
listen_addr: &str,
249-
) -> Result<Vec<ChunkDownloadObject>, GitLFSError> {
250-
let config = storage.config().lfs.clone();
251-
252-
if !config.local.enable_split {
253-
return Err(GitLFSError::GeneralError(
254-
"Server didn't run in `split` mode, didn't support chunk ids".to_string(),
255-
));
256-
}
257-
let db_storage = storage.lfs_db_storage();
258-
259-
let meta = lfs_get_meta(db_storage.clone(), oid)
260-
.await
261-
.map_err(|_| GitLFSError::GeneralError("".to_string()))?
262-
.unwrap();
263-
assert!(meta.splited, "database didn't match the split mode");
264-
265-
let relations = db_storage
266-
.get_lfs_relations(oid)
267-
.await
268-
.map_err(|_| GitLFSError::GeneralError("".to_string()))?;
269-
270-
if relations.is_empty() {
271-
return Err(GitLFSError::GeneralError(
272-
"oid didn't have chunks".to_string(),
273-
));
274-
}
275-
let mut response_objects = Vec::<ChunkDownloadObject>::new();
276-
277-
for relation in relations {
278-
// Reuse RequestArgs to create a link
279-
let req_obj = RequestObject {
280-
oid: relation.sub_oid.clone(),
281-
size: relation.size,
282-
..Default::default()
283-
};
284-
let download_url = storage
285-
.lfs_file_storage()
286-
.download_url(&req_obj.oid, listen_addr)
287-
.await
288-
.unwrap();
289-
response_objects.push(ChunkDownloadObject {
290-
sub_oid: relation.sub_oid,
291-
size: relation.size,
292-
offset: relation.offset,
293-
link: Link::new(&download_url),
294-
});
295-
}
296-
Ok(response_objects)
297-
}
298-
299241
/// Upload object to storage.
300242
/// if server enable split, split the object and upload each part to storage, save the relationship to database.
301243
pub async fn lfs_upload_object(
302244
storage: &Storage,
303245
req_obj: &RequestObject,
304246
body_bytes: Vec<u8>,
305247
) -> Result<(), GitLFSError> {
306-
let config = storage.config().lfs.clone();
307248
let db_storage = storage.lfs_db_storage();
308249
let file_storage = storage.lfs_file_storage();
309250

@@ -313,36 +254,14 @@ pub async fn lfs_upload_object(
313254
} else {
314255
return Err(GitLFSError::GeneralError(String::from("Not found ")));
315256
};
316-
let split_size = match PackConfig::get_size_from_str(&config.local.split_size, || Ok(0)) {
317-
Ok(split_size) => split_size,
318-
Err(err) => return Err(GitLFSError::GeneralError(err)),
319-
};
320257

321-
if config.local.enable_split && meta.splited {
322-
// assert!(request_vars.size == body_bytes.len() as i64, "size didn't match: {} != {}", request_vars.size, body_bytes.len()); // TODO: git client, request_vars.size is `0`!!
323-
// split object to blocks
324-
match file_storage
325-
.put_object_with_chunk(&meta.oid, &body_bytes, split_size)
326-
.await
327-
{
328-
Ok(_) => {}
329-
Err(_) => {
330-
lfs_delete_meta(&db_storage, req_obj).await?;
331-
// TODO: whether/how to delete the uploaded blocks.
332-
return Err(GitLFSError::GeneralError(String::from(
333-
"Header not acceptable!",
334-
)));
335-
}
336-
};
337-
} else {
338-
// normal mode
339-
let res = file_storage.put_object(&meta.oid, body_bytes).await;
340-
if res.is_err() {
341-
lfs_delete_meta(&db_storage, req_obj).await.unwrap();
342-
return Err(GitLFSError::GeneralError(String::from(
343-
"Header not acceptable!",
344-
)));
345-
}
258+
// normal mode
259+
let res = file_storage.put_object(&meta.oid, body_bytes).await;
260+
if res.is_err() {
261+
lfs_delete_meta(&db_storage, req_obj).await.unwrap();
262+
return Err(GitLFSError::GeneralError(String::from(
263+
"Header not acceptable!",
264+
)));
346265
}
347266
Ok(())
348267
}
@@ -358,53 +277,14 @@ pub async fn lfs_download_object(
358277

359278
let meta = lfs_get_meta(db_storage.clone(), &oid).await?;
360279
match meta {
361-
Some(meta) => {
362-
if meta.splited {
363-
// client didn't support split, splice the object and return it.
364-
let relations = db_storage.get_lfs_relations(&meta.oid).await.unwrap();
365-
if relations.is_empty() {
366-
return Err(GitLFSError::GeneralError(
367-
"oid didn't have chunks".to_string(),
368-
));
369-
}
370-
let (tx, rx) = tokio::sync::mpsc::channel(1);
371-
let oid = oid.clone();
372-
tokio::spawn(async move {
373-
let chunks_len = relations.len();
374-
for relation in relations {
375-
let sub_bytes = file_storage.get_object(&relation.sub_oid).await.unwrap();
376-
if let Err(err) = tx.send(Ok(sub_bytes)).await {
377-
tracing::error!(
378-
"lfs object download failed, failed to send chunk [{}], error: {}",
379-
relation.offset,
380-
err
381-
);
382-
break;
383-
}
384-
}
385-
tracing::debug!(
386-
"lfs object download completed for oid: {}, {} chunks",
387-
oid,
388-
chunks_len
389-
);
390-
});
391-
Ok(ReceiverStream::new(rx))
392-
} else {
393-
let meta = lfs_get_meta(db_storage, &oid).await?.unwrap();
394-
let bytes = file_storage.get_object(&meta.oid).await.unwrap();
395-
let (tx, rx) = tokio::sync::mpsc::channel(1);
396-
tx.send(Ok(bytes)).await.unwrap();
397-
Ok(ReceiverStream::new(rx))
398-
}
280+
Some(_) => {
281+
let meta = lfs_get_meta(db_storage, &oid).await?.unwrap();
282+
let bytes = file_storage.get_object(&meta.oid).await.unwrap();
283+
let (tx, rx) = tokio::sync::mpsc::channel(1);
284+
tx.send(Ok(bytes)).await.unwrap();
285+
Ok(ReceiverStream::new(rx))
399286
}
400287
None => {
401-
// check if the oid is a part of a split object, if so, return the part.
402-
if !lfs_check_sub_oid_exist(db_storage, &oid).await.unwrap() {
403-
return Err(GitLFSError::GeneralError(
404-
"oid didn't belong to any object".to_string(),
405-
));
406-
}
407-
408288
let bytes = file_storage.get_object(&oid).await.unwrap();
409289
// because return type must be `ReceiverStream`, so we need to wrap the bytes into a stream.
410290
let (tx, rx) = tokio::sync::mpsc::channel(1);
@@ -414,46 +294,6 @@ pub async fn lfs_download_object(
414294
}
415295
}
416296

417-
/// Download a chunk from a large object.
418-
/// It's used when server didn't have splited chunk, but client request a chunk.
419-
/// If the server enable split, then the chunk must be a splited chunk, rather than a random part of the object.
420-
pub async fn lfs_download_chunk(
421-
storage: Storage,
422-
origin_oid: &str,
423-
chunk_oid: &String,
424-
offset: u64,
425-
size: u64,
426-
) -> Result<Bytes, GitLFSError> {
427-
let config = &storage.config().lfs;
428-
let db_storage = storage.lfs_db_storage();
429-
let file_storage = storage.lfs_file_storage();
430-
431-
// check if the chunk is already exist.
432-
if config.local.enable_split {
433-
let relations = db_storage.get_lfs_relations(origin_oid).await.unwrap();
434-
let chunk_relation = relations.iter().find(|r| &r.sub_oid == chunk_oid);
435-
if chunk_relation.is_none() {
436-
return Err(GitLFSError::GeneralError(
437-
"Chunk not found in split object".to_string(),
438-
));
439-
}
440-
let chunk = file_storage.get_object(chunk_oid).await.unwrap();
441-
Ok(chunk)
442-
} else {
443-
// return part of the original object.
444-
let bytes = file_storage.get_object(origin_oid).await.unwrap();
445-
let chunk_bytes = bytes[offset as usize..(offset + size) as usize].to_vec();
446-
// check hash
447-
let chunk_hash = hex::encode(ring::digest::digest(&ring::digest::SHA256, &chunk_bytes));
448-
if chunk_hash != *chunk_oid {
449-
return Err(GitLFSError::GeneralError(
450-
"Chunk hash not match".to_string(),
451-
));
452-
}
453-
Ok(Bytes::from(chunk_bytes))
454-
}
455-
}
456-
457297
async fn lfs_get_filtered_locks(
458298
storage: LfsDbStorage,
459299
refspec: &str,
@@ -591,7 +431,6 @@ async fn lfs_delete_meta(
591431
req_obj: &RequestObject,
592432
) -> Result<(), GitLFSError> {
593433
let res = storage.delete_lfs_object(req_obj.oid.to_owned()).await;
594-
storage.delete_lfs_relations(&req_obj.oid).await.unwrap();
595434
match res {
596435
Ok(_) => Ok(()),
597436
Err(_) => Err(GitLFSError::GeneralError("".to_string())),
@@ -669,14 +508,6 @@ async fn delete_lock(
669508
}
670509
}
671510

672-
async fn lfs_check_sub_oid_exist(
673-
storage: LfsDbStorage,
674-
sub_oid: &str,
675-
) -> Result<bool, GitLFSError> {
676-
let result = storage.get_lfs_relations_ori_oid(sub_oid).await.unwrap();
677-
Ok(!result.is_empty())
678-
}
679-
680511
#[cfg(test)]
681512
mod tests {
682513
use super::*;

‎jupiter/callisto/src/lfs_split_relations.rs‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎jupiter/callisto/src/mod.rs‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ pub mod item_labels;
2525
pub mod label;
2626
pub mod lfs_locks;
2727
pub mod lfs_objects;
28-
pub mod lfs_split_relations;
2928
pub mod mega_blob;
3029
pub mod mega_cl;
3130
pub mod mega_cl_commits;
@@ -37,17 +36,9 @@ pub mod mega_refs;
3736
pub mod mega_tag;
3837
pub mod mega_tree;
3938
pub mod merge_queue;
40-
pub mod mq_storage;
4139
pub mod notes;
4240
pub mod path_check_configs;
43-
pub mod raw_blob;
4441
pub mod reactions;
45-
pub mod relay_lfs_info;
46-
pub mod relay_node;
47-
pub mod relay_nostr_event;
48-
pub mod relay_nostr_req;
49-
pub mod relay_path_mapping;
50-
pub mod relay_repo_info;
5142
pub mod sea_orm_active_enums;
5243
pub mod ssh_keys;
5344
pub mod tasks;

‎jupiter/callisto/src/mq_storage.rs‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎jupiter/callisto/src/prelude.rs‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ pub use super::item_labels::Entity as ItemLabels;
2222
pub use super::label::Entity as Label;
2323
pub use super::lfs_locks::Entity as LfsLocks;
2424
pub use super::lfs_objects::Entity as LfsObjects;
25-
pub use super::lfs_split_relations::Entity as LfsSplitRelations;
2625
pub use super::mega_blob::Entity as MegaBlob;
2726
pub use super::mega_cl::Entity as MegaCl;
2827
pub use super::mega_cl_commits::Entity as MegaClCommits;
@@ -34,17 +33,9 @@ pub use super::mega_refs::Entity as MegaRefs;
3433
pub use super::mega_tag::Entity as MegaTag;
3534
pub use super::mega_tree::Entity as MegaTree;
3635
pub use super::merge_queue::Entity as MergeQueue;
37-
pub use super::mq_storage::Entity as MqStorage;
3836
pub use super::notes::Entity as Notes;
3937
pub use super::path_check_configs::Entity as PathCheckConfigs;
40-
pub use super::raw_blob::Entity as RawBlob;
4138
pub use super::reactions::Entity as Reactions;
42-
pub use super::relay_lfs_info::Entity as RelayLfsInfo;
43-
pub use super::relay_node::Entity as RelayNode;
44-
pub use super::relay_nostr_event::Entity as RelayNostrEvent;
45-
pub use super::relay_nostr_req::Entity as RelayNostrReq;
46-
pub use super::relay_path_mapping::Entity as RelayPathMapping;
47-
pub use super::relay_repo_info::Entity as RelayRepoInfo;
4839
pub use super::ssh_keys::Entity as SshKeys;
4940
pub use super::tasks::Entity as Tasks;
5041
pub use super::vault::Entity as Vault;

‎jupiter/callisto/src/raw_blob.rs‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)