chore: remove some config about split chunk (#1778)

* chore: remove some config about split chunk

* chore/remove related config in common

* chore/remove storage_type_enum

* fix:PostgreSQL-only migration

Author: ztygod

ceres/src/lfs/handler.rs

@@ -183,15 +183,14 @@ pub async fn lfs_process_batch(
     let mut response_objects = Vec::new();
     let file_storage = storage.lfs_file_storage();
     let db_storage = storage.lfs_db_storage();
-    let config = storage.config().lfs.clone();
     for object in objects {
         let meta_res = lfs_get_meta(db_storage.clone(), &object.oid).await.unwrap();
         let meta = match meta_res {
             Some(meta) => meta,
             None => {
                 if request.operation == Operation::Upload {
                     // Save to database if not exist.
-                    let meta = MetaObject::new(&object, &config);
+                    let meta = MetaObject::new(&object);
                     db_storage
                         .new_lfs_object(meta.clone().into())
                         .await
@@ -519,7 +518,6 @@ mod tests {
             oid: "oid1".into(),
             size: 10,
             exist: true,
-            splited: false,
         };
         let res = ResponseObject::new(
             &meta,
@@ -543,7 +541,6 @@ mod tests {
             oid: "oid2".into(),
             size: 20,
             exist: false,
-            splited: false,
         };
         let res = ResponseObject::new(
             &meta,
@@ -566,7 +563,6 @@ mod tests {
             oid: "oid3".into(),
             size: 30,
             exist: false,
-            splited: false,
         };
         let res = ResponseObject::new(
             &meta,

ceres/src/lfs/lfs_structs.rs

@@ -1,7 +1,5 @@
 use callisto::lfs_objects;
-use callisto::sea_orm_active_enums::StorageTypeEnum;
 use chrono::{DateTime, Duration, Utc};
-use common::config::LFSConfig;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use utoipa::ToSchema;
@@ -44,7 +42,6 @@ pub struct MetaObject {
     pub oid: String,
     pub size: i64,
     pub exist: bool,
-    pub splited: bool,
 }
 
 impl From<lfs_objects::Model> for MetaObject {
@@ -53,7 +50,6 @@ impl From<lfs_objects::Model> for MetaObject {
             oid: value.oid,
             size: value.size,
             exist: value.exist,
-            splited: value.splited,
         }
     }
 }
@@ -64,23 +60,16 @@ impl From<MetaObject> for lfs_objects::Model {
             oid: value.oid,
             size: value.size,
             exist: value.exist,
-            splited: value.splited,
         }
     }
 }
 
 impl MetaObject {
-    pub fn new(req_obj: &RequestObject, config: &LFSConfig) -> Self {
-        let splited = config.local.enable_split;
+    pub fn new(req_obj: &RequestObject) -> Self {
         Self {
             oid: req_obj.oid.to_string(),
             size: req_obj.size,
             exist: true,
-            splited: if StorageTypeEnum::AwsS3 == config.storage_type.clone().into() {
-                false
-            } else {
-                splited
-            },
         }
     }
 }

common/src/config.rs

@@ -13,8 +13,6 @@ use config::builder::DefaultState;
 use config::{Source, ValueKind};
 use serde::{Deserialize, Deserializer, Serialize};
 
-use callisto::sea_orm_active_enums::StorageTypeEnum;
-
 use crate::utils;
 
 /// Retrieves the base directory path for Mega
@@ -531,26 +529,6 @@ pub enum StorageType {
     S3,
 }
 
-impl From<StorageTypeEnum> for StorageType {
-    fn from(value: StorageTypeEnum) -> Self {
-        match value {
-            StorageTypeEnum::Database => StorageType::Database,
-            StorageTypeEnum::LocalFs => StorageType::LocalFs,
-            StorageTypeEnum::AwsS3 => StorageType::S3,
-        }
-    }
-}
-
-impl From<StorageType> for StorageTypeEnum {
-    fn from(value: StorageType) -> Self {
-        match value {
-            StorageType::Database => StorageTypeEnum::Database,
-            StorageType::LocalFs => StorageTypeEnum::LocalFs,
-            StorageType::S3 => StorageTypeEnum::AwsS3,
-        }
-    }
-}
-
 impl Default for LFSConfig {
     fn default() -> Self {
         Self {
@@ -564,17 +542,12 @@ impl Default for LFSConfig {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct LFSLocalConfig {
     pub lfs_file_path: PathBuf,
-    pub enable_split: bool,
-    #[serde(deserialize_with = "string_or_usize")]
-    pub split_size: String,
 }
 
 impl Default for LFSLocalConfig {
     fn default() -> Self {
         Self {
             lfs_file_path: mega_base().join("lfs"),
-            enable_split: true,
-            split_size: "20M".to_string(),
         }
     }
 }

config/config-workflow.toml

@@ -110,13 +110,6 @@ http_url = "http://localhost:8000"
 # set the local path of the lfs storage
 lfs_file_path = "${base_dir}/lfs"
 
-## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature.
-# Enable or disable splitting large files into smaller chunks
-enable_split = true # Default is disabled. Set to true to enable file splitting.
-
-# Size of each file chunk when splitting is enabled. Ignored if splitting is disabled.
-split_size = "20M"
-
 [lfs.aws]
 s3_bucket = "gitmono"
 s3_region = "ap-southeast-2"
@@ -138,4 +131,8 @@ cookie_domain = "localhost"
 campsite_api_domain = "http://api.gitmega.com"
 
 # allowed cors origins
-allowed_cors_origins = ["http://local.gitmega.com", "https://app.gitmega.com", "http://app.gitmono.test"]
+allowed_cors_origins = [
+    "http://local.gitmega.com",
+    "https://app.gitmega.com",
+    "http://app.gitmono.test",
+]

config/config.toml

@@ -114,14 +114,6 @@ http_url = "http://localhost:8000"
 # set the local path of the lfs storage
 lfs_file_path = "${base_dir}/lfs"
 
-## IMPORTANT: The 'enable_split' feature can only be enabled for new databases. Existing databases do not support this feature.
-# Enable or disable splitting large files into smaller chunks
-enable_split = true # Default is disabled. Set to true to enable file splitting.
-
-# Size of each file chunk when splitting is enabled. Ignored if splitting is disabled.
-split_size = "20M"
-
-
 [s3]
 # AWS region name.
 # For AWS S3 this must match the bucket's region.

jupiter/callisto/src/lfs_objects.rs

@@ -10,7 +10,6 @@ pub struct Model {
     pub oid: String,
     pub size: i64,
     pub exist: bool,
-    pub splited: bool,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

jupiter/callisto/src/sea_orm_active_enums.rs

@@ -123,13 +123,3 @@ pub enum ReferenceTypeEnum {
     #[sea_orm(string_value = "blocks")]
     Blocks,
 }
-#[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "storage_type_enum")]
-pub enum StorageTypeEnum {
-    #[sea_orm(string_value = "database")]
-    Database,
-    #[sea_orm(string_value = "local_fs")]
-    LocalFs,
-    #[sea_orm(string_value = "aws_s3")]
-    AwsS3,
-}

jupiter/src/migration/m20260108_085945_remove_splited_in_lfs_objects.rs

@@ -0,0 +1,40 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(LfsObjects::Table)
+                    .drop_column(LfsObjects::Splited)
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(LfsObjects::Table)
+                    .add_column(
+                        ColumnDef::new(LfsObjects::Splited)
+                            .boolean()
+                            .not_null()
+                            .default(false),
+                    )
+                    .to_owned(),
+            )
+            .await
+    }
+}
+
+#[derive(Iden)]
+enum LfsObjects {
+    Table,
+    Splited,
+}

jupiter/src/migration/m20260108_105158_remove_storage_type_enum.rs

@@ -0,0 +1,55 @@
+use sea_orm_migration::{
+    prelude::*,
+    sea_orm::{DatabaseBackend, Statement},
+};
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        let backend = manager.get_database_backend();
+
+        match backend {
+            DatabaseBackend::Postgres => {
+                manager
+                    .get_connection()
+                    .execute(Statement::from_string(
+                        DatabaseBackend::Postgres,
+                        "DROP TYPE IF EXISTS storage_type_enum;".to_owned(),
+                    ))
+                    .await?;
+            }
+            DatabaseBackend::Sqlite | DatabaseBackend::MySql => {}
+        }
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        let backend = manager.get_database_backend();
+
+        match backend {
+            DatabaseBackend::Postgres => {
+                manager
+                    .get_connection()
+                    .execute(Statement::from_string(
+                        DatabaseBackend::Postgres,
+                        r#"
+                            CREATE TYPE storage_type_enum AS ENUM (
+                                'database',
+                                'local_fs',
+                                'aws_s3'
+                            );
+                            "#
+                        .to_owned(),
+                    ))
+                    .await?;
+            }
+            DatabaseBackend::Sqlite | DatabaseBackend::MySql => {}
+        }
+
+        Ok(())
+    }
+}

jupiter/src/migration/mod.rs

@@ -76,6 +76,8 @@ mod m20251203_013745_add_dynamic_sidebar;
 mod m20251210_113942_remove_unique_constraint_from_order_index;
 mod m20260106_070511_add_retry_time;
 mod m20260106_070515_remove_relay_mq_lfs_raw_table;
+mod m20260108_085945_remove_splited_in_lfs_objects;
+mod m20260108_105158_remove_storage_type_enum;
 
 /// Creates a primary key column definition with big integer type.
 ///
@@ -140,6 +142,8 @@ impl MigratorTrait for Migrator {
             Box::new(m20251210_113942_remove_unique_constraint_from_order_index::Migration),
             Box::new(m20260106_070511_add_retry_time::Migration),
             Box::new(m20260106_070515_remove_relay_mq_lfs_raw_table::Migration),
+            Box::new(m20260108_085945_remove_splited_in_lfs_objects::Migration),
+            Box::new(m20260108_105158_remove_storage_type_enum::Migration),
         ]
     }
 }