feat: fix migration and add note table (#2033)

* feat: fix migration and add note table

* fix: merge clippy

Author: benjamin-747

.github/workflows/base.yml

@@ -8,15 +8,18 @@
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - main
   pull_request:
     paths-ignore:
-      - 'config/**'
-      - 'docker/**'
-      - 'docs/**'
-      - 'moon/**'
-      - 'scripts/**'
-      - 'toolchains/**'
-      - '.github/workflows/web-**'
+      - "config/**"
+      - "docker/**"
+      - "docs/**"
+      - "moon/**"
+      - "scripts/**"
+      - "toolchains/**"
+      - ".github/workflows/web-**"
 
 name: Base GitHub Action for Check, Test and Lints
 
@@ -26,7 +29,7 @@ concurrency:
 
 jobs:
   format:
-    if: ${{ github.repository == 'web3infra-foundation/mega' }}
+    if: ${{ !(github.repository == 'web3infra-foundation/mega' && github.event_name == 'push') }}
     name: Rustfmt Check
     runs-on: ubuntu-latest
     steps:
@@ -44,12 +47,14 @@ jobs:
 
   #
   clippy:
-    if: ${{ github.repository == 'web3infra-foundation/mega' }}
+    if: ${{ !(github.repository == 'web3infra-foundation/mega' && github.event_name == 'push') }}
     name: Clippy Check
     runs-on: ubuntu-latest
     env:
       CARGO_TERM_COLOR: always
       RUSTUP_TOOLCHAIN: stable
+      RUSTC_WRAPPER: sccache
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -61,26 +66,42 @@ jobs:
         with:
           cache-key: sysdeps
           platform: ubuntu
+      - name: Install sccache
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y sccache
+      - name: Prepare sccache directory
+        run: |
+          mkdir -p "$SCCACHE_DIR"
+      - name: Cache sccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-${{ runner.os }}-stable-${{ hashFiles('**/Cargo.lock') }}-clippy
+          restore-keys: |
+            sccache-${{ runner.os }}-stable-${{ hashFiles('**/Cargo.lock') }}-
+            sccache-${{ runner.os }}-stable-
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: base-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
+          cache-on-failure: true
       - name: Run cargo clippy
         run: |
+          sccache --start-server || true
           cargo +stable clippy --all-targets --all-features -- -D warnings
+          sccache --show-stats || true
 
   test:
+    if: ${{ !(github.repository == 'web3infra-foundation/mega' && github.event_name == 'push') }}
     name: Full Test
-    if: ${{ github.repository == 'web3infra-foundation/mega' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu
-            runs-on: [self-hosted, Linux, X64]
-            home: /home/github
-
-    runs-on: ${{ matrix.runs-on }}
+    runs-on: ${{ fromJson(github.repository == 'web3infra-foundation/mega' && '["self-hosted","Linux","X64"]' || '"ubuntu-latest"') }}
     env:
       CARGO_TERM_COLOR: always
       RUSTUP_TOOLCHAIN: stable
-      HOME: ${{ matrix.home }}
+      HOME: ${{ github.repository == 'web3infra-foundation/mega' && '/home/github' || '/home/runner' }}
+      RUSTC_WRAPPER: sccache
+      SCCACHE_DIR: ${{ github.workspace }}/.sccache
     steps:
       - name: Install Redis
         run: sudo apt-get update && sudo apt-get install -y redis-server
@@ -95,25 +116,48 @@ jobs:
         uses: ./.github/install-dep
         with:
           cache-key: sysdeps
-          platform: ${{ matrix.os }}
-          self-hosted: true
+          platform: ubuntu
+          self-hosted: ${{ github.repository == 'web3infra-foundation/mega' }}
           use-gtk: false
+      - name: Install sccache
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y sccache
+      - name: Prepare sccache directory
+        run: |
+          mkdir -p "$SCCACHE_DIR"
+      - name: Cache sccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-${{ runner.os }}-stable-${{ hashFiles('**/Cargo.lock') }}-test
+          restore-keys: |
+            sccache-${{ runner.os }}-stable-${{ hashFiles('**/Cargo.lock') }}-
+            sccache-${{ runner.os }}-stable-
+      - name: Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: base-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
+          cache-on-failure: true
 
       - name: Set up git lfs
         run: |
-          echo "GPG_TTY=${{ matrix.os == 'windows' && 'CON' || '$(tty)' }}" >> $GITHUB_ENV
+          echo "GPG_TTY=$(tty)" >> $GITHUB_ENV
           git lfs install
           git config --global user.email "mega@github.com"
           git config --global user.name "Mega"
           git config --global lfs.url http://localhost:8000
 
       - name: Run cargo test
         run: |
+          sccache --start-server || true
           cargo test --manifest-path common/Cargo.toml --all-features --no-fail-fast -- --nocapture
           cargo test --manifest-path jupiter/Cargo.toml --all-features --no-fail-fast -- --nocapture
           cargo test --manifest-path ceres/Cargo.toml --all-features --no-fail-fast -- --nocapture
           cargo test --manifest-path vault/Cargo.toml --all-features --no-fail-fast -- --nocapture
           cargo test --manifest-path saturn/Cargo.toml --all-features --no-fail-fast -- --nocapture
+          cargo test --manifest-path orion-server/Cargo.toml --all-features --no-fail-fast -- --nocapture
+          sccache --show-stats || true
 
   # Note: The fuse/scorpio job has been removed as scorpio has been moved
   # to its own repository: https://github.com/web3infra-foundation/scorpiofs

.github/workflows/mono-engine-deploy.yml

@@ -85,13 +85,19 @@ jobs:
 
           ECR_IMAGE="$ECR_REGISTRY/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}:$IMAGE_TAG"
           GCP_IMAGE="us-central1-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPOSITORY }}:$IMAGE_TAG"
+          ECR_CACHE_IMAGE="$ECR_REGISTRY/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}:buildcache-${ARCH_SUFFIX}"
+          CACHE_SCOPE="mono-engine-${ARCH_SUFFIX}"
 
           echo "ECR_IMAGE=$ECR_IMAGE"
           echo "GCP_IMAGE=$GCP_IMAGE"
+          echo "ECR_CACHE_IMAGE=$ECR_CACHE_IMAGE"
+          echo "CACHE_SCOPE=$CACHE_SCOPE"
 
           docker buildx build \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
+            --cache-from type=gha,scope=$CACHE_SCOPE \
+            --cache-from type=registry,ref=$ECR_CACHE_IMAGE \
+            --cache-to type=gha,mode=max,scope=$CACHE_SCOPE \
+            --cache-to type=registry,ref=$ECR_CACHE_IMAGE,mode=max \
             --provenance=false \
             --sbom=false \
             -f ./mono/Dockerfile \

.github/workflows/orion-server-deploy.yml

@@ -79,11 +79,15 @@ jobs:
           GCP_IMAGE_BASE="us-central1-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.REPOSITORY }}"
 
           TAG="${{ env.IMAGE_TAG_BASE }}-$ARCH_SUFFIX"
+          AWS_CACHE_IMAGE="$AWS_IMAGE_BASE:buildcache-$ARCH_SUFFIX"
+          CACHE_SCOPE="orion-server-$ARCH_SUFFIX"
 
           docker buildx build \
             --platform "$PLATFORM" \
-            --cache-from type=gha \
-            --cache-to type=gha,mode=max \
+            --cache-from type=gha,scope=$CACHE_SCOPE \
+            --cache-from type=registry,ref=$AWS_CACHE_IMAGE \
+            --cache-to type=gha,mode=max,scope=$CACHE_SCOPE \
+            --cache-to type=registry,ref=$AWS_CACHE_IMAGE,mode=max \
             --provenance=false \
             --sbom=false \
             -f orion-server/Dockerfile \

ceres/Cargo.toml

@@ -37,8 +37,6 @@ async-recursion = { workspace = true }
 rand = { workspace = true }
 sysinfo = { workspace = true }
 utoipa = { workspace = true }
-base64 = { workspace = true }
-http = { workspace = true }
 regex = { workspace = true }
 tokio-util = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }

ceres/src/protocol/mod.rs

@@ -1,14 +1,12 @@
 use core::fmt;
 use std::{path::PathBuf, str::FromStr, sync::Arc};
 
-use base64::{engine::general_purpose, prelude::*};
 use bellatrix::Bellatrix;
 use callisto::sea_orm_active_enums::RefTypeEnum;
 use common::{
     errors::{MegaError, ProtocolError},
     utils::ZERO_ID,
 };
-use http::{HeaderMap, HeaderValue};
 use import_refs::RefCommand;
 use jupiter::redis::lock::RedLock;
 use repo::Repo;
@@ -225,62 +223,6 @@ impl SmartProtocol {
             Ok(Arc::new(res))
         }
     }
-
-    pub fn enable_http_auth(&self, state: &ProtocolApiState) -> bool {
-        state.storage.config().enable_http_auth()
-    }
-
-    pub async fn http_auth(
-        &mut self,
-        state: &ProtocolApiState,
-        header: &HeaderMap<HeaderValue>,
-    ) -> bool {
-        for (k, v) in header {
-            if k == http::header::AUTHORIZATION {
-                let decoded = general_purpose::STANDARD
-                    .decode(
-                        v.to_str()
-                            .unwrap()
-                            .strip_prefix("Basic ")
-                            .unwrap()
-                            .as_bytes(),
-                    )
-                    .unwrap();
-                let credentials = String::from_utf8(decoded).unwrap_or_default();
-                let mut parts = credentials.splitn(2, ':');
-                let username = parts.next().unwrap_or("");
-                self.username = Some(username.to_owned());
-                let token = parts.next().unwrap_or("");
-                let auth_config = state.storage.config().authentication.clone();
-                if auth_config.enable_test_user
-                    && username == auth_config.test_user_name
-                    && token == auth_config.test_user_token
-                {
-                    self.authenticated_user = Some(PushUserInfo {
-                        username: username.to_string(),
-                    });
-                    return true;
-                }
-                let token_valid = state
-                    .storage
-                    .user_storage()
-                    .check_token(username, token)
-                    .await
-                    .unwrap_or(false);
-
-                if token_valid {
-                    // Valid token: set minimal authenticated user info
-                    self.authenticated_user = Some(PushUserInfo {
-                        username: username.to_string(),
-                    });
-                    return true;
-                }
-
-                return token_valid;
-            }
-        }
-        false
-    }
 }
 
 #[cfg(test)]

common/src/config/mod.rs

@@ -84,7 +84,6 @@ pub struct Config {
     pub database: DbConfig,
     pub monorepo: MonoConfig,
     pub pack: PackConfig,
-    pub authentication: AuthConfig,
     pub lfs: LFSConfig,
     #[serde(default)]
     pub blame: BlameConfig,
@@ -129,7 +128,6 @@ impl Config {
             database: DbConfig::default(),
             monorepo: MonoConfig::default(),
             pack: PackConfig::default(),
-            authentication: AuthConfig::default(),
             lfs: LFSConfig::default(),
             blame: BlameConfig::default(),
             oauth: OauthConfig::default(),
@@ -173,10 +171,6 @@ impl Config {
     pub fn from_config(config: c::Config) -> Result<Self, ConfigError> {
         config.try_deserialize::<Config>()
     }
-
-    pub fn enable_http_auth(&self) -> bool {
-        self.authentication.enable_http_auth
-    }
 }
 
 /// supports braces-delimited variables (i.e. ${foo}) in config.
@@ -351,25 +345,6 @@ impl Default for RenameConfig {
         }
     }
 }
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct AuthConfig {
-    pub enable_http_auth: bool,
-    pub enable_test_user: bool,
-    pub test_user_name: String,
-    pub test_user_token: String,
-}
-
-impl Default for AuthConfig {
-    fn default() -> Self {
-        Self {
-            enable_http_auth: false,
-            enable_test_user: false,
-            test_user_name: String::from("mega"),
-            test_user_token: String::from("mega"),
-        }
-    }
-}
-
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct PackConfig {
     #[serde(deserialize_with = "string_or_usize")]

config/config.toml

@@ -47,22 +47,6 @@ connect_timeout = 3
 # Whether to enable SQLx logging
 sqlx_logging = false
 
-[authentication]
-# Support http authentication, login in with github and generate token before push
-enable_http_auth = false
-
-# Enable a test user for debugging and development purposes.
-# If set to true, the service allows using a predefined test user for authentication.
-enable_test_user = true
-
-# Specify the name of the test user.
-# This is only relevant if `enable_test_user` is set to true.
-test_user_name = "mega"
-
-# Specify the token for the test user.
-# This is used for authentication when `enable_test_user` is set to true.
-test_user_token = "mega"
-
 [monorepo]
 ## Only import directory support multi-branch commit and tag, monorepo only support main branch
 ## Mega treats files under this directory as import repo and other directories as monorepo

jupiter/callisto/src/build_targets.rs

@@ -41,11 +41,3 @@ impl Related<super::target_state_histories::Entity> for Entity {
 }
 
 impl ActiveModelBehavior for ActiveModel {}
-//:path1 //:path2 //:path3
-
-//target log:
-//  - id
-//  - target id
-//  - task
-//  - state
-//  - createdAt

jupiter/callisto/src/mega_cl.rs

@@ -17,12 +17,12 @@ pub struct Model {
     pub status: MergeStatusEnum,
     #[sea_orm(column_type = "Text")]
     pub path: String,
-    pub base_branch: String,
     pub from_hash: String,
     pub to_hash: String,
     pub created_at: DateTime,
     pub updated_at: DateTime,
     pub username: String,
+    pub base_branch: String,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]