update readme

.gitignore

@@ -152,13 +152,17 @@ examples/multi_modality/joint_embedding/models/*.pth
 !examples/result_analysis/get_important_pattern.py
 !examples/result_analysis/get_num.py
 
+
 # =============================================================================
 # Model files
 # =============================================================================
 models/model_joint_embedding_1.pth
 examples/tuning/joint_embedding_scmogcn/models/model_joint_embedding_1.pth
 examples/multi_modality/joint_embedding/models/*.pth
 
+
+!examples/result_analysis/celltype_annotation/cta_scdeepsort/main.py
+
 # =============================================================================
 # Others
 # =============================================================================

.pre-commit-config.yaml

@@ -23,11 +23,13 @@ repos:
     rev: v3.16.0
     hooks:
       - id: pyupgrade
+        language_version: python3.10
         args: [--py3-plus]
   - repo: https://github.com/google/yapf
     rev: v0.40.2
     hooks:
       - id: yapf
+        language_version: python3.10
         name: Format code
         additional_dependencies: [toml]
   - repo: https://github.com/pycqa/isort
@@ -41,6 +43,7 @@ repos:
     hooks:
       - id: docformatter
         name: Format docstring
+        language_version: python3.10
         additional_dependencies: [tomli]
         args: [--config, pyproject.toml]
   - repo: https://github.com/executablebooks/mdformat

dance/datasets/multimodality.py

@@ -16,6 +16,27 @@
 from dance.data import Data
 from dance.datasets.base import BaseDataset
 from dance.registry import register_dataset
+from dance.transforms import (
+    AlignMod,
+    ColumnSumNormalize,
+    Compose,
+    FeatureCellPlaceHolder,
+    FilterCellsCommonMod,
+    FilterCellsPlaceHolder,
+    FilterCellsScanpyOrder,
+    FilterGenesPercentile,
+    FilterGenesPlaceHolder,
+    FilterGenesRegression,
+    FilterGenesScanpyOrder,
+    FilterGenesTopK,
+    GaussRandProjFeature,
+    HighlyVariableGenesRawCount,
+    NormalizePlaceHolder,
+    NormalizeTotal,
+    NormalizeTotalLog1P,
+    SetConfig,
+    tfidfTransform,
+)
 from dance.transforms.preprocess import lsiTransformer
 from dance.typing import List
 from dance.utils import is_numeric
@@ -776,6 +797,221 @@ def _maybe_preprocess(self, raw_data):
             meta_common_cells = list(set(meta1.obs.index) & set(meta2.obs.index))
             meta1 = meta1[meta_common_cells, :]
             meta2 = meta2[meta_common_cells, :]
+        elif self.preprocess == "dcca_optim":
+            data = Data(md.MuData({
+                "mod1": mod1,
+                "mod2": mod2,
+                "meta1": meta1,
+                "meta2": meta2,
+                "test_sol": test_sol,
+            }), train_size=train_size)
+
+            preprocessing_pipeline = Compose(
+                # pipeline.0.filter.gene: mod1
+                FilterGenesPlaceHolder(mod="mod1"),
+
+                # pipeline.1.filter.gene: mod2
+                FilterGenesPercentile(mod="mod2"),
+
+                # pipeline.2.misc
+                AlignMod(),
+
+                # pipeline.3.normalize: mod1, candidate group [NormalizePlaceHolder, tfidfTransform]
+                NormalizePlaceHolder(mod="mod1"),
+
+                # pipeline.4.normalize: mod2, candidate group [NormalizePlaceHolder, tfidfTransform]
+                NormalizePlaceHolder(mod="mod2"),
+
+                # pipeline.5.normalize: mod1
+                NormalizePlaceHolder(mod="mod1"),
+
+                # pipeline.6.normalize: mod2
+                NormalizeTotalLog1P(mod="mod2"),
+
+                # pipeline.7.filter.gene: mod1, channel=counts/layers
+                FilterGenesRegression(
+                    mod="mod1",
+                    channel="counts",
+                    channel_type="layers",
+                    num_genes=self.selection_threshold,
+                ),
+
+                # pipeline.8.filter.gene: mod2, channel=counts/layers
+                FilterGenesRegression(
+                    mod="mod2",
+                    channel="counts",
+                    channel_type="layers",
+                    num_genes=self.selection_threshold,
+                ),
+
+                # pipeline.9.filter.cell: mod1
+                FilterCellsPlaceHolder(mod="mod1"),
+
+                # # pipeline.10.filter.cell: mod2
+                FilterCellsPlaceHolder(mod="mod2"),
+                # FilterCellsScanpyOrder(
+                #     mod="mod2",
+                #     order=["min_counts", "min_genes", "max_counts", "max_genes"],
+                #     min_counts=0.01,
+                #     max_counts=0.99,
+                #     min_genes=0.01,
+                #     max_genes=0.99,
+                # ),
+
+                # pipeline.11.filter.cell
+                FilterCellsCommonMod(mod1="mod1", mod2="mod2", sol="test_sol"),
+
+                # pipeline.12.filter.cell: meta1
+                FilterCellsPlaceHolder(mod="meta1"),
+
+                # pipeline.13.filter.cell: meta2
+                FilterCellsPlaceHolder(mod="meta2"),
+
+                # pipeline.14.filter.cell
+                FilterCellsCommonMod(mod1="meta1", mod2="meta2"),
+
+                # pipeline.15.feature.cell: mod1
+                FeatureCellPlaceHolder(out="feature.cell", log_level="INFO", mod="mod1"),
+
+                # pipeline.16.feature.cell: mod2
+                FeatureCellPlaceHolder(out="feature.cell", log_level="INFO", mod="mod2"),
+
+                # pipeline.17.misc
+                SetConfig({
+                    "feature_mod": ["mod1", "mod2", "mod1", "mod2", "mod1", "mod2"],
+                    "label_mod":
+                    "mod1",
+                    "feature_channel_type": ["obsm", "obsm", "layers", "layers", "obsm", "obsm"],
+                    "feature_channel":
+                    ["feature.cell", "feature.cell", "counts", "counts", "size_factors", "size_factors"],
+                    "label_channel":
+                    "labels",
+                }),
+            )
+            preprocessing_pipeline(data)
+
+            mod1 = data.data.mod["mod1"]
+            mod2 = data.data.mod["mod2"]
+            meta1 = data.data.mod["meta1"]
+            meta2 = data.data.mod["meta2"]
+            test_sol = data.data.mod["test_sol"]
+        elif self.preprocess == "dcca_optim_v2":
+            data = Data(md.MuData({
+                "mod1": mod1,
+                "mod2": mod2,
+                "meta1": meta1,
+                "meta2": meta2,
+                "test_sol": test_sol,
+            }), train_size=train_size)
+
+            preprocessing_pipeline = Compose(
+                # pipeline.0.filter.gene: mod1
+                FilterGenesScanpyOrder(
+                    mod="mod1",
+                    order=["min_counts", "max_counts", "min_cells", "max_cells"],
+                    min_counts=0.01,
+                    max_counts=0.99,
+                    min_cells=0.01,
+                    max_cells=0.99,
+                ),
+
+                # pipeline.1.filter.gene: mod2
+                FilterGenesScanpyOrder(
+                    mod="mod2",
+                    order=["min_counts", "max_counts", "min_cells", "max_cells"],
+                    min_counts=0.01,
+                    max_counts=0.99,
+                    min_cells=0.01,
+                    max_cells=0.99,
+                ),
+
+                # pipeline.2.misc
+                AlignMod(),
+
+                # pipeline.3.normalize: mod1
+                tfidfTransform(mod="mod1"),
+
+                # pipeline.4.normalize: mod2
+                NormalizePlaceHolder(mod="mod2"),
+
+                # pipeline.5.normalize: mod1
+                ColumnSumNormalize(mod="mod1"),  # pyright: ignore[reportUndefinedVariable]
+
+                # pipeline.6.normalize: mod2
+                NormalizeTotal(mod="mod2"),
+
+                # pipeline.7.filter.gene: mod1
+                FilterGenesTopK(
+                    mod="mod1",
+                    channel="counts",
+                    channel_type="layers",
+                    num_genes=self.selection_threshold,
+                ),
+
+                # pipeline.8.filter.gene: mod2
+                HighlyVariableGenesRawCount(
+                    mod="mod2",
+                    channel="counts",
+                    channel_type="layers",
+                    n_top_genes=self.selection_threshold,
+                ),
+
+                # pipeline.9.filter.cell: mod1
+                FilterCellsPlaceHolder(mod="mod1"),
+
+                # pipeline.10.filter.cell: mod2
+                FilterCellsPlaceHolder(mod="mod2"),
+
+                # pipeline.11.filter.cell
+                FilterCellsCommonMod(mod1="mod1", mod2="mod2", sol="test_sol"),
+
+                # pipeline.12.filter.cell: meta1
+                FilterCellsPlaceHolder(mod="meta1"),
+
+                # pipeline.13.filter.cell: meta2
+                FilterCellsScanpyOrder(
+                    mod="meta2",
+                    order=["min_counts", "min_genes", "max_counts", "max_genes"],
+                    min_counts=0.01,
+                    max_counts=0.99,
+                    min_genes=0.01,
+                    max_genes=0.99,
+                ),
+
+                # pipeline.14.filter.cell
+                FilterCellsCommonMod(mod1="meta1", mod2="meta2"),
+
+                # pipeline.15.feature.cell: mod1
+                GaussRandProjFeature(
+                    mod="mod1",
+                    out="feature.cell",
+                    log_level="INFO",
+                    n_components=400,
+                ),
+
+                # pipeline.16.feature.cell: mod2
+                FeatureCellPlaceHolder(out="feature.cell", log_level="INFO", mod="mod2"),
+
+                # pipeline.17.misc
+                SetConfig({
+                    "feature_mod": ["mod1", "mod2", "mod1", "mod2", "mod1", "mod2"],
+                    "label_mod":
+                    "mod1",
+                    "feature_channel_type": ["obsm", "obsm", "layers", "layers", "obsm", "obsm"],
+                    "feature_channel":
+                    ["feature.cell", "feature.cell", "counts", "counts", "size_factors", "size_factors"],
+                    "label_channel":
+                    "labels",
+                }),
+            )
+            preprocessing_pipeline(data)
+
+            mod1 = data.data.mod["mod1"]
+            mod2 = data.data.mod["mod2"]
+            meta1 = data.data.mod["meta1"]
+            meta2 = data.data.mod["meta2"]
+            test_sol = data.data.mod["test_sol"]
+
         else:
             logger.info(f"Preprocessing method {self.preprocess!r} not supported.")
 

dance/modules/single_modality/cell_type_annotation/celltypist.py

@@ -11,7 +11,7 @@
 
 from dance import logger
 from dance.modules.base import BaseClassificationMethod
-from dance.transforms import SetConfig
+from dance.transforms import AnnDataTransform, Compose, SetConfig
 from dance.typing import LogLevel, Optional, Union
 
 
@@ -549,8 +549,26 @@ def __init__(self, majority_voting: bool = False, clf=None, scaler=None, descrip
         self.description = description
 
     @staticmethod
-    def preprocessing_pipeline(log_level: LogLevel = "INFO"):
-        return SetConfig({"label_channel": "cell_type"}, log_level=log_level)
+    def preprocessing_pipeline(normalize: bool = False, log_level: LogLevel = "INFO"):
+        """Create the preprocessing pipeline for CellTypist.
+
+        Parameters
+        ----------
+        normalize
+            Whether to transform ``.X`` to log1p-normalized expression at 10,000 counts per cell.
+        log_level
+            Logging level for the composed preprocessing pipeline.
+
+        """
+        transforms = []
+
+        if normalize:
+            transforms.append(AnnDataTransform(sc.pp.normalize_total, target_sum=1e4))
+            transforms.append(AnnDataTransform(sc.pp.log1p))
+
+        transforms.append(SetConfig({"label_channel": "cell_type"}))
+
+        return Compose(*transforms, log_level=log_level)
 
     def fit(self, indata: np.array, labels: Optional[Union[str, list, tuple, np.ndarray, pd.Series, pd.Index]] = None,
             C: float = 1.0, solver: Optional[str] = None, max_iter: int = 1000, n_jobs: Optional[int] = None,

dance/modules/single_modality/cell_type_annotation/scdeepsort.py

@@ -12,13 +12,14 @@
 from pathlib import Path
 
 import dgl
+import scanpy as sc
 import torch
 import torch.nn as nn
 from dgl.dataloading import DataLoader, NeighborSampler
 
 from dance.models.nn import AdaptiveSAGE
 from dance.modules.base import BaseClassificationMethod
-from dance.transforms import Compose, SetConfig
+from dance.transforms import AnnDataTransform, Compose, SetConfig
 from dance.transforms.graph import PCACellFeatureGraph
 from dance.typing import LogLevel, Optional
 
@@ -132,12 +133,31 @@ def __init__(self, dim_in: int, dim_hid: int, num_layers: int, species: str, tis
             self.save_path.mkdir(parents=True)
 
     @staticmethod
-    def preprocessing_pipeline(n_components: int = 400, log_level: LogLevel = "INFO"):
-        return Compose(
+    def preprocessing_pipeline(n_components: int = 400, normalize: bool = False, log_level: LogLevel = "INFO"):
+        """Create the preprocessing pipeline for scDeepSort.
+
+        Parameters
+        ----------
+        n_components
+            Number of PCA components used to initialize cell and gene features.
+        normalize
+            Whether to apply the Scanpy equivalent of Seurat ``NormalizeData`` before graph construction.
+        log_level
+            Logging level for the composed preprocessing pipeline.
+
+        """
+        transforms = []
+
+        if normalize:
+            transforms.append(AnnDataTransform(sc.pp.normalize_total, target_sum=1e4))
+            transforms.append(AnnDataTransform(sc.pp.log1p))
+
+        transforms.extend([
             PCACellFeatureGraph(n_components=n_components, split_name="train"),
             SetConfig({"label_channel": "cell_type"}),
-            log_level=log_level,
-        )
+        ])
+
+        return Compose(*transforms, log_level=log_level)
 
     def fit(self, graph: dgl.DGLGraph, labels: torch.Tensor, epochs: int = 300, lr: float = 1e-3,
             weight_decay: float = 0, val_ratio: float = 0.2):

dance/transforms/graph/cell_feature_graph.py

@@ -50,7 +50,7 @@ def __call__(self, data):
         edata = torch.FloatTensor(edata)
 
         # Initialize cell-gene graph
-        g = dgl.graph((row, col))
+        g = dgl.graph((row, col), num_nodes=num_feats + num_cells)
         g.edata["weight"] = edata
         # FIX: change to feat_id
         g.ndata["cell_id"] = torch.concat((torch.arange(num_feats, dtype=torch.int32),