diff --git a/dsi/core.py b/dsi/core.py index f19c6c92..154a92a3 100644 --- a/dsi/core.py +++ b/dsi/core.py @@ -1445,3 +1445,39 @@ def can_create_file_here(self, dir = "."): return True except (PermissionError, OSError): return False + + def vcs_init(self, repo_path): + """Initialize VCS repo.""" + pass + + def vcs_status(self): + """Return vcs status metadata for the first loaded backend.""" + pass + + def vcs_commit(self, message, author_name=None, author_email=None): + """Commit the current versioned backend state.""" + pass + + def vcs_diff(self, old_ref=None, new_ref=None, cached=False): + """Return the git diff for the first loaded backend.""" + pass + + def vcs_log(self, limit=None, ref='HEAD'): + """Return commit history for the first loaded backend.""" + pass + + def vcs_current_branch(self): + """Return the current branch for the first loaded backend.""" + pass + + def vcs_list_branches(self): + """Return local branches for the first loaded backend.""" + pass + + def vcs_create_branch(self, name, start_point='HEAD', checkout=False, force=False): + """Create a branch on the first loaded backend.""" + pass + + def vcs_checkout_branch(self, name, force=False): + """Checkout a branch on the first loaded backend.""" + pass diff --git a/dsi/dsi.py b/dsi/dsi.py index 45d6514e..52014242 100644 --- a/dsi/dsi.py +++ b/dsi/dsi.py @@ -16,6 +16,8 @@ import inspect import warnings + +from dsi.utils.version_control.dsi_vcs import Version warnings.filterwarnings("ignore", category=FutureWarning) logger = logging.getLogger(__name__) @@ -201,7 +203,7 @@ def __init__(self, filename = ".temp_dsi.db", backend_name = "Sqlite", **kwargs) msg = f"Created an instance of DSI with the {backend_name} backend: {filename}" else: msg = "Created an instance of DSI" - + self.vcs = None logger.log(logging.INFO, msg) if self.silence_messages else print(msg) @@ -1277,3 +1279,68 @@ def move(self, filepath): def fetch(self, fname): pass + + def version(self, command: str, args: str = None): + """ + Internal DSI Versioning. + + `command` : str + - init # initialize a versioning repository in a root folder + - add # add file(s) to the staging area for the next commit + - remove # remove file(s) from the staging area without touching the actual files + - delete # delete file(s) from the staging area for the next commit + - commit # commit a new version with the staged file(s) and an optional message describing the version + - log # list versions + - diff # diff between two versions. If no version is provided, diff the current version with the previous version + - restore # restore a version with commit hash + + `args` : str + - init: A required argument with the name of the root folder for the versioning repository. + - add: A required argument with the file(s) to add to the staging area for the next commit, specified as a space-separated string or list of file paths. + - remove: A required argument with the file(s) to remove from the staging area without touching the actual files, specified as a space-separated string or list of file paths. + - delete: A required argument with the file(s) to delete from the staging area for the next commit, specified as a space-separated string or list of file paths. + - commit: An optional message describing the version being committed, specified as a string. + - log: An optional argument to specify the number of recent versions to display, specified as an integer. If not provided, recent 5 versions are displayed. + - diff: An optional argument to specify the versions to compare, specified as a space-separated string or list of commit hashes. + - restore: A required argument to specify the version to restore, specified by its commit hash. + """ + if command == "init": + if args is None: + raise RuntimeError("version() ERROR: 'init' command requires a 'root_folder' argument specifying the name of the root folder for the versioning repository.") + + self.vcs = Version(args) + elif command == "add" and self.vcs is not None: + if args is None: + raise RuntimeError("version() ERROR: 'add' command requires a 'files' argument specifying the file(s) to add to the staging area for the next commit.") + print("-->" + self.vcs.root_folder) + self.vcs.cmd_add(args.split()) + elif command == "remove" and self.vcs is not None: + if args is None: + raise RuntimeError("version() ERROR: 'remove' command requires a 'files' argument specifying the file(s) to remove from the staging area for the next commit.") + self.vcs.cmd_remove(args.split()) + elif command == "delete" and self.vcs is not None: + if args is None: + raise RuntimeError("version() ERROR: 'delete' command requires a 'files' argument specifying the file(s) to delete from the repository.") + self.vcs.cmd_delete(args.split()) + elif command == "commit" and self.vcs is not None: + self.vcs.cmd_commit(args) + elif command == "diff" and self.vcs is not None: + c1 = None + c2 = None + if args is not None: + arg_list = args.split() + if len(arg_list) == 1: + c1 = arg_list[0] + elif len(arg_list) == 2: + c1, c2 = arg_list + elif len(arg_list) > 2: + raise RuntimeError("version() ERROR: 'diff' command requires zero, one, or two commit hashes as arguments.") + self.vcs.cmd_diff(c1, c2) + elif command == "log" and self.vcs is not None: + self.vcs.cmd_log() + elif command == "restore" and self.vcs is not None: + if args is None: + raise RuntimeError("version() ERROR: 'restore' command requires a 'commit_hash' argument specifying the version to restore.") + self.vcs.cmd_restore(args) + else: + raise RuntimeError("version() ERROR: Invalid command or versioning repository not initialized. Please check the command and ensure 'init' has been called with a root folder argument.") \ No newline at end of file diff --git a/dsi/tests/test_core.py b/dsi/tests/test_core.py index 6b8dd2c1..4d755442 100644 --- a/dsi/tests/test_core.py +++ b/dsi/tests/test_core.py @@ -1400,4 +1400,4 @@ def test_terminal_ndp_close(): # After close assert backend._loaded is False - assert len(backend._cache) == 0 \ No newline at end of file + assert len(backend._cache) == 0 diff --git a/dsi/tests/test_dsi.py b/dsi/tests/test_dsi.py index 77cd6b8a..61fd2baa 100644 --- a/dsi/tests/test_dsi.py +++ b/dsi/tests/test_dsi.py @@ -6,6 +6,7 @@ from pandas import DataFrame from collections import OrderedDict import hashlib +import random def test_list_functions(): test = DSI() @@ -1795,6 +1796,39 @@ def test_fail_overwrite_schema_duckdb_backend(): except Exception as e: expected = "schema() ERROR: A complex schema with a circular dependency cannot be ingested into a DuckDB backend." assert str(e) == expected + +def test_versioning(): + test = DSI() + wpath = os.getcwd() + test.version("init", wpath) + assert os.path.exists(wpath + "/.dsi_vcs_snapshots/.dsi_vcs.db") + + # Create a file with ten random integers + dummy_file_path = os.path.join(wpath, "a_dummy_file") + with open(dummy_file_path, 'w') as f: + for _ in range(10): + f.write(str(random.randint(0, 100)) + '\n') + test.version("add", os.path.join(wpath, "a_dummy_file")) + + dummy_file_path = os.path.join(wpath, "second_dummy_file") + with open(dummy_file_path, 'w') as f: + for _ in range(10): + f.write(str(random.randint(0, 100)) + '\n') + test.version("add", os.path.join(wpath, "second_dummy_file")) + + dummy_file_path = os.path.join(wpath, "third_dummy_file") + with open(dummy_file_path, 'w') as f: + for _ in range(10): + f.write(str(random.randint(0, 100)) + '\n') + test.version("add", os.path.join(wpath, "third_dummy_file")) + + test.version("remove", os.path.join(wpath, "second_dummy_file")) + test.version("delete", os.path.join(wpath, "third_dummy_file")) + + test.version("commit", "Tester Commits") + test.version("log") + test.version("diff") + assert True # NDP @@ -1865,4 +1899,115 @@ def test_close_ndp_backend(): assert df is not None dsi.close() - assert True \ No newline at end of file + assert True + +def test_different_keywords_ndp_backend(): + """Test NDP with different keyword searches""" + keywords_list = ["ocean", "climate", "water"] + + for keyword in keywords_list: + dsi = DSI(backend_name="NDP", keywords=keyword, limit=5) + + # Verify we can get datasets table + df = dsi.get_table("datasets", collection=True) + assert isinstance(df, DataFrame) + assert len(df) > 0 + + dsi.close() + + assert True + +def test_limit_parameter_ndp_backend(): + """Test NDP with different limit values""" + # Test with limit=3 + dsi1 = DSI(backend_name="NDP", keywords="data", limit=3) + df1 = dsi1.get_table("datasets", collection=True) + dsi1.close() + + # Test with limit=10 + dsi2 = DSI(backend_name="NDP", keywords="data", limit=10) + df2 = dsi2.get_table("datasets", collection=True) + dsi2.close() + + # Verify both return dataframes + assert isinstance(df1, DataFrame) + assert isinstance(df2, DataFrame) + + # Note: Can't guarantee df2 > df1 due to API behavior + assert len(df1) > 0 + assert len(df2) > 0 + + assert True + +def test_datasets_structure_ndp_backend(): + """Test that datasets table has expected structure""" + dsi = DSI(backend_name="NDP", keywords="climate", limit=5) + + df = dsi.get_table("datasets", collection=True) + + # Verify it's a DataFrame + assert isinstance(df, DataFrame) + assert len(df) > 0 + + # Verify key columns exist + expected_columns = ['title', 'num_resources'] + for col in expected_columns: + assert col in df.columns, f"Missing expected column: {col}" + + # Verify data types + assert df['num_resources'].dtype in ['int64', 'int32', 'float64'] + assert df['title'].dtype == object # String column + + dsi.close() + +def test_error_invalid_backend_ndp(): + """Test error handling for invalid NDP parameters""" + # Test with empty keywords + try: + dsi = DSI(backend_name="NDP", keywords="", limit=5) + dsi.close() + # Empty keywords might work, just verify connection + assert True + except: + # Or it might error - either is acceptable + assert True + +def test_close_ndp_backend(): + """Test proper closing of NDP connections""" + dsi = DSI(backend_name="NDP", keywords="test", limit=3) + + # Verify connection is active + tables = dsi.list(collection=True) + assert len(tables) > 0 + + # Close connection + dsi.close() + + # Test is successful if close() doesn't raise exception + assert True + +# def main(): +def test_versioning(): + test = DSI() + # test.version("init", os.getcwd()) + # wpath = "/Users/ssakin/Public/versioning-test/clover-demo" + wpath = os.getcwd() + test.version("init", wpath) + assert os.path.exists(wpath + "/.dsi_vcs_snapshots/.dsi_vcs.db") + + test.version("add", os.path.join(wpath, "a_dummy_file")) + print(">Single file added.") + test.version("add", os.path.join(wpath, "schema.json") + " " + os.path.join(wpath, "schema2.json")) + print(">Multi file added.") + print(">Versioning initialized.") + + test.version("remove", os.path.join(wpath, "schema2.json")) + print(">Single file removed.") + + test.version("commit", "Tester Commits") + test.version("log") + test.version("diff", "77345f1115d94c69a1255b9fb0524378 4af9e3d4dc854d699b96b5a84f913ac0") + assert True + +# if __name__ == "__main__": +# main() \ No newline at end of file diff --git a/dsi/utils/version_control/README.md b/dsi/utils/version_control/README.md new file mode 100644 index 00000000..1c35401e --- /dev/null +++ b/dsi/utils/version_control/README.md @@ -0,0 +1,355 @@ +# dsi-vcs: rsync-based File Version Control with Full Linux Metadata + +## Overview + +**dsi-vcs** is a lightweight file version control system designed for capturing and preserving complete Linux file metadata. Unlike traditional VCS tools, dsi-vcs focuses on: + +- **Full metadata capture**: permissions, ownership, ACLs, extended attributes, and SELinux contexts +- **Rsync-based snapshots**: efficient storage with hard-link deduplication +- **Merkle commit chain**: SHA-256 commit IDs with per-path Merkle nodes for pruning unchanged subtrees +- **SQLite database**: structured metadata storage for querying and diffing +- **Complete file history**: MD5 hashes, file stats, and all metadata changes tracked + +--- + +## CLI Commands + +Initialize dsi-vcs in the current directory: + +```bash +dsi-vcs init +``` + +### Stage Files for Commit + +**Add files/directories:** + +```bash +dsi-vcs add [ ...] +dsi-vcs add ./data +dsi-vcs add file1.txt file2.txt +``` + +**Stage files for deletion:** + +```bash +dsi-vcs delete [ ...] +dsi-vcs delete ./old_data/ +``` + +**Remove files from staging (without deleting):** + +```bash +dsi-vcs remove [ ...] +dsi-vcs remove file1.txt +``` + +### Commit Changes + +Create a snapshot with an optional message: + +```bash +dsi-vcs commit +dsi-vcs commit "Initial data import" +``` + +### View History + +List all commits: + +```bash +dsi-vcs log +``` + +Example output: + +```shell +COMMIT HASH OWNER DATE/TIME (UTC) FILES BYTES MESSAGE +──────────────────────────────────────────────────────────────────────────────────────────────────── +f826177ae78f4e48a8c08054e2bb9a71 owner1 2026-04-28T21:10:05.339321+00:00 7 15,557 first commit +4af9e3d4dc854d699b96b5a84f913ac0 owner2 2026-04-28T21:19:47.167081+00:00 7 15,559 second commit +``` + +### Compare Versions + +Diff two commits (shows added, modified, deleted files). + +Parameters: + +- Provide upto two commit hash as paramerter. +- No parameter will compare with the latest changes with the last commit. +- One parameter will compare with the latest change with the provided commit version. +- Two parameters will compare between the provided versions. + +```bash +dsi-vcs diff +dsi-vcs diff f826177ae78f4e48a8c08054e2bb9a71 +``` + +Example output: + +```shell +Diff f826177ae78f4e48a8c08054e2bb9a71 → None (./root_folder) + +STATUS PATH +────────────────────────────────────────────────────────────────────── +MODIFIED file_new [owner] +MODIFIED file_schema.json [owner] +diff result: 2c2 +< "genesis_datacard": { +--- +> 2"genesis_datacard": { +26c26 +< } +\ No newline at end of file +--- +> } +MODIFIED schema2.json [content, size] + +Summary: +0 added -0 deleted ~3 modified =4 unchanged +``` + +### Restore a Version + +Restore the entire repository to a previous commit: + +```bash +dsi-vcs restore +dsi-vcs restore abc123def456 +``` + +## Python API + +Integrated within DSI. User versioning with the function `version(command, args)` + +### Usage + +```python +obj.version(command: str, args: str = None) +``` + +### Parameters + +| Parameter | Type | Description | +| ----------- | ------- | -------------------------------------------------------- | +| `command` | `str` | The versioning operation to perform (see commands below) | +| `args` | `str` | Optional or required arguments depending on the command | + +--- + +### `init` + +Initializes a versioning repository in a root folder. + +```python +obj.version("init", "my_project_folder") +``` + +**Args (required):** Name of the root folder for the versioning repository. + +--- + +### `add` + +Adds one or more files to the staging area for the next commit. + +```python +obj.version("add", "file1.py file2.py") +``` + +**Args (required):** Space-separated file paths to stage. + +--- + +### `remove` + +Removes one or more files from the staging area **without** deleting the actual files. + +```python +obj.version("remove", "file1.py") +``` + +**Args (required):** Space-separated file paths to unstage. + +--- + +### `delete` + +Marks one or more files for deletion in the next commit. + +```python +obj.version("delete", "file1.py file2.py") +``` + +**Args (required):** Space-separated file paths to delete. + +--- + +### `commit` + +Commits all staged changes as a new version snapshot. + +```python +obj.version("commit", "Initial release") # with message +obj.version("commit") # without message +``` + +**Args (optional):** A descriptive message for the version being committed. + +--- + +### `log` + +Lists recent committed versions. + +```python +obj.version("log") # shows last 5 versions (default) +obj.version("log", "10") # shows last 10 versions +``` + +**Args (optional):** Number of recent versions to display. Defaults to `5`. + +--- + +### `diff` + +Shows the differences between two versions. + +```python +obj.version("diff") # current vs previous +obj.version("diff", "abc123") # specific commit vs its previous +obj.version("diff", "abc123 def456") # between two specific commits +``` + +**Args (optional):** Zero, one, or two commit hashes separated by a space. + +| # of hashes | Behavior | +| ----------- | -------------------------------------------------- | +| 0 | Diffs current version against previous | +| 1 | Diffs specified commit against its previous | +| 2 | Diffs the two specified commits against each other | + +--- + +### `restore` + +Restores the repository to a previously committed version. + +```python +obj.version("restore", "abc123def456") +``` + +**Args (required):** The commit hash of the version to restore. + +Example : Basic Workflow + +```python +from dsi.utils.version_control import Version + +# Initialize dsi +dsi = DSI() + +# Initialize repo +dsi.version("init", "/data/archive") + +# Stage files +dsi.version("add", ["./documents", "config.json"]) + +# Commit +dsi.version("commit", "Initial archive") + +# View history +dsi.version("log") + +# Modify files and commit again +dsi.version(["./documents"]) +dsi.version("commit", "Updated documents") + +# Compare two versions +dsi.version("diff") +``` + +## Database Schema + +### `versions` Table + +| Column | Type | Description | +| ------------- | ---------- | --------------------------- | +| id | INTEGER PK | Auto-increment ID | +| root_folder | TEXT | Repository root path | +| commit_hash | TEXT | SHA-256 Merkle commit hash | +| root_tree_hash | TEXT | SHA-256 hash of root tree | +| parent_commit_hash | TEXT | Previous commit hash | +| hash_algorithm | TEXT | Merkle format identifier | +| committed_at | TEXT | ISO-8601 timestamp | +| owner_name | TEXT | Username of the committer | +| message | TEXT | Optional commit message | +| snapshot_path | TEXT | Path to rsync snapshot copy | +| file_count | INTEGER | Number of files in commit | +| total_bytes | INTEGER | Total size in bytes | + +### `file_entries` Table + +Stores metadata for each file in each commit. + +| Column | Type | Description | +| ---------------- | ---------- | -------------------------------- | +| id | INTEGER PK | Auto-increment | +| version_id | INTEGER FK | References versions(id) | +| root_folder | TEXT | Partition key | +| relative_path | TEXT | Path relative to root | +| absolute_path | TEXT | Full path | +| file_name | TEXT | Filename only | +| file_type | TEXT | file/dir/symlink/etc | +| md5_hash | TEXT | Content hash (files only) | +| lstat | TEXT | JSON of os.lstat() result | +| permissions_int | INTEGER | e.g. 755 | +| owner_name | TEXT | Username | +| group_name | TEXT | Group name | +| acl_text | TEXT | Raw getfacl output | +| xattrs | TEXT | JSON dict of extended attributes | +| security_context | TEXT | SELinux context | +| symlink_target | TEXT | Target of symlink | + +### `merkle_nodes` Table + +Stores the content-addressed tree node for each committed path, including the synthetic root path `.`. + +| Column | Type | Description | +| ------------------- | ---------- | --------------------------------- | +| id | INTEGER PK | Auto-increment | +| version_id | INTEGER FK | References versions(id) | +| root_folder | TEXT | Partition key | +| relative_path | TEXT | Path relative to root | +| file_type | TEXT | file/dir/symlink/etc | +| node_hash | TEXT | SHA-256 hash for this path node | +| metadata_hash | TEXT | SHA-256 hash of stable metadata | +| content_hash_sha256 | TEXT | SHA-256 file content hash | +| subtree_file_count | INTEGER | File count below this node | +| subtree_total_bytes | INTEGER | File bytes below this node | +| child_count | INTEGER | Direct child count for this node | + +### `staging` Table + +Temporary storage for files to be committed. + +| Column | Type | Description | +| ------------- | ---------- | ----------------------- | +| id | INTEGER PK | Auto-increment | +| root_folder | TEXT | Partition key | +| absolute_path | TEXT | Full file path (UNIQUE) | +| action | TEXT | "add" or "delete" | +| added_at | TEXT | ISO-8601 timestamp | + +--- + +## Directory Structure + +``` +root_folder/ +└── .dsi_vcs_snapshots/ # rsync snapshot directory + ├── .dsi_vcs.db # SQLite metadata database + ├── abc123def456/ # Snapshot for commit abc123def456 + ├── xyz789abc123/ # Snapshot for commit xyz789abc123 + └── ... +``` diff --git a/dsi/utils/version_control/__init__.py b/dsi/utils/version_control/__init__.py new file mode 100644 index 00000000..d22f487d --- /dev/null +++ b/dsi/utils/version_control/__init__.py @@ -0,0 +1,3 @@ +from .dsi_vcs import Version + +__all__ = ["Version"] diff --git a/dsi/utils/version_control/__main__.py b/dsi/utils/version_control/__main__.py new file mode 100644 index 00000000..2efe6923 --- /dev/null +++ b/dsi/utils/version_control/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + +if __name__ == "__main__": + main() + diff --git a/dsi/utils/version_control/cli.py b/dsi/utils/version_control/cli.py new file mode 100644 index 00000000..06007da2 --- /dev/null +++ b/dsi/utils/version_control/cli.py @@ -0,0 +1,69 @@ +import argparse +import datetime +import sys +import os + +from .dsi_vcs import Version + +def main(): + parser = argparse.ArgumentParser( + description="dsi-vcs — rsync-based file version control with full Linux metadata" + ) + sub = parser.add_subparsers(dest="command", required=True) + + sub.add_parser("init", help="Initialize a new repo") + # p_init.add_argument("root_folder") + + p_add = sub.add_parser("add", help="add file or directory to staging") + p_add.add_argument('files', nargs='+', type=str) + + p_delete = sub.add_parser("delete", help="stage file or directory deletion") + p_delete.add_argument('files', nargs='+', type=str) + + p_remove = sub.add_parser("remove", help="remove file or directory from staging") + p_remove.add_argument('files', nargs='+', type=str) + + p_commit = sub.add_parser("commit", help="Snapshot and commit staged paths") + p_commit.add_argument("message", nargs="?", default="Committed at " + datetime.datetime.now().isoformat()) + + sub.add_parser("log", help="List all commits") + + p_diff = sub.add_parser("diff", help="Diff two commits") + p_diff.add_argument("c1", nargs="?", default=None) + p_diff.add_argument("c2", nargs="?", default=None) + + p_restore = sub.add_parser("restore", help="Restore a version") + p_restore.add_argument("version") + + args = parser.parse_args(args=None if sys.argv[1:] else ["-h"]) + + # parser.print_help() + # parser.format_help() + if args.command == "init": + vcs = Version(os.getcwd()) + elif args.command == "add": + vcs = Version(os.getcwd()) + vcs.cmd_add(args.files) + elif args.command == "delete": + vcs = Version(os.getcwd()) + vcs.cmd_delete(args.files) + elif args.command == "remove": + vcs = Version(os.getcwd()) + vcs.cmd_remove(args.files) + elif args.command == "commit": + vcs = Version(os.getcwd()) + vcs.cmd_commit(args.message) + elif args.command == "diff": + vcs = Version(os.getcwd()) + vcs.cmd_diff(args.c1, args.c2) + elif args.command == "log": + vcs = Version(os.getcwd()) + vcs.cmd_log() + elif args.command == "restore": + vcs = Version(os.getcwd()) + vcs.cmd_restore(args.version) + +if __name__ == "__main__": + # print("\n=== dsi-vcs: rsync-based file version control ===\n") + # print(os.getcwd()) + main() diff --git a/dsi/utils/version_control/dsi_vcs.py b/dsi/utils/version_control/dsi_vcs.py new file mode 100755 index 00000000..fbcd672f --- /dev/null +++ b/dsi/utils/version_control/dsi_vcs.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python3 +""" +dsi_vcs.py — rsync-based file version control system +Captures full Linux file metadata (stat, ACL, xattrs), MD5 hash, +and stores versioned snapshots in SQLite. + +Usage: + python dsi_vcs.py init # init repo in current directory + python dsi_vcs.py add ... # stage paths for the next commit + python dsi_vcs.py delete ... # stage paths for deletion + python dsi_vcs.py remove ... # unstage paths + python dsi_vcs.py commit [message] # commit a new version + python dsi_vcs.py log # list versions + python dsi_vcs.py diff # diff two versions + python dsi_vcs.py restore # restore a version + +Requirements: + pip install pyxattr # for extended attributes + sudo apt install acl # for getfacl (ACL support) + rsync must be installed # for snapshot copies +""" + +import os +import sys +import subprocess +import json +import datetime +import shutil +import tempfile +from typing import Optional + +from .vcs_db import DB_NAME, SNAPSHOTS_DIR, open_db +from .vcs_metadata_helper import collect_metadata, collect_tree_metadata, owner_name +from .merkle import HASH_ALGORITHM, build_merkle_tree, commit_hash as merkle_commit_hash, parent_path + +# ─────────────────────────── RSYNC SNAPSHOT ────────────────────────────────── + +def rsync_snapshot( + root_folder: str, + dest_path: str, + prev_snapshot: Optional[str] = None, +) -> bool: + """ + Copy the full root_folder tree to dest_path using rsync hard-link deduplication. + If prev_snapshot is provided, unchanged files are hard-linked (saves disk). + """ + os.makedirs(dest_path, exist_ok=True) + cmd = ["rsync", "-aAXH"] # for linux + if sys.platform == "darwin": + cmd = ["rsync", "-aEH"] # for macOS + + if prev_snapshot and os.path.isdir(prev_snapshot): + cmd += ["--link-dest", os.path.abspath(prev_snapshot)] + + cmd += [ + "--delete", + "--exclude", DB_NAME, + "--exclude", SNAPSHOTS_DIR, + ] + + cmd += [ + root_folder.rstrip("/") + "/", + dest_path.rstrip("/") + "/", + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode not in (0, 24): # 24 = some files vanished (ok) + print(f"[rsync error] {result.stderr.strip()}", file=sys.stderr) + return False + return True + + +def snapshot_target(snapshot_path: str, relative_path: str) -> str: + target = os.path.abspath(os.path.join(snapshot_path, relative_path)) + snapshot_root = os.path.abspath(snapshot_path) + if os.path.commonpath([snapshot_root, target]) != snapshot_root: + raise ValueError(f"Snapshot path escapes snapshot root: {relative_path}") + return target + + +def apply_snapshot_deletes(snapshot_path: str, root_folder: str, staged_deletes: list[str]) -> None: + for abs_path in staged_deletes: + rel_path = os.path.relpath(abs_path, root_folder) + if rel_path in ("", "."): + raise ValueError("Refusing to stage repository root for deletion.") + target = snapshot_target(snapshot_path, rel_path) + if os.path.isdir(target) and not os.path.islink(target): + shutil.rmtree(target) + elif os.path.lexists(target): + os.unlink(target) + + +# ─────────────────────────── COMMANDS ──────────────────────────────────────── +class Version(): + + def __init__(self, folder: str): + self.root_folder = os.path.abspath(folder) + if not os.path.isdir(self.root_folder): + sys.exit(f"Error: '{self.root_folder}' is not a directory.") + + conn = open_db(self.root_folder) + conn.close() + print(f"Initialized dsi-vcs repository in: {self.root_folder}") + print(f" Snapshots: {self.root_folder}/{SNAPSHOTS_DIR}/") + + + def cmd_add(self, paths: list[str]): + """ + Stage one or more files/directories for the next commit. + Directories are expanded recursively; each resolved file is inserted into + the staging table. Adding an already-staged path is a silent no-op. + paths should be a list of relative paths to root_folder. + """ + # print(f"Staging {len(paths)} path(s) for commit…") + # print(paths) + db_path = os.path.join(self.root_folder, SNAPSHOTS_DIR, DB_NAME) + if not os.path.isfile(db_path): + sys.exit("No dsi-vcs repo found. Run 'init' first.") + + conn = open_db(self.root_folder) + cur = conn.cursor() + added_at = datetime.datetime.now(datetime.timezone.utc).isoformat() + staged = 0 + skip_names = {DB_NAME, SNAPSHOTS_DIR} + + def stage_path(abs_path: str): + nonlocal staged + abs_path = os.path.abspath(abs_path) + + if not os.path.lexists(abs_path): + print(f" [skip] {abs_path}: path does not exist") + return + + cur.execute( + "INSERT OR REPLACE INTO staging (root_folder, absolute_path, action, added_at) " + "VALUES (?, ?, ?, ?)", + (self.root_folder, abs_path, "add", added_at) + ) + if cur.rowcount: + staged += 1 + + for raw in paths: + abs_path = os.path.abspath(raw if os.path.isabs(raw) else os.path.join(self.root_folder, raw)) + + if os.path.isdir(abs_path): + # Expand directory recursively + for dirpath, dirnames, filenames in os.walk(abs_path, followlinks=False): + dirnames[:] = [d for d in dirnames if d not in skip_names] + stage_path(dirpath) + for fname in filenames: + if fname in skip_names: + continue + stage_path(os.path.join(dirpath, fname)) + else: + stage_path(abs_path) + + conn.commit() + conn.close() + print(f" {staged} path(s) added to staging.") + """Show files currently in the staging area.""" + root_folder = os.path.abspath(self.root_folder) + conn = open_db(root_folder) + rows = conn.execute( + "SELECT absolute_path, action, added_at FROM staging " + "WHERE root_folder=? ORDER BY absolute_path", + (root_folder,) + ).fetchall() + conn.close() + + if not rows: + print("Nothing staged. Use 'add ...' or 'delete ...' to stage paths.") + return + + print(f"Staged paths ({len(rows)}):") + for r in rows: + rel = os.path.relpath(r["absolute_path"], root_folder) + print(f" {rel} [{r['action']}]") + + def cmd_delete(self, paths: list[str]): + """Stage path(s) for deletion in the next commit.""" + db_path = os.path.join(self.root_folder, SNAPSHOTS_DIR, DB_NAME) + if not os.path.isfile(db_path): + sys.exit("No dsi-vcs repo found. Run 'init' first.") + + conn = open_db(self.root_folder) + cur = conn.cursor() + added_at = datetime.datetime.now(datetime.timezone.utc).isoformat() + staged = 0 + + for raw in paths: + abs_path = os.path.abspath(raw if os.path.isabs(raw) else os.path.join(self.root_folder, raw)) + cur.execute( + "INSERT OR REPLACE INTO staging (root_folder, absolute_path, action, added_at) " + "VALUES (?, ?, ?, ?)", + (self.root_folder, abs_path, "delete", added_at) + ) + if cur.rowcount: + staged += 1 + + conn.commit() + conn.close() + print(f" {staged} path(s) staged for deletion.") + """Show files currently in the staging area.""" + root_folder = os.path.abspath(self.root_folder) + conn = open_db(root_folder) + rows = conn.execute( + "SELECT absolute_path, action, added_at FROM staging " + "WHERE root_folder=? ORDER BY absolute_path", + (root_folder,) + ).fetchall() + conn.close() + + if not rows: + print("Nothing staged. Use 'add ...' or 'delete ...' to stage paths.") + return + + print(f"Staged paths ({len(rows)}):") + for r in rows: + rel = os.path.relpath(r["absolute_path"], root_folder) + print(f" {rel} [{r['action']}]") + + def cmd_remove(self, paths: list[str]): + """Remove path(s) from the staging area without touching the actual files.""" + root_folder = os.path.abspath(self.root_folder) + db_path = os.path.join(self.root_folder, SNAPSHOTS_DIR, DB_NAME) + if not os.path.isfile(db_path): + sys.exit("No dsi-vcs repo found. Run 'init' first.") + + conn = open_db(self.root_folder) + cur = conn.cursor() + removed = 0 + + for raw in paths: + abs_path = os.path.abspath(raw if os.path.isabs(raw) else os.path.join(root_folder, raw)) + cur.execute( + "DELETE FROM staging WHERE root_folder=? AND absolute_path=?", + (root_folder, abs_path) + ) + if cur.rowcount: + rel = os.path.relpath(abs_path, root_folder) + print(f" Unstaged: {rel}") + removed += 1 + else: + rel = os.path.relpath(abs_path, root_folder) + print(f" [not staged] {rel}") + + conn.commit() + conn.close() + print(f" {removed} path(s) removed from staging.") + """Show files currently in the staging area.""" + root_folder = os.path.abspath(root_folder) + conn = open_db(root_folder) + rows = conn.execute( + "SELECT absolute_path, action, added_at FROM staging " + "WHERE root_folder=? ORDER BY absolute_path", + (root_folder,) + ).fetchall() + conn.close() + + if not rows: + print("Nothing staged. Use 'add ...' or 'delete ...' to stage paths.") + return + + print(f"Staged paths ({len(rows)}):") + for r in rows: + rel = os.path.relpath(r["absolute_path"], root_folder) + print(f" {rel} [{r['action']}]") + + def cmd_commit(self, message: str = ""): + db_path = os.path.join(self.root_folder, SNAPSHOTS_DIR, DB_NAME) + if not os.path.isfile(db_path): + sys.exit("No dsi-vcs repo found. Run 'init' first.") + + conn = open_db(self.root_folder) + cur = conn.cursor() + + # ── Load staged paths ──────────────────────────────────────────────────── + staged_rows = cur.execute( + "SELECT absolute_path, action FROM staging WHERE root_folder = ? ORDER BY absolute_path", + (self.root_folder,) + ).fetchall() + + if not staged_rows: + conn.close() + sys.exit("Nothing staged. Use 'add' or 'delete' before committing.") + + staged_adds = [r["absolute_path"] for r in staged_rows if r["action"] == "add"] + staged_deletes = [r["absolute_path"] for r in staged_rows if r["action"] == "delete"] + + # ── Previous snapshot for hard-link deduplication ──────────────────────── + prev_row = cur.execute( + "SELECT commit_hash, snapshot_path FROM versions " + "WHERE root_folder=? ORDER BY id DESC LIMIT 1", + (self.root_folder,) + ).fetchone() + prev_snapshot = prev_row["snapshot_path"] if prev_row else None + parent_commit_hash = prev_row["commit_hash"] if prev_row else None + + committed_at = datetime.datetime.now(datetime.timezone.utc).isoformat() + running_user = owner_name(os.getuid()) + snapshots_root = os.path.join(self.root_folder, SNAPSHOTS_DIR) + + # ── Validate staged paths before creating the snapshot ───────────────── + print(f"Validating {len(staged_rows)} staged path(s)…") + valid_staged = len(staged_deletes) + for abs_path in staged_adds: + e = collect_metadata(abs_path, self.root_folder) + if "error" in e: + print(f" [skip] {e['relative_path']}: {e['error']}") + else: + valid_staged += 1 + + if valid_staged == 0: + conn.close() + sys.exit("No readable staged paths — commit aborted.") + + # ── Create a complete snapshot of the current repository tree ─────────── + tmp_snapshot_path = tempfile.mkdtemp(prefix=".tmp-", dir=snapshots_root) + print(f" Creating rsync snapshot → {tmp_snapshot_path}") + + ok = rsync_snapshot(self.root_folder, tmp_snapshot_path, prev_snapshot) + if not ok: + conn.close() + shutil.rmtree(tmp_snapshot_path, ignore_errors=True) + sys.exit("rsync failed — commit aborted.") + + try: + apply_snapshot_deletes(tmp_snapshot_path, self.root_folder, staged_deletes) + except ValueError as e: + conn.close() + shutil.rmtree(tmp_snapshot_path, ignore_errors=True) + sys.exit(str(e)) + + # ── Collect metadata for the complete committed tree ─────────────────── + entries = collect_tree_metadata(tmp_snapshot_path, self.root_folder) + + total_bytes = sum(e.get("_st_size") or 0 for e in entries if e.get("file_type") == "file") + file_count = sum(1 for e in entries if e.get("file_type") == "file") + print(f" {file_count} file(s), {total_bytes:,} bytes") + + root_tree_hash, merkle_nodes = build_merkle_tree(entries, tmp_snapshot_path) + commit_hash = merkle_commit_hash( + root_tree_hash=root_tree_hash, + parent_commit_hash=parent_commit_hash, + committed_at=committed_at, + owner_name=running_user, + message=message, + file_count=file_count, + total_bytes=total_bytes, + ) + snapshot_path = os.path.join(snapshots_root, commit_hash[:12]) + if os.path.exists(snapshot_path): + conn.close() + shutil.rmtree(tmp_snapshot_path, ignore_errors=True) + sys.exit(f"Snapshot path already exists for commit prefix: {snapshot_path}") + os.rename(tmp_snapshot_path, snapshot_path) + + # ── Insert version row ─────────────────────────────────────────────────── + cur.execute( + """INSERT INTO versions + (root_folder, commit_hash, root_tree_hash, parent_commit_hash, hash_algorithm, + committed_at, owner_name, message, snapshot_path, file_count, total_bytes) + VALUES (?,?,?,?,?,?,?,?,?,?,?)""", + (self.root_folder, commit_hash, root_tree_hash, parent_commit_hash, HASH_ALGORITHM, + committed_at, running_user, message, snapshot_path, file_count, total_bytes) + ) + version_id = cur.lastrowid + + # ── Bulk-insert file entries ───────────────────────────────────────────── + cols = [ + "version_id", "root_folder", "relative_path", "absolute_path", + "file_name", "file_type", "md5_hash", + "lstat", + "permissions_int", "owner_name", "group_name", + "acl_text", "xattrs", "security_context", "symlink_target", + ] + placeholders = ",".join("?" * len(cols)) + col_str = ",".join(cols) + + cur.executemany( + f"INSERT INTO file_entries ({col_str}) VALUES ({placeholders})", + [ + tuple( + version_id if c == "version_id" else + self.root_folder if c == "root_folder" else + e.get(c) + for c in cols + ) + for e in entries + ] + ) + + merkle_cols = [ + "version_id", "root_folder", "relative_path", "file_type", + "node_hash", "metadata_hash", "content_hash_sha256", + "subtree_file_count", "subtree_total_bytes", "child_count", + ] + merkle_placeholders = ",".join("?" * len(merkle_cols)) + merkle_col_str = ",".join(merkle_cols) + + cur.executemany( + f"INSERT INTO merkle_nodes ({merkle_col_str}) VALUES ({merkle_placeholders})", + [ + tuple( + version_id if c == "version_id" else + self.root_folder if c == "root_folder" else + node.get(c) + for c in merkle_cols + ) + for node in merkle_nodes + ] + ) + + # ── Clear staging after a successful commit ────────────────────────────── + cur.execute("DELETE FROM staging WHERE root_folder=?", (self.root_folder,)) + + conn.commit() + conn.close() + + print(f"\n Committed {commit_hash} at {committed_at}") + print(f" Short hash : {commit_hash[:12]}") + print(f" Owner : {running_user}") + if message: + print(f" Message : {message}") + + + def cmd_log(self): + root_folder = os.path.abspath(self.root_folder) + conn = open_db(root_folder) + rows = conn.execute( + "SELECT commit_hash, committed_at, owner_name, message, file_count, total_bytes " + "FROM versions WHERE root_folder=? ORDER BY id", + (root_folder,) + ).fetchall() + conn.close() + + if not rows: + print("No versions yet. Run 'commit' first.") + return + + print(f"{'COMMIT HASH':<66} {'OWNER':<16} {'DATE/TIME (UTC)':<28} {'FILES':>7} {'BYTES':>15} MESSAGE") + print("─" * 132) + for r in rows: + msg = (r["message"] or "")[:35] + print(f"{r['commit_hash']:<66} {r['owner_name']:<16} {r['committed_at']:<28}" + f"{r['file_count']:>7} {r['total_bytes']:>15,} {msg}") + + + def cmd_diff(self, c1: str, c2: str): + root_folder = os.path.abspath(self.root_folder) + conn = open_db(root_folder) + + def get_files_in_root_folder(): + files = {} + for dirpath, dirnames, filenames in os.walk(root_folder, followlinks=False): + dirnames[:] = [d for d in dirnames if d not in {DB_NAME, SNAPSHOTS_DIR}] + if dirpath != root_folder: + abs_path = dirpath + rel_path = os.path.relpath(abs_path, root_folder) + files[rel_path] = collect_metadata(abs_path, self.root_folder) + files[rel_path]["absolute_path"] = abs_path + files[rel_path]["lstat"] = json.loads(files[rel_path]["lstat"]) if files[rel_path]["lstat"] else {} + for fname in filenames: + if fname not in {DB_NAME, SNAPSHOTS_DIR}: + abs_path = os.path.join(dirpath, fname) + rel_path = os.path.relpath(abs_path, root_folder) + files[rel_path] = collect_metadata(abs_path, self.root_folder) + files[rel_path]["absolute_path"] = abs_path + files[rel_path]["lstat"] = json.loads(files[rel_path]["lstat"]) if files[rel_path]["lstat"] else {} + return files + + def get_version(chash): + if chash == "latest": + row = conn.execute( + "SELECT id, commit_hash, snapshot_path, root_tree_hash FROM versions " + "WHERE root_folder=? ORDER BY id DESC LIMIT 1", + (root_folder,) + ).fetchone() + else: + row = conn.execute( + "SELECT id, commit_hash, snapshot_path, root_tree_hash FROM versions " + "WHERE root_folder = ? AND commit_hash LIKE ?", + (root_folder, chash + "%") + ).fetchone() + if not row: + conn.close() + sys.exit(f"Commit '{chash}' not found.") + return row + + def get_files(chash): + vid = get_version(chash) + return get_files_for_version(vid) + + def get_files_for_version(version, only_paths: Optional[set[str]] = None): + path_filter = None if only_paths is None else sorted(only_paths) + if path_filter == []: + return {} + + rows = [] + base_sql = ( + "SELECT relative_path, absolute_path, file_type, md5_hash, permissions_int, " + " owner_name, group_name, lstat " + "FROM file_entries WHERE version_id=?" + ) + if path_filter is None: + rows = conn.execute(base_sql, (version["id"],)).fetchall() + else: + for i in range(0, len(path_filter), 500): + chunk = path_filter[i:i + 500] + placeholders = ",".join("?" * len(chunk)) + rows.extend( + conn.execute( + f"{base_sql} AND relative_path IN ({placeholders})", + (version["id"], *chunk), + ).fetchall() + ) + + snapshot_path = version["snapshot_path"] + result = {} + for r in rows: + rec = dict(r) + # Unpack the lstat JSON so callers can access st_size, st_mtime, etc. + rec["absolute_path"] = snapshot_target(snapshot_path, r["relative_path"]) + rec["lstat"] = json.loads(r["lstat"]) if r["lstat"] else {} + result[r["relative_path"]] = rec + return result + + def get_merkle_nodes(version_id): + rows = conn.execute( + "SELECT relative_path, file_type, node_hash, metadata_hash " + "FROM merkle_nodes WHERE version_id=?", + (version_id,), + ).fetchall() + return {r["relative_path"]: dict(r) for r in rows} + + def children_index(nodes): + children = {path: [] for path in nodes} + for path in nodes: + if path == ".": + continue + children.setdefault(parent_path(path), []).append(path) + return children + + def add_subtree(changed_paths, path, children): + if path != ".": + changed_paths.add(path) + for child_path in children.get(path, []): + add_subtree(changed_paths, child_path, children) + + def changed_paths_from_merkle(v1, v2): + nodes1 = get_merkle_nodes(v1["id"]) + nodes2 = get_merkle_nodes(v2["id"]) + children1 = children_index(nodes1) + children2 = children_index(nodes2) + changed_paths = set() + + def walk(path): + n1 = nodes1.get(path) + n2 = nodes2.get(path) + if n1 and n2 and n1["node_hash"] == n2["node_hash"]: + return + if n1 is None: + add_subtree(changed_paths, path, children2) + return + if n2 is None: + add_subtree(changed_paths, path, children1) + return + + if path != ".": + if n1["file_type"] != n2["file_type"] or n1["metadata_hash"] != n2["metadata_hash"]: + changed_paths.add(path) + elif n1["file_type"] != "dir": + changed_paths.add(path) + + if n1["file_type"] == "dir" or n2["file_type"] == "dir": + child_paths = set(children1.get(path, [])) | set(children2.get(path, [])) + for child_path in sorted(child_paths): + walk(child_path) + + walk(".") + return changed_paths + + def unchanged_count_from_merkle(v1, v2): + row = conn.execute( + "SELECT COUNT(*) AS count " + "FROM merkle_nodes a " + "JOIN merkle_nodes b " + " ON a.relative_path = b.relative_path " + " AND a.node_hash = b.node_hash " + "WHERE a.version_id=? AND b.version_id=? AND a.relative_path <> '.'", + (v1["id"], v2["id"]), + ).fetchone() + return row["count"] if row else 0 + + files1 = files2 = {} + unchanged = 0 + if c1 is None and c2 is None: + files1 = get_files_in_root_folder() + files2 = get_files("latest") + elif c1 is None: + files1 = get_files_in_root_folder() + files2 = get_files(c2) + elif c2 is None: + files1 = get_files(c1) + files2 = get_files_in_root_folder() + else: + version1 = get_version(c1) + version2 = get_version(c2) + changed_paths = changed_paths_from_merkle(version1, version2) + unchanged = unchanged_count_from_merkle(version1, version2) + files1 = get_files_for_version(version1, changed_paths) + files2 = get_files_for_version(version2, changed_paths) + conn.close() + + all_paths = sorted(set(files1) | set(files2)) + added = deleted = modified = 0 + + print(f"Diff {c1} → {c2} ({root_folder})\n") + print(f"{'STATUS':<10} {'PATH'}") + print("─" * 70) + + for p in all_paths: + # if p in files1 and p not in files2: + # print(f"{'DELETED':<10} {p:<40} in {c2}") + # deleted += 1 + # elif p in files2 and p not in files1: + # print(f"{'ADDED':<10} {p:<40} in {c2}") + # added += 1 + if p not in files1: + print(f"{'ADDED':<10} {p:<40} in {c2}") + added += 1 + elif p not in files2: + print(f"{'DELETED':<10} {p:<40} in {c2}") + deleted += 1 + else: + f1, f2 = files1[p], files2[p] + changes = [] + if f1["file_type"] != f2["file_type"]: + changes.append("type") + if f1["md5_hash"] != f2["md5_hash"]: + if f1["md5_hash"] or f2["md5_hash"]: + changes.append("content") + if f1["md5_hash"] and f2["md5_hash"]: + result = subprocess.run(['diff', f1["absolute_path"], f2["absolute_path"]], capture_output=True, text=True) + print(f"diff result: {result.stdout.strip()}") + if f1["permissions_int"] != f2["permissions_int"]: + changes.append("perms") + if f1["owner_name"] != f2["owner_name"] or f1["group_name"] != f2["group_name"]: + changes.append("owner") + if f1["lstat"].get("st_size") != f2["lstat"].get("st_size"): + changes.append("size") + if changes: + print(f"{'MODIFIED':<10} {p} [{', '.join(changes)}]") + modified += 1 + else: + unchanged += 1 + + print(f"\nSummary: +{added} added -{deleted} deleted ~{modified} modified ={unchanged} unchanged") + + + def cmd_restore(self, commit_hash: str): + root_folder = os.path.abspath(self.root_folder) + conn = open_db(root_folder) + row = conn.execute( + "SELECT commit_hash, snapshot_path FROM versions " + "WHERE root_folder = ? AND commit_hash LIKE ?", + (root_folder, commit_hash + "%") + ).fetchone() + conn.close() + + if not row: + sys.exit(f"Commit '{commit_hash}' not found.") + + full_hash = row["commit_hash"] + snapshot = row["snapshot_path"] + if not os.path.isdir(snapshot): + sys.exit(f"Snapshot directory missing: {snapshot}") + + print(f"Restoring {commit_hash} from {snapshot} → {root_folder}") + cmd = [ + "rsync", "-aAXH", "--delete", + "--exclude", DB_NAME, + "--exclude", SNAPSHOTS_DIR, + snapshot.rstrip("/") + "/", + root_folder.rstrip("/") + "/", + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode not in (0, 24): + sys.exit(f"rsync restore failed:\n{result.stderr}") + print(f" Restored to {full_hash}") diff --git a/dsi/utils/version_control/merkle.py b/dsi/utils/version_control/merkle.py new file mode 100644 index 00000000..3ea1c9e5 --- /dev/null +++ b/dsi/utils/version_control/merkle.py @@ -0,0 +1,162 @@ +import hashlib +import json +import os +from typing import Any, Optional + + +HASH_ALGORITHM = "sha256-merkle-v1" + + +def _canonical_json(data: dict[str, Any]) -> bytes: + return json.dumps(data, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def hash_payload(data: dict[str, Any]) -> str: + return hashlib.sha256(_canonical_json(data)).hexdigest() + + +def sha256_file(path: str, chunk_size: int = 1 << 20) -> Optional[str]: + h = hashlib.sha256() + try: + with open(path, "rb") as f: + while chunk := f.read(chunk_size): + h.update(chunk) + return h.hexdigest() + except (PermissionError, OSError): + return None + + +def parent_path(relative_path: str) -> str: + parent = os.path.dirname(relative_path) + return parent if parent else "." + + +def _stable_metadata(entry: dict[str, Any], relative_path: str) -> dict[str, Any]: + if relative_path == ".": + return { + "relative_path": ".", + "file_type": "dir", + "root": True, + } + + return { + "relative_path": relative_path, + "file_name": os.path.basename(relative_path), + "file_type": entry.get("file_type"), + "permissions_int": entry.get("permissions_int"), + "owner_name": entry.get("owner_name"), + "group_name": entry.get("group_name"), + "acl_text": entry.get("acl_text"), + "xattrs": entry.get("xattrs"), + "security_context": entry.get("security_context"), + "symlink_target": entry.get("symlink_target"), + } + + +def build_merkle_tree(entries: list[dict[str, Any]], snapshot_path: str) -> tuple[str, list[dict[str, Any]]]: + entries_by_path = {entry["relative_path"]: entry for entry in entries} + children_by_parent: dict[str, list[tuple[str, str]]] = {".": []} + + for relative_path in entries_by_path: + parent = parent_path(relative_path) + children_by_parent.setdefault(parent, []).append((os.path.basename(relative_path), relative_path)) + children_by_parent.setdefault(relative_path, []) + + nodes: dict[str, dict[str, Any]] = {} + + def build_node(relative_path: str) -> dict[str, Any]: + if relative_path in nodes: + return nodes[relative_path] + + entry = entries_by_path.get(relative_path, {}) + file_type = "dir" if relative_path == "." else entry.get("file_type") + metadata_hash = hash_payload( + { + "object": "dsi-vcs-metadata-v1", + "metadata": _stable_metadata(entry, relative_path), + } + ) + child_rows = sorted(children_by_parent.get(relative_path, [])) + content_hash = None + subtree_file_count = 0 + subtree_total_bytes = 0 + + if file_type == "dir": + child_payload = [] + for name, child_path in child_rows: + child = build_node(child_path) + subtree_file_count += child["subtree_file_count"] + subtree_total_bytes += child["subtree_total_bytes"] + child_payload.append( + { + "name": name, + "file_type": child["file_type"], + "node_hash": child["node_hash"], + } + ) + node_hash = hash_payload( + { + "object": "dsi-vcs-tree-v1", + "metadata_hash": metadata_hash, + "children": child_payload, + } + ) + elif file_type == "file": + content_hash = sha256_file(os.path.join(snapshot_path, relative_path)) + subtree_file_count = 1 + subtree_total_bytes = entry.get("_st_size") or 0 + node_hash = hash_payload( + { + "object": "dsi-vcs-file-v1", + "metadata_hash": metadata_hash, + "content_hash_sha256": content_hash, + } + ) + else: + node_hash = hash_payload( + { + "object": "dsi-vcs-special-v1", + "file_type": file_type, + "metadata_hash": metadata_hash, + } + ) + + node = { + "relative_path": relative_path, + "file_type": file_type, + "node_hash": node_hash, + "metadata_hash": metadata_hash, + "content_hash_sha256": content_hash, + "subtree_file_count": subtree_file_count, + "subtree_total_bytes": subtree_total_bytes, + "child_count": len(child_rows), + } + nodes[relative_path] = node + return node + + root = build_node(".") + return root["node_hash"], [nodes[path] for path in sorted(nodes)] + + +def commit_hash( + root_tree_hash: str, + parent_commit_hash: Optional[str], + committed_at: str, + owner_name: str, + message: str, + file_count: int, + total_bytes: int, +) -> str: + return hash_payload( + { + "object": "dsi-vcs-commit-v1", + "hash_algorithm": HASH_ALGORITHM, + "root_tree_hash": root_tree_hash, + "parent_commit_hash": parent_commit_hash, + "committed_at": committed_at, + "owner_name": owner_name, + "message": message or "", + "file_count": file_count, + "total_bytes": total_bytes, + } + ) diff --git a/dsi/utils/version_control/requirements.txt b/dsi/utils/version_control/requirements.txt new file mode 100644 index 00000000..11258685 --- /dev/null +++ b/dsi/utils/version_control/requirements.txt @@ -0,0 +1 @@ +pyxattr>=0.8.1 \ No newline at end of file diff --git a/dsi/utils/version_control/tests/test_vcs.py b/dsi/utils/version_control/tests/test_vcs.py new file mode 100644 index 00000000..d34009f2 --- /dev/null +++ b/dsi/utils/version_control/tests/test_vcs.py @@ -0,0 +1,169 @@ +import os +import re +import sqlite3 +from pathlib import Path +from shutil import which +import pytest +import stat + +from dsi.dsi import DSI +from dsi.utils.version_control.dsi_vcs import Version +from dsi.utils.version_control.vcs_db import DB_NAME, SNAPSHOTS_DIR + +def require_rsync(): + if which("rsync") is None: + pytest.skip("rsync is required for dsi_vcs.") + + + +def connect_repo(repo_path): + return sqlite3.connect(repo_path / SNAPSHOTS_DIR / DB_NAME) + +def commits(repo_path): + with connect_repo(repo_path) as conn: + conn.row_factory = sqlite3.Row + return conn.execute( + "SELECT id, commit_hash, root_tree_hash, parent_commit_hash, " + "hash_algorithm, snapshot_path FROM versions ORDER BY id" + ).fetchall() + +def merkle_nodes(repo_path, version_id): + with connect_repo(repo_path) as conn: + conn.row_factory = sqlite3.Row + return { + row["relative_path"]: row + for row in conn.execute( + "SELECT relative_path, file_type, node_hash, metadata_hash, " + "content_hash_sha256, subtree_file_count, subtree_total_bytes, child_count " + "FROM merkle_nodes WHERE version_id=? ORDER BY relative_path", + (version_id,), + ).fetchall() + } + +def latest_entries(repo_path): + with connect_repo(repo_path) as conn: + conn.row_factory = sqlite3.Row + return conn.execute( + "SELECT relative_path, file_type, permissions_int " + "FROM file_entries WHERE version_id=(SELECT MAX(id) from versions) " + "ORDER BY relative_path").fetchall() + +def test_add(tmp_path): + require_rsync() + repo = Version(str(tmp_path)) + + empty_dir = tmp_path / "empty" + nested_dir = tmp_path / "nested" / "child" + empty_dir.mkdir() + nested_dir.mkdir(parents=True) + (tmp_path / "a.txt").write_text("foo") + (nested_dir / "b.txt").write_text("bar") + + empty_dir.chmod(0o2775) + + repo.cmd_add(["a.txt", "nested", "empty"]) + repo.cmd_commit("alpha") + + (tmp_path / "c.txt").write_text("baz") + + repo.cmd_add(["c.txt"]) + repo.cmd_commit("beta") + + alpha, beta = commits(tmp_path) + assert re.fullmatch(r"[0-9a-f]{64}", alpha["commit_hash"]) + assert re.fullmatch(r"[0-9a-f]{64}", beta["commit_hash"]) + assert alpha["hash_algorithm"] == "sha256-merkle-v1" + assert beta["parent_commit_hash"] == alpha["commit_hash"] + assert Path(alpha["snapshot_path"]).name == alpha["commit_hash"][:12] + assert Path(beta["snapshot_path"]).name == beta["commit_hash"][:12] + + alpha_path = Path(alpha["snapshot_path"]) + beta_path = Path(beta["snapshot_path"]) + assert (alpha_path / "a.txt").read_text() == "foo" + assert (alpha_path / "nested" / "child" / "b.txt").read_text() == "bar" + assert (beta_path / "c.txt").read_text() == "baz" + assert (alpha_path / "empty").exists() + assert stat.S_IMODE((alpha_path / "empty").stat().st_mode) == 0o2775 + + rows = {row["relative_path"]: row for row in latest_entries(tmp_path)} + empty_entry = rows["empty"] + assert empty_entry["file_type"] == "dir" + assert empty_entry["permissions_int"] == 0o2775 + + beta_nodes = merkle_nodes(tmp_path, beta["id"]) + assert "." in beta_nodes + assert beta_nodes["."]["node_hash"] == beta["root_tree_hash"] + assert beta_nodes["a.txt"]["file_type"] == "file" + assert re.fullmatch(r"[0-9a-f]{64}", beta_nodes["a.txt"]["content_hash_sha256"]) + + +def test_merkle_tracks_content_metadata_symlink_and_deletes(tmp_path): + require_rsync() + repo = Version(str(tmp_path)) + + nested = tmp_path / "nested" + nested.mkdir() + file_path = nested / "data.txt" + file_path.write_text("one") + link_path = tmp_path / "link.txt" + link_path.symlink_to("nested/data.txt") + + repo.cmd_add(["nested", "link.txt"]) + repo.cmd_commit("initial") + first = commits(tmp_path)[0] + first_nodes = merkle_nodes(tmp_path, first["id"]) + + file_path.write_text("two more") + repo.cmd_add(["nested/data.txt"]) + repo.cmd_commit("content") + second = commits(tmp_path)[1] + second_nodes = merkle_nodes(tmp_path, second["id"]) + assert second_nodes["nested/data.txt"]["content_hash_sha256"] != first_nodes["nested/data.txt"]["content_hash_sha256"] + assert second_nodes["nested/data.txt"]["metadata_hash"] == first_nodes["nested/data.txt"]["metadata_hash"] + assert second["root_tree_hash"] != first["root_tree_hash"] + + file_path.chmod(0o640) + repo.cmd_add(["nested/data.txt"]) + repo.cmd_commit("metadata") + third = commits(tmp_path)[2] + third_nodes = merkle_nodes(tmp_path, third["id"]) + assert third_nodes["nested/data.txt"]["content_hash_sha256"] == second_nodes["nested/data.txt"]["content_hash_sha256"] + assert third_nodes["nested/data.txt"]["metadata_hash"] != second_nodes["nested/data.txt"]["metadata_hash"] + + link_path.unlink() + link_path.symlink_to("missing.txt") + repo.cmd_add(["link.txt"]) + repo.cmd_commit("symlink") + fourth = commits(tmp_path)[3] + fourth_nodes = merkle_nodes(tmp_path, fourth["id"]) + assert fourth_nodes["link.txt"]["metadata_hash"] != third_nodes["link.txt"]["metadata_hash"] + + repo.cmd_delete(["nested/data.txt"]) + repo.cmd_commit("delete") + fifth = commits(tmp_path)[4] + fifth_nodes = merkle_nodes(tmp_path, fifth["id"]) + assert "nested/data.txt" not in fifth_nodes + assert fifth["parent_commit_hash"] == fourth["commit_hash"] + + +def test_diff_short_circuits_equal_root_trees(tmp_path, capsys): + require_rsync() + repo = Version(str(tmp_path)) + + (tmp_path / "a.txt").write_text("same") + repo.cmd_add(["a.txt"]) + repo.cmd_commit("first") + + repo.cmd_delete(["does-not-exist.txt"]) + repo.cmd_commit("same tree") + + first, second = commits(tmp_path) + assert first["root_tree_hash"] == second["root_tree_hash"] + assert first["commit_hash"] != second["commit_hash"] + + repo.cmd_diff(first["commit_hash"][:12], second["commit_hash"][:12]) + out = capsys.readouterr().out + assert "ADDED" not in out + assert "DELETED" not in out + assert "MODIFIED" not in out + assert "Summary: +0 added -0 deleted ~0 modified" in out diff --git a/dsi/utils/version_control/vcs_db.py b/dsi/utils/version_control/vcs_db.py new file mode 100644 index 00000000..2724cd7a --- /dev/null +++ b/dsi/utils/version_control/vcs_db.py @@ -0,0 +1,113 @@ +import sqlite3 +import os + +# ─────────────────────────── CONFIG ────────────────────────────────────────── + +DB_NAME = ".dsi_vcs.db" # SQLite DB stored inside the root folder +SNAPSHOTS_DIR = ".dsi_vcs_snapshots" # rsync snapshot copies live here + +# ─────────────────────────── DATABASE ──────────────────────────────────────── + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + root_folder TEXT NOT NULL, + commit_hash TEXT NOT NULL, -- SHA-256 Merkle commit hash + root_tree_hash TEXT NOT NULL, + parent_commit_hash TEXT, + hash_algorithm TEXT NOT NULL, + committed_at TEXT NOT NULL, -- ISO-8601 timestamp + owner_name TEXT NOT NULL, -- Username of the committer + message TEXT, + snapshot_path TEXT NOT NULL, -- path to rsync copy + file_count INTEGER NOT NULL, + total_bytes INTEGER NOT NULL, + UNIQUE(root_folder, commit_hash) +); + +CREATE TABLE IF NOT EXISTS file_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version_id INTEGER NOT NULL REFERENCES versions(id) ON DELETE CASCADE, + root_folder TEXT NOT NULL, -- partition key / lookup key + relative_path TEXT NOT NULL, -- path relative to root + absolute_path TEXT NOT NULL, + + -- ── Identity ────────────────────────────────────────────────────────── + file_name TEXT NOT NULL, + file_type TEXT NOT NULL, -- file/dir/symlink/block/char/fifo/socket + + -- ── Content hash ────────────────────────────────────────────────────── + md5_hash TEXT, -- NULL for non-regular files + + -- ── lstat(2) — all raw os.lstat() fields packed as a JSON object ────── + -- Keys: st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, + -- st_atime, st_mtime, st_ctime, st_blocks, st_blksize + lstat TEXT NOT NULL, + + -- ── Human-readable permission strings ───────────────────────────────── + permissions_int INTEGER, -- e.g. "755" as an integer + owner_name TEXT, + group_name TEXT, + + -- ── ACL (POSIX) ─────────────────────────────────────────────────────── + acl_text TEXT, -- raw getfacl output + + -- ── Extended attributes ─────────────────────────────────────────────── + xattrs TEXT, -- JSON dict of xattr key→value + + -- ── Symlink target ─────────────────────────────────────────────────── + symlink_target TEXT, + + -- ── SELinux / AppArmor context ──────────────────────────────────────── + security_context TEXT +); + +CREATE INDEX IF NOT EXISTS idx_file_entries_root + ON file_entries(root_folder, version_id); + +CREATE INDEX IF NOT EXISTS idx_file_entries_path + ON file_entries(root_folder, relative_path); + +CREATE TABLE IF NOT EXISTS merkle_nodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version_id INTEGER NOT NULL REFERENCES versions(id) ON DELETE CASCADE, + root_folder TEXT NOT NULL, + relative_path TEXT NOT NULL, + file_type TEXT NOT NULL, + node_hash TEXT NOT NULL, + metadata_hash TEXT NOT NULL, + content_hash_sha256 TEXT, + subtree_file_count INTEGER NOT NULL, + subtree_total_bytes INTEGER NOT NULL, + child_count INTEGER NOT NULL, + UNIQUE(version_id, relative_path) +); + +CREATE INDEX IF NOT EXISTS idx_merkle_nodes_root_path + ON merkle_nodes(root_folder, version_id, relative_path); + +CREATE INDEX IF NOT EXISTS idx_merkle_nodes_hash + ON merkle_nodes(root_folder, node_hash); + +CREATE TABLE IF NOT EXISTS staging ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + root_folder TEXT NOT NULL, + absolute_path TEXT NOT NULL UNIQUE, -- absolute path must be unique in staging + action TEXT NOT NULL DEFAULT 'add', + added_at TEXT NOT NULL, -- ISO-8601 timestamp + UNIQUE(root_folder, absolute_path) +); +""" + + +def open_db(root_folder: str) -> sqlite3.Connection: + snaps = os.path.join(root_folder, SNAPSHOTS_DIR) + os.makedirs(snaps, exist_ok=True) + db_path = os.path.join(snaps, DB_NAME) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA journal_mode = WAL") + conn.executescript(SCHEMA) + conn.commit() + return conn diff --git a/dsi/utils/version_control/vcs_metadata_helper.py b/dsi/utils/version_control/vcs_metadata_helper.py new file mode 100644 index 00000000..6bf4a417 --- /dev/null +++ b/dsi/utils/version_control/vcs_metadata_helper.py @@ -0,0 +1,288 @@ +import os +import stat +import hashlib +import json +import subprocess +import pwd +import grp +import sys +from typing import Optional + +from .vcs_db import DB_NAME, SNAPSHOTS_DIR + +# ─────────────────────────── METADATA HELPERS ──────────────────────────────── + +def file_type_str(mode: int) -> str: + if stat.S_ISREG(mode): + return "file" + if stat.S_ISDIR(mode): + return "dir" + if stat.S_ISLNK(mode): + return "symlink" + if stat.S_ISBLK(mode): + return "block" + if stat.S_ISCHR(mode): + return "char" + if stat.S_ISFIFO(mode): + return "fifo" + if stat.S_ISSOCK(mode): + return "socket" + return "unknown" + + +def permission_str(mode: int) -> str: + """Convert mode bits → rwxrwxrwx string (no file-type prefix).""" + chars = [] + for bit, char in ( + (stat.S_IRUSR, "r"), (stat.S_IWUSR, "w"), (stat.S_IXUSR, "x"), + (stat.S_IRGRP, "r"), (stat.S_IWGRP, "w"), (stat.S_IXGRP, "x"), + (stat.S_IROTH, "r"), (stat.S_IWOTH, "w"), (stat.S_IXOTH, "x"), + ): + chars.append(char if mode & bit else "-") + if mode & stat.S_ISUID: + chars[2] = "s" if chars[2] == "x" else "S" + if mode & stat.S_ISGID: + chars[5] = "s" if chars[5] == "x" else "S" + if mode & stat.S_ISVTX: + chars[8] = "t" if chars[8] == "x" else "T" + return "".join(chars) + + +def owner_name(uid: int) -> str: + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + return str(uid) + + +def group_name(gid: int) -> str: + try: + return grp.getgrgid(gid).gr_name + except KeyError: + return str(gid) + + +def get_acl(path: str) -> Optional[str]: + """Return raw getfacl output, or None if unavailable.""" + try: + if sys.platform == "darwin": + result = subprocess.run( + # ["getfacl", "--omit-header", "--absolute-names", path], + ["ls", "-le", path], + capture_output=True, text=True, timeout=5 + ) + lines = result.stdout.splitlines() + + acl_entries = [] + + for line in lines: + line = line.strip() + if line.startswith(tuple(str(i) + ":" for i in range(10))): + # Example line: "0: user:azad allow read,write" + parts = line.split("user:") + if len(parts) > 1: + acl_part = parts[1].strip() + acl_entries.append(acl_part) + + return ";".join(acl_entries) if acl_entries else None + elif sys.platform == "linux": + result = subprocess.run( + ["getfacl", "--omit-header", "--absolute-names", path], + capture_output=True, text=True, timeout=5 + ) + return result.stdout.strip() or None + else: + return None + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + + +def set_acl(path: str, acl_string: str) -> Optional[str]: + """Return raw getfacl output, or None if unavailable.""" + acl_list = acl_string.split(";") + for acl_entry in acl_list: + try: + if len(acl_entry) == 0: + continue + acls = acl_entry.split() + acl_perm = "user:" + acls[0] + " allow " + acls[2] + print(acl_perm) + result = subprocess.run( + ["chmod", "+a", acl_perm, path], + capture_output=True, text=True, timeout=5 + ) + if result.returncode != 0: + print(f"Failed to set ACL entry '{acl_entry}' on {path}: {result.stderr}") + except Exception as e: + print(f"Error setting ACL entry '{acl_entry}' on {path}: {e}") + + +def get_xattrs(path: str) -> Optional[str]: + return None + # """Return JSON dict of extended attributes, or None.""" + # try: + # import xattr # pyxattr + # attrs = {} + # for key in xattr.listxattr(path, symlink=True): + # try: + # val = xattr.getxattr(path, key, symlink=True) + # attrs[key] = val.decode("utf-8", errors="replace") + # except OSError: + # attrs[key] = "" + # return json.dumps(attrs) if attrs else None + # except ImportError: + # # Fallback: use getfattr + # try: + # result = subprocess.run( + # ["getfattr", "-d", "-m", "-", "--absolute-names", path], + # capture_output=True, text=True, timeout=5 + # ) + # return result.stdout.strip() or None + # except FileNotFoundError: + # return None + # except OSError: + # return None + + +def get_security_context(path: str) -> Optional[str]: + """Return SELinux/AppArmor security context if available.""" + try: + result = subprocess.run( + ["ls", "-Z", path], + capture_output=True, text=True, timeout=5 + ) + if result.returncode == 0: + parts = result.stdout.strip().split() + # ls -Z format: context file (or just file if no context) + if len(parts) >= 2 and ":" in parts[0]: + return parts[0] + return None + except Exception: + return None + + +def md5_hash(path: str, chunk_size: int = 1 << 20) -> Optional[str]: + """Stream-hash a file in 1 MB chunks. Returns None on error.""" + h = hashlib.md5() + try: + with open(path, "rb") as f: + while chunk := f.read(chunk_size): + h.update(chunk) + return h.hexdigest() + except (PermissionError, OSError): + return None + + +def collect_metadata(abs_path: str, root_folder: str) -> dict: + """ + Gather all available metadata for a single path. + + Returns a dict with two logical groups: + • lstat — raw os.lstat() result stored as a nested dict (→ JSON column) + • all other keys — each stored in its own column + """ + rel_path = os.path.relpath(abs_path, root_folder) + + try: + s = os.lstat(abs_path) + except OSError as e: + return {"relative_path": rel_path, "error": str(e)} + + mode = s.st_mode + ftype = file_type_str(mode) + + # ── lstat dict (raw kernel values only) ────────────────────────────────── + lstat_dict = { + "st_mode": mode, + "st_ino": s.st_ino, + "st_dev": s.st_dev, + "st_nlink": s.st_nlink, + "st_uid": s.st_uid, + "st_gid": s.st_gid, + "st_size": s.st_size, + "st_atime": s.st_atime, + "st_mtime": s.st_mtime, + "st_ctime": s.st_ctime, + "st_blocks": getattr(s, "st_blocks", None), + "st_blksize": getattr(s, "st_blksize", None), + } + + # ── all other (derived / external) attributes — own columns ────────────── + entry = { + "relative_path": rel_path, + "absolute_path": abs_path, + "file_name": os.path.basename(abs_path), + "file_type": ftype, + + # serialised lstat dict → stored as TEXT (JSON) in a single column + "lstat": json.dumps(lstat_dict), + + # human-readable permission strings + "permissions_int": stat.S_IMODE(mode), + "owner_name": owner_name(s.st_uid), + "group_name": group_name(s.st_gid), + + # external / subprocess-derived attributes + "acl_text": get_acl(abs_path), + # "xattrs": get_xattrs(abs_path), + "security_context": get_security_context(abs_path), + "symlink_target": os.readlink(abs_path) if ftype == "symlink" else None, + + # content hash — regular files only + "md5_hash": md5_hash(abs_path) if ftype == "file" else None, + + # convenience shortcut used by walk_folder aggregation (not stored) + "_st_size": s.st_size, + } + return entry + + +def collect_tree_metadata(scan_root: str, identity_root: str) -> list[dict]: + """Collect metadata under scan_root while storing paths under identity_root.""" + entries = [] + skip_names = {DB_NAME, SNAPSHOTS_DIR} + + for dirpath, dirnames, filenames in os.walk(scan_root, followlinks=False): + dirnames[:] = [d for d in dirnames if d not in skip_names] + if dirpath != scan_root: + entry = collect_metadata(dirpath, scan_root) + if "error" in entry: + print(f" [skip] {entry['relative_path']}: {entry['error']}") + else: + entry["absolute_path"] = os.path.join(identity_root, entry["relative_path"]) + entries.append(entry) + + for fname in filenames: + if fname in skip_names: + continue + entry = collect_metadata(os.path.join(dirpath, fname), scan_root) + if "error" in entry: + print(f" [skip] {entry['relative_path']}: {entry['error']}") + else: + entry["absolute_path"] = os.path.join(identity_root, entry["relative_path"]) + entries.append(entry) + + return entries + + + +# def walk_folder(root_folder: str) -> list[dict]: +# """Recursively collect metadata for every entry under root_folder.""" +# entries = [] +# skip_dirs = {DB_NAME, SNAPSHOTS_DIR} + +# for dirpath, dirnames, filenames in os.walk(root_folder, followlinks=False): +# # Prune internal bookkeeping dirs +# dirnames[:] = [d for d in dirnames if d not in skip_dirs] + +# # Include the directory entry itself (except root) +# if dirpath != root_folder: +# entries.append(collect_metadata(dirpath, root_folder)) + +# for fname in filenames: +# if fname == DB_NAME: +# continue +# abs_path = os.path.join(dirpath, fname) +# entries.append(collect_metadata(abs_path, root_folder)) + +# return entries diff --git a/pyproject.toml b/pyproject.toml index 13424634..d82e6a2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,9 @@ include-package-data = true [project.scripts] dsi = "dsi.cli:main" +dsi-vcs = "dsi.utils.version_control.cli:main" [tool.setuptools.packages.find] where = ["."] include = ["dsi*"] -exclude = ["clover3d"] \ No newline at end of file +exclude = ["clover3d"] diff --git a/tools/perf_analyzer/fly_server.py b/tools/perf_analyzer/fly_server.py index e3465720..717b0452 100755 --- a/tools/perf_analyzer/fly_server.py +++ b/tools/perf_analyzer/fly_server.py @@ -353,16 +353,24 @@ def generateParCordChart(sorted_df, git_nodes, mk_data=None, perf_filter=list()) combined_all_df = merged_df[merged_df.cname != None].sort_values(by=['date'], ascending=True) combined_all_df["formatted_date"] = pd.to_datetime(combined_all_df['date']).dt.strftime("%b-%d,%Y(%H:%M:%S)") - combined_all_df = combined_all_df.dropna(subset=perf_filter.extend(["date"])).reset_index(drop=True) + combined_all_df = combined_all_df\ + .dropna(subset=perf_filter.extend(["date"])).reset_index(drop=True)\ + .drop_duplicates(subset=['formatted_date'], keep='first') + + # dims = [] + # labels = [] - # print(combined_all_df["formatted_date"]) + print(combined_all_df["formatted_date"]) + # print(combined_all_df["branch"]) parCordValues = list() - if sorted_df is not None and len(sorted_df) > 0: + if combined_all_df is not None and len(combined_all_df) > 0: f_col = combined_all_df.index.values + # labels.append("Commit
Time") + # dims.append(f_col) parCordValues.append( dict(range = [f_col.min(), f_col.max()], values = f_col, - label = "Commit
Time", tickvals = f_col, ticktext = combined_all_df["formatted_date"].astype(str)) + label = "
Commit
Time", tickvals = f_col, ticktext = combined_all_df["formatted_date"].astype(str)) ) for col_name in perf_filter:# ["pdv", "cell_advection", "mpi_halo_exchange", "self_halo_exchange", "momentum_advection", "total"]: if col_name == "date": @@ -371,7 +379,7 @@ def generateParCordChart(sorted_df, git_nodes, mk_data=None, perf_filter=list()) # Replace the second to last underscore with a white space in the label parts = col_name.split('_') csum = 0 - label = '' + label = '
' for part in parts: if csum > 0: label += '_' @@ -380,11 +388,13 @@ def generateParCordChart(sorted_df, git_nodes, mk_data=None, perf_filter=list()) if csum > len(col_name) // 2: label += "
" csum = 0 + # labels.append(label) + # dims.append(f_col) parCordValues.append( dict(range = [f_col.min(), f_col.max()], label = label, values = f_col) ) - + # if len(dims) == 0: if len(parCordValues) == 0: fig = go.Figure().add_annotation( x=2, y=2, @@ -395,10 +405,11 @@ def generateParCordChart(sorted_df, git_nodes, mk_data=None, perf_filter=list()) fig.update_layout( xaxis = { "visible": False }, yaxis = { "visible": False }) + # return fig else: fig = go.Figure(data=go.Parcoords( line = dict(color = parCordValues[0]['values'], - colorscale = 'Electric', + colorscale = [[0, 'blue'], [1, 'blue']], showscale = False, colorbar = dict(x=-0.15, showticklabels=False), cmin = parCordValues[0]['range'][0], @@ -406,9 +417,12 @@ def generateParCordChart(sorted_df, git_nodes, mk_data=None, perf_filter=list()) dimensions = parCordValues, labelangle = 0, labelside = "bottom", + labelfont = dict(size=16, color='black'), + rangefont = dict(size=14, color='black'), + tickfont = dict(size=12, color='black'), ) ) - fig.update_layout(margin=dict(l=150)) + fig.update_layout(margin=dict(l=160)) return fig # @callback( @@ -546,68 +560,75 @@ def main(perf_data, git_nodes): # 'textOverflow': 'ellipsis', } ), - ], style={'width': 790,'marginTop': 0,'marginLeft': 0}) + ], style={'width': '100%','marginTop': 0,'marginLeft': 0}), # html.Div( # html.Img(src='assets/image.svg', # style={'marginLeft': 15, 'marginRight': 15, 'marginTop': 30, 'width': 310}) # ) + # ], style={ + # 'width': '60%', + # 'height': '1000px', + # 'marginLeft': 10, + # 'marginTop': 5, + # 'marginRight': 10 + # }), + html.Div([ + html.Div([ + html.H4("Search any variable, (regex or plaintext):"), + dcc.Input(id='custom-var-search', value='pragma, define', type='text', style={'marginBottom': 10}), + html.Div(id='search-var-info', children='Choose files types to filter'), + dcc.Dropdown( + options={r"\.c":".c", r"\.cc": ".cc", r"\.py":".py", r"\.f90":".f90", r"\.ipynb": ".ipynb"}, + value = [r"\.c", r"\.cc",], + id="filter-file-multi-options", + clearable=True, + optionHeight=40, + multi=True + ), + html.Div(id='file-filter-selection-text', children='Filtered Files', style={'marginBottom': 10}), + html.Button('Search', id='submit-var-search', n_clicks=0, disabled=True), + dash_table.DataTable(id='var-search-table', + columns=[ + {'name': 'variable', 'id': 'var_name', 'type': 'text'}, + {'name': 'file name', 'id': 'file_name', 'type': 'text'}, + # {'name': 'occurance', 'id': 'occ', 'type': 'numeric'} + ], + filter_action='native', + editable=False, + sort_action="native", + row_selectable="multi", + row_deletable=True, + style_cell={'textAlign': 'left'}, + style_table={ + 'height': '250px', 'minHeight': '200px', 'maxHeight': '250px', + 'overflow': 'auto', + 'font-size': '12px', + }, + style_data={ + # 'width': '100px', 'minWidth': '100px', + 'maxWidth': '100px', + 'whiteSpace': 'normal', + 'overflow': 'auto', + # 'textOverflow': 'ellipsis', + } + ), + # html.Div(id='var-results-table', style={'marginTop': 10}), + html.Div(id='code-view', style={'marginTop': 10, + 'height': '1000px', 'maxHeight': '1000px', + 'overflow': 'auto', + }), + ], style={'marginLeft': 0, 'marginTop': 5}), + ], style={ + 'width': '100%', + 'marginLeft': 0, + }), ], style={ - 'width': '60%', + 'width': '100%', 'height': '1000px', 'marginLeft': 10, 'marginTop': 5, 'marginRight': 10 }), - html.Div([ - html.Div([ - html.H4("Search any variable, (regex or plaintext):"), - dcc.Input(id='custom-var-search', value='pragma, define', type='text', style={'marginBottom': 10}), - html.Div(id='search-var-info', children='Choose files types to filter'), - dcc.Dropdown( - options={r"\.c":".c", r"\.cc": ".cc", r"\.py":".py", r"\.f90":".f90", r"\.ipynb": ".ipynb"}, - value = [r"\.c", r"\.cc",], - id="filter-file-multi-options", - clearable=True, - optionHeight=40, - multi=True - ), - html.Div(id='file-filter-selection-text', children='Filtered Files', style={'marginBottom': 10}), - html.Button('Search', id='submit-var-search', n_clicks=0, disabled=True), - dash_table.DataTable(id='var-search-table', - columns=[ - {'name': 'variable', 'id': 'var_name', 'type': 'text'}, - {'name': 'file name', 'id': 'file_name', 'type': 'text'}, - # {'name': 'occurance', 'id': 'occ', 'type': 'numeric'} - ], - filter_action='native', - editable=False, - sort_action="native", - row_selectable="multi", - row_deletable=True, - style_cell={'textAlign': 'left'}, - style_table={ - 'height': '250px', 'minHeight': '200px', 'maxHeight': '250px', - 'overflow': 'auto', - 'font-size': '12px', - }, - style_data={ - # 'width': '100px', 'minWidth': '100px', - 'maxWidth': '100px', - 'whiteSpace': 'normal', - 'overflow': 'auto', - # 'textOverflow': 'ellipsis', - } - ), - # html.Div(id='var-results-table', style={'marginTop': 10}), - html.Div(id='code-view', style={'marginTop': 10, - 'height': '1000px', 'maxHeight': '1000px', - 'overflow': 'auto', - }), - ], style={'marginLeft': 0, 'marginTop': 5}), - ], style={ - 'width': '40%', - 'marginLeft': 0, - }), ], fluid=True, style={'display': 'flex'}, @@ -744,12 +765,18 @@ def action_on_selected_vars(rows, derived_virtual_selected_rows, selected_commit return mk_component elif len(derived_selected_commits_list) > 2: return dcc.Markdown("##### Please select upto two commits", id='actual-source-block') + elif len(derived_selected_commits_list) == 1: + return dcc.Markdown("##### Please select upto two commits", id='actual-source-block') if derived_virtual_selected_rows is not None and len(derived_virtual_selected_rows) > 0 and len(rows) >= derived_virtual_selected_rows[0]: mk_data = list() for i in range(len(derived_virtual_selected_rows)): mk_data.append(rows[derived_virtual_selected_rows[i]]['file_name']) + # if len(derived_selected_commits_list) > 0 and derived_selected_commits_list[0] not in selected_commits_row: + # return dcc.Markdown("##### Please select two commits to compare", id='actual-source-block') + # if len(derived_selected_commits_list) > 1 and derived_selected_commits_list[1] not in selected_commits_row: + # return dcc.Markdown("##### Please select two commits to compare", id='actual-source-block') git_repo = getGitRepo(perf_runner.git_user_repo) first_commit_hash = selected_commits_row[derived_selected_commits_list[0]]["long_hash"] second_commit_hash = selected_commits_row[derived_selected_commits_list[1]]["long_hash"] @@ -835,7 +862,7 @@ def find_interesting_perf_metric(df): if column == 'testname' or column.startswith("git_") or 'min' in column or 'max' in column or 'x_cells' in column or 'y_cells' in column: continue df[column] = pd.to_numeric(df[column].fillna(value=np.nan), errors='coerce') - another = df.select_dtypes(include='number').std() > 0.015 + another = df.select_dtypes(include='number').std() > 0.03# 0.015 return list(another[another].index) @callback( diff --git a/tools/perf_analyzer/runner_script.sh b/tools/perf_analyzer/runner_script.sh index 61d3b52b..2a63a90f 100644 --- a/tools/perf_analyzer/runner_script.sh +++ b/tools/perf_analyzer/runner_script.sh @@ -19,9 +19,9 @@ cp ../clover.in . make clean; make COMPILER=GNU; -echo "================================ Compile Done ================================ "l - -echo "============================= Running CloverLeaf ============================= "j +echo "================================ Compile Done ================================ " + +echo "============================= Running CloverLeaf ============================= " mpirun -np 2 tau_exec $SOURCE_BASE_DIRECTORY/clover_leaf pprof > tau_results cd -