Skip to content

Commit 1aa84d8

Browse files
committed
Materialize accessed rootfs files into stable private inodes
1 parent a308423 commit 1aa84d8

15 files changed

Lines changed: 535 additions & 22 deletions

‎containers/transport/proot/README.md‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,40 @@
1-
# PRoot read-only canonical-image transport correction
1+
# PRoot canonical-image reads and lazy private rootfs
2+
3+
## Lazy private files (bms2)
4+
5+
`cow.c` materializes a sparse **private inode** before its first translated
6+
content/descriptor access, including O_PATH and executable/interpreter loading.
7+
No tracee receives a donor descriptor. Existing descriptors, shared mappings,
8+
hardlinks and subsequent writers therefore use the same inode. Stat/access-only
9+
queries leave untouched files sparse; after materialization, normal kernel
10+
metadata reports actual private blocks. Each file tries FICLONE and falls back
11+
to copying on unsupported filesystems. No read-only restriction, storage cap,
12+
concurrency reduction or change to scientific/checkpoint materialization is added.
13+
14+
`lazy-files.patch` wires the extension and adds x86 openat2 path interception.
15+
`bms_container.execute` requests sparse views and supplies the invocation-owned
16+
NUL-delimited map to the engine. udocker's pre-engine auth/shell/OS-identification
17+
inputs are populated during projection, resolving symlinks inside the image
18+
manifest; all remaining files materialize on first use. Mapping lifetime is the
19+
existing private-view/lease lifetime. Source metadata checks remain in place.
20+
21+
Local qualification uses the actual pinned engine and a real generated SIF,
22+
unsquashfs, pinned udocker installation, and cold/warm `bms_container.execute`.
23+
Both runs measured **2752 -> 7968 512-byte blocks** for regular private-rootfs
24+
inodes (hardlinks counted once): **1,409,024 -> 4,079,616 bytes**. The untouched
25+
32 MiB member retained zero blocks. This host returns EOPNOTSUPP for FICLONE,
26+
so these runs exercised the actual byte-copy fallback. These are fixture
27+
measurements, not a Protenix working-set estimate. Actual worker measurement
28+
and deployment remain separate; no remote changes were made.
29+
30+
Tests: `tests/test_proot_lazy.py`, `tests/test_proot_lazy_container.py`,
31+
`tests/test_runtime_image_views.py`, and the canonical-read tests below.
32+
The real-SIF test requires `BMS_TEST_UDOCKER_WHEEL` and
33+
`BMS_TEST_UDOCKER_ENGINES` pointing to independently downloaded pinned archives.
34+
Clean frozen-toolchain rebuild matches the shipped binary. The following
35+
canonical-read history describes the retained bms1 correction, not remote
36+
qualification of the new lazy implementation.
37+
238

339
This is a small patch to the existing udocker PRoot owner, **not** a udocker
440
upgrade, a guest-UID change, a new admission gate, or an image-guard exception.
@@ -79,7 +115,7 @@ The critical-runtime producer bundles this artifact as
79115
`lib/proot-x86_64-4_8_0`, alongside the changed bootstrap and container launcher.
80116
Its normal managed installation invokes bootstrap's udocker mode, which verifies
81117
upstream archives and the bundled engine and publishes a **new** tool directory:
82-
`<worker-root>/tools/udocker-1.3.17-bms1`. Existing tool directories are not edited.
118+
`<worker-root>/tools/udocker-1.3.17-bms2`. Existing tool directories are not edited.
83119
`bms_managed_runtime.container_environment` selects that directory.
84120

85121
A controller-only update is insufficient. Deployment must include these source

‎containers/transport/proot/build.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,15 @@ def main():
3030
run('git', '-C', str(source), 'submodule', 'update', '--init', 'lib/uthash')
3131
assert subprocess.check_output(['git', '-C', str(source / 'lib/uthash'), 'rev-parse', 'HEAD'], text=True).strip() == UTHASH
3232
run('git', '-C', str(source), 'apply', str(HERE / 'readonly-open.patch'))
33+
assert hashlib.sha256((HERE / 'lazy-files.patch').read_bytes()).hexdigest() == PINS['lazy_patch_sha256']
34+
assert hashlib.sha256((HERE / 'cow.c').read_bytes()).hexdigest() == PINS['cow_source_sha256']
35+
run('git', '-C', str(source), 'apply', str(HERE / 'lazy-files.patch'))
36+
shutil.copyfile(HERE / 'cow.c', source / 'src/extension/cow.c')
3337
run('docker', 'build', '-t', 'bms-proot-readonly-builder', str(HERE))
3438
run('docker', 'run', '--rm', '--network=none', '--user', f'{os.getuid()}:{os.getgid()}',
3539
'-v', f'{source}:/proot', '-w', '/proot/src', 'bms-proot-readonly-builder',
3640
'make', '-j4', 'HAS_SWIG=', 'HAS_PYTHON_CONFIG=', 'GIT_VERSION=', 'GIT_COMMIT=',
37-
'VERSION=udocker-fe64598-bms-readonly-1')
41+
'VERSION=udocker-fe64598-bms-lazy-1')
3842
run('docker', 'run', '--rm', '--network=none', '--user', f'{os.getuid()}:{os.getgid()}',
3943
'-v', f'{source}:/proot', 'bms-proot-readonly-builder',
4044
'strip', '--strip-debug', '/proot/src/proot')

‎containers/transport/proot/cow.c‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/* SPDX-License-Identifier: GPL-2.0-or-later
2+
* Lazy private-inode materialization. Never hand a donor FD to a tracee.
3+
* PRoot's single event loop completes materialization before resuming a guest.
4+
* Inode keys preserve hardlink/rename identity. No per-file descriptor inventory.
5+
*/
6+
#include <errno.h>
7+
#include <fcntl.h>
8+
#include <stdint.h>
9+
#include <stdio.h>
10+
#include <stdlib.h>
11+
#include <string.h>
12+
#include <sys/ioctl.h>
13+
#include <sys/stat.h>
14+
#include <unistd.h>
15+
#include <uthash.h>
16+
#include "extension/extension.h"
17+
#include "tracee/reg.h"
18+
#include "syscall/sysnum.h"
19+
20+
#define BMS_FICLONE 0x40049409
21+
22+
typedef struct { uint64_t dev, ino; } Key;
23+
typedef struct CowFile {
24+
Key key;
25+
struct stat donor_stat;
26+
char *lower;
27+
int copied;
28+
UT_hash_handle hh;
29+
} CowFile;
30+
static CowFile *files;
31+
32+
static Key key_of(const struct stat *s) {
33+
Key k = {s->st_dev, s->st_ino};
34+
return k;
35+
}
36+
static int unchanged(const struct stat *a, const struct stat *b) {
37+
return a->st_dev == b->st_dev && a->st_ino == b->st_ino &&
38+
a->st_mode == b->st_mode && a->st_nlink == b->st_nlink &&
39+
a->st_size == b->st_size &&
40+
a->st_mtim.tv_sec == b->st_mtim.tv_sec && a->st_mtim.tv_nsec == b->st_mtim.tv_nsec &&
41+
a->st_ctim.tv_sec == b->st_ctim.tv_sec && a->st_ctim.tv_nsec == b->st_ctim.tv_nsec;
42+
}
43+
static int copy_up(CowFile *f, const char *upper_path) {
44+
int src = -1, dst = -1, result = 0;
45+
struct stat before, after, upper;
46+
struct timespec times[2];
47+
char data[131072];
48+
off_t offset = 0;
49+
ssize_t n;
50+
if (f->copied) return 0;
51+
src = open(f->lower, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
52+
if (src < 0) return -errno;
53+
if (fstat(src, &before) || !unchanged(&before, &f->donor_stat)) {
54+
result = -ESTALE; goto out;
55+
}
56+
if (stat(upper_path, &upper)) { result = -errno; goto out; }
57+
if (chmod(upper_path, (upper.st_mode & 07777) | 0600)) { result = -errno; goto out; }
58+
dst = open(upper_path, O_WRONLY | O_CLOEXEC);
59+
if (dst < 0) { result = -errno; chmod(upper_path, upper.st_mode & 07777); goto out; }
60+
if (ioctl(dst, BMS_FICLONE, src)) {
61+
if (errno != EXDEV && errno != EOPNOTSUPP && errno != ENOTTY &&
62+
errno != EINVAL && errno != ENOSYS) { result = -errno; goto restore; }
63+
while (offset < before.st_size) {
64+
n = pread(src, data, sizeof(data), offset);
65+
if (n < 0 && errno == EINTR) continue;
66+
if (n <= 0) { result = -(n < 0 ? errno : EIO); goto restore; }
67+
ssize_t done = 0;
68+
while (done < n) {
69+
ssize_t wrote = pwrite(dst, data + done, n - done, offset + done);
70+
if (wrote < 0 && errno == EINTR) continue;
71+
if (wrote <= 0) { result = -(wrote < 0 ? errno : EIO); goto restore; }
72+
done += wrote;
73+
}
74+
offset += n;
75+
}
76+
if (ftruncate(dst, before.st_size)) { result = -errno; goto restore; }
77+
}
78+
if (fstat(src, &after) || !unchanged(&before, &after)) { result = -ESTALE; goto restore; }
79+
f->copied = 1;
80+
restore:
81+
times[0] = upper.st_atim; times[1] = upper.st_mtim;
82+
if (futimens(dst, times) && !result) result = -errno;
83+
if (fchmod(dst, upper.st_mode & 07777) && !result) result = -errno;
84+
out:
85+
if (dst >= 0) close(dst);
86+
if (src >= 0) close(src);
87+
return result;
88+
}
89+
static int load_map(const char *path) {
90+
FILE *stream = fopen(path, "rb");
91+
char *upper = NULL, *lower = NULL;
92+
size_t a = 0, b = 0;
93+
int result = 0;
94+
if (!stream) return -errno;
95+
while (getdelim(&upper, &a, 0, stream) >= 0) {
96+
struct stat s;
97+
CowFile *f;
98+
if (getdelim(&lower, &b, 0, stream) < 0 || upper[0] != '/' || lower[0] != '/') {
99+
result = -EINVAL; break;
100+
}
101+
f = calloc(1, sizeof(*f));
102+
if (!f) { result = -ENOMEM; break; }
103+
if (lstat(upper, &s) || !S_ISREG(s.st_mode) || lstat(lower, &f->donor_stat) ||
104+
!S_ISREG(f->donor_stat.st_mode) || s.st_size != f->donor_stat.st_size ||
105+
(s.st_dev == f->donor_stat.st_dev && s.st_ino == f->donor_stat.st_ino)) {
106+
free(f); result = -EINVAL; break;
107+
}
108+
f->key = key_of(&s);
109+
f->lower = strdup(lower);
110+
HASH_ADD(hh, files, key, sizeof(f->key), f);
111+
}
112+
if (ferror(stream) && !result) result = -EIO;
113+
free(upper); free(lower); fclose(stream);
114+
return result;
115+
}
116+
int cow_callback(Extension *extension, ExtensionEvent event, intptr_t data1, intptr_t data2) {
117+
(void)extension; (void)data2;
118+
switch (event) {
119+
case INITIALIZATION: return load_map((const char *)data1);
120+
case INHERIT_PARENT: return 1;
121+
case TRANSLATED_PATH: {
122+
struct stat s;
123+
CowFile *f;
124+
Key key;
125+
if (stat((char *)data1, &s) || !S_ISREG(s.st_mode)) return 0;
126+
key = key_of(&s);
127+
HASH_FIND(hh, files, &key, sizeof(key), f);
128+
if (!f || f->copied) return 0;
129+
/* Stat/access alone need no content. All descriptor-producing paths,
130+
* including O_PATH, and PRoot's ELF/shebang translations come here. */
131+
switch (get_sysnum(TRACEE(extension), ORIGINAL)) {
132+
case PR_stat: case PR_lstat: case PR_stat64: case PR_lstat64:
133+
case PR_newfstatat: case PR_statx: case PR_access: case PR_faccessat:
134+
case PR_readlink: case PR_readlinkat: return 0;
135+
default: return copy_up(f, (const char *)data1);
136+
}
137+
}
138+
default: return 0;
139+
}
140+
}
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
diff --git a/src/GNUmakefile b/src/GNUmakefile
2+
index 64347ef..4591c2b 100644
3+
--- a/src/GNUmakefile
4+
+++ b/src/GNUmakefile
5+
@@ -65,6 +65,7 @@ OBJECTS += \
6+
ptrace/user.o \
7+
ptrace/wait.o \
8+
extension/extension.o \
9+
+ extension/cow.o \
10+
extension/kompat/kompat.o \
11+
extension/fake_id0/fake_id0.o \
12+
extension/link2symlink/link2symlink.o \
13+
diff --git a/src/cli/cli.c b/src/cli/cli.c
14+
index 4e79066..2f57c18 100644
15+
--- a/src/cli/cli.c
16+
+++ b/src/cli/cli.c
17+
@@ -475,6 +475,11 @@ int main(int argc, char *const argv[])
18+
goto error;
19+
20+
/* Start the first tracee. */
21+
+ if (getenv("BMS_PROOT_COW_MAP")) {
22+
+ extern int cow_callback(Extension *, ExtensionEvent, intptr_t, intptr_t);
23+
+ if (initialize_extension(tracee, cow_callback, getenv("BMS_PROOT_COW_MAP")) < 0)
24+
+ goto error;
25+
+ }
26+
status = launch_process(tracee, &argv[status]);
27+
if (status < 0) {
28+
print_execve_help(tracee, tracee->exe, status);
29+
diff --git a/src/syscall/enter.c b/src/syscall/enter.c
30+
index 729700e..e692ee8 100644
31+
--- a/src/syscall/enter.c
32+
+++ b/src/syscall/enter.c
33+
@@ -502,9 +502,16 @@ int translate_syscall_enter(Tracee *tracee)
34+
status = translate_sysarg(tracee, SYSARG_2, REGULAR);
35+
break;
36+
37+
+ case PR_openat2:
38+
case PR_openat:
39+
dirfd = peek_reg(tracee, CURRENT, SYSARG_1);
40+
flags = peek_reg(tracee, CURRENT, SYSARG_3);
41+
+ if (get_sysnum(tracee, ORIGINAL) == PR_openat2) {
42+
+ uint64_t how_flags;
43+
+ status = read_data(tracee, &how_flags, peek_reg(tracee, CURRENT, SYSARG_3), sizeof(how_flags));
44+
+ if (status < 0) break;
45+
+ flags = how_flags;
46+
+ }
47+
48+
status = get_sysarg_path(tracee, path, SYSARG_2);
49+
if (status < 0)
50+
diff --git a/src/syscall/seccomp.c b/src/syscall/seccomp.c
51+
index efbbee1..94dc8d8 100644
52+
--- a/src/syscall/seccomp.c
53+
+++ b/src/syscall/seccomp.c
54+
@@ -375,6 +375,7 @@ static FilteredSysnum proot_sysnums[] = {
55+
{ PR_oldlstat, 0 },
56+
{ PR_oldstat, 0 },
57+
{ PR_open, 0 },
58+
+ { PR_openat2, 0 },
59+
{ PR_openat, 0 },
60+
{ PR_pivot_root, 0 },
61+
{ PR_prctl, 0 },
62+
diff --git a/src/syscall/sysnums-i386.h b/src/syscall/sysnums-i386.h
63+
index c50ae56..3a8f558 100644
64+
--- a/src/syscall/sysnums-i386.h
65+
+++ b/src/syscall/sysnums-i386.h
66+
@@ -293,6 +293,7 @@ static const Sysnum sysnums_i386[] = {
67+
[ 293 ] = PR_inotify_rm_watch,
68+
[ 294 ] = PR_migrate_pages,
69+
[ 295 ] = PR_openat,
70+
+ [ 437 ] = PR_openat2,
71+
[ 296 ] = PR_mkdirat,
72+
[ 297 ] = PR_mknodat,
73+
[ 298 ] = PR_fchownat,
74+
diff --git a/src/syscall/sysnums-x32.h b/src/syscall/sysnums-x32.h
75+
index 6388229..42354ae 100644
76+
--- a/src/syscall/sysnums-x32.h
77+
+++ b/src/syscall/sysnums-x32.h
78+
@@ -228,6 +228,7 @@ static const Sysnum sysnums_x32[] = {
79+
[ 255 ] = PR_inotify_rm_watch,
80+
[ 256 ] = PR_migrate_pages,
81+
[ 257 ] = PR_openat,
82+
+ [ 437 ] = PR_openat2,
83+
[ 258 ] = PR_mkdirat,
84+
[ 259 ] = PR_mknodat,
85+
[ 260 ] = PR_fchownat,
86+
diff --git a/src/syscall/sysnums-x86_64.h b/src/syscall/sysnums-x86_64.h
87+
index 333d058..7bc9a8b 100644
88+
--- a/src/syscall/sysnums-x86_64.h
89+
+++ b/src/syscall/sysnums-x86_64.h
90+
@@ -259,6 +259,7 @@ static const Sysnum sysnums_x86_64[] = {
91+
[ 255 ] = PR_inotify_rm_watch,
92+
[ 256 ] = PR_migrate_pages,
93+
[ 257 ] = PR_openat,
94+
+ [ 437 ] = PR_openat2,
95+
[ 258 ] = PR_mkdirat,
96+
[ 259 ] = PR_mknodat,
97+
[ 260 ] = PR_fchownat,
98+
diff --git a/src/syscall/sysnums.list b/src/syscall/sysnums.list
99+
index 2b6c9b2..f1cae02 100644
100+
--- a/src/syscall/sysnums.list
101+
+++ b/src/syscall/sysnums.list
102+
@@ -213,6 +213,7 @@ SYSNUM(olduname)
103+
SYSNUM(open)
104+
SYSNUM(open_by_handle_at)
105+
SYSNUM(openat)
106+
+SYSNUM(openat2)
107+
SYSNUM(pause)
108+
SYSNUM(pciconfig_iobase)
109+
SYSNUM(pciconfig_read)

‎containers/transport/proot/pins.json‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33
"source_commit": "fe64598e08ca3b4e9884c4cf8ecb83f6dc9720b8",
44
"uthash_commit": "e493aa90a2833b4655927598f169c31cfcdf7861",
55
"patch_sha256": "24abd8d211a780bdb1f54175f757db02cbf16e63501c354b9a0744f2982fb19c",
6-
"artifact_sha256": "5ff4fd4367f2cdf3fcbb8088f5e36eed2070a97fc0e401f97ca30b2f90facd5a",
7-
"artifact_size": 1157920,
6+
"artifact_sha256": "dc9012cf46fb1f4d1e168273a22f15077693ebe94cd09fa3247204a726187efd",
7+
"artifact_size": 1166840,
88
"original_englib_sha256": "2a4804ba82e087ca3e99305fce9887227925d2b56aaefcad6474c4cb86b7a157",
99
"original_modern_engine_sha256": "a9a3161b733b7114c87fd81e0d50381b5ec2cccad7f2b981257a834b8220cc34",
10-
"installed_manifest_sha256": "b1bafbf468885612e118d8e85302f83ce6bba540e795ea4e5c3c49fb08e988ae"
10+
"installed_manifest_sha256": "1d632e2163680c4ac6d82a931f706dc82c0d4629bd85ea6ce4068ed6948f28bc",
11+
"lazy_patch_sha256": "95d7a5bd3e2e04b6f4f4498a87e14a4b7994234d6b1b8389b1cb26190be14751",
12+
"cow_source_sha256": "12982e4be54ed2c2724b9c5bbba12bc41f225c047b67c8847f18c1900e01a07f"
1113
}
8.71 KB
Binary file not shown.

‎platform/api/services/remote_execution/bootstrap_worker.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,14 @@ apptainer --version >/dev/null || fail 'Apptainer installation failed'
6969
python3 - "$2" "$(dirname "$0")/proot-x86_64-4_8_0" <<'PY_INSTALL'
7070
import hashlib, json, os, pathlib, shutil, tarfile, tempfile, urllib.request, zipfile
7171
root = pathlib.Path(__import__('sys').argv[1])
72-
final = root / 'tools/udocker-1.3.17-bms1'
72+
final = root / 'tools/udocker-1.3.17-bms2'
7373
final.parent.mkdir(parents=True, exist_ok=True)
7474
if final.parent.is_symlink():
7575
raise ValueError('udocker tool root must not be a symlink')
7676
# Derived from both checksum-pinned upstream archives, bundled PRoot and launcher.
7777
# This independently authenticates the receipt; a locally rewritten receipt
7878
# must never be able to bless altered installed files.
79-
INSTALLED_MANIFEST_SHA256 = 'b1bafbf468885612e118d8e85302f83ce6bba540e795ea4e5c3c49fb08e988ae'
79+
INSTALLED_MANIFEST_SHA256 = '1d632e2163680c4ac6d82a931f706dc82c0d4629bd85ea6ce4068ed6948f28bc'
8080
8181
def installed_file(name, *, retain=False):
8282
import stat
@@ -154,7 +154,7 @@ if not installed():
154154
# Only the modern-kernel engine changes; udocker and the legacy engine
155155
# remain byte-pinned upstream. Recipe/source: containers/transport/proot.
156156
patched = pathlib.Path(__import__('sys').argv[2]).read_bytes()
157-
if hashlib.sha256(patched).hexdigest() != '5ff4fd4367f2cdf3fcbb8088f5e36eed2070a97fc0e401f97ca30b2f90facd5a':
157+
if hashlib.sha256(patched).hexdigest() != 'dc9012cf46fb1f4d1e168273a22f15077693ebe94cd09fa3247204a726187efd':
158158
raise ValueError('bundled PRoot checksum mismatch')
159159
(stage / 'engines/bin/proot-x86_64-4_8_0').write_bytes(patched)
160160
launcher = stage / 'bin/udocker'

‎platform/api/tests/test_attempt_materialization_isolation.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ def legacy(fd, parent, name, item, mode):
7676
monkeypatch.chdir(tmp_path)
7777
monkeypatch.setattr(driver, 'canonical_image', lambda *a: (tmp_path / 'image', 'a' * 64))
7878
@contextmanager
79-
def private(*args):
79+
def private(*args, shared_files=None):
80+
assert shared_files == []
8081
yield dict(rootfs=rootfs, image_fd=0)
8182
monkeypatch.setattr(driver.views, 'private_image_view', private)
8283
monkeypatch.setattr(driver, 'host_system_bindings', lambda _: [])

0 commit comments

Comments
 (0)