From 6018fe60da99106192dbf5581c80d340fed99ea4 Mon Sep 17 00:00:00 2001 From: Masala Date: Thu, 7 Mar 2024 14:37:20 +0000 Subject: [PATCH 01/14] update Dockerfile to golang:1.20 and go mod tidy --- Dockerfile | 2 +- go.mod | 2 -- go.sum | 8 +++----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index d54c9a6..5a149ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Dockerfile for talek -FROM golang:1.12 +FROM golang:1.20 MAINTAINER Raymond Cheng USER root diff --git a/go.mod b/go.mod index e22a079..66e85b1 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,12 @@ go 1.12 require ( github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 - github.com/barnex/cuda5 v0.0.0-20170131213030-57cec7ab46da github.com/coreos/etcd v0.0.0-20170414193847-e2d0db95ebbc github.com/coreos/go-systemd v0.0.0-20170324095819-1f9909e51b2d // indirect github.com/coreos/pkg v0.0.0-20170405072653-099530d80109 // indirect github.com/dchest/siphash v1.2.1 github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 github.com/go-gl/cl v0.0.0-20160402050751-283e73a0ca2a - github.com/google/zopfli v0.0.0-20190118173051-ef109ddf1649 github.com/gorilla/rpc v1.1.0 github.com/spf13/pflag v0.0.0-20170412152249-e453343e6260 github.com/willscott/bloom v0.0.0-20190611115233-60e4b211444f diff --git a/go.sum b/go.sum index f12393f..3a17ece 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,27 @@ github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= -github.com/barnex/cuda5 v0.0.0-20170131213030-57cec7ab46da/go.mod h1:GnBnFz4V/+kxwKFnquvOOi+IjZoVJsIUbcAVOXLCxCo= github.com/coreos/etcd v0.0.0-20170414193847-e2d0db95ebbc h1:VR+2Zdu4LwGrVDMUdVphibZfMHoFEnb5t8xQDWi1IAk= github.com/coreos/etcd v0.0.0-20170414193847-e2d0db95ebbc/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-systemd v0.0.0-20170324095819-1f9909e51b2d h1:JmJsvY9kF4oqiNrI+3FEoTtpRu8HvlxPgCSIrxxoZTQ= github.com/coreos/go-systemd v0.0.0-20170324095819-1f9909e51b2d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20170405072653-099530d80109 h1:8RxHvQFXd1AgM3q7iG92w06tQKiuqpHCNf5lVKs9bbs= github.com/coreos/pkg v0.0.0-20170405072653-099530d80109/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI= -github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4= github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2 h1:VA6jElpcJ+wkwEBufbnVkSBCA2TEnxdRppjRT5Kvh0A= github.com/foobaz/go-zopfli v0.0.0-20140122214029-7432051485e2/go.mod h1:Yi95+RbwKz7uGndSuUhoq7LJKh8qH8DT9fnL4ewU30k= github.com/go-gl/cl v0.0.0-20160402050751-283e73a0ca2a h1:XIGMyilyw1fCjQW2XBAQYNSqBjz6ZEDSQcU24zjC/TI= github.com/go-gl/cl v0.0.0-20160402050751-283e73a0ca2a/go.mod h1:x9JFrvJwNd4nJdwEzeF+68Bul1G/WftfhxdnJF85OUc= -github.com/google/zopfli v0.0.0-20190118173051-ef109ddf1649 h1:zDqfvNfJRhocVF/Ul+M/hhhewIw9R8xJwDOGrHDKzzI= -github.com/google/zopfli v0.0.0-20190118173051-ef109ddf1649/go.mod h1:s2ULOAKrLKt7BL/w7MwT3F8bAXHc792qVNGEd5Wp7OQ= github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A= github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/pflag v0.0.0-20170412152249-e453343e6260 h1:Rel8ggqtJ8xEfIJKPRDcF/f8a5Ukr6Wsyen4uGcYh+k= github.com/spf13/pflag v0.0.0-20170412152249-e453343e6260/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/willscott/bloom v0.0.0-20190611115233-60e4b211444f h1:HDruOClXOci/Duq+8QQZZTt2S2BvgT70pyX3nsVZbPQ= github.com/willscott/bloom v0.0.0-20190611115233-60e4b211444f/go.mod h1:ioaxSY0uYBnMt/u6zoCy87QnCVbch+cfpqo9rtSfK2s= From 018de5383bf8f055b609ca21d41945c825d298f4 Mon Sep 17 00:00:00 2001 From: Masala Date: Fri, 8 Mar 2024 07:08:13 +0000 Subject: [PATCH 02/14] Makefile, Dockerfile: add talek testnet make targets This adds Makefile targets for running a local deployment of talek with podman and docker-compose. testnet-start: starts a talek deployment testnet-stop: stops the deployment testnet-build-config: builds configuration with talekutil testnet-clean: cleans images and stampfiles testnet-cli: starts a shell with talek cli programs --- Makefile | 44 ++++++++++++++++++++++++++++++++---- cli/Dockerfile | 13 +++++++++++ talek_net/docker-compose.yml | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 cli/Dockerfile create mode 100644 talek_net/docker-compose.yml diff --git a/Makefile b/Makefile index dc4a9a3..451c03f 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,8 @@ +docker_compose?= $(shell if which podman|grep -q .; then echo DOCKER_HOST="unix://$$XDG_RUNTIME_DIR/podman/podman.sock" docker-compose; else echo docker-compose; fi) +docker_user?=$(shell if echo ${docker}|grep -q podman; then echo 0:0; else echo ${uid}:${gid}; fi) +docker=$(shell if which podman|grep -q .; then echo podman; else echo docker; fi) +net_name=talek_net + .PHONY: get-tools test lint unit get-tools: @@ -25,9 +30,40 @@ ci: lint goveralls -coverprofile=overalls.coverprofile -service=travis-ci; \ fi -docker-build: - docker build -t talek-base:latest ./ - docker build -t talek-replica:latest ./cli/talekreplica/ +docker-build.stamp: + $(docker) build -t talek-base:latest ./ + #$(docker) build -t talek-replica:latest ./cli/talekreplica/ + touch $@ docker-bash: - docker run -it talek-base:latest bash + $(docker) run -it talek-base:latest bash + +testnet-build-config: docker-build-cli.stamp + $(docker) run --rm -v ./$(net_name):/$(net_name) talek-cli bash -c "cd /$(net_name) && talekutil --common --outfile common.json && \ + talekutil --replica --incommon common.json --private --index 1 --name replica1 --address :8081 --outfile replica1.json && \ + talekutil --replica --incommon common.json --private --index 2 --name replica2 --address :8082 --outfile replica2.json && \ + talekutil --replica --incommon common.json --private --index 3 --name replica3 --address :8083 --outfile replica3.json && \ + talekutil --trustdomain --infile replica1.json --outfile replica1.pub.json && \ + talekutil --trustdomain --infile replica2.json --outfile replica2.pub.json && \ + talekutil --trustdomain --infile replica3.json --outfile replica3.pub.json && \ + talekutil --client --infile common.json --trustdomains replica1.pub.json,replica2.pub.json,replica3.pub.json --outfile talek.json" + +docker-build-cli.stamp: docker-build.stamp + $(docker) build -t talek-cli:latest ./cli/ + touch $@ + +testnet-start: testnet-build-config + cd $(net_name); DOCKER_USER=${docker_user} $(docker_compose) up --remove-orphans -d; $(docker_compose) top + touch $(net_name)/running.stamp + +testnet-stop: + cd $(net_name) && DOCKER_USER=${docker_user} $(docker_compose) down --remove-orphans; rm -fv running.stamp + +testnet-clean: + $(docker) rmi talek-cli + $(docker) rmi talek-base + rm docker-build-cli.stamp + rm docker-build.stamp + +testnet-cli: + $(docker) run --rm --network host -it -v ./$(net_name):/$(net_name) talek-cli:latest bash diff --git a/cli/Dockerfile b/cli/Dockerfile new file mode 100644 index 0000000..9604491 --- /dev/null +++ b/cli/Dockerfile @@ -0,0 +1,13 @@ +# Dockerfile for talek +FROM talek-base:latest +MAINTAINER Masala +USER root + +# build and install talek cli tools +RUN go mod download +RUN cd cli/talekutil && go install +RUN cd cli/clinfo && go install +RUN cd cli/talekclient && go install +RUN cd cli/talekfrontend && go install +RUN cd cli/talekreplica && go install +RUN cd cli/talekutil && go install diff --git a/talek_net/docker-compose.yml b/talek_net/docker-compose.yml new file mode 100644 index 0000000..e2c602b --- /dev/null +++ b/talek_net/docker-compose.yml @@ -0,0 +1,39 @@ +version: "2" + +services: + + frontend: + restart: "no" + image: talek + volumes: + - ./:/talek_net + command: talekfrontend --client /talek_net/talek.json --common /talek_net/common.json -l ":8080" --verbose + network_mode: host + depends_on: + - replica1 + - replica2 + - replica3 + + replica1: + restart: "no" + image: talek + volumes: + - ./:/talek_net + command: talekreplica -f /talek_net/common.json -l ":8081" -c /talek_net/replica1.json + network_mode: host + + replica2: + restart: "no" + image: talek + volumes: + - ./:/talek_net + command: talekreplica -f /talek_net/common.json -l ":8082" -c /talek_net/replica2.json + network_mode: host + + replica3: + restart: "no" + image: talek + volumes: + - ./:/talek_net + command: talekreplica -f /talek_net/common.json -l ":8083" -c /talek_net/replica3.json + network_mode: host From 0161abf39edc3f6a0e39ad2c2363efe4d139277b Mon Sep 17 00:00:00 2001 From: Masala Date: Mon, 11 Mar 2024 10:48:04 +0000 Subject: [PATCH 03/14] make volume mount path talek_shared, use talek-cli image --- talek_net/docker-compose.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/talek_net/docker-compose.yml b/talek_net/docker-compose.yml index e2c602b..2ef6b7e 100644 --- a/talek_net/docker-compose.yml +++ b/talek_net/docker-compose.yml @@ -4,10 +4,10 @@ services: frontend: restart: "no" - image: talek + image: talek-cli volumes: - - ./:/talek_net - command: talekfrontend --client /talek_net/talek.json --common /talek_net/common.json -l ":8080" --verbose + - ./:/talek_shared + command: talekfrontend --client /talek_shared/talek.json --common /talek_shared/common.json -l ":8080" --verbose network_mode: host depends_on: - replica1 @@ -16,24 +16,24 @@ services: replica1: restart: "no" - image: talek + image: talek-cli volumes: - - ./:/talek_net - command: talekreplica -f /talek_net/common.json -l ":8081" -c /talek_net/replica1.json + - ./:/talek_shared + command: talekreplica -f /talek_shared/common.json -l ":8081" -c /talek_shared/replica1.json network_mode: host replica2: restart: "no" - image: talek + image: talek-cli volumes: - - ./:/talek_net - command: talekreplica -f /talek_net/common.json -l ":8082" -c /talek_net/replica2.json + - ./:/talek_shared + command: talekreplica -f /talek_shared/common.json -l ":8082" -c /talek_shared/replica2.json network_mode: host replica3: restart: "no" - image: talek + image: talek-cli volumes: - - ./:/talek_net - command: talekreplica -f /talek_net/common.json -l ":8083" -c /talek_net/replica3.json + - ./:/talek_shared + command: talekreplica -f /talek_shared/common.json -l ":8083" -c /talek_shared/replica3.json network_mode: host From b7b4dd86131e6355b8edd86923a056ee3bc32a6a Mon Sep 17 00:00:00 2001 From: Masala Date: Mon, 11 Mar 2024 10:56:13 +0000 Subject: [PATCH 04/14] fix 2 volume mount paths, mount to talek-shared --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 451c03f..c3e821d 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ docker-bash: $(docker) run -it talek-base:latest bash testnet-build-config: docker-build-cli.stamp - $(docker) run --rm -v ./$(net_name):/$(net_name) talek-cli bash -c "cd /$(net_name) && talekutil --common --outfile common.json && \ + $(docker) run --rm -v ./$(net_name):/talek_shared talek-cli bash -c "cd /talek_shared && talekutil --common --outfile common.json && \ talekutil --replica --incommon common.json --private --index 1 --name replica1 --address :8081 --outfile replica1.json && \ talekutil --replica --incommon common.json --private --index 2 --name replica2 --address :8082 --outfile replica2.json && \ talekutil --replica --incommon common.json --private --index 3 --name replica3 --address :8083 --outfile replica3.json && \ @@ -66,4 +66,4 @@ testnet-clean: rm docker-build.stamp testnet-cli: - $(docker) run --rm --network host -it -v ./$(net_name):/$(net_name) talek-cli:latest bash + $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared talek-cli:latest bash From 2e3b625e31eb9a481b456a83c482923e8aa7734d Mon Sep 17 00:00:00 2001 From: Masala Date: Mon, 11 Mar 2024 11:21:12 +0000 Subject: [PATCH 05/14] set FrontendAddr in client configuration --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c3e821d..d59bb58 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ testnet-build-config: docker-build-cli.stamp talekutil --trustdomain --infile replica1.json --outfile replica1.pub.json && \ talekutil --trustdomain --infile replica2.json --outfile replica2.pub.json && \ talekutil --trustdomain --infile replica3.json --outfile replica3.pub.json && \ - talekutil --client --infile common.json --trustdomains replica1.pub.json,replica2.pub.json,replica3.pub.json --outfile talek.json" + talekutil --client --infile common.json --trustdomains replica1.pub.json,replica2.pub.json,replica3.pub.json --outfile tmp.json" + $(docker) run --rm -v ./$(net_name):/talek_shared python bash -c "cd /talek_shared && python -c 'import json,sys;c = json.load(sys.stdin); c[\"FrontendAddr\"] = \"127.0.0.1:8080/rpc\"; print(json.dumps(c))' > talek.json < tmp.json && rm tmp.json" docker-build-cli.stamp: docker-build.stamp $(docker) build -t talek-cli:latest ./cli/ From 260c5e36d6c04b64dada3b2b1a15148e1446fed5 Mon Sep 17 00:00:00 2001 From: Masala Date: Mon, 11 Mar 2024 12:58:49 +0000 Subject: [PATCH 06/14] index replicas from 0 --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index d59bb58..dd8c7f6 100644 --- a/Makefile +++ b/Makefile @@ -40,14 +40,14 @@ docker-bash: testnet-build-config: docker-build-cli.stamp $(docker) run --rm -v ./$(net_name):/talek_shared talek-cli bash -c "cd /talek_shared && talekutil --common --outfile common.json && \ - talekutil --replica --incommon common.json --private --index 1 --name replica1 --address :8081 --outfile replica1.json && \ - talekutil --replica --incommon common.json --private --index 2 --name replica2 --address :8082 --outfile replica2.json && \ - talekutil --replica --incommon common.json --private --index 3 --name replica3 --address :8083 --outfile replica3.json && \ - talekutil --trustdomain --infile replica1.json --outfile replica1.pub.json && \ - talekutil --trustdomain --infile replica2.json --outfile replica2.pub.json && \ - talekutil --trustdomain --infile replica3.json --outfile replica3.pub.json && \ - talekutil --client --infile common.json --trustdomains replica1.pub.json,replica2.pub.json,replica3.pub.json --outfile tmp.json" - $(docker) run --rm -v ./$(net_name):/talek_shared python bash -c "cd /talek_shared && python -c 'import json,sys;c = json.load(sys.stdin); c[\"FrontendAddr\"] = \"127.0.0.1:8080/rpc\"; print(json.dumps(c))' > talek.json < tmp.json && rm tmp.json" + talekutil --replica --incommon common.json --private --index 0 --name replica0 --address http://127.0.0.1:8081 --outfile replica0.json && \ + talekutil --replica --incommon common.json --private --index 1 --name replica1 --address http://127.0.0.1:8082 --outfile replica1.json && \ + talekutil --replica --incommon common.json --private --index 2 --name replica2 --address http://127.0.0.1:8083 --outfile replica2.json && \ + talekutil --trustdomain --index 0 --name replica0 --address http://127.0.0.1:8081 --infile replica0.json --outfile replica0.pub.json && \ + talekutil --trustdomain --index 1 --name replica1 --address http://127.0.0.1:8082 --infile replica1.json --outfile replica1.pub.json && \ + talekutil --trustdomain --index 2 --name replica2 --address http://127.0.0.1:8083 --infile replica2.json --outfile replica2.pub.json && \ + talekutil --client --infile common.json --trustdomains replica0.pub.json,replica1.pub.json,replica2.pub.json --outfile tmp.json" + $(docker) run --rm -v ./$(net_name):/talek_shared python bash -c "cd /talek_shared && python -c 'import json,sys;c = json.load(sys.stdin); c[\"FrontendAddr\"] = \"http://127.0.0.1:8080/rpc\"; print(json.dumps(c))' > talek.json < tmp.json && rm tmp.json" docker-build-cli.stamp: docker-build.stamp $(docker) build -t talek-cli:latest ./cli/ From ebb2509800d2f4befa069ee875476268c63719ac Mon Sep 17 00:00:00 2001 From: Masala Date: Mon, 11 Mar 2024 13:33:10 +0000 Subject: [PATCH 07/14] use correct replica configuration names --- talek_net/docker-compose.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/talek_net/docker-compose.yml b/talek_net/docker-compose.yml index 2ef6b7e..def83cd 100644 --- a/talek_net/docker-compose.yml +++ b/talek_net/docker-compose.yml @@ -10,30 +10,30 @@ services: command: talekfrontend --client /talek_shared/talek.json --common /talek_shared/common.json -l ":8080" --verbose network_mode: host depends_on: + - replica0 - replica1 - replica2 - - replica3 - replica1: + replica0: restart: "no" image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8081" -c /talek_shared/replica1.json + command: talekreplica -f /talek_shared/common.json -l ":8081" -c /talek_shared/replica0.json network_mode: host - replica2: + replica1: restart: "no" image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8082" -c /talek_shared/replica2.json + command: talekreplica -f /talek_shared/common.json -l ":8082" -c /talek_shared/replica1.json network_mode: host - replica3: + replica2: restart: "no" image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8083" -c /talek_shared/replica3.json + command: talekreplica -f /talek_shared/common.json -l ":8083" -c /talek_shared/replica2.json network_mode: host From c5b277fe18803c871951b960485491527a2c0432 Mon Sep 17 00:00:00 2001 From: Masala Date: Tue, 12 Mar 2024 11:54:58 +0000 Subject: [PATCH 08/14] talekclient: parse read-only share fixes #108 --- cli/talekclient/main.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cli/talekclient/main.go b/cli/talekclient/main.go index ddc32d4..27a013b 100644 --- a/cli/talekclient/main.go +++ b/cli/talekclient/main.go @@ -56,8 +56,18 @@ func main() { } topic = *nt } else { - if err = topic.UnmarshalText(topicdata); err != nil { - panic(err) + err = topic.UnmarshalText(topicdata) + if err != nil { + // if it is a read-only share, unmarshal only the Handle part + if *read { + topic.SigningPrivateKey = new([64]byte) + err = topic.Handle.UnmarshalText(topicdata) + if err != nil { + panic(err) + } + } else { + panic(err) + } } } From e9c0942e9f48ea7508883eaa4b0bbf2ec88012ed Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 14:10:26 +0200 Subject: [PATCH 09/14] docker testnet: use sed to edit json config instead of requiring a python docker image --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dd8c7f6..0fc2773 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,8 @@ testnet-build-config: docker-build-cli.stamp talekutil --trustdomain --index 0 --name replica0 --address http://127.0.0.1:8081 --infile replica0.json --outfile replica0.pub.json && \ talekutil --trustdomain --index 1 --name replica1 --address http://127.0.0.1:8082 --infile replica1.json --outfile replica1.pub.json && \ talekutil --trustdomain --index 2 --name replica2 --address http://127.0.0.1:8083 --infile replica2.json --outfile replica2.pub.json && \ - talekutil --client --infile common.json --trustdomains replica0.pub.json,replica1.pub.json,replica2.pub.json --outfile tmp.json" - $(docker) run --rm -v ./$(net_name):/talek_shared python bash -c "cd /talek_shared && python -c 'import json,sys;c = json.load(sys.stdin); c[\"FrontendAddr\"] = \"http://127.0.0.1:8080/rpc\"; print(json.dumps(c))' > talek.json < tmp.json && rm tmp.json" + talekutil --client --infile common.json --trustdomains replica0.pub.json,replica1.pub.json,replica2.pub.json --outfile talek.json && \ + sed -i -e 's/"FrontendAddr": ""/"FrontendAddr": "http:\/\/127.0.0.1:880\/rpc"/' talek.json" docker-build-cli.stamp: docker-build.stamp $(docker) build -t talek-cli:latest ./cli/ From adcbbf6f4a3c7aec01382b8328bea437a9898c3a Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 14:23:05 +0200 Subject: [PATCH 10/14] Makefile: robustify cleanup --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0fc2773..87e82cd 100644 --- a/Makefile +++ b/Makefile @@ -58,13 +58,13 @@ testnet-start: testnet-build-config touch $(net_name)/running.stamp testnet-stop: - cd $(net_name) && DOCKER_USER=${docker_user} $(docker_compose) down --remove-orphans; rm -fv running.stamp + -cd $(net_name) && DOCKER_USER=${docker_user} $(docker_compose) down --remove-orphans; rm -fv running.stamp -testnet-clean: - $(docker) rmi talek-cli - $(docker) rmi talek-base - rm docker-build-cli.stamp - rm docker-build.stamp +testnet-clean: testnet-stop + -$(docker) rmi talek-cli + -$(docker) rmi talek-base + rm -f docker-build-cli.stamp + rm -f docker-build.stamp testnet-cli: $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared talek-cli:latest bash From 7149a3458dfa67a4b7db04aa851a30dfad683862 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 14:25:55 +0200 Subject: [PATCH 11/14] docker testnet: cleanup - use default config file names - use docker workdir - escape quotes so sed line actually works - fix port number --- Makefile | 14 +++++++------- talek_net/docker-compose.yml | 12 ++++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 87e82cd..69a0917 100644 --- a/Makefile +++ b/Makefile @@ -39,15 +39,15 @@ docker-bash: $(docker) run -it talek-base:latest bash testnet-build-config: docker-build-cli.stamp - $(docker) run --rm -v ./$(net_name):/talek_shared talek-cli bash -c "cd /talek_shared && talekutil --common --outfile common.json && \ - talekutil --replica --incommon common.json --private --index 0 --name replica0 --address http://127.0.0.1:8081 --outfile replica0.json && \ - talekutil --replica --incommon common.json --private --index 1 --name replica1 --address http://127.0.0.1:8082 --outfile replica1.json && \ - talekutil --replica --incommon common.json --private --index 2 --name replica2 --address http://127.0.0.1:8083 --outfile replica2.json && \ + $(docker) run --rm -v ./$(net_name):/talek_shared talek-cli bash -c "cd /talek_shared && talekutil --common --outfile common.conf && \ + talekutil --replica --incommon common.conf --private --index 0 --name replica0 --address http://127.0.0.1:8081 --outfile replica0.json && \ + talekutil --replica --incommon common.conf --private --index 1 --name replica1 --address http://127.0.0.1:8082 --outfile replica1.json && \ + talekutil --replica --incommon common.conf --private --index 2 --name replica2 --address http://127.0.0.1:8083 --outfile replica2.json && \ talekutil --trustdomain --index 0 --name replica0 --address http://127.0.0.1:8081 --infile replica0.json --outfile replica0.pub.json && \ talekutil --trustdomain --index 1 --name replica1 --address http://127.0.0.1:8082 --infile replica1.json --outfile replica1.pub.json && \ talekutil --trustdomain --index 2 --name replica2 --address http://127.0.0.1:8083 --infile replica2.json --outfile replica2.pub.json && \ - talekutil --client --infile common.json --trustdomains replica0.pub.json,replica1.pub.json,replica2.pub.json --outfile talek.json && \ - sed -i -e 's/"FrontendAddr": ""/"FrontendAddr": "http:\/\/127.0.0.1:880\/rpc"/' talek.json" + talekutil --client --infile common.conf --trustdomains replica0.pub.json,replica1.pub.json,replica2.pub.json --outfile talek.conf && \ + sed -i -e 's/\"FrontendAddr\": \"\"/\"FrontendAddr\": \"http:\/\/127.0.0.1:8080\/rpc\"/' talek.conf" docker-build-cli.stamp: docker-build.stamp $(docker) build -t talek-cli:latest ./cli/ @@ -67,4 +67,4 @@ testnet-clean: testnet-stop rm -f docker-build.stamp testnet-cli: - $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared talek-cli:latest bash + $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash diff --git a/talek_net/docker-compose.yml b/talek_net/docker-compose.yml index def83cd..162987e 100644 --- a/talek_net/docker-compose.yml +++ b/talek_net/docker-compose.yml @@ -7,7 +7,8 @@ services: image: talek-cli volumes: - ./:/talek_shared - command: talekfrontend --client /talek_shared/talek.json --common /talek_shared/common.json -l ":8080" --verbose + working_dir: /talek_shared + command: talekfrontend -l ":8080" --verbose network_mode: host depends_on: - replica0 @@ -19,7 +20,8 @@ services: image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8081" -c /talek_shared/replica0.json + working_dir: /talek_shared + command: talekreplica -l ":8081" -c /talek_shared/replica0.json network_mode: host replica1: @@ -27,7 +29,8 @@ services: image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8082" -c /talek_shared/replica1.json + working_dir: /talek_shared + command: talekreplica -l ":8082" -c /talek_shared/replica1.json network_mode: host replica2: @@ -35,5 +38,6 @@ services: image: talek-cli volumes: - ./:/talek_shared - command: talekreplica -f /talek_shared/common.json -l ":8083" -c /talek_shared/replica2.json + working_dir: /talek_shared + command: talekreplica -l ":8083" -c /talek_shared/replica2.json network_mode: host From 0c38005a4721c5df12e75837d75974d69b5270ae Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 14:59:51 +0200 Subject: [PATCH 12/14] Makefile: add testnet-test-write-and-read target This works for me right now, but I don't know why I need to fork the log handle into a reader and a writer the way I am here before using it. If I read and write using the same handle (without making a copy of it first), the read operations always read from the locations --write would write to *next*, rather than the previous locations which have actually been written to. So, there is either an off-by-one error somewhere or I am confused about how this is supposed to work. --- Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Makefile b/Makefile index 69a0917..e9ede84 100644 --- a/Makefile +++ b/Makefile @@ -68,3 +68,19 @@ testnet-clean: testnet-stop testnet-cli: $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash + +testnet-test-write-and-read: + $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash -c " \ + rm -f test_output.log && \ + talekclient --verbose --create --topic writehandle && \ + cp writehandle readhandle && \ + talekclient --verbose --topic writehandle --write Hello1 && \ + talekclient --verbose --topic writehandle --write Hello2 && \ + talekclient --verbose --topic writehandle --write Hello3 && \ + talekclient --verbose --topic readhandle --read | tee >(grep Hello >> test_output.log) && \ + talekclient --verbose --topic readhandle --read | tee >(grep Hello >> test_output.log) && \ + talekclient --verbose --topic readhandle --read | tee >(grep Hello >> test_output.log) && \ + cat test_output.log && \ + sha256sum -c <<<'81ff3368be913d26d96879a0215e4fbace25449f029c7aabf07350207f0aa980 test_output.log' \ + && echo 'Successfully wrote and read three items!' \ + || echo 'Failed to read something.'" From 4587f579beb16ab727aa68fdb19445c8a0d6cbb0 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 15:19:45 +0200 Subject: [PATCH 13/14] Makefile: cleanup - allow stopping and restarting without regenerating configs - make testnet-test-write-and-read target depend on the network being up --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e9ede84..058e14a 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ docker-build.stamp: docker-bash: $(docker) run -it talek-base:latest bash -testnet-build-config: docker-build-cli.stamp +$(net_name)/talek.conf: docker-build-cli.stamp $(docker) run --rm -v ./$(net_name):/talek_shared talek-cli bash -c "cd /talek_shared && talekutil --common --outfile common.conf && \ talekutil --replica --incommon common.conf --private --index 0 --name replica0 --address http://127.0.0.1:8081 --outfile replica0.json && \ talekutil --replica --incommon common.conf --private --index 1 --name replica1 --address http://127.0.0.1:8082 --outfile replica1.json && \ @@ -53,7 +53,9 @@ docker-build-cli.stamp: docker-build.stamp $(docker) build -t talek-cli:latest ./cli/ touch $@ -testnet-start: testnet-build-config +testnet-start: $(net_name)/running.stamp + +$(net_name)/running.stamp: $(net_name)/talek.conf cd $(net_name); DOCKER_USER=${docker_user} $(docker_compose) up --remove-orphans -d; $(docker_compose) top touch $(net_name)/running.stamp @@ -69,7 +71,7 @@ testnet-clean: testnet-stop testnet-cli: $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash -testnet-test-write-and-read: +testnet-test-write-and-read: $(net_name)/running.stamp $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash -c " \ rm -f test_output.log && \ talekclient --verbose --create --topic writehandle && \ From e8c6925a1d4ec8971edaa229449cedf0db1e4375 Mon Sep 17 00:00:00 2001 From: Leif Ryge Date: Sat, 4 May 2024 18:22:36 +0200 Subject: [PATCH 14/14] testnet-test-write-and-read: use --share After merging masala's fix for parsing read-only handles from https://github.com/privacylab/talek/pull/110 this works now. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 058e14a..86b5df0 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ testnet-test-write-and-read: $(net_name)/running.stamp $(docker) run --rm --network host -it -v ./$(net_name):/talek_shared -w /talek_shared talek-cli:latest bash -c " \ rm -f test_output.log && \ talekclient --verbose --create --topic writehandle && \ - cp writehandle readhandle && \ + talekclient --verbose --topic writehandle --share readhandle && \ talekclient --verbose --topic writehandle --write Hello1 && \ talekclient --verbose --topic writehandle --write Hello2 && \ talekclient --verbose --topic writehandle --write Hello3 && \