Bump zookeeper version to fix apache/openwhisk#5564

ansible/group_vars/all

@@ -187,8 +187,8 @@ kafka_protocol_for_setup: "{{ kafka_protocol | default('PLAINTEXT') }}"
 
 zookeeper:
   image:
-    amd64: zookeeper:3.4
-    arm64: arm64v8/zookeeper:3.4
+    amd64: zookeeper:3.5.9
+    arm64: arm64v8/zookeeper:3.5.9
   port: 2181
 
 zookeeper_connect_string: "{% set ret = [] %}\

ansible/roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -20,4 +20,4 @@ gateway.expected_nodes: {{ host_group|length }}
 gateway.recover_after_time: 5m
 
 xpack.security.enabled: false
-bootstrap.memory_lock: true
+bootstrap.memory_lock: true

ansible/roles/kafka/tasks/deploy.yml

@@ -100,11 +100,10 @@
     pull: "{{ kafka.pull_kafka | default(true) }}"
 
 - name: wait until the kafka server started up
-  shell:
-    (echo dump; sleep 1) |
-    nc {{hostvars[groups['zookeepers']|first].ansible_host}} {{zookeeper.port}} |
-    grep /brokers/ids/{{ groups['kafkas'].index(inventory_hostname) }}
+  shell: |
+    docker exec zookeeper0 /bin/bash -c "echo dump | nc 127.0.0.1 2181" |
+    grep "/brokers/ids/{{ groups['kafkas'].index(inventory_hostname) }}"
   register: result
   until: (result.rc == 0)
-  retries: 10
+  retries: 20
   delay: 5

ansible/roles/zookeeper/tasks/deploy.yml

@@ -37,12 +37,21 @@
     env:
         TZ: "{{ docker.timezone }}"
         ZOO_MY_ID: "{{ groups['zookeepers'].index(inventory_hostname) + 1 }}"
+        # CRITICAL FIX 1: Turn off standalone mode capabilities completely
+        ZOO_STANDALONE_ENABLED: "false"
+        # Disable the embedded Jetty AdminServer to prevent port 8080 conflicts globally
+        ZOO_ADMINSERVER_ENABLED: "false"
+        # Disable 3.5 dynamic reconfiguration to keep OpenWhisk's static host lists working
+        ZOO_RECONFIG_ENABLED: "false"
+        # Whitelist the health status command so the netcat step passes
+        ZOO_4LW_COMMANDS_WHITELIST: "ruok,dump"
+        # CRITICAL FIX 2: Append ';2181' client configuration suffix to server entries for 3.5 quorum rules
         ZOO_SERVERS: "{% set zhosts = [] %}
                       {% for host in groups['zookeepers'] %}
                         {% if host == inventory_hostname %}
-                          {{ zhosts.append('server.' + (loop.index|string) + '=' + '0.0.0.0:2888:3888') }}
+                          {{ zhosts.append('server.' + (loop.index|string) + '=0.0.0.0:2888:3888;2181') }}
                         {% else %}
-                          {{ zhosts.append('server.' + (loop.index|string) + '=' + hostvars[host].ansible_host + ':' + ((2888+loop.index-1)|string) + ':' + ((3888+loop.index-1)|string) ) }}
+                          {{ zhosts.append('server.' + (loop.index|string) + '=' + hostvars[host].ansible_host + ':' + ((2888+loop.index-1)|string) + ':' + ((3888+loop.index-1)|string) + ';2181' ) }}
                         {% endif %}
                       {% endfor %}
                       {{ zhosts | join(' ') }}"
@@ -53,8 +62,8 @@
     pull: "{{ zookeeper.pull_zookeeper | default(true) }}"
 
 - name: wait until the Zookeeper in this host is up and running
-  action: shell (echo ruok; sleep 1) | nc {{ ansible_host }} {{ zookeeper.port + groups['zookeepers'].index(inventory_hostname) }}
+  shell: docker exec zookeeper{{ groups['zookeepers'].index(inventory_hostname) }} zkServer.sh status
   register: result
-  until: (result.rc == 0) and (result.stdout == 'imok')
+  until: result.rc == 0
   retries: 36
   delay: 5

common/scala/Dockerfile

@@ -18,6 +18,7 @@
 # NOTE:
 # OpenWhisk will use a 21-jre multi arch image, compilation will be done with a jdk 17 temurin based image.
 # as wsk CLI is compiled against glibc we need touse a GLIBC based JRE Image (alpine it is not GLIBC based)
+# Use Eclipse Temurin 17 JRE
 FROM eclipse-temurin:21-jre
 
 ENV LANG=en_US.UTF-8

core/invoker/build.gradle

@@ -48,10 +48,17 @@ dependencies {
     implementation ("org.apache.curator:curator-recipes:${gradle.curator.version}") {
         exclude group: 'org.apache.zookeeper', module:'zookeeper'
     }
-    implementation ("org.apache.zookeeper:zookeeper:3.4.14") {
-        exclude group: 'org.slf4j'
-        exclude group: 'log4j'
-        exclude group: 'jline'
+
+    // Force absolute isolation so ZooKeeper cannot impact Elasticsearch client configurations globally
+    implementation("org.apache.zookeeper:zookeeper:3.5.9") {
+        transitive = false
+        exclude group: 'io.netty'
+        exclude group: 'com.fasterxml.jackson.core'
+    }
+
+    // Manually add only the companion Jute data framework if the Invoker fails class lookups for it
+    implementation("org.apache.zookeeper:zookeeper-jute:3.5.9") {
+        transitive = false
     }
 }
 

gradle/docker.gradle

@@ -69,6 +69,10 @@ if( !project.hasProperty('dockerBuildArgs') && dockerMultiArchBuild ) {
     dockerBuildArg += ['--build-arg','BASE='+scalaBaseImageName]
 }
 
+if( project.hasProperty('dockerTargetPlatformsArgs') ) {
+    dockerBuildArg += ['--build-arg','TARGETPLATFORM='+project.dockerTargetPlatformsArgs]
+}
+
 def builDockerCommand(dockerFile) {
     def cmd = dockerBinary
 
@@ -134,12 +138,14 @@ task distDockerCoverage() {
 
 def prepareBuildArgs(List buildArgs) {
     def result = ['build']
+
     if(project.hasProperty('dockerBuildArgs')) {
         buildArgs.addAll(dockerBuildArgs)
     }
     buildArgs.each {arg ->
         result += ['--build-arg', arg]
     }
+
     result
 }
 

tests/build.gradle

@@ -256,8 +256,8 @@ dependencies {
 
     implementation "software.amazon.awssdk:s3:2.46.17"
     implementation "com.microsoft.azure:azure-cosmos:3.7.6"
-    implementation 'org.testcontainers:elasticsearch:1.17.6'
-    implementation 'org.testcontainers:mongodb:1.17.1'
+    implementation 'org.testcontainers:elasticsearch:1.19.8'
+    implementation 'org.testcontainers:mongodb:1.19.8'
     implementation project(':common:scala')
     implementation project(':core:controller')
     implementation project(':core:scheduler')

tests/src/test/scala/org/apache/openwhisk/core/database/elasticsearch/ElasticSearchActivationStoreTests.scala

@@ -52,7 +52,10 @@ class ElasticSearchActivationStoreTests
                                      upto: Option[Instant] = None,
                                      context: UserContext,
                                      expected: IndexedSeq[WhiskActivation])(implicit transid: TransactionId): Unit = {
-    retry(super.checkQueryActivations(namespace, name, skip, limit, includeDocs, since, upto, context, expected), 10)
+    retry(
+      super.checkQueryActivations(namespace, name, skip, limit, includeDocs, since, upto, context, expected),
+      10,
+      Some(500.milliseconds))
   }
 
   override def checkCountActivations(namespace: String,
@@ -62,6 +65,9 @@ class ElasticSearchActivationStoreTests
                                      upto: Option[Instant] = None,
                                      context: UserContext,
                                      expected: Long)(implicit transid: TransactionId): Unit = {
-    retry(super.checkCountActivations(namespace, name, skip, since, upto, context, expected), 10)
+    retry(
+      super.checkCountActivations(namespace, name, skip, since, upto, context, expected),
+      10,
+      Some(500.milliseconds))
   }
 }

tests/src/test/scala/org/apache/openwhisk/core/database/mongodb/MongoDBAsyncStreamGraphTests.scala

@@ -59,15 +59,16 @@ class MongoDBAsyncStreamGraphTests
 
   it should "close the stream when done" in {
     val bytes = randomBytes(4000)
-    val inputStream = new ByteArrayInputStream(bytes)
-    val spiedStream = spy(inputStream)
-    val asyncStream = AsyncStreamHelper.toAsyncInputStream(spiedStream)
+    // Replace the mockito spy with our safe native recording stream
+    val safeStream = new CloseRecordingInputStream(bytes)
+    val asyncStream = AsyncStreamHelper.toAsyncInputStream(safeStream)
 
     val readStream = MongoDBAsyncStreamSource(asyncStream, 42).runWith(StreamConverters.asInputStream())
     val readBytes = IOUtils.toByteArray(readStream)
 
     bytes shouldBe readBytes
-    verify(spiedStream).close()
+    // Replace mockito verify() with a bulletproof boolean assertion
+    safeStream.closed shouldBe true
   }
 
   it should "onError with failure and return a failed IOResult when reading from failed stream" in {
@@ -132,4 +133,12 @@ class MongoDBAsyncStreamGraphTests
     var closed: Boolean = _
     override def close() = { super.close(); closed = true }
   }
+
+  private class CloseRecordingInputStream(bytes: Array[Byte]) extends ByteArrayInputStream(bytes) {
+    @volatile var closed: Boolean = false
+    override def close(): Unit = {
+      super.close()
+      closed = true
+    }
+  }
 }

tools/macos/README.md

@@ -102,9 +102,12 @@ Bellow are the ansible commands required to prepare your machine:
 cd ./ansible
 
 ansible-playbook setup.yml -e mode=HA
+ansible-playbook prereq.yml
 ansible-playbook couchdb.yml
 ansible-playbook initdb.yml
 ansible-playbook wipe.yml
+ansible-playbook elasticsearch.yml
+ansible-playbook etcd.yml
 ansible-playbook downloadcli-github.yml
 
 ansible-playbook properties.yml
@@ -114,5 +117,8 @@ To run the unit tests execute the command bellow from the project's root folder:
 ```bash
 # go back to project's root folder
 cd ../
+
+export TESTCONTAINERS_RYUK_DISABLED="true"
+
 ./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean
 ```

tools/macos/runUnitTests.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+SCRIPTDIR=$(cd $(dirname "$0") && pwd)
+ROOTDIR="$SCRIPTDIR/../.."
+
+cd $ROOTDIR/ansible
+
+ansible-playbook setup.yml -e mode=HA
+ansible-playbook prereq.yml
+ansible-playbook couchdb.yml
+ansible-playbook initdb.yml
+ansible-playbook wipe.yml
+ansible-playbook elasticsearch.yml
+ansible-playbook etcd.yml
+ansible-playbook properties.yml
+
+
+cd $ROOTDIR
+# ./gradlew distDocker
+
+./gradlew -PtestSetName="REQUIRE_ONLY_DB" :tests:testCoverageLean

tools/ubuntu-setup/docker.sh

@@ -19,24 +19,31 @@
 set -e
 set -x
 
-sudo sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo apt-key fingerprint 0EBFCD88
+sudo apt-get update
+sudo apt-get install -y ca-certificates curl gnupg
 
-sudo add-apt-repository \
-   "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+# Add the GPG key signatures
+sudo install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
 
-sudo add-apt-repository \
-    "deb [arch=$(uname -m | sed -e 's/x86_64/amd64/g')] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-sudo apt-get -y update
+# Register the stable Ubuntu Jammy APT source line channel
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
-# AUFS
-# Use '-virtual' package to support docker tests of the script
-sudo apt-get --no-install-recommends -y install linux-image-extra-virtual
+sudo apt-get update
 
 # DOCKER
-sudo apt-get install -y docker-ce=18.06.3~ce~3-0~ubuntu containerd.io
-sudo apt-mark hold docker-ce
+# Force the exact target version
+sudo apt-get install -y \
+  docker-ce=5:27.3.1-1~ubuntu.22.04~jammy \
+  docker-ce-cli=5:27.3.1-1~ubuntu.22.04~jammy \
+  containerd.io
+
+# CRITICAL: Lock the packages so 'apt upgrade' cannot update them automatically later
+sudo apt-mark hold docker-ce docker-ce-cli
 
 # enable (security - use 127.0.0.1)
 sudo -E bash -c 'echo '\''DOCKER_OPTS="-H unix:///var/run/docker.sock --storage-driver=aufs"'\'' >> /etc/default/docker'