fix: DH-23199: Propagate loader details from package.json to the manifest.json (#8299)

docker/registry/server-base/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:bf7d30a09484c40b85f9c2edfa4082dfd8df276295ac636204d42addab6080a5
+deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:f774bf3349210a1d44988bde3b6b61c858e4e01f11de9b0b2040b46eceede2c2

docker/registry/slim-base/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-slim-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:6159422839b101500be29884788342201a58ce7b0461179164867b43a0cbfa1a
+deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:1cdcda42f911ca0dbaf2baf20a8cd17c2136bdaffd9149135c5ce9074913ba95

docker/server-jetty/src/main/server-jetty/requirements.txt

@@ -1,7 +1,7 @@
-adbc-driver-manager==1.11.0
-adbc-driver-postgresql==1.11.0
+adbc-driver-manager==1.12.0
+adbc-driver-postgresql==1.12.0
 connectorx==0.4.5; platform.machine == 'x86_64'
-deephaven-plugin==0.7.0
+deephaven-plugin==0.8.0
 docstring_parser==0.18.0
 importlib_resources==7.1.0
 java-utilities==0.3.0
@@ -10,10 +10,10 @@ jpy==2.1.0
 llvmlite==0.48.0
 numba==0.66.0
 numpy==2.4.6
-packaging==26.2
-pandas==3.0.3
+packaging==26.3
+pandas==3.0.5
 parso==0.8.7
-pyarrow==24.0.0
+pyarrow==25.0.1
 python-dateutil==2.9.0.post0
 six==1.17.0
-typing_extensions==4.15.0
+typing_extensions==4.16.0

docker/server/src/main/server-netty/requirements.txt

@@ -1,7 +1,7 @@
-adbc-driver-manager==1.11.0
-adbc-driver-postgresql==1.11.0
+adbc-driver-manager==1.12.0
+adbc-driver-postgresql==1.12.0
 connectorx==0.4.5; platform.machine == 'x86_64'
-deephaven-plugin==0.7.0
+deephaven-plugin==0.8.0
 docstring_parser==0.18.0
 importlib_resources==7.1.0
 java-utilities==0.3.0
@@ -10,10 +10,10 @@ jpy==2.1.0
 llvmlite==0.48.0
 numba==0.66.0
 numpy==2.4.6
-packaging==26.2
-pandas==3.0.3
+packaging==26.3
+pandas==3.0.5
 parso==0.8.7
-pyarrow==24.0.0
+pyarrow==25.0.1
 python-dateutil==2.9.0.post0
 six==1.17.0
-typing_extensions==4.15.0
+typing_extensions==4.16.0

docker/web-plugin-packager/src/main/docker/files/pack-plugins.sh

@@ -16,23 +16,16 @@ TMP_PACKAGE_DIR=${TMP_DIR}/package
 # Make a output folder
 mkdir "${OUTPUT_DIR}"
 
-# Start the manifest.json file. It will add all the dependencies automatically
-echo "{\"plugins\":[" > "${MANIFEST_FILE}"
-
-# Keep track of the count so we add a comma when necessary
-PLUGIN_COUNT=0
+# Keep track of the directories of all the packed plugins so we can build the manifest.json once everything has
+# been downloaded and extracted
+PACKAGE_DIRS=()
 
 echo "Packing plugins $@..."
 
-# Iterate through each plugin defined in the plugin list, download the package and adding info to the manifest
+# Iterate through each plugin defined in the plugin list, download the package and move it into place
 # Can/should include the version number in the plugin line item
 for PACKAGE in "$@"
 do
-  # Add a comma to the manifest.json if this is not the first plugin
-  if [ $PLUGIN_COUNT -gt 0 ]; then
-    echo "," >> "${MANIFEST_FILE}"
-  fi
-
   # Make a temporary directory for downloading/extracting the package into
   mkdir -p "${TMP_DIR}"
   cd "${TMP_DIR}"
@@ -53,30 +46,38 @@ do
 
   echo "In working dir $(pwd)"
 
-  # It always unzips to the folder "package". Get the name and version info from the package so we can move it to
-  # the correct location and add the info to the manifest
+  # It always unzips to the folder "package". Get the name from the package so we can move it to the correct
+  # location
   PACKAGE_NAME=$(npm pkg get name --prefix="${TMP_PACKAGE_DIR}")
-  PACKAGE_INFO=$(npm pkg get name version main --prefix="${TMP_PACKAGE_DIR}")
   # Need to remove quotes from the package name
   PACKAGE_NAME="${PACKAGE_NAME%\"}"
   PACKAGE_NAME="${PACKAGE_NAME#\"}"
 
   echo "Got package name ${PACKAGE_NAME}"
 
-  # Add the info to the manifest file
-  echo "${PACKAGE_INFO}" >> "${MANIFEST_FILE}"
-
   # Move the plugin to the correct directory
   # Need to make the directory based on the name first
-  mkdir -p "${OUTPUT_DIR}/${PACKAGE_NAME}"
-  mv "${TMP_PACKAGE_DIR}/"* "${OUTPUT_DIR}/${PACKAGE_NAME}"
+  PACKAGE_DIR=${OUTPUT_DIR}/${PACKAGE_NAME}
+  mkdir -p "${PACKAGE_DIR}"
+  mv "${TMP_PACKAGE_DIR}/"* "${PACKAGE_DIR}"
   rm --recursive "${TMP_DIR}"
-
-  # Increment the plugin count
-  PLUGIN_COUNT=$((PLUGIN_COUNT + 1))
+  PACKAGE_DIRS+=("${PACKAGE_DIR}")
 done
 
-echo "Done!"
+# Assemble the full manifest in one shot by reading each plugin's package.json. The optional "loader" field is
+# preserved as-is so plugin authors cannot shadow core fields like name/version/main.
+"${NODE_EXE:-node}" -e '
+  const fs = require("fs");
+  const path = require("path");
+  const plugins = process.argv.slice(1).map((dir) => {
+    const pkg = JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8"));
+    const entry = { name: pkg.name, version: pkg.version, main: pkg.main };
+    if (pkg.loader != null) {
+      entry.loader = pkg.loader;
+    }
+    return entry;
+  });
+  process.stdout.write(JSON.stringify({ plugins }, null, 2));
+' ${PACKAGE_DIRS[@]+"${PACKAGE_DIRS[@]}"} > "${MANIFEST_FILE}"
 
-# Close out the manifest file
-echo "]}" >> "${MANIFEST_FILE}"
+echo "Done!"

plugin/src/main/java/io/deephaven/plugin/js/JsPlugin.java

@@ -12,6 +12,7 @@
 
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Optional;
 
 /**
  * A JS plugin is a {@link Plugin} that allows for custom javascript and related content to be served, see
@@ -100,6 +101,15 @@ public Paths paths() {
         return Paths.all();
     }
 
+    /**
+     * The JS plugin loader configuration, as sourced from the "loader" field of the plugin's package.json (if present).
+     * This is an arbitrary, plugin-specific JSON object that is opaque to the server and is simply passed through as
+     * the "loader" field for the manifest entry in "js-plugins/manifest.json".
+     *
+     * @return the loader configuration
+     */
+    public abstract Optional<Object> loader();
+
     @Override
     public final <T, V extends Plugin.Visitor<T>> T walk(V visitor) {
         return visitor.visit(this);
@@ -142,6 +152,8 @@ public interface Builder {
 
         Builder paths(Paths paths);
 
+        Builder loader(Object loader);
+
         JsPlugin build();
     }
 }

py/server/deephaven/__init__.py

@@ -7,6 +7,11 @@
 
 """
 
+# The deephaven package is split across multiple distributions (e.g. deephaven-core and deephaven-plugin); extend the
+# search path so subpackages installed elsewhere (such as deephaven.plugin) remain importable when this copy shadows
+# site-packages.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)
+
 import importlib.metadata
 
 from deephaven_internal import jvm

py/server/deephaven/jcompat.py

@@ -49,6 +49,8 @@
     "io.deephaven.integrations.common.PrimitiveArrayConversionUtility"
 )
 _JTableDefinition = jpy.get_type("io.deephaven.engine.table.TableDefinition")
+_JLinkedHashMap = jpy.get_type("java.util.LinkedHashMap")
+_JArrayList = jpy.get_type("java.util.ArrayList")
 
 _DH_PANDAS_NULLABLE_TYPE_MAP: dict[DType, type[ExtensionDtype]] = {
     bool_: pd.BooleanDtype,
@@ -90,6 +92,42 @@ def j_hashmap(d: Optional[dict] = None) -> Optional[jpy.JType]:
     return r
 
 
+JsonValue = Union[
+    None,
+    bool,
+    int,
+    float,
+    str,
+    dict[Any, "JsonValue"],
+    list["JsonValue"],
+    tuple["JsonValue", ...],
+]
+"""A JSON-compatible Python value: a scalar, or a dict/list/tuple of them."""
+
+
+def j_json_value(value: JsonValue) -> Union[jpy.JType, None, bool, int, float, str]:
+    """Recursively converts a JSON-compatible Python value into Java objects.
+
+    A dict is converted to a java.util.LinkedHashMap and a list or tuple is converted to a
+    java.util.ArrayList, with their keys and values converted recursively. Any other value is
+    unwrapped and left for jpy to convert.
+
+    Nested containers must be converted explicitly; otherwise jpy passes them through as opaque
+    org.jpy.PyObject values.
+    """
+    if isinstance(value, dict):
+        r = _JLinkedHashMap(len(value))
+        for k, v in value.items():
+            r.put(j_json_value(k), j_json_value(v))
+        return r
+    if isinstance(value, (list, tuple)):
+        r = _JArrayList(len(value))
+        for v in value:
+            r.add(j_json_value(v))
+        return r
+    return unwrap(value)
+
+
 def j_hashset(s: Optional[Union[set, Sequence]] = None) -> Optional[jpy.JType]:
     """Creates a Java HashSet from a set."""
     if s is None:

py/server/deephaven_internal/plugin/js/__init__.py

@@ -5,6 +5,7 @@
 import pathlib
 
 import jpy
+from deephaven.jcompat import j_json_value
 from deephaven.plugin.js import JsPlugin
 
 _JJsPlugin = jpy.get_type("io.deephaven.plugin.js.JsPlugin")
@@ -26,4 +27,10 @@ def to_j_js_plugin(js_plugin: JsPlugin) -> jpy.JType:
     builder.version(js_plugin.version)
     builder.main(main_path)
     builder.path(j_path)
+    # "loader" is not required field on the deephaven.plugin.js.JsPlugin interface, so we duck-type check for it
+    # here to allow plugins to opt-in to providing loader configuration without requiring a new release of the
+    # deephaven-plugin package.
+    loader = getattr(js_plugin, "loader", None)
+    if loader is not None:
+        builder.loader(j_json_value(loader))
     return builder.build()

py/server/tests/test_jcompat.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending
 #
 
+import json
 import unittest
 
 import jpy
@@ -13,11 +14,15 @@
     j_collection_to_list,
     j_function,
     j_hashset,
+    j_json_value,
     j_lambda,
 )
 from tests.testbase import BaseTestCase
 
 _JSharedContext = jpy.get_type("io.deephaven.engine.table.SharedContext")
+_JMap = jpy.get_type("java.util.Map")
+_JList = jpy.get_type("java.util.List")
+_JObjectMapper = jpy.get_type("com.fasterxml.jackson.databind.ObjectMapper")
 
 
 class JCompatTestCase(BaseTestCase):
@@ -56,6 +61,91 @@ def test_j_collection_to_list(self):
         j_set = j_hashset(s)
         self.assertEqual(s, set(j_collection_to_list(j_set)))
 
+    def assert_json_equals(self, j_object, expected) -> None:
+        """Serializes a Java object with Jackson and asserts the resulting JSON matches expected."""
+        actual = json.loads(_JObjectMapper().writeValueAsString(j_object))
+        self.assertEqual(actual, expected)
+
+    def assert_round_trip(self, value) -> None:
+        """Asserts that value survives conversion to Java and JSON serialization unchanged."""
+        self.assert_json_equals(j_json_value(value), value)
+
+    def test_j_json_value_basic_values_pass_through(self):
+        for value in [None, True, False, 0, 42, -7, 0.0, 1.5, -2.25, "", "a string"]:
+            with self.subTest(value=value):
+                self.assertIs(j_json_value(value), value)
+
+    def test_j_json_value_dict(self):
+        j_map = j_json_value({"key": "value", "count": 3})
+        self.assertTrue(_JMap.jclass.isInstance(j_map))
+        self.assertEqual(j_map.size(), 2)
+        self.assertEqual(j_map.get("key"), "value")
+        self.assertEqual(j_map.get("count"), 3)
+
+    def test_j_json_value_dict_of_basic_values(self):
+        self.assert_round_trip(
+            {
+                "none": None,
+                "true": True,
+                "false": False,
+                "int": 42,
+                "negative_int": -7,
+                "float": 1.5,
+                "str": "a string",
+            }
+        )
+
+    def test_j_json_value_nested_dict(self):
+        j_map = j_json_value({"outer": {"inner": {"deepest": "value"}}})
+        j_outer = j_map.get("outer")
+        self.assertTrue(_JMap.jclass.isInstance(j_outer))
+        j_inner = j_outer.get("inner")
+        self.assertTrue(_JMap.jclass.isInstance(j_inner))
+        self.assertEqual(j_inner.get("deepest"), "value")
+
+    def test_j_json_value_list(self):
+        j_list = j_json_value(["a", "b", "c"])
+        self.assertTrue(_JList.jclass.isInstance(j_list))
+        self.assertEqual(j_list.size(), 3)
+        self.assertEqual(j_list.get(0), "a")
+        self.assertEqual(j_list.get(2), "c")
+
+    def test_j_json_value_tuple(self):
+        j_list = j_json_value(("a", "b"))
+        self.assertTrue(_JList.jclass.isInstance(j_list))
+        self.assert_json_equals(j_list, ["a", "b"])
+
+    def test_j_json_value_empty_containers(self):
+        self.assert_round_trip({})
+        self.assert_round_trip([])
+        self.assert_round_trip({"empty_map": {}, "empty_list": []})
+
+    def test_j_json_value_nested_containers(self):
+        self.assert_round_trip(
+            {
+                "list_of_dicts": [{"a": 1}, {"b": [2, 3]}],
+                "dict_of_lists": {"x": [[1, 2], []], "y": [{"z": None}]},
+                "deep": {"a": {"b": {"c": {"d": ["e", {"f": True}]}}}},
+            }
+        )
+
+    def test_j_json_value_list_of_lists(self):
+        j_list = j_json_value([[1, 2], [3]])
+        self.assertTrue(_JList.jclass.isInstance(j_list.get(0)))
+        self.assert_json_equals(j_list, [[1, 2], [3]])
+
+    def test_j_json_value_non_string_keys(self):
+        # Jackson serializes non-string map keys as strings, which is the same behavior as json.dumps
+        self.assert_json_equals(
+            j_json_value({1: "a", 2.5: "b"}), {"1": "a", "2.5": "b"}
+        )
+
+    def test_j_json_value_unwraps_wrapped_objects(self):
+        j_map = j_json_value(
+            {"context": AutoCloseable(_JSharedContext.makeSharedContext())}
+        )
+        self.assertTrue(_JSharedContext.jclass.isInstance(j_map.get("context")))
+
 
 if __name__ == "__main__":
     unittest.main()