diff --git a/pom.xml b/pom.xml index 65e2e0dbf..de3452302 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 1.6 2.16.1 1.4 - 3.0 + 3.1 2.6 3.5 3.6.1 diff --git a/wrangler-api/pom.xml b/wrangler-api/pom.xml index e97464a64..847b4c1c6 100644 --- a/wrangler-api/pom.xml +++ b/wrangler-api/pom.xml @@ -39,6 +39,11 @@ ${cdap.version} provided - + + com.google.guava + guava + ${guava.version} + provided + diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DefaultJexlAllowlist.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DefaultJexlAllowlist.java new file mode 100644 index 000000000..b47009c29 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DefaultJexlAllowlist.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Defines the set of classes allowed in the JEXL Sandbox by default. + */ +public final class DefaultJexlAllowlist { + + private static final List DEFAULT_CLASSES = Arrays.asList( + // Data types + "java.lang.Boolean", "java.lang.Byte", "java.lang.Character", "java.lang.Double", "java.lang.Float", + "java.lang.Integer", "java.lang.Long", "java.lang.Short", + + // Strings + "java.lang.String", "java.lang.StringBuilder", "java.util.StringJoiner", + + // Math + "java.lang.Math", "java.math.BigDecimal", "java.math.BigInteger", + + // Time + "java.time.ZonedDateTime", "java.time.LocalDate", "java.time.LocalDateTime", "java.time.Instant", + "java.time.Duration", "java.time.format.DateTimeFormatter", + + // Utilities + "java.util.Arrays", "java.util.Collections", "java.util.UUID", "java.util.Base64"); + + private static final List ALLOWLIST = Collections.unmodifiableList( + DEFAULT_CLASSES.stream() + .map(className -> { + return new JexlAllowlist( + className, + Collections.singletonList(JexlAllowlist.INCLUDE_ALL_WILDCARD), + Collections.singletonList(JexlAllowlist.INCLUDE_ALL_WILDCARD)); + }) + .collect(Collectors.toList())); + + private DefaultJexlAllowlist() { + } + + /** + * @return the list of default allowed classes. + */ + public static List get() { + return ALLOWLIST; + } +} diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java index d139594cc..075724690 100644 --- a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfig.java @@ -16,19 +16,24 @@ package io.cdap.wrangler.api; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.Nullable; /** - * This class {@link DirectiveConfig} defines the configuration for the Wrangler. + * This class {@link DirectiveConfig} defines the configuration for the + * Wrangler. * It specifies the directive exclusions -- meaning directives that should * not be accessible to the users and as well as directive aliases. * @@ -42,18 +47,63 @@ * "aliases" : { * "json-parser" : "parse-as-json", * "js-parser" : "parse-as-json" - * } + * }, + * "jexlAllowlist" : [ + * { + * "className": "java.lang.Runtime", + * "methods": ["*"], + * "properties": ["*"] + * } + * ] * } */ @Deprecated public final class DirectiveConfig { - public static final DirectiveConfig EMPTY = new DirectiveConfig(); - // RecipeParser to be excluded or made non-accessible. - private final Set exclusions = new HashSet<>(); + public static final DirectiveConfig EMPTY = new DirectiveConfig(null, null, Collections.emptyList()); + public static final String EXCLUSIONS_KEY = "exclusions"; + public static final String ALIASES_KEY = "aliases"; + public static final String JEXL_ALLOWLIST_KEY = "jexlAllowlist"; - // RecipeParser to be aliased. - private final Map aliases = new HashMap<>(); + private final ImmutableSet exclusions; + private final ImmutableMap aliases; + @Nullable private final ImmutableList jexlAllowlist; + public DirectiveConfig( + @Nullable Set exclusions, + @Nullable Map aliases, + @Nullable List jexlAllowlist) { + this.exclusions = exclusions != null ? ImmutableSet.copyOf(exclusions) : ImmutableSet.of(); + this.aliases = aliases != null ? ImmutableMap.copyOf(aliases) : ImmutableMap.of(); + this.jexlAllowlist = jexlAllowlist != null ? ImmutableList.copyOf(jexlAllowlist) : null; + } + + /** + * Gets the set of excluded directives. + * + * @return the set of excluded directives + */ + public Set getExclusions() { + return exclusions; + } + + /** + * Gets the directive alias mappings. + * + * @return map of alias to directive name + */ + public Map getAliases() { + return aliases; + } + + /** + * Gets the list of JEXL inclusions. + * + * @return the list of JEXL inclusions + */ + @Nullable + public List getJexlAllowlist() { + return jexlAllowlist; + } /** * Checks if a directive is aliased. @@ -108,8 +158,9 @@ public boolean isExcluded(String directive) { public JsonElement toJson() { Gson gson = new Gson(); JsonObject object = new JsonObject(); - object.add("exclusions", gson.toJsonTree(exclusions)); - object.add("aliases", gson.toJsonTree(aliases)); + object.add(EXCLUSIONS_KEY, gson.toJsonTree(exclusions)); + object.add(ALIASES_KEY, gson.toJsonTree(aliases)); + object.add(JEXL_ALLOWLIST_KEY, gson.toJsonTree(jexlAllowlist)); return object; } } diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfigDeserializer.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfigDeserializer.java new file mode 100644 index 000000000..814de8bb3 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveConfigDeserializer.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.reflect.TypeToken; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; + +/** + * Custom GSON deserializer for {@link DirectiveConfig}. + */ +public final class DirectiveConfigDeserializer implements JsonDeserializer { + private static final Type STRING_SET_TYPE = new TypeToken>() { }.getType(); + private static final Type STRING_MAP_TYPE = new TypeToken>() { }.getType(); + private static final Type JEXL_ALLOWLIST_LIST_TYPE = new TypeToken>() { }.getType(); + + @Override + public DirectiveConfig deserialize(JsonElement configJson, Type typeOfT, JsonDeserializationContext ctx) + throws JsonParseException { + JsonObject configJsonObj = configJson.getAsJsonObject(); + + return new DirectiveConfig( + deserializeProperty(configJsonObj, DirectiveConfig.EXCLUSIONS_KEY, STRING_SET_TYPE, ctx), + deserializeProperty(configJsonObj, DirectiveConfig.ALIASES_KEY, STRING_MAP_TYPE, ctx), + deserializeProperty(configJsonObj, DirectiveConfig.JEXL_ALLOWLIST_KEY, JEXL_ALLOWLIST_LIST_TYPE, ctx)); + } + + @Nullable + private static T deserializeProperty(JsonObject obj, String key, Type type, JsonDeserializationContext ctx) { + JsonElement element = obj.get(key); + return (element != null && !element.isJsonNull()) ? ctx.deserialize(element, type) : null; + } +} diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java index 78df981d6..d128e47d6 100644 --- a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveContext.java @@ -20,5 +20,5 @@ * {@link DirectiveContext} provides the context object to the processing of * directives. */ -public interface DirectiveContext extends DirectiveEnforcer, DirectiveAlias { +public interface DirectiveContext extends DirectiveEnforcer, DirectiveAlias, DirectiveJexlAllowlist { } diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveJexlAllowlist.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveJexlAllowlist.java new file mode 100644 index 000000000..79f096e53 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/DirectiveJexlAllowlist.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import java.util.List; + +import javax.annotation.Nullable; + +/** + * This interface {@link DirectiveJexlAllowlist} provides a way to get JEXL + * configuration. + */ +public interface DirectiveJexlAllowlist { + + /** + * Gets the list of JEXL inclusions. + * + * @return the list of JEXL inclusions + */ + @Nullable + List getJexlAllowlist(); + + /** + * Checks if JEXL allowlisting is enabled. + * + * @return true if JEXL allowlisting is enabled, false otherwise. + */ + boolean isJexlAllowlistEnabled(); +} diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlist.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlist.java new file mode 100644 index 000000000..bcd3e61c2 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlist.java @@ -0,0 +1,158 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import com.google.common.base.Strings; +import io.cdap.wrangler.api.annotations.PublicEvolving; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.lang.model.SourceVersion; + +/** + * Defines custom class, method, and property inclusion rules. + */ +@PublicEvolving +public final class JexlAllowlist { + public static final String INCLUDE_ALL_WILDCARD = "*"; + + /** + * The fully qualified name of the class to include (e.g. java.lang.Math). + */ + @Nonnull + private final String className; + + /** + * The list of allowed methods for the class. + */ + private final List methods; + + /** + * The list of allowed properties for the class. + */ + private final List properties; + + public JexlAllowlist(@Nonnull String className, List methods, List properties) { + if (!isValidClassName(className)) { + throw new IllegalArgumentException("className cannot be null, empty, or an invalid Java name: " + className); + } + this.className = className; + this.methods = sanitizeList(methods, "method"); + this.properties = sanitizeList(properties, "property"); + + if (this.methods.isEmpty() && this.properties.isEmpty()) { + throw new IllegalArgumentException( + "Both methods and properties cannot be empty at the same time for an allowlisted class: " + className); + } + } + + private static List sanitizeList(List list, String type) { + if (list == null) { + throw new IllegalArgumentException("The " + type + " list cannot be null."); + } + return Collections.unmodifiableList(list.stream() + .map(item -> { + if (!INCLUDE_ALL_WILDCARD.equals(item) && !isValidIdentifier(item)) { + throw new IllegalArgumentException("Invalid " + type + " name: " + item); + } + return item; + }) + .collect(Collectors.toList())); + } + + private static boolean isValidClassName(String className) { + if (Strings.isNullOrEmpty(className)) { + return false; + } + return SourceVersion.isName(className); + } + + private static boolean isValidIdentifier(String name) { + if (Strings.isNullOrEmpty(name)) { + return false; + } + return SourceVersion.isIdentifier(name); + } + + /** + * Gets the class name. + * + * @return the class name + */ + @Nonnull + public String getClassName() { + return className; + } + + /** + * Gets the list of allowed methods. + * + * @return the allowed methods + */ + @Nonnull + public List getMethods() { + return methods; + } + + /** + * Gets the list of allowed properties. + * + * @return the allowed properties + */ + @Nonnull + public List getProperties() { + return properties; + } + + /** + * Checks if all methods are allowed. + * + * @return true if all methods are allowed + */ + public boolean allowAllMethods() { + return methods.contains(INCLUDE_ALL_WILDCARD); + } + + /** + * Checks if all properties are allowed. + * + * @return true if all properties are allowed + */ + public boolean allowAllProperties() { + return properties.contains(INCLUDE_ALL_WILDCARD); + } + + /** + * Checks if all methods are blocked for this class. + * + * @return true if all methods are blocked + */ + public boolean blockAllMethods() { + return methods.isEmpty(); + } + + /** + * Checks if all properties are blocked for this class. + * + * @return true if all properties are blocked + */ + public boolean blockAllProperties() { + return properties.isEmpty(); + } +} diff --git a/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlistDeserializer.java b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlistDeserializer.java new file mode 100644 index 000000000..907c0cfa1 --- /dev/null +++ b/wrangler-api/src/main/java/io/cdap/wrangler/api/JexlAllowlistDeserializer.java @@ -0,0 +1,67 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.reflect.TypeToken; +import io.cdap.wrangler.api.annotations.PublicEvolving; + +import java.lang.reflect.Type; +import java.util.List; + +/** + * Custom GSON deserializer for {@link JexlAllowlist}. + * Invokes the parameterized constructor to guarantee validation is executed. + */ +@PublicEvolving +public final class JexlAllowlistDeserializer implements JsonDeserializer { + + private static final String CLASS_NAME_KEY = "className"; + private static final String METHODS_KEY = "methods"; + private static final String PROPERTIES_KEY = "properties"; + + private static final Type STRING_LIST_TYPE = new TypeToken>() { }.getType(); + + @Override + public JexlAllowlist deserialize(JsonElement allowlistJson, Type typeOfT, JsonDeserializationContext ctx) + throws JsonParseException { + JsonObject allowlistJsonObject = allowlistJson.getAsJsonObject(); + + try { + return new JexlAllowlist( + deserializeProperty(allowlistJsonObject, CLASS_NAME_KEY, String.class, "", ctx), + deserializeProperty(allowlistJsonObject, METHODS_KEY, STRING_LIST_TYPE, null, ctx), + deserializeProperty(allowlistJsonObject, PROPERTIES_KEY, STRING_LIST_TYPE, null, ctx)); + } catch (IllegalArgumentException e) { + throw new JsonParseException(e.getMessage(), e); + } + } + + private static T deserializeProperty( + JsonObject obj, String key, Type type, T defaultValue, JsonDeserializationContext ctx) { + JsonElement element = obj.get(key); + if (element == null || element.isJsonNull()) { + return defaultValue; + } + T result = ctx.deserialize(element, type); + return result != null ? result : defaultValue; + } +} diff --git a/wrangler-api/src/test/java/io/cdap/wrangler/api/DirectiveConfigDeserializerTest.java b/wrangler-api/src/test/java/io/cdap/wrangler/api/DirectiveConfigDeserializerTest.java new file mode 100644 index 000000000..8f03f9120 --- /dev/null +++ b/wrangler-api/src/test/java/io/cdap/wrangler/api/DirectiveConfigDeserializerTest.java @@ -0,0 +1,144 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +/** + * Tests for {@link DirectiveConfigDeserializer}. + */ +public class DirectiveConfigDeserializerTest { + + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); + + @Test + public void testDirectiveConfigDeserialization() { + String json = "{\n" + + " \"exclusions\": [\"drop\"],\n" + + " \"aliases\": {\"p\": \"parse-as-json\"},\n" + + " \"jexlAllowlist\": [\n" + + " {\n" + + " \"className\": \"java.lang.String\",\n" + + " \"methods\": [\"*\"],\n" + + " \"properties\": [\"*\"]\n" + + " }\n" + + " ]\n" + + "}"; + + DirectiveConfig config = GSON.fromJson(json, DirectiveConfig.class); + Assert.assertNotNull(config); + Assert.assertTrue(config.isExcluded("drop")); + Assert.assertEquals("parse-as-json", config.getAliasName("p")); + Assert.assertEquals(Collections.singleton("drop"), config.getExclusions()); + Assert.assertEquals(Collections.singletonMap("p", "parse-as-json"), config.getAliases()); + Assert.assertNotNull(config.getJexlAllowlist()); + Assert.assertEquals(1, config.getJexlAllowlist().size()); + + JexlAllowlist allowlist = config.getJexlAllowlist().get(0); + Assert.assertEquals("java.lang.String", allowlist.getClassName()); + Assert.assertTrue(allowlist.allowAllMethods()); + Assert.assertTrue(allowlist.allowAllProperties()); + } + + @Test + public void testDirectiveConfigDeserializationWithSpecificMethodsAndProperties() { + String json = "{\n" + + " \"exclusions\": [\"drop\"],\n" + + " \"aliases\": {\"p\": \"parse-as-json\"},\n" + + " \"jexlAllowlist\": [\n" + + " {\n" + + " \"className\": \"java.lang.String\",\n" + + " \"methods\": [\"trim\", \"substring\"],\n" + + " \"properties\": [\"bytes\"]\n" + + " }\n" + + " ]\n" + + "}"; + + DirectiveConfig config = GSON.fromJson(json, DirectiveConfig.class); + Assert.assertNotNull(config); + Assert.assertTrue(config.isExcluded("drop")); + Assert.assertEquals("parse-as-json", config.getAliasName("p")); + Assert.assertEquals(Collections.singleton("drop"), config.getExclusions()); + Assert.assertEquals(Collections.singletonMap("p", "parse-as-json"), config.getAliases()); + Assert.assertNotNull(config.getJexlAllowlist()); + Assert.assertEquals(1, config.getJexlAllowlist().size()); + + JexlAllowlist allowlist = config.getJexlAllowlist().get(0); + Assert.assertEquals("java.lang.String", allowlist.getClassName()); + Assert.assertEquals(Arrays.asList("trim", "substring"), allowlist.getMethods()); + Assert.assertEquals(Arrays.asList("bytes"), allowlist.getProperties()); + Assert.assertFalse(allowlist.allowAllMethods()); + Assert.assertFalse(allowlist.allowAllProperties()); + } + + @Test + public void testDirectiveConfigDeserializationEmpty() { + String json = "{}"; + + DirectiveConfig config = GSON.fromJson(json, DirectiveConfig.class); + Assert.assertNotNull(config); + Assert.assertTrue(config.getExclusions().isEmpty()); + Assert.assertTrue(config.getAliases().isEmpty()); + Assert.assertNull(config.getJexlAllowlist()); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeInvalidJexlAllowlistEntry() { + String json = "{\n" + + " \"jexlAllowlist\": [\n" + + " {\n" + + " \"className\": \"123InvalidClass\",\n" + + " \"methods\": [\"*\"],\n" + + " \"properties\": [\"*\"]\n" + + " }\n" + + " ]\n" + + "}"; + GSON.fromJson(json, DirectiveConfig.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeMalformedExclusions() { + String json = "{\n" + + " \"exclusions\": \"not-a-list\"\n" + + "}"; + GSON.fromJson(json, DirectiveConfig.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeMalformedAliases() { + String json = "{\n" + + " \"aliases\": \"not-a-map\"\n" + + "}"; + GSON.fromJson(json, DirectiveConfig.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeNonObjectJson() { + String json = "[\"drop\"]"; + GSON.fromJson(json, DirectiveConfig.class); + } +} diff --git a/wrangler-api/src/test/java/io/cdap/wrangler/api/JexlAllowlistDeserializerTest.java b/wrangler-api/src/test/java/io/cdap/wrangler/api/JexlAllowlistDeserializerTest.java new file mode 100644 index 000000000..98af68342 --- /dev/null +++ b/wrangler-api/src/test/java/io/cdap/wrangler/api/JexlAllowlistDeserializerTest.java @@ -0,0 +1,149 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + * Tests for {@link JexlAllowlistDeserializer}. + */ +public class JexlAllowlistDeserializerTest { + + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); + + @Test + public void testJexlAllowlistDeserialization() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [\"max\", \"min\"],\n" + + " \"properties\": [\"*\"]\n" + + "}"; + + JexlAllowlist allowlist = GSON.fromJson(json, JexlAllowlist.class); + Assert.assertNotNull(allowlist); + Assert.assertEquals("java.lang.Math", allowlist.getClassName()); + Assert.assertEquals(Arrays.asList("max", "min"), allowlist.getMethods()); + Assert.assertEquals(Arrays.asList("*"), allowlist.getProperties()); + Assert.assertFalse(allowlist.allowAllMethods()); + Assert.assertTrue(allowlist.allowAllProperties()); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeMissingClassName() { + String json = "{\n" + + " \"methods\": [\"*\"],\n" + + " \"properties\": [\"*\"]\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeInvalidClassName() { + String json = "{\n" + + " \"className\": \"123InvalidClass\",\n" + + " \"methods\": [\"*\"],\n" + + " \"properties\": [\"*\"]\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeMissingMethods() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"properties\": [\"*\"]\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test + public void testDeserializeEmptyMethods() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [],\n" + + " \"properties\": [\"*\"]\n" + + "}"; + JexlAllowlist allowlist = GSON.fromJson(json, JexlAllowlist.class); + Assert.assertNotNull(allowlist); + Assert.assertEquals("java.lang.Math", allowlist.getClassName()); + Assert.assertTrue(allowlist.getMethods().isEmpty()); + Assert.assertTrue(allowlist.blockAllMethods()); + Assert.assertFalse(allowlist.allowAllMethods()); + Assert.assertFalse(allowlist.blockAllProperties()); + Assert.assertTrue(allowlist.allowAllProperties()); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeMissingProperties() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [\"*\"]\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test + public void testDeserializeEmptyProperties() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [\"*\"],\n" + + " \"properties\": []\n" + + "}"; + JexlAllowlist allowlist = GSON.fromJson(json, JexlAllowlist.class); + Assert.assertNotNull(allowlist); + Assert.assertEquals("java.lang.Math", allowlist.getClassName()); + Assert.assertTrue(allowlist.getProperties().isEmpty()); + Assert.assertTrue(allowlist.blockAllProperties()); + Assert.assertFalse(allowlist.allowAllProperties()); + Assert.assertFalse(allowlist.blockAllMethods()); + Assert.assertTrue(allowlist.allowAllMethods()); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeBothEmptyMethodsAndProperties() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [],\n" + + " \"properties\": []\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeInvalidMethodName() { + String json = "{\n" + + " \"className\": \"java.lang.Math\",\n" + + " \"methods\": [\"invalid method!\"],\n" + + " \"properties\": [\"*\"]\n" + + "}"; + GSON.fromJson(json, JexlAllowlist.class); + } + + @Test(expected = JsonParseException.class) + public void testDeserializeNonObjectJson() { + String json = "[\"java.lang.Math\"]"; + GSON.fromJson(json, JexlAllowlist.class); + } +} diff --git a/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java b/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java index fbe89793d..5d5845806 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java +++ b/wrangler-core/src/main/java/io/cdap/directives/aggregates/IncrementTransientVariable.java @@ -34,6 +34,7 @@ import io.cdap.wrangler.api.parser.Numeric; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -71,7 +72,7 @@ public void initialize(Arguments args) throws DirectiveParseException { this.incrementBy = ((Numeric) args.value("value")).value().longValue(); String expression = ((Expression) args.value("condition")).value(); try { - el = EL.compile(expression); + el = EL.compile(expression, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java b/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java index becf12cc6..0476245d4 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java +++ b/wrangler-core/src/main/java/io/cdap/directives/aggregates/SetTransientVariable.java @@ -33,6 +33,7 @@ import io.cdap.wrangler.api.parser.Identifier; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -70,7 +71,7 @@ public void initialize(Arguments args) throws DirectiveParseException { this.variable = ((Identifier) args.value("variable")).value(); String expression = ((Expression) args.value("condition")).value(); try { - el = EL.compile(expression); + el = EL.compile(expression, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java b/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java index 880ffc63c..5c3e7060b 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/Fail.java @@ -33,6 +33,7 @@ import io.cdap.wrangler.api.parser.Expression; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -70,7 +71,7 @@ public void initialize(Arguments args) throws DirectiveParseException { } condition = expression.value(); try { - el = EL.compile(condition); + el = EL.compile(condition, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java b/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java index b3eb4adb2..c74070561 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/RecordConditionFilter.java @@ -35,6 +35,7 @@ import io.cdap.wrangler.api.parser.Expression; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -78,7 +79,7 @@ public void initialize(Arguments args) throws DirectiveParseException { } String condition = ((Expression) args.value("condition")).value(); try { - el = EL.compile(condition); + el = EL.compile(condition, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java b/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java index 7cbd4bf82..61270e41e 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/SendToError.java @@ -37,6 +37,7 @@ import io.cdap.wrangler.api.parser.Text; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -80,7 +81,7 @@ public UsageDefinition define() { public void initialize(Arguments args) throws DirectiveParseException { condition = ((Expression) args.value("condition")).value(); try { - el = EL.compile(condition); + el = EL.compile(condition, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException( NAME, String.format(" Invalid condition '%s'.", condition) diff --git a/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java b/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java index 20922d1e8..1724a0220 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java +++ b/wrangler-core/src/main/java/io/cdap/directives/row/SendToErrorAndContinue.java @@ -38,6 +38,7 @@ import io.cdap.wrangler.api.parser.Text; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -81,7 +82,7 @@ public UsageDefinition define() { public void initialize(Arguments args) throws DirectiveParseException { condition = ((Expression) args.value("condition")).value(); try { - el = EL.compile(condition); + el = EL.compile(condition, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException( NAME, String.format("Invalid condition '%s'.", condition), e); diff --git a/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java b/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java index 25c9c895b..5f3d37e63 100644 --- a/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java +++ b/wrangler-core/src/main/java/io/cdap/directives/transformation/ColumnExpression.java @@ -35,6 +35,7 @@ import io.cdap.wrangler.api.parser.Expression; import io.cdap.wrangler.api.parser.TokenType; import io.cdap.wrangler.api.parser.UsageDefinition; +import io.cdap.wrangler.expression.CompileOptions; import io.cdap.wrangler.expression.EL; import io.cdap.wrangler.expression.ELContext; import io.cdap.wrangler.expression.ELException; @@ -81,7 +82,7 @@ public void initialize(Arguments args) throws DirectiveParseException { this.column = ((ColumnName) args.value("column")).value(); this.expression = ((Expression) args.value("expression")).value(); try { - el = EL.compile(expression); + el = EL.compile(expression, CompileOptions.fromArguments(args)); } catch (ELException e) { throw new DirectiveParseException(NAME, e.getMessage(), e); } diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/clients/DataPrepServiceClient.java b/wrangler-core/src/main/java/io/cdap/wrangler/clients/DataPrepServiceClient.java index 6cc0afb64..1aec04766 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/clients/DataPrepServiceClient.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/clients/DataPrepServiceClient.java @@ -27,9 +27,13 @@ import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import io.cdap.cdap.etl.api.StageContext; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.proto.ServiceResponse; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -51,7 +55,10 @@ */ public class DataPrepServiceClient { private static final Logger LOG = LoggerFactory.getLogger(DataPrepServiceClient.class); - private static final Gson GSON = new Gson(); + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); private static final String SYSTEM_NAMESPACE = "system"; private static final String APPLICATION_NAME = "dataprep"; diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/expression/CompileOptions.java b/wrangler-core/src/main/java/io/cdap/wrangler/expression/CompileOptions.java new file mode 100644 index 000000000..d4c60ea17 --- /dev/null +++ b/wrangler-core/src/main/java/io/cdap/wrangler/expression/CompileOptions.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.expression; + +import com.google.common.collect.ImmutableList; +import io.cdap.wrangler.api.Arguments; +import io.cdap.wrangler.api.DirectiveContext; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.parser.MapArgumentsWithContext; + +import java.util.List; +import javax.annotation.Nullable; + +/** + * Holds options used during JEXL script compilation. + */ +public class CompileOptions { + private static final CompileOptions DEFAULT = new CompileOptions(false, null); + + private final boolean allowlistEnabled; + private final ImmutableList jexlAllowlist; + + private CompileOptions(boolean allowlistEnabled, @Nullable List jexlAllowlist) { + this.allowlistEnabled = allowlistEnabled; + this.jexlAllowlist = jexlAllowlist != null ? ImmutableList.copyOf(jexlAllowlist) : null; + } + + public static CompileOptions getDefaultCompileOptions() { + return DEFAULT; + } + + public boolean isAllowlistEnabled() { + return allowlistEnabled; + } + + @Nullable + public List getJexlAllowlist() { + return jexlAllowlist; + } + + public static CompileOptions fromArguments(Arguments args) { + if (args instanceof MapArgumentsWithContext) { + DirectiveContext context = ((MapArgumentsWithContext) args).getDirectiveContext(); + if (context != null) { + return new CompileOptions(context.isJexlAllowlistEnabled(), context.getJexlAllowlist()); + } + } + return DEFAULT; + } +} diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java b/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java index 28baad940..698d6a77f 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/expression/EL.java @@ -16,7 +16,6 @@ package io.cdap.wrangler.expression; -import com.google.common.base.Joiner; import com.google.common.base.Strings; import io.cdap.cdap.api.common.Bytes; import io.cdap.functions.DDL; @@ -28,6 +27,7 @@ import io.cdap.functions.JsonFunctions; import io.cdap.functions.Logical; import io.cdap.functions.NumberFunctions; +import io.cdap.wrangler.api.JexlAllowlist; import io.cdap.wrangler.utils.ArithmeticOperations; import io.cdap.wrangler.utils.DecimalTransform; import org.apache.commons.jexl3.JexlBuilder; @@ -35,6 +35,7 @@ import org.apache.commons.jexl3.JexlException; import org.apache.commons.jexl3.JexlInfo; import org.apache.commons.jexl3.JexlScript; +import org.apache.commons.jexl3.introspection.JexlSandbox; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.logging.Log; @@ -42,10 +43,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * This class EL is a Expression Language Handler. @@ -65,23 +67,44 @@ public static boolean isUsed() { } /** - * Same as calling {@link #compile(ELRegistration, String)} using {@link DefaultFunctions}. + * Same as calling {@link #compile(ELRegistration, String, CompileOptions)} + * using + * {@link DefaultFunctions}. + * Note: This defaults to allowlist disabled. */ public static EL compile(String expression) throws ELException { - return compile(new DefaultFunctions(), expression); + return compile(new DefaultFunctions(), expression, CompileOptions.getDefaultCompileOptions()); } /** - * Compiles the given expressions and return an {@link EL} for script execution. + * Compiles with specific JEXL allowlist settings. * - * @param registration extra objects available for the script to use - * @param expression the JEXL expresion - * @return an {@link EL} instance - * @throws ELException if failed to compile the expression + * @param expression the JEXL expression + * @param options the compilation options + * @return the compiled EL + * @throws ELException if failed to compile */ - public static EL compile(ELRegistration registration, String expression) throws ELException { + public static EL compile(final String expression, final CompileOptions options) throws ELException { + return compile(new DefaultFunctions(), expression, options); + } + + /** + * Compiles the expression and returns a executable expression. + * + * @param registration to be registered with the JEXL context. + * @param expression to be compiled. + * @param options the compilation options + * @return a compiled {@link EL} object + * @throws ELException if failed to compile + */ + public static EL compile(final ELRegistration registration, + final String expression, + final CompileOptions options) + throws ELException { used = true; + JexlSandbox sandbox = createSandbox(options.getJexlAllowlist(), options.isAllowlistEnabled()); JexlEngine engine = new JexlBuilder() + .sandbox(sandbox) .namespaces(registration.functions()) .silent(false) .cache(1024) @@ -90,28 +113,109 @@ public static EL compile(ELRegistration registration, String expression) throws .create(); try { - Set variables = new HashSet<>(); JexlScript script = engine.createScript(expression); - Set> varSet = script.getVariables(); - for (List vars : varSet) { - variables.add(Joiner.on(".").join(vars)); - } - + Set variables = extractVariables(script); return new EL(script, variables); - } catch (JexlException e) { - // JexlException.getMessage() uses 'io.cdap.wrangler.expression.EL' class name in the error message. - // So instead use info object to get information about error message and create custom error message. - JexlInfo info = e.getInfo(); - throw new ELException( - String.format("Error encountered while compiling '%s' at line '%d' and column '%d'. " + - "Make sure a valid jexl transformation is provided.", - // here the detail can be null since there are multiple subclasses which extends this - // JexlException, not all of them has this detail information - info.getDetail() == null ? expression : info.getDetail(), info.getLine(), info.getColumn()), e); } catch (Exception e) { - throw new ELException(e); + throw handleException(e, expression); } + } + /** + * Extracts variables from the script. + * + * @param script the script + * @return the variables + */ + private static Set extractVariables(final JexlScript script) { + return script.getVariables().stream() + .map(vars -> String.join(".", vars)) + .collect(Collectors.toSet()); + } + + private static ELException handleException(Exception e, String expression) { + if (e instanceof JexlException) { + JexlException jexlException = (JexlException) e; + JexlInfo info = jexlException.getInfo(); + return new ELException( + String.format("Error encountered while evaluating '%s', at line '%d' and column '%d'. " + + "Make sure the JEXL transformation is valid and uses only allowlisted classes, methods, and properties.", + info == null || info.getDetail() == null ? expression : info.getDetail(), + info == null ? 0 : info.getLine(), info == null ? 0 : info.getColumn()), + e); + } else if (e instanceof NumberFormatException) { + return new ELException("Type mismatch. Change type of constant " + + "or convert to right data type using conversion functions available. Reason : " + + e.getMessage(), e); + } else { + if (e.getCause() != null) { + return new ELException(e.getCause().getMessage(), e); + } else { + return new ELException(e); + } + } + } + + /** + * Creates a JEXL sandbox. + * + * @param allowlist the allowlist + * @return the sandbox + */ + public static JexlSandbox createSandbox( + @Nullable List allowlist, boolean allowlistEnabled) { + if (!allowlistEnabled) { + return null; + } + + JexlSandbox sandbox = new JexlSandbox(false); + if (allowlist != null && !allowlist.isEmpty()) { + allowlist.forEach(rule -> applyAllowlistRule(sandbox, rule)); + } + + return sandbox; + } + + /** + * Applies an allowlist rule to the sandbox. + * + * @param sandbox the sandbox + * @param rule the rule + */ + private static void applyAllowlistRule(JexlSandbox sandbox, JexlAllowlist rule) { + String className = rule.getClassName(); + + if (rule.allowAllMethods() && rule.allowAllProperties()) { + sandbox.white(className); + } else { + // configure sandbox permissions: blocklist or allowlist is configured for properties and methods. + // if blockAllMethods = true, a blocklist is configured to block all methods for the class. + // if blockAllMethods = false, a allowlist is configured which allows only mentioned methods. + // Similarly for properties. + JexlSandbox.Permissions permissions = sandbox.permissions( + className, + !rule.blockAllProperties(), + !rule.blockAllProperties(), + !rule.blockAllMethods()); + + applyMethodPermissions(rule, permissions); + applyPropertyPermissions(rule, permissions); + } + } + + private static void applyMethodPermissions(JexlAllowlist rule, JexlSandbox.Permissions permissions) { + if (!rule.allowAllMethods() && !rule.blockAllMethods()) { + rule.getMethods().forEach(method -> permissions.execute(method)); + } + } + + private static void applyPropertyPermissions(JexlAllowlist rule, JexlSandbox.Permissions permissions) { + if (!rule.allowAllProperties() && !rule.blockAllProperties()) { + rule.getProperties().forEach(property -> { + permissions.read(property); + permissions.write(property); + }); + } } private EL(JexlScript script, Set variables) { @@ -137,27 +241,8 @@ public ELResult execute(ELContext context) throws ELException { } Object value = script.execute(context); return new ELResult(value); - } catch (JexlException e) { - // JexlException.getMessage() uses 'io.cdap.wrangler.expression.EL' class name in the error message. - // So instead use info object to get information about error message and create custom error message. - JexlInfo info = e.getInfo(); - throw new ELException( - String.format("Error encountered while executing '%s', at line '%d' and column '%d'. " + - "Make sure a valid jexl transformation is provided.", - // here the detail can be null since there are multiple subclasses which extends this - // JexlException, not all of them has this detail information - info.getDetail() == null ? script.getSourceText() : info.getDetail(), - info.getLine(), info.getColumn()), e); - } catch (NumberFormatException e) { - throw new ELException("Type mismatch. Change type of constant " + - "or convert to right data type using conversion functions available. Reason : " - + e.getMessage(), e); } catch (Exception e) { - if (e.getCause() != null) { - throw new ELException(e.getCause().getMessage(), e); - } else { - throw new ELException(e); - } + throw handleException(e, script.getSourceText()); } } diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java index 20d100077..33ba45cf0 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/ConfigDirectiveContext.java @@ -18,6 +18,9 @@ import io.cdap.wrangler.api.DirectiveConfig; import io.cdap.wrangler.api.DirectiveContext; +import io.cdap.wrangler.api.JexlAllowlist; + +import java.util.List; /** * This class {@link ConfigDirectiveContext} manages the context for directive @@ -26,9 +29,21 @@ */ public class ConfigDirectiveContext implements DirectiveContext { private final DirectiveConfig config; + private final boolean jexlAllowlistEnabled; - public ConfigDirectiveContext(DirectiveConfig config) { + public ConfigDirectiveContext(DirectiveConfig config, boolean jexlAllowlistEnabled) { this.config = config; + this.jexlAllowlistEnabled = jexlAllowlistEnabled; + } + + @Override + public List getJexlAllowlist() { + return config.getJexlAllowlist(); + } + + @Override + public boolean isJexlAllowlistEnabled() { + return jexlAllowlistEnabled; } /** diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java index 21ac03ca1..15778f316 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/GrammarBasedParser.java @@ -84,7 +84,7 @@ public List parse() throws RecipeException { try { Directive directive = info.instance(); UsageDefinition definition = directive.define(); - Arguments arguments = new MapArguments(definition, tokenGroup); + Arguments arguments = new MapArgumentsWithContext(definition, tokenGroup, context); directive.initialize(arguments); result.add(directive); diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArgumentsWithContext.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArgumentsWithContext.java new file mode 100644 index 000000000..a301f26ae --- /dev/null +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/MapArgumentsWithContext.java @@ -0,0 +1,39 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.parser; + +import io.cdap.wrangler.api.DirectiveContext; +import io.cdap.wrangler.api.DirectiveParseException; +import io.cdap.wrangler.api.TokenGroup; +import io.cdap.wrangler.api.parser.UsageDefinition; + +/** + * A {@link MapArguments} that also holds a {@link DirectiveContext}. + */ +public class MapArgumentsWithContext extends MapArguments { + private final DirectiveContext context; + + public MapArgumentsWithContext(UsageDefinition definition, TokenGroup group, DirectiveContext context) + throws DirectiveParseException { + super(definition, group); + this.context = context; + } + + public DirectiveContext getDirectiveContext() { + return context; + } +} diff --git a/wrangler-core/src/main/java/io/cdap/wrangler/parser/NoOpDirectiveContext.java b/wrangler-core/src/main/java/io/cdap/wrangler/parser/NoOpDirectiveContext.java index 2db9b0b3f..daee2553f 100644 --- a/wrangler-core/src/main/java/io/cdap/wrangler/parser/NoOpDirectiveContext.java +++ b/wrangler-core/src/main/java/io/cdap/wrangler/parser/NoOpDirectiveContext.java @@ -17,6 +17,8 @@ package io.cdap.wrangler.parser; import io.cdap.wrangler.api.DirectiveContext; +import io.cdap.wrangler.api.JexlAllowlist; +import java.util.List; /** * This class {@link NoOpDirectiveContext} is a pass through implementation of @@ -24,6 +26,16 @@ */ public class NoOpDirectiveContext implements DirectiveContext { + @Override + public List getJexlAllowlist() { + return null; + } + + @Override + public boolean isJexlAllowlistEnabled() { + return false; + } + /** * Checks if the directive is aliased. * diff --git a/wrangler-core/src/test/java/io/cdap/wrangler/config/DirectiveConfigTest.java b/wrangler-core/src/test/java/io/cdap/wrangler/config/DirectiveConfigTest.java index 8d624412d..616bfe993 100644 --- a/wrangler-core/src/test/java/io/cdap/wrangler/config/DirectiveConfigTest.java +++ b/wrangler-core/src/test/java/io/cdap/wrangler/config/DirectiveConfigTest.java @@ -17,7 +17,11 @@ package io.cdap.wrangler.config; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import org.junit.Assert; import org.junit.Test; @@ -63,9 +67,14 @@ public class DirectiveConfigTest { private static final String EMPTY = "{}"; + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); + @Test public void testParsingOfConfiguration() { - DirectiveConfig config = new Gson().fromJson(SPECIFICATION, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(SPECIFICATION, DirectiveConfig.class); Assert.assertNotNull(config); Assert.assertTrue(config.isExcluded("parse-as-csv")); Assert.assertFalse(config.isExcluded("parse-as-json")); @@ -74,7 +83,7 @@ public void testParsingOfConfiguration() { @Test public void testParsingOnlyExclusions() { - DirectiveConfig config = new Gson().fromJson(ONLY_EXCLUSIONS, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(ONLY_EXCLUSIONS, DirectiveConfig.class); Assert.assertNotNull(config); Assert.assertTrue(config.isExcluded("parse-as-csv")); Assert.assertFalse(config.isExcluded("parse-as-json")); @@ -83,7 +92,7 @@ public void testParsingOnlyExclusions() { @Test public void testParsingOnlyAliases() { - DirectiveConfig config = new Gson().fromJson(ONLY_ALIASES, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(ONLY_ALIASES, DirectiveConfig.class); Assert.assertNotNull(config); Assert.assertFalse(config.isExcluded("parse-as-csv")); Assert.assertEquals("parse-as-json", config.getAliasName("json-parser")); @@ -91,7 +100,7 @@ public void testParsingOnlyAliases() { @Test public void testParsingEmpty() { - DirectiveConfig config = new Gson().fromJson(EMPTY, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(EMPTY, DirectiveConfig.class); Assert.assertNotNull(config); Assert.assertFalse(config.isExcluded("parse-as-csv")); Assert.assertNull(config.getAliasName("json-parser")); diff --git a/wrangler-core/src/test/java/io/cdap/wrangler/parser/ConfigDirectiveContextTest.java b/wrangler-core/src/test/java/io/cdap/wrangler/parser/ConfigDirectiveContextTest.java index f9bafae7c..27c6c2b4e 100644 --- a/wrangler-core/src/test/java/io/cdap/wrangler/parser/ConfigDirectiveContextTest.java +++ b/wrangler-core/src/test/java/io/cdap/wrangler/parser/ConfigDirectiveContextTest.java @@ -17,8 +17,12 @@ package io.cdap.wrangler.parser; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import io.cdap.wrangler.api.Directive; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.api.RecipeException; import io.cdap.wrangler.api.RecipeParser; import io.cdap.wrangler.proto.Contexts; @@ -51,18 +55,22 @@ public class ConfigDirectiveContextTest { private static final String EMPTY = "{}"; + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); + @Test(expected = RecipeException.class) public void testBasicExclude() throws Exception { String[] text = new String[] { "parse-as-csv body , true" }; - Gson gson = new Gson(); - DirectiveConfig config = gson.fromJson(CONFIG, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(CONFIG, DirectiveConfig.class); RecipeParser directives = new GrammarBasedParser(Contexts.SYSTEM, text, new CompositeDirectiveRegistry(SystemDirectiveRegistry.INSTANCE), - new ConfigDirectiveContext(config)); + new ConfigDirectiveContext(config, false)); directives.parse(); } @@ -72,12 +80,11 @@ public void testAliasedAndExcluded() throws Exception { "js-parser body" }; - Gson gson = new Gson(); - DirectiveConfig config = gson.fromJson(CONFIG, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(CONFIG, DirectiveConfig.class); RecipeParser directives = new GrammarBasedParser(Contexts.SYSTEM, text, new CompositeDirectiveRegistry(SystemDirectiveRegistry.INSTANCE), - new ConfigDirectiveContext(config)); + new ConfigDirectiveContext(config, false)); directives.parse(); } @@ -87,12 +94,11 @@ public void testAliasing() throws Exception { "json-parser :body;" }; - Gson gson = new Gson(); - DirectiveConfig config = gson.fromJson(CONFIG, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(CONFIG, DirectiveConfig.class); RecipeParser directives = new GrammarBasedParser(Contexts.SYSTEM, text, new CompositeDirectiveRegistry(SystemDirectiveRegistry.INSTANCE), - new ConfigDirectiveContext(config)); + new ConfigDirectiveContext(config, false)); List steps = directives.parse(); Assert.assertEquals(1, steps.size()); } @@ -103,12 +109,11 @@ public void testEmptyAliasingShouldFail() throws Exception { "json-parser :body;" }; - Gson gson = new Gson(); - DirectiveConfig config = gson.fromJson(EMPTY, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(EMPTY, DirectiveConfig.class); RecipeParser directives = new GrammarBasedParser(Contexts.SYSTEM, text, new CompositeDirectiveRegistry(SystemDirectiveRegistry.INSTANCE), - new ConfigDirectiveContext(config)); + new ConfigDirectiveContext(config, false)); List steps = directives.parse(); Assert.assertEquals(1, steps.size()); } @@ -119,12 +124,11 @@ public void testWithNoAliasingNoExclusion() throws Exception { "parse-as-json :body;" }; - Gson gson = new Gson(); - DirectiveConfig config = gson.fromJson(EMPTY, DirectiveConfig.class); + DirectiveConfig config = GSON.fromJson(EMPTY, DirectiveConfig.class); RecipeParser directives = new GrammarBasedParser(Contexts.SYSTEM, text, new CompositeDirectiveRegistry(SystemDirectiveRegistry.INSTANCE), - new ConfigDirectiveContext(config)); + new ConfigDirectiveContext(config, false)); List steps = directives.parse(); Assert.assertEquals(1, steps.size()); } diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/RequestExtractor.java b/wrangler-service/src/main/java/io/cdap/wrangler/RequestExtractor.java index 717cc5abf..e048e87f4 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/RequestExtractor.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/RequestExtractor.java @@ -20,6 +20,10 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; import io.cdap.cdap.api.service.http.HttpServiceRequest; +import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.dataset.workspace.RequestDeserializer; import io.cdap.wrangler.proto.BadRequestException; import io.cdap.wrangler.proto.Request; @@ -38,6 +42,8 @@ public final class RequestExtractor { private static final Gson GSON = new GsonBuilder() .registerTypeAdapter(Request.class, new RequestDeserializer()) + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) .create(); private final HttpServiceRequest request; public static final String CONTENT_TYPE_HEADER = PropertyIds.CONTENT_TYPE; diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/DataPrepService.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/DataPrepService.java index 2dbc86bf4..78789d312 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/DataPrepService.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/DataPrepService.java @@ -47,6 +47,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.concurrent.TimeUnit; /** @@ -92,12 +93,14 @@ protected void configure() { } @Override - public void initialize(SystemServiceContext context) { + public void initialize(SystemServiceContext context) throws IOException { // only do the upgrade on first instance to avoid transaction conflict if (context.getInstanceId() != 0) { return; } + new ConfigStore(context).initialize(); + UpgradeStore upgradeStore = new UpgradeStore(context); WorkspaceStore wsStore = new WorkspaceStore(context); UpgradeState connState = upgradeStore.getEntityUpgradeState(UpgradeEntityType.CONNECTION); diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java index 4debd5eee..82a70730f 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/AbstractDirectiveHandler.java @@ -25,6 +25,7 @@ import io.cdap.directives.aggregates.DefaultTransientStore; import io.cdap.wrangler.api.CompileException; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.ErrorRecordBase; import io.cdap.wrangler.api.ExecutorContext; @@ -90,6 +91,7 @@ public class AbstractDirectiveHandler extends AbstractWranglerHandler { protected DirectiveRegistry composite; protected boolean schemaManagementEnabled; + protected boolean jexlAllowlistEnabled; protected ConfigStore configStore; @Override @@ -101,6 +103,7 @@ public void initialize(SystemHttpServiceContext context) throws Exception { new UserDirectiveRegistry(context) ); schemaManagementEnabled = Feature.WRANGLER_SCHEMA_MANAGEMENT.isEnabled(context); + jexlAllowlistEnabled = Feature.WRANGLER_JEXL_ALLOWLIST.isEnabled(context); } /** @@ -132,15 +135,16 @@ protected List executeDirectives( // Parse and call grammar visitor DirectiveConfig config = getDirectiveConfig(); + DirectiveContext directiveContext = new ConfigDirectiveContext(config, jexlAllowlistEnabled); try { - GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), new ConfigDirectiveContext(config)); + GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), directiveContext); walker.walk(recipe, grammarVisitor); } catch (CompileException e) { throw new BadRequestException(e.getMessage(), e); } RecipeParser parser = new GrammarBasedParser(namespace, recipe, composite, - new ConfigDirectiveContext(config)); + directiveContext); try (RecipePipelineExecutor executor = new RecipePipelineExecutor(parser, new ServicePipelineContext( namespace, ExecutorContext.Environment.SERVICE, diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/DirectivesHandler.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/DirectivesHandler.java index 78ab70d3a..e6cfcc20d 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/DirectivesHandler.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/DirectivesHandler.java @@ -44,7 +44,10 @@ import io.cdap.wrangler.ServiceUtils; import io.cdap.wrangler.api.Directive; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; import io.cdap.wrangler.api.DirectiveParseException; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.api.Row; import io.cdap.wrangler.datamodel.DataModelGlossary; import io.cdap.wrangler.dataset.workspace.DataType; @@ -113,8 +116,11 @@ @Deprecated public class DirectivesHandler extends AbstractDirectiveHandler { private static final Logger LOG = LoggerFactory.getLogger(DirectivesHandler.class); - private static final Gson GSON = - new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create(); + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); private static final String DATA_MODEL_PROPERTY = "dataModel"; private static final String DATA_MODEL_REVISION_PROPERTY = "dataModelRevision"; @@ -129,9 +135,8 @@ public class DirectivesHandler extends AbstractDirectiveHandler { public void initialize(SystemHttpServiceContext context) throws Exception { super.initialize(context); composite = new CompositeDirectiveRegistry( - SystemDirectiveRegistry.INSTANCE, - new UserDirectiveRegistry(context) - ); + SystemDirectiveRegistry.INSTANCE, + new UserDirectiveRegistry(context)); contextAccessEnforcer = context.getContextAccessEnforcer(); isWorkspaceAuthEnforcementEnabled = Feature.WRANGLER_WORKSPACE_AUTH_CHECK.isEnabled(context); isDirectiveConfigAuthEnforcementEnabled = Feature.WRANGLER_DIRECTIVE_CONFIG_AUTH_CHECK.isEnabled(context); @@ -177,8 +182,8 @@ public void healthCheck(HttpServiceRequest request, HttpServiceResponder respond @Path("contexts/{context}/workspaces/{id}") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void create(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("context") String namespace, - @PathParam("id") String id, @QueryParam("name") String name, - @QueryParam("scope") @DefaultValue(WorkspaceDataset.DEFAULT_SCOPE) String scope) { + @PathParam("id") String id, @QueryParam("name") String name, + @QueryParam("scope") @DefaultValue(WorkspaceDataset.DEFAULT_SCOPE) String scope) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.CREATE); String workspaceName = name == null || name.isEmpty() ? id : name; @@ -187,9 +192,9 @@ public void create(HttpServiceRequest request, HttpServiceResponder responder, @ properties.put(PropertyIds.NAME, workspaceName); NamespacedId workspaceId = new NamespacedId(ns, id); WorkspaceMeta workspaceMeta = WorkspaceMeta.builder(workspaceName) - .setScope(scope) - .setProperties(properties) - .build(); + .setScope(scope) + .setProperties(properties) + .build(); TransactionRunners.run(getContext(), context -> { WorkspaceDataset ws = WorkspaceDataset.get(context); ws.writeWorkspaceMeta(workspaceId, workspaceMeta); @@ -222,7 +227,7 @@ public void create(HttpServiceRequest request, HttpServiceResponder responder, @ @Path("contexts/{context}/workspaces") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void list(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @QueryParam("scope") @DefaultValue("default") String scope) { + @PathParam("context") String namespace, @QueryParam("scope") @DefaultValue("default") String scope) { respond(request, responder, namespace, ns -> { enforceOnParentNamespace(ns.getName(), StandardPermission.LIST); List workspaces = TransactionRunners.run(getContext(), context -> { @@ -251,7 +256,7 @@ public void list(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/workspaces/{id}") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void delete(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.DELETE); TransactionRunners.run(getContext(), context -> { @@ -280,7 +285,7 @@ public void delete(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/workspaces/") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void deleteGroup(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @QueryParam("group") String group) { + @PathParam("context") String namespace, @QueryParam("group") String group) { respond(request, responder, namespace, ns -> { enforceNamespacePermission(ns.getName(), StandardPermission.DELETE); TransactionRunners.run(getContext(), context -> { @@ -324,7 +329,7 @@ public void deleteGroup(HttpServiceRequest request, HttpServiceResponder respond @Path("contexts/{context}/workspaces/{id}") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void get(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.GET); Workspace workspace = getWorkspace(new NamespacedId(ns, id)); @@ -355,12 +360,12 @@ public void get(HttpServiceRequest request, HttpServiceResponder responder, private JsonObject merge(JsonObject first, JsonObject second) { JsonObject merged = new JsonObject(); if (first != null && !first.isJsonNull()) { - for (Map.Entry entry: first.entrySet()) { + for (Map.Entry entry : first.entrySet()) { merged.add(entry.getKey(), entry.getValue()); } } if (second != null && !second.isJsonNull()) { - for (Map.Entry entry: second.entrySet()) { + for (Map.Entry entry : second.entrySet()) { merged.add(entry.getKey(), entry.getValue()); } } @@ -377,7 +382,7 @@ private JsonObject merge(JsonObject first, JsonObject second) { @Path("contexts/{context}/workspaces") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void upload(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace) { + @PathParam("context") String namespace) { respond(request, responder, namespace, ns -> { String name = request.getHeader(PropertyIds.FILE_NAME); if (name == null) { @@ -411,14 +416,14 @@ public void upload(HttpServiceRequest request, HttpServiceResponder responder, byte[] content = handler.getContent(); if (content == null) { throw new BadRequestException("Body not present, please post the file containing the " - + "records to be wrangled."); + + "records to be wrangled."); } // Depending on content type, load data. DataType type = DataType.fromString(contentType); if (type == null) { throw new BadRequestException("Invalid content type. Must be 'text/plain', 'application/octet-stream' " + - "or 'application/data-prep'"); + "or 'application/data-prep'"); } switch (type) { case TEXT: @@ -454,8 +459,8 @@ public void upload(HttpServiceRequest request, HttpServiceResponder responder, ws.updateWorkspaceProperties(id, properties); WorkspaceInfo workspaceInfo = new WorkspaceInfo(id.getId(), name, delimiter, charset, contentType, - ConnectionType.UPLOAD.getType(), - SamplingMethod.NONE.getMethod()); + ConnectionType.UPLOAD.getType(), + SamplingMethod.NONE.getMethod()); return new ServiceResponse<>(workspaceInfo); }); }); @@ -472,7 +477,7 @@ public void upload(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/workspaces/{id}/upload") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void uploadData(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.USE); RequestExtractor handler = new RequestExtractor(request); @@ -493,7 +498,7 @@ public void uploadData(HttpServiceRequest request, HttpServiceResponder responde DataType type = DataType.fromString(contentType); if (type == null) { throw new BadRequestException("Invalid content type. Must be 'text/plain', 'application/octet-stream' " + - "or 'application/data-prep'"); + "or 'application/data-prep'"); } NamespacedId namespaceId = new NamespacedId(ns, id); @@ -564,7 +569,7 @@ public void uploadData(HttpServiceRequest request, HttpServiceResponder responde @Path("contexts/{context}/workspaces/{id}/execute") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void execute(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.USE); composite.reload(namespace); @@ -588,8 +593,8 @@ public void execute(HttpServiceRequest request, HttpServiceResponder responder, }, userDirectivesCollector); userDirectivesCollector.addLoadDirectivesPragma(directives); - io.cdap.wrangler.proto.workspace.v2.DirectiveExecutionResponse response = - generateExecutionResponse(rows, directiveRequest.getWorkspace().getResults()); + io.cdap.wrangler.proto.workspace.v2.DirectiveExecutionResponse response = generateExecutionResponse(rows, + directiveRequest.getWorkspace().getResults()); // Save the recipes being executed. TransactionRunners.run(getContext(), context -> { @@ -598,7 +603,7 @@ public void execute(HttpServiceRequest request, HttpServiceResponder responder, }); return new DirectiveExecutionResponse(response.getValues(), response.getHeaders(), - response.getTypes(), directives); + response.getTypes(), directives); } catch (JsonParseException e) { throw new BadRequestException(e.getMessage(), e); } @@ -616,7 +621,7 @@ public void execute(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/workspaces/{id}/summary") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void summary(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.USE); try { @@ -647,7 +652,7 @@ public void summary(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/workspaces/{id}/schema") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void schema(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.USE); composite.reload(namespace); @@ -677,8 +682,8 @@ public void schema(HttpServiceRequest request, HttpServiceResponder responder, // the current contract with the UI is not to pass the // entire schema string, but just the fields. return new JsonParser().parse(schemaJson) - .getAsJsonObject() - .get("fields").getAsJsonArray(); + .getAsJsonObject() + .get("fields").getAsJsonArray(); }); } @@ -692,7 +697,7 @@ public void schema(HttpServiceRequest request, HttpServiceResponder responder, @POST @Path("contexts/{context}/workspaces/{id}/datamodels") public void addDataModel(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.UPDATE); NamespacedId namespacedId = new NamespacedId(ns, id); @@ -706,10 +711,10 @@ public void addDataModel(HttpServiceRequest request, HttpServiceResponder respon throw new BadRequestException("There is no data model initialized."); } org.apache.avro.Schema schema = DataModelGlossary.getGlossary() - .get(dataModelInfo.getId(), dataModelInfo.getRevision()); + .get(dataModelInfo.getId(), dataModelInfo.getRevision()); if (schema == null) { throw new BadRequestException(String.format("Unable to find data model %s revision %d", dataModelInfo.getId(), - dataModelInfo.getRevision())); + dataModelInfo.getRevision())); } Map properties = new HashMap<>(workspace.getProperties()); @@ -741,7 +746,7 @@ public void addDataModel(HttpServiceRequest request, HttpServiceResponder respon @Path("contexts/{context}/workspaces/{id}/datamodels") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void removeDataModel(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.UPDATE); NamespacedId namespacedId = new NamespacedId(ns, id); @@ -770,7 +775,7 @@ public void removeDataModel(HttpServiceRequest request, HttpServiceResponder res @POST @Path("contexts/{context}/workspaces/{id}/models") public void addModels(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id) { + @PathParam("context") String namespace, @PathParam("id") String id) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.UPDATE); NamespacedId namespacedId = new NamespacedId(ns, id); @@ -801,14 +806,14 @@ public void addModels(HttpServiceRequest request, HttpServiceResponder responder throw new BadRequestException("There is no data model initialized."); } org.apache.avro.Schema schema = DataModelGlossary.getGlossary() - .get(dataModelInfo.getId(), dataModelInfo.getRevision()); + .get(dataModelInfo.getId(), dataModelInfo.getRevision()); List fieldMatch = schema.getFields().stream() - .filter(field -> field.name().equals(model.getId())) - .collect(Collectors.toList()); + .filter(field -> field.name().equals(model.getId())) + .collect(Collectors.toList()); if (fieldMatch.isEmpty()) { throw new NotFoundException( - String.format("Unable to find model %s in data model %s revision %d.", model.getId(), dataModelInfo.getId(), - dataModelInfo.getRevision())); + String.format("Unable to find model %s in data model %s revision %d.", model.getId(), dataModelInfo.getId(), + dataModelInfo.getRevision())); } properties.put(DATA_MODEL_MODEL_PROPERTY, model.getId()); @@ -830,8 +835,8 @@ public void addModels(HttpServiceRequest request, HttpServiceResponder responder @Path("contexts/{context}/workspaces/{id}/models/{modelid}") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void removeModels(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace, @PathParam("id") String id, - @PathParam("modelid") String modelId) { + @PathParam("context") String namespace, @PathParam("id") String id, + @PathParam("modelid") String modelId) { respond(request, responder, namespace, ns -> { enforceWorkspacePermission(ns.getName(), id, StandardPermission.UPDATE); NamespacedId namespacedId = new NamespacedId(ns, id); @@ -893,7 +898,7 @@ public void capabilities(HttpServiceRequest request, HttpServiceResponder respon @Path("contexts/{context}/usage") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void usage(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace) { + @PathParam("context") String namespace) { respond(request, responder, namespace, ns -> { enforceNamespacePermission(namespace, StandardPermission.USE); // CDAP-15397 - reload must be called before it can be safely used @@ -904,9 +909,9 @@ public void usage(HttpServiceRequest request, HttpServiceResponder responder, for (DirectiveInfo directive : composite.list(namespace)) { DirectiveUsage directiveUsage = new DirectiveUsage(directive.name(), directive.usage(), directive.description(), - config.isExcluded(directive.name()), false, - directive.scope().name(), directive.definition(), - directive.categories()); + config.isExcluded(directive.name()), false, + directive.scope().name(), directive.definition(), + directive.categories()); values.add(directiveUsage); // For this directive we find all aliases and add them to the @@ -915,9 +920,9 @@ public void usage(HttpServiceRequest request, HttpServiceResponder responder, List list = aliases.get(directive.name()); for (String alias : list) { directiveUsage = new DirectiveUsage(alias, directive.usage(), directive.description(), - config.isExcluded(directive.name()), true, - directive.scope().name(), directive.definition(), - directive.categories()); + config.isExcluded(directive.name()), true, + directive.scope().name(), directive.definition(), + directive.categories()); values.add(directiveUsage); } } @@ -936,7 +941,7 @@ public void usage(HttpServiceRequest request, HttpServiceResponder responder, @Path("contexts/{context}/artifacts") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void artifacts(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace) { + @PathParam("context") String namespace) { respond(request, responder, namespace, ns -> { enforceOnParentNamespace(ns.getName(), StandardPermission.LIST); List values = new ArrayList<>(); @@ -953,6 +958,7 @@ public void artifacts(HttpServiceRequest request, HttpServiceResponder responder return new ServiceResponse<>(values); }); } + /** * This HTTP endpoint is used to retrieve plugins that are * of type Directive.Type (directive). Artifact will be reported @@ -964,15 +970,15 @@ public void artifacts(HttpServiceRequest request, HttpServiceResponder responder @Path("contexts/{context}/directives") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void directives(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace) { + @PathParam("context") String namespace) { respond(request, responder, namespace, ns -> { enforceOnParentNamespace(ns.getName(), StandardPermission.LIST); List values = new ArrayList<>(); List artifacts = getContext().listArtifacts(namespace); for (ArtifactInfo artifact : artifacts) { Set plugins = artifact.getClasses().getPlugins(); - DirectiveArtifact directiveArtifact = - new DirectiveArtifact(artifact.getName(), artifact.getVersion(), artifact.getScope().name()); + DirectiveArtifact directiveArtifact = new DirectiveArtifact(artifact.getName(), artifact.getVersion(), + artifact.getScope().name()); for (PluginClass plugin : plugins) { if (Directive.TYPE.equalsIgnoreCase(plugin.getType())) { values.add(new DirectiveDescriptor(plugin, directiveArtifact)); @@ -992,7 +998,7 @@ public void directives(HttpServiceRequest request, HttpServiceResponder responde @Path("contexts/{context}/directives/reload") @TransactionPolicy(value = TransactionControl.EXPLICIT) public void directivesReload(HttpServiceRequest request, HttpServiceResponder responder, - @PathParam("context") String namespace) { + @PathParam("context") String namespace) { respond(request, responder, namespace, ns -> { enforceNamespacePermission(ns.getName(), StandardPermission.USE); composite.reload(namespace); @@ -1034,6 +1040,22 @@ public void uploadConfig(HttpServiceRequest request, HttpServiceResponder respon if (config == null) { throw new BadRequestException("Config is empty. Please check if the request is sent as HTTP POST body."); } + + List userAllowlist = config.getJexlAllowlist(); + if (jexlAllowlistEnabled && userAllowlist == null) { + throw new BadRequestException("JEXL allowlist cannot be null. Please provide a valid JEXL allowlist."); + } + if (!jexlAllowlistEnabled && userAllowlist != null) { + throw new BadRequestException( + "Updates to JEXL allowlist in the wrangler directives config is disabled."); + } + // No updates to JexlAllowlist when JEXL Allowlist feature is disabled + if (!jexlAllowlistEnabled) { + DirectiveConfig existingConfig = configStore.getConfig(); + List existingAllowlist = existingConfig != null ? existingConfig.getJexlAllowlist() : null; + config = new DirectiveConfig(config.getExclusions(), config.getAliases(), existingAllowlist); + } + configStore.updateConfig(config); return new ServiceResponse("Successfully updated configuration."); }); @@ -1050,8 +1072,14 @@ public void uploadConfig(HttpServiceRequest request, HttpServiceResponder respon @TransactionPolicy(value = TransactionControl.EXPLICIT) public void getConfig(HttpServiceRequest request, HttpServiceResponder responder) { respond(request, responder, () -> { + DirectiveConfig config = configStore.getConfig(); + if (!jexlAllowlistEnabled) { + JsonObject configJson = config.toJson().getAsJsonObject(); + configJson.remove(DirectiveConfig.JEXL_ALLOWLIST_KEY); + return new ServiceResponse<>(configJson); + } enforceDirectiveConfigPermission(StandardPermission.GET); - return new ServiceResponse<>(configStore.getConfig()); + return new ServiceResponse<>(config); }); } @@ -1065,7 +1093,7 @@ public static List fromWorkspace(Workspace workspace) throws IOException, C DataType type = workspace.getType(); List rows = new ArrayList<>(); - switch(type) { + switch (type) { case TEXT: { String data = Bytes.toString(workspace.getData()); if (data != null) { @@ -1103,8 +1131,9 @@ public static List fromWorkspace(Workspace workspace) throws IOException, C * @return records generated from the directives. */ private List executeDirectives(NamespacedId id, List directives, - Function, List> sample) { - return executeDirectives(id, directives, sample, (a, b) -> { }); + Function, List> sample) { + return executeDirectives(id, directives, sample, (a, b) -> { + }); } /** @@ -1117,8 +1146,8 @@ private List executeDirectives(NamespacedId id, List directives, * @return records generated from the directives. */ private List executeDirectives(NamespacedId id, List directives, - Function, List> sample, - Visitor grammarVisitor) { + Function, List> sample, + Visitor grammarVisitor) { return TransactionRunners.run(getContext(), ctx -> { WorkspaceDataset ws = WorkspaceDataset.get(ctx); @@ -1126,7 +1155,7 @@ private List executeDirectives(NamespacedId id, List< // Extract rows from the workspace. List rows = fromWorkspace(workspace); return executeDirectives(id.getNamespace().getName(), directives, sample.apply(rows), - grammarVisitor); + grammarVisitor); }); } diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java index 65a0ebb12..d330354c8 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/RemoteExecutionTask.java @@ -28,10 +28,14 @@ import io.cdap.wrangler.api.CompileException; import io.cdap.wrangler.api.Directive; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveLoadException; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.ErrorRecordBase; import io.cdap.wrangler.api.ExecutorContext; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.api.RecipeException; import io.cdap.wrangler.api.RemoteDirectiveResponse; import io.cdap.wrangler.api.Row; @@ -43,7 +47,7 @@ import io.cdap.wrangler.parser.ConfigDirectiveContext; import io.cdap.wrangler.parser.DirectiveClass; import io.cdap.wrangler.parser.GrammarWalker; -import io.cdap.wrangler.parser.MapArguments; +import io.cdap.wrangler.parser.MapArgumentsWithContext; import io.cdap.wrangler.parser.RecipeCompiler; import io.cdap.wrangler.proto.BadRequestException; import io.cdap.wrangler.proto.ErrorRecordsException; @@ -67,6 +71,8 @@ public class RemoteExecutionTask implements RunnableTask { private static final Gson GSON = new GsonBuilder() .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) .create(); @Override @@ -83,7 +89,10 @@ public void run(RunnableTaskContext runnableTaskContext) throws Exception { try (UserDirectiveRegistry userDirectiveRegistry = new UserDirectiveRegistry(systemAppContext)) { List directives = new ArrayList<>(); DirectiveConfig config = directiveRequest.getDirectiveConfig(); - GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), new ConfigDirectiveContext(config)); + boolean jexlAllowlistEnabled = systemAppContext != null + && Feature.WRANGLER_JEXL_ALLOWLIST.isEnabled(systemAppContext); + DirectiveContext directiveContext = new ConfigDirectiveContext(config, jexlAllowlistEnabled); + GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), directiveContext); walker.walk(directiveRequest.getRecipe(), (command, tokenGroup) -> { DirectiveInfo info; DirectiveClass directiveClass = systemDirectives.get(command); @@ -101,7 +110,7 @@ public void run(RunnableTaskContext runnableTaskContext) throws Exception { Directive directive = info.instance(); UsageDefinition definition = directive.define(); - Arguments arguments = new MapArguments(definition, tokenGroup); + Arguments arguments = new MapArgumentsWithContext(definition, tokenGroup, directiveContext); directive.initialize(arguments); directives.add(directive); }); diff --git a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/WorkspaceHandler.java b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/WorkspaceHandler.java index 564cda657..c1b9845d3 100644 --- a/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/WorkspaceHandler.java +++ b/wrangler-service/src/main/java/io/cdap/wrangler/service/directive/WorkspaceHandler.java @@ -660,7 +660,8 @@ private List executeRemotely(String namespace, List { DirectiveInfo info = SystemDirectiveRegistry.INSTANCE.get(command); diff --git a/wrangler-storage/src/main/java/io/cdap/wrangler/dataset/workspace/ConfigStore.java b/wrangler-storage/src/main/java/io/cdap/wrangler/dataset/workspace/ConfigStore.java index 30efc206e..bd31c1050 100644 --- a/wrangler-storage/src/main/java/io/cdap/wrangler/dataset/workspace/ConfigStore.java +++ b/wrangler-storage/src/main/java/io/cdap/wrangler/dataset/workspace/ConfigStore.java @@ -17,6 +17,8 @@ package io.cdap.wrangler.dataset.workspace; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; import io.cdap.cdap.spi.data.StructuredRow; import io.cdap.cdap.spi.data.StructuredTable; import io.cdap.cdap.spi.data.table.StructuredTableId; @@ -26,7 +28,13 @@ import io.cdap.cdap.spi.data.table.field.Fields; import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; +import io.cdap.wrangler.api.DefaultJexlAllowlist; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -44,7 +52,11 @@ */ @Deprecated public class ConfigStore { - private static final Gson GSON = new Gson(); + private static final Logger LOG = LoggerFactory.getLogger(ConfigStore.class); + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .create(); private static final String KEY_COL = "key"; private static final String VAL_COL = "value"; private static final Field keyField = Fields.stringField(KEY_COL, "directives"); @@ -60,6 +72,19 @@ public ConfigStore(TransactionRunner transactionRunner) { this.transactionRunner = transactionRunner; } + // This is one time bootstrap of ConfigStore to initialize default directive config + public void initialize() throws IOException { + DirectiveConfig config = getConfig(); + + if (config == null) { + LOG.info("Initializing Directive config with default values"); + updateConfig(new DirectiveConfig(null, null, DefaultJexlAllowlist.get())); + } else if (config.getJexlAllowlist() == null) { + LOG.info("Directive config is configured without JEXL allowlist, adding default JEXL allowlist."); + updateConfig(new DirectiveConfig(config.getExclusions(), config.getAliases(), DefaultJexlAllowlist.get())); + } + } + public void updateConfig(DirectiveConfig config) throws IOException { TransactionRunners.run(transactionRunner, context -> { StructuredTable table = context.getTable(TABLE_ID); @@ -74,8 +99,9 @@ public DirectiveConfig getConfig() throws IOException { return TransactionRunners.run(transactionRunner, context -> { StructuredTable table = context.getTable(TABLE_ID); Optional row = table.read(Collections.singletonList(keyField)); - String configStr = row.map(r -> r.getString(VAL_COL)).orElse("{}"); - return GSON.fromJson(configStr, DirectiveConfig.class); + return row.map(r -> r.getString(VAL_COL)) + .map(str -> GSON.fromJson(str, DirectiveConfig.class)) + .orElse(null); }, IOException.class); } } diff --git a/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java b/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java index b2d03c46a..4656c72ef 100644 --- a/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java +++ b/wrangler-transform/src/main/java/io/cdap/wrangler/Wrangler.java @@ -20,6 +20,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonSyntaxException; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -55,12 +56,15 @@ import io.cdap.wrangler.api.Compiler; import io.cdap.wrangler.api.Directive; import io.cdap.wrangler.api.DirectiveConfig; +import io.cdap.wrangler.api.DirectiveConfigDeserializer; import io.cdap.wrangler.api.DirectiveContext; import io.cdap.wrangler.api.DirectiveLoadException; import io.cdap.wrangler.api.DirectiveParseException; import io.cdap.wrangler.api.EntityCountMetric; import io.cdap.wrangler.api.ErrorRecord; import io.cdap.wrangler.api.ExecutorContext; +import io.cdap.wrangler.api.JexlAllowlist; +import io.cdap.wrangler.api.JexlAllowlistDeserializer; import io.cdap.wrangler.api.RecipeParser; import io.cdap.wrangler.api.RecipePipeline; import io.cdap.wrangler.api.RecipeSymbol; @@ -114,8 +118,10 @@ @Description("Wrangler - A interactive tool for data cleansing and transformation.") public class Wrangler extends Transform implements LinearRelationalTransform { private static final Logger LOG = LoggerFactory.getLogger(Wrangler.class); - private static final Gson GSON = new Gson(); - + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(DirectiveConfig.class, new DirectiveConfigDeserializer()) + .registerTypeAdapter(JexlAllowlist.class, new JexlAllowlistDeserializer()) + .create(); private static final String ON_ERROR_DEFAULT = "fail-pipeline"; private static final String ON_ERROR_FAIL_PIPELINE = "fail-pipeline"; private static final String ON_ERROR_PROCEED = "send-to-error-port"; @@ -240,7 +246,7 @@ public void configurePipeline(PipelineConfigurer configurer) { // service/arguments are not yet available. DirectiveConfig.EMPTY is used for // compile-time // grammar validation. - DirectiveContext directiveContext = new ConfigDirectiveContext(DirectiveConfig.EMPTY); + DirectiveContext directiveContext = new ConfigDirectiveContext(DirectiveConfig.EMPTY, false); GrammarWalker walker = new GrammarWalker(new RecipeCompiler(), directiveContext); walker.walk(new MigrateToV2(directives).migrate(), (command, tokenGroup) -> { DirectiveInfo directiveInfo = registry.get("", command); @@ -624,7 +630,8 @@ private RecipeParser getRecipeParser(StageContext context) { } DirectiveConfig directiveConfig = getSystemDirectiveConfigFromRuntimeArgs(context); - DirectiveContext directiveContext = new ConfigDirectiveContext(directiveConfig); + boolean jexlAllowlistEnabled = context != null && Feature.WRANGLER_JEXL_ALLOWLIST.isEnabled(context); + DirectiveContext directiveContext = new ConfigDirectiveContext(directiveConfig, jexlAllowlistEnabled); try { return new GrammarBasedParser(context.getNamespace(), new MigrateToV2(directives).migrate(),