[ML] Convert encodeIds flag in TreeEncoder to configuration property

pom.xml

@@ -1108,6 +1108,7 @@
                         <exclude>**/.rat-excludes</exclude>
                         <exclude>**/*.csv</exclude>
                         <exclude>**/*.properties</exclude>
+                        <exclude>**/*.onnx</exclude>
                     </excludes>
                 </configuration>
             </plugin>

wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/Configuration.java

@@ -726,6 +726,12 @@ public void setProperty(String key, String value) {
         this.properties.set(key, value);
     }
 
+    public void removeProperty(String key) {
+        if (this.properties != null) {
+            this.properties.remove(key);
+        }
+    }
+
     public String getStringProperty(String key) {
         return this.properties.provideFor(key);
     }
@@ -738,6 +744,14 @@ public String getStringProperty(String key, String fallback) {
         return this.getOptionalStringProperty(key).orElse(fallback);
     }
 
+    public boolean isPropertyExplicitLocally(String key) {
+        return this.properties != null && this.properties.isExplicitLocally(key);
+    }
+
+    public String getExplicitLocalStringProperty(String key) {
+        return this.properties != null ? this.properties.provideExplicitLocally(key) : null;
+    }
+
     public KeyValueProvider<Platform, LoadProfileToTimeConverter> getLoadProfileToTimeConverterProvider() {
         return this.loadProfileToTimeConverterProvider;
     }

wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/KeyValueProvider.java

@@ -46,7 +46,7 @@ public NoSuchKeyException(String message) {
     private String warningSlf4jFormat;
 
     protected KeyValueProvider(KeyValueProvider<Key, Value> parent) {
-        this(parent, parent.configuration);
+        this(parent, parent != null ? parent.configuration : null);
     }
 
     protected KeyValueProvider(KeyValueProvider<Key, Value> parent, Configuration configuration) {
@@ -105,6 +105,28 @@ public Value provideLocally(Key key) {
         return this.tryToProvide(key, this);
     }
 
+    /**
+     * Checks if this provider explicitly defines the value for the given key (as opposed to caching
+     * an inherited parent value or falling back).
+     *
+     * @param key the key
+     * @return {@code true} if this provider explicitly defines the key
+     */
+    public boolean isExplicitLocally(Key key) {
+        return false;
+    }
+
+    /**
+     * Provide the value associated to the given key only if it was explicitly defined on this very instance,
+     * ignoring any cached inherited values.
+     *
+     * @param key the key
+     * @return the value or {@code null} if this instance does not explicitly associate a value with the key
+     */
+    public Value provideExplicitLocally(Key key) {
+        return this.isExplicitLocally(key) ? this.provideLocally(key) : null;
+    }
+
     /**
      * React to a value that has been delivered by a {@link #parent}. Does nothing be default.
      *
@@ -134,6 +156,10 @@ public void set(Key key, Value value) {
         throw new WayangException(String.format("Setting values not supported for %s.", this.getClass().getSimpleName()));
     }
 
+    public void remove(Key key) {
+        throw new WayangException(String.format("Removing values not supported for %s.", this.getClass().getSimpleName()));
+    }
+
     public Configuration getConfiguration() {
         return this.configuration;
     }

wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProvider.java

@@ -21,8 +21,10 @@
 import org.apache.wayang.core.api.Configuration;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * Implementation of {@link KeyValueProvider} that uses a {@link Map} to provide a value.
@@ -31,6 +33,8 @@ public class MapBasedKeyValueProvider<Key, Value> extends KeyValueProvider<Key,
 
     private final Map<Key, Value> storedValues = new HashMap<>();
 
+    private final Set<Key> explicitKeys = new HashSet<>();
+
     private final boolean isCaching;
 
     /**
@@ -94,6 +98,18 @@ protected void processParentEntry(Key key, Value value) {
     public void set(Key key, Value value) {
         Objects.requireNonNull(key);
         this.storedValues.put(key, value);
+        this.explicitKeys.add(key);
+    }
+
+    @Override
+    public boolean isExplicitLocally(Key key) {
+        return this.explicitKeys.contains(key);
     }
 
+    @Override
+    public void remove(Key key) {
+        Objects.requireNonNull(key);
+        this.storedValues.remove(key);
+        this.explicitKeys.remove(key);
+    }
 }

wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProviderTest.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.wayang.core.api.configuration;
+
+import org.apache.wayang.core.api.Configuration;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link MapBasedKeyValueProvider} explicit tracking and caching behavior.
+ */
+public class MapBasedKeyValueProviderTest {
+
+    @Test
+    public void testExplicitSetAndProvide() {
+        final MapBasedKeyValueProvider<String, String> provider = new MapBasedKeyValueProvider<>((KeyValueProvider<String, String>) null, false);
+        Assertions.assertFalse(provider.isExplicitLocally("k1"));
+        Assertions.assertNull(provider.provideExplicitLocally("k1"));
+        Assertions.assertNull(provider.provideLocally("k1"));
+
+        provider.set("k1", "v1");
+        Assertions.assertTrue(provider.isExplicitLocally("k1"));
+        Assertions.assertEquals("v1", provider.provideExplicitLocally("k1"));
+        Assertions.assertEquals("v1", provider.provideLocally("k1"));
+    }
+
+    @Test
+    public void testCachingDoesNotMarkKeyAsExplicit() {
+        final MapBasedKeyValueProvider<String, String> parent = new MapBasedKeyValueProvider<>((KeyValueProvider<String, String>) null, false);
+        parent.set("inheritedKey", "parentVal");
+
+        final MapBasedKeyValueProvider<String, String> child = new MapBasedKeyValueProvider<>(parent, true);
+
+        // Before read: not explicit and not present locally
+        Assertions.assertFalse(child.isExplicitLocally("inheritedKey"));
+        Assertions.assertNull(child.provideExplicitLocally("inheritedKey"));
+        Assertions.assertNull(child.provideLocally("inheritedKey"));
+
+        // Normal read causes caching into child
+        final String resolved = child.provideFor("inheritedKey");
+        Assertions.assertEquals("parentVal", resolved);
+
+        // After caching: provideLocally returns cached value, but isExplicitLocally must remain FALSE
+        Assertions.assertEquals("parentVal", child.provideLocally("inheritedKey"));
+        Assertions.assertFalse(child.isExplicitLocally("inheritedKey"));
+        Assertions.assertNull(child.provideExplicitLocally("inheritedKey"));
+    }
+
+    @Test
+    public void testExplicitOverrideAfterCachedRead() {
+        final MapBasedKeyValueProvider<String, String> parent = new MapBasedKeyValueProvider<>((KeyValueProvider<String, String>) null, false);
+        parent.set("key", "parentVal");
+
+        final MapBasedKeyValueProvider<String, String> child = new MapBasedKeyValueProvider<>(parent, true);
+        child.provideFor("key"); // cache parentVal locally
+
+        Assertions.assertFalse(child.isExplicitLocally("key"));
+        Assertions.assertNull(child.provideExplicitLocally("key"));
+
+        // Now explicitly set child
+        child.set("key", "childVal");
+        Assertions.assertTrue(child.isExplicitLocally("key"));
+        Assertions.assertEquals("childVal", child.provideExplicitLocally("key"));
+        Assertions.assertEquals("childVal", child.provideLocally("key"));
+    }
+
+    @Test
+    public void testConfigurationExplicitPropertyTracking() {
+        final Configuration parent = new Configuration();
+        parent.setProperty("foo", "bar");
+
+        final Configuration child = parent.fork();
+        Assertions.assertFalse(child.isPropertyExplicitLocally("foo"));
+        Assertions.assertNull(child.getExplicitLocalStringProperty("foo"));
+
+        // Query property through child (triggers caching in MapBasedKeyValueProvider)
+        Assertions.assertEquals("bar", child.getStringProperty("foo"));
+
+        // Caching must not mark property as explicit locally
+        Assertions.assertFalse(child.isPropertyExplicitLocally("foo"));
+        Assertions.assertNull(child.getExplicitLocalStringProperty("foo"));
+
+        // Explicitly setting property on child makes it explicit
+        child.setProperty("foo", "baz");
+        Assertions.assertTrue(child.isPropertyExplicitLocally("foo"));
+        Assertions.assertEquals("baz", child.getExplicitLocalStringProperty("foo"));
+    }
+
+    @Test
+    public void testRemoveProperty() {
+        final Configuration parent = new Configuration();
+        parent.setProperty("foo", "parentVal");
+
+        final Configuration child = parent.fork();
+        child.setProperty("foo", "childVal");
+        Assertions.assertTrue(child.isPropertyExplicitLocally("foo"));
+        Assertions.assertEquals("childVal", child.getStringProperty("foo"));
+
+        child.removeProperty("foo");
+        Assertions.assertFalse(child.isPropertyExplicitLocally("foo"));
+        Assertions.assertEquals("parentVal", child.getStringProperty("foo"));
+    }
+}

wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java

@@ -63,8 +63,8 @@ public void execute(final WayangPlan wayangPlan, final String... udfJars) {
             final Optional<String> originalOption = config.getOptionalStringProperty("wayang.ml.experience.original");
 
             final OneHotMappings mappings = new OneHotMappings();
-            final TreeEncoder encoder = new TreeEncoder(mappings);
-            final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext(), false).toString());
+            final TreeEncoder encoder = new TreeEncoder(mappings, config);
+            final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext()).toString());
 
             final Optional<String> choicesOption = config
                     .getOptionalStringProperty("wayang.ml.experience.with-platforms");

wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java

@@ -18,6 +18,7 @@
 
 package org.apache.wayang.ml;
 
+import java.io.InputStream;
 import java.util.Collection;
 import java.util.Collections;
 
@@ -26,14 +27,18 @@
 import org.apache.wayang.core.optimizer.channels.ChannelConversion;
 import org.apache.wayang.core.platform.Platform;
 import org.apache.wayang.core.plugin.Plugin;
+import org.apache.wayang.core.util.ReflectionUtils;
 import org.apache.wayang.java.platform.JavaPlatform;
+import org.apache.wayang.ml.encoding.TreeEncoder;
 import org.apache.wayang.spark.platform.SparkPlatform;
 
 /**
  * Provides {@link Plugin}s that enable usage of the xxxx.
  */
 public class MachineLearning {
 
+    private static final String DEFAULT_CONFIG_FILE = "wayang-ml-defaults.properties";
+
     /**
      * Enables use with the {@link JavaPlatform} and {@link SparkPlatform}.
      */
@@ -56,9 +61,42 @@ public Collection<ChannelConversion> getChannelConversions() {
 
         @Override
         public void setProperties(final Configuration configuration) {
+            try (final InputStream is = ReflectionUtils.loadResource(DEFAULT_CONFIG_FILE)) {
+                if (is != null) {
+                    final java.util.Properties properties = new java.util.Properties();
+                    properties.load(is);
+                    for (final java.util.Map.Entry<Object, Object> propertyEntry : properties.entrySet()) {
+                        final String key = propertyEntry.getKey().toString();
+                        final String value = propertyEntry.getValue().toString();
+                        if (isConfigured(configuration, key)) {
+                            continue;
+                        }
+                        configuration.setProperty(key, value);
+                    }
+                }
+            } catch (java.io.IOException e) {
+                throw new org.apache.wayang.core.api.exception.WayangException("Could not load ML default configuration.", e);
+            }
         }
     };
 
+    private static boolean isConfigured(final Configuration configuration, final String key) {
+        if (TreeEncoder.ENCODE_IDS_PROPERTY.equals(key) || TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY.equals(key)) {
+            return isExplicitlyConfigured(configuration, TreeEncoder.ENCODE_IDS_PROPERTY)
+                    || isExplicitlyConfigured(configuration, TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY);
+        }
+        return isExplicitlyConfigured(configuration, key);
+    }
+
+    private static boolean isExplicitlyConfigured(final Configuration configuration, final String key) {
+        for (Configuration current = configuration; current != null; current = current.getParent()) {
+            if (current.getProperties() != null && current.getProperties().provideExplicitLocally(key) != null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Retrieve a {@link Plugin} to use xxx on the
      *