diff --git a/pom.xml b/pom.xml index 4d08cc103..24a767a03 100644 --- a/pom.xml +++ b/pom.xml @@ -1108,6 +1108,7 @@ **/.rat-excludes **/*.csv **/*.properties + **/*.onnx diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/Configuration.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/Configuration.java index 9431343b2..a9f41ac35 100644 --- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/Configuration.java +++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/Configuration.java @@ -726,6 +726,12 @@ public void setProperty(String key, String value) { this.properties.set(key, value); } + public void removeProperty(String key) { + if (this.properties != null) { + this.properties.remove(key); + } + } + public String getStringProperty(String key) { return this.properties.provideFor(key); } @@ -738,6 +744,14 @@ public String getStringProperty(String key, String fallback) { return this.getOptionalStringProperty(key).orElse(fallback); } + public boolean isPropertyExplicitLocally(String key) { + return this.properties != null && this.properties.isExplicitLocally(key); + } + + public String getExplicitLocalStringProperty(String key) { + return this.properties != null ? this.properties.provideExplicitLocally(key) : null; + } + public KeyValueProvider getLoadProfileToTimeConverterProvider() { return this.loadProfileToTimeConverterProvider; } diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/KeyValueProvider.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/KeyValueProvider.java index 512d14ca0..5aaecc61c 100644 --- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/KeyValueProvider.java +++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/KeyValueProvider.java @@ -46,7 +46,7 @@ public NoSuchKeyException(String message) { private String warningSlf4jFormat; protected KeyValueProvider(KeyValueProvider parent) { - this(parent, parent.configuration); + this(parent, parent != null ? parent.configuration : null); } protected KeyValueProvider(KeyValueProvider parent, Configuration configuration) { @@ -105,6 +105,28 @@ public Value provideLocally(Key key) { return this.tryToProvide(key, this); } + /** + * Checks if this provider explicitly defines the value for the given key (as opposed to caching + * an inherited parent value or falling back). + * + * @param key the key + * @return {@code true} if this provider explicitly defines the key + */ + public boolean isExplicitLocally(Key key) { + return false; + } + + /** + * Provide the value associated to the given key only if it was explicitly defined on this very instance, + * ignoring any cached inherited values. + * + * @param key the key + * @return the value or {@code null} if this instance does not explicitly associate a value with the key + */ + public Value provideExplicitLocally(Key key) { + return this.isExplicitLocally(key) ? this.provideLocally(key) : null; + } + /** * React to a value that has been delivered by a {@link #parent}. Does nothing be default. * @@ -134,6 +156,10 @@ public void set(Key key, Value value) { throw new WayangException(String.format("Setting values not supported for %s.", this.getClass().getSimpleName())); } + public void remove(Key key) { + throw new WayangException(String.format("Removing values not supported for %s.", this.getClass().getSimpleName())); + } + public Configuration getConfiguration() { return this.configuration; } diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProvider.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProvider.java index 6ee70e04e..abe128d0f 100644 --- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProvider.java +++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProvider.java @@ -21,8 +21,10 @@ import org.apache.wayang.core.api.Configuration; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; +import java.util.Set; /** * Implementation of {@link KeyValueProvider} that uses a {@link Map} to provide a value. @@ -31,6 +33,8 @@ public class MapBasedKeyValueProvider extends KeyValueProvider storedValues = new HashMap<>(); + private final Set explicitKeys = new HashSet<>(); + private final boolean isCaching; /** @@ -94,6 +98,18 @@ protected void processParentEntry(Key key, Value value) { public void set(Key key, Value value) { Objects.requireNonNull(key); this.storedValues.put(key, value); + this.explicitKeys.add(key); + } + + @Override + public boolean isExplicitLocally(Key key) { + return this.explicitKeys.contains(key); } + @Override + public void remove(Key key) { + Objects.requireNonNull(key); + this.storedValues.remove(key); + this.explicitKeys.remove(key); + } } diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProviderTest.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProviderTest.java new file mode 100644 index 000000000..07e422afe --- /dev/null +++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/api/configuration/MapBasedKeyValueProviderTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.wayang.core.api.configuration; + +import org.apache.wayang.core.api.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link MapBasedKeyValueProvider} explicit tracking and caching behavior. + */ +public class MapBasedKeyValueProviderTest { + + @Test + public void testExplicitSetAndProvide() { + final MapBasedKeyValueProvider provider = new MapBasedKeyValueProvider<>((KeyValueProvider) null, false); + Assertions.assertFalse(provider.isExplicitLocally("k1")); + Assertions.assertNull(provider.provideExplicitLocally("k1")); + Assertions.assertNull(provider.provideLocally("k1")); + + provider.set("k1", "v1"); + Assertions.assertTrue(provider.isExplicitLocally("k1")); + Assertions.assertEquals("v1", provider.provideExplicitLocally("k1")); + Assertions.assertEquals("v1", provider.provideLocally("k1")); + } + + @Test + public void testCachingDoesNotMarkKeyAsExplicit() { + final MapBasedKeyValueProvider parent = new MapBasedKeyValueProvider<>((KeyValueProvider) null, false); + parent.set("inheritedKey", "parentVal"); + + final MapBasedKeyValueProvider child = new MapBasedKeyValueProvider<>(parent, true); + + // Before read: not explicit and not present locally + Assertions.assertFalse(child.isExplicitLocally("inheritedKey")); + Assertions.assertNull(child.provideExplicitLocally("inheritedKey")); + Assertions.assertNull(child.provideLocally("inheritedKey")); + + // Normal read causes caching into child + final String resolved = child.provideFor("inheritedKey"); + Assertions.assertEquals("parentVal", resolved); + + // After caching: provideLocally returns cached value, but isExplicitLocally must remain FALSE + Assertions.assertEquals("parentVal", child.provideLocally("inheritedKey")); + Assertions.assertFalse(child.isExplicitLocally("inheritedKey")); + Assertions.assertNull(child.provideExplicitLocally("inheritedKey")); + } + + @Test + public void testExplicitOverrideAfterCachedRead() { + final MapBasedKeyValueProvider parent = new MapBasedKeyValueProvider<>((KeyValueProvider) null, false); + parent.set("key", "parentVal"); + + final MapBasedKeyValueProvider child = new MapBasedKeyValueProvider<>(parent, true); + child.provideFor("key"); // cache parentVal locally + + Assertions.assertFalse(child.isExplicitLocally("key")); + Assertions.assertNull(child.provideExplicitLocally("key")); + + // Now explicitly set child + child.set("key", "childVal"); + Assertions.assertTrue(child.isExplicitLocally("key")); + Assertions.assertEquals("childVal", child.provideExplicitLocally("key")); + Assertions.assertEquals("childVal", child.provideLocally("key")); + } + + @Test + public void testConfigurationExplicitPropertyTracking() { + final Configuration parent = new Configuration(); + parent.setProperty("foo", "bar"); + + final Configuration child = parent.fork(); + Assertions.assertFalse(child.isPropertyExplicitLocally("foo")); + Assertions.assertNull(child.getExplicitLocalStringProperty("foo")); + + // Query property through child (triggers caching in MapBasedKeyValueProvider) + Assertions.assertEquals("bar", child.getStringProperty("foo")); + + // Caching must not mark property as explicit locally + Assertions.assertFalse(child.isPropertyExplicitLocally("foo")); + Assertions.assertNull(child.getExplicitLocalStringProperty("foo")); + + // Explicitly setting property on child makes it explicit + child.setProperty("foo", "baz"); + Assertions.assertTrue(child.isPropertyExplicitLocally("foo")); + Assertions.assertEquals("baz", child.getExplicitLocalStringProperty("foo")); + } + + @Test + public void testRemoveProperty() { + final Configuration parent = new Configuration(); + parent.setProperty("foo", "parentVal"); + + final Configuration child = parent.fork(); + child.setProperty("foo", "childVal"); + Assertions.assertTrue(child.isPropertyExplicitLocally("foo")); + Assertions.assertEquals("childVal", child.getStringProperty("foo")); + + child.removeProperty("foo"); + Assertions.assertFalse(child.isPropertyExplicitLocally("foo")); + Assertions.assertEquals("parentVal", child.getStringProperty("foo")); + } +} diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java index 534f873e5..cf799a23a 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java @@ -63,8 +63,8 @@ public void execute(final WayangPlan wayangPlan, final String... udfJars) { final Optional originalOption = config.getOptionalStringProperty("wayang.ml.experience.original"); final OneHotMappings mappings = new OneHotMappings(); - final TreeEncoder encoder = new TreeEncoder(mappings); - final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext(), false).toString()); + final TreeEncoder encoder = new TreeEncoder(mappings, config); + final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext()).toString()); final Optional choicesOption = config .getOptionalStringProperty("wayang.ml.experience.with-platforms"); diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java index 80ebc1f67..8ad29697c 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java @@ -18,6 +18,7 @@ package org.apache.wayang.ml; +import java.io.InputStream; import java.util.Collection; import java.util.Collections; @@ -26,7 +27,9 @@ import org.apache.wayang.core.optimizer.channels.ChannelConversion; import org.apache.wayang.core.platform.Platform; import org.apache.wayang.core.plugin.Plugin; +import org.apache.wayang.core.util.ReflectionUtils; import org.apache.wayang.java.platform.JavaPlatform; +import org.apache.wayang.ml.encoding.TreeEncoder; import org.apache.wayang.spark.platform.SparkPlatform; /** @@ -34,6 +37,8 @@ */ public class MachineLearning { + private static final String DEFAULT_CONFIG_FILE = "wayang-ml-defaults.properties"; + /** * Enables use with the {@link JavaPlatform} and {@link SparkPlatform}. */ @@ -56,9 +61,42 @@ public Collection getChannelConversions() { @Override public void setProperties(final Configuration configuration) { + try (final InputStream is = ReflectionUtils.loadResource(DEFAULT_CONFIG_FILE)) { + if (is != null) { + final java.util.Properties properties = new java.util.Properties(); + properties.load(is); + for (final java.util.Map.Entry propertyEntry : properties.entrySet()) { + final String key = propertyEntry.getKey().toString(); + final String value = propertyEntry.getValue().toString(); + if (isConfigured(configuration, key)) { + continue; + } + configuration.setProperty(key, value); + } + } + } catch (java.io.IOException e) { + throw new org.apache.wayang.core.api.exception.WayangException("Could not load ML default configuration.", e); + } } }; + private static boolean isConfigured(final Configuration configuration, final String key) { + if (TreeEncoder.ENCODE_IDS_PROPERTY.equals(key) || TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY.equals(key)) { + return isExplicitlyConfigured(configuration, TreeEncoder.ENCODE_IDS_PROPERTY) + || isExplicitlyConfigured(configuration, TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY); + } + return isExplicitlyConfigured(configuration, key); + } + + private static boolean isExplicitlyConfigured(final Configuration configuration, final String key) { + for (Configuration current = configuration; current != null; current = current.getParent()) { + if (current.getProperties() != null && current.getProperties().provideExplicitLocally(key) != null) { + return true; + } + } + return false; + } + /** * Retrieve a {@link Plugin} to use xxx on the * diff --git a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java index cff4430f2..48909e780 100644 --- a/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java +++ b/wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.Queue; +import org.apache.wayang.core.api.Configuration; +import org.apache.wayang.core.api.configuration.KeyValueProvider; import org.apache.wayang.core.api.exception.WayangException; import org.apache.wayang.core.optimizer.OptimizationContext; import org.apache.wayang.core.optimizer.enumeration.PlanImplementation; @@ -40,17 +42,77 @@ import org.apache.wayang.core.platform.Junction; public class TreeEncoder { + + public static final String ENCODE_IDS_PROPERTY = "wayang.ml.encoding.encode-ids"; + + public static final String ENCODE_IDS_SHORTHAND_PROPERTY = "wayang.ml.encode-ids"; + private final OneHotMappings mappings; + private Configuration configuration; + public TreeEncoder(final OneHotMappings mappings) { + this(mappings, null); + } + + public TreeEncoder(final OneHotMappings mappings, final Configuration configuration) { this.mappings = mappings; + this.configuration = configuration; } public OneHotMappings getMappings() { return this.mappings; } + public Configuration getConfiguration() { + return this.configuration; + } + + public void setConfiguration(final Configuration configuration) { + this.configuration = configuration; + } + + /** + * Resolves the effective {@code encode-ids} setting for the given {@link Configuration}. + * + *

Traverses the configuration hierarchy from child to parent until an explicitly configured level is + * encountered. If a level defines both the canonical and shorthand properties explicitly, the canonical + * property takes precedence. If neither is explicitly defined at any level, defaults to {@code false}.

+ * + * @param config the configuration to resolve for, or {@code null} + * @return {@code true} if ID encoding is enabled, {@code false} otherwise + */ + public static boolean resolveEncodeIds(final Configuration config) { + for (Configuration level = config; level != null; level = level.getParent()) { + final KeyValueProvider properties = level.getProperties(); + if (properties == null) { + continue; + } + final String canonical = properties.provideExplicitLocally(ENCODE_IDS_PROPERTY); + final String shorthand = properties.provideExplicitLocally(ENCODE_IDS_SHORTHAND_PROPERTY); + if (canonical == null && shorthand == null) { + continue; + } + if (canonical != null) { + return Boolean.parseBoolean(canonical); + } + return Boolean.parseBoolean(shorthand); + } + return false; + } + + private boolean resolveEncodeIds(final OptimizationContext optimizationContext) { + final Configuration config = this.configuration != null + ? this.configuration + : (optimizationContext != null ? optimizationContext.getConfiguration() : null); + return resolveEncodeIds(config); + } + public TreeNode encode(final PlanImplementation plan) { + return this.encode(plan, this.resolveEncodeIds(plan.getOptimizationContext())); + } + + public TreeNode encode(final PlanImplementation plan, final boolean encodeIds) { final List result = new ArrayList(); final HashMap> tree = new HashMap<>(); @@ -58,9 +120,6 @@ public TreeNode encode(final PlanImplementation plan) { final Map, Junction> junctions = plan.getJunctions(); - // TODO: convert to config - final boolean encodeIds = false; - for (final Operator sink : sinks) { final TreeNode sinkNode = traversePIOperator(sink, plan.getOptimizationContext(), encodeIds, junctions, tree); @@ -77,6 +136,10 @@ public TreeNode encode(final PlanImplementation plan) { return resultNode; } + public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext) { + return this.encode(plan, optimizationContext, this.resolveEncodeIds(optimizationContext)); + } + public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext, final boolean encodeIds) { final List result = new ArrayList(); diff --git a/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties b/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties index 822169eb8..7cbc683f6 100644 --- a/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties +++ b/wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties @@ -73,4 +73,7 @@ wayang.ml.experience.enabled = false wayang.ml.executions.file = /var/www/html/data/executions.txt wayang.ml.optimizations.file = /var/www/html/data/optmizations.txt wayang.ml.experience.file = /var/www/html/data/experience/experience-vae.txt +# Not set on purpose (TreeEncoder defaults to false); a default here would shadow inherited settings. +# Use wayang.ml.encoding.encode-ids or the shorthand wayang.ml.encode-ids. +# wayang.ml.encoding.encode-ids = false org.apache.logging.log4j.level = INFO diff --git a/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java b/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java index 31ea7de01..bfb130f4b 100644 --- a/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java +++ b/wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -28,9 +29,11 @@ import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.api.Job; import org.apache.wayang.core.api.WayangContext; +import org.apache.wayang.core.optimizer.enumeration.PlanImplementation; import org.apache.wayang.core.plan.executionplan.ExecutionPlan; import org.apache.wayang.core.plan.wayangplan.WayangPlan; import org.apache.wayang.java.Java; +import org.apache.wayang.ml.MachineLearning; import org.apache.wayang.ml.encoding.OneHotMappings; import org.apache.wayang.ml.encoding.TreeEncoder; import org.apache.wayang.ml.encoding.TreeNode; @@ -57,5 +60,485 @@ public void testTreeEncoding() throws IOException, URISyntaxException { Assertions.assertNotNull(exPlan); Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); + } + + @Test + public void testEncodePlanImplementationWithDefaultConfig() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Collection planImplementations = buildPlanImplementations(wayangPlan, wayangContext); + Assertions.assertFalse(planImplementations.isEmpty()); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + for (final PlanImplementation planImplementation : planImplementations) { + final TreeNode encoded = encoder.encode(planImplementation); + Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); + } + } + + @Test + public void testEncodePlanImplementationWithEncodeIdsEnabled() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Collection planImplementations = buildPlanImplementations(wayangPlan, wayangContext); + Assertions.assertFalse(planImplementations.isEmpty()); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + for (final PlanImplementation planImplementation : planImplementations) { + final TreeNode encoded = encoder.encode(planImplementation); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + } + + @Test + public void testEncodeWayangPlanWithConfiguration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty("wayang.ml.encode-ids", "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testTreeEncoderConstructorWithConfiguration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final WayangContext wayangContext = new WayangContext(config); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings(), config); + Assertions.assertEquals(config, encoder.getConfiguration()); + + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testMachineLearningPluginSetProperties() { + final Configuration config = new Configuration(); + org.apache.wayang.ml.MachineLearning.plugin().setProperties(config); + Assertions.assertFalse(config.getOptionalBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY).isPresent()); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config)); + Assertions.assertEquals(100L, config.getLongProperty("wayang.ml.tuple.average-size")); + } + + @Test + public void testPluginRegistrationPreservesUserSettings() { + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config.setProperty("wayang.ml.model.file", "/custom/model/path.onnx"); + config.setProperty("wayang.ml.experience.enabled", "true"); + config.setProperty("wayang.ml.experience.file", "/custom/experience.txt"); + config.setProperty("wayang.core.optimizer.reoptimize", "true"); + config.setProperty("wayang.core.optimizer.reoptimize.proactive", "true"); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + final Configuration contextConfig = wayangContext.getConfiguration(); + + // User settings must remain intact + Assertions.assertTrue(contextConfig.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + Assertions.assertEquals("/custom/model/path.onnx", contextConfig.getStringProperty("wayang.ml.model.file")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.ml.experience.enabled")); + Assertions.assertEquals("/custom/experience.txt", contextConfig.getStringProperty("wayang.ml.experience.file")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.core.optimizer.reoptimize")); + Assertions.assertTrue(contextConfig.getBooleanProperty("wayang.core.optimizer.reoptimize.proactive")); + + // Unconfigured defaults should still be populated + Assertions.assertEquals(100L, contextConfig.getLongProperty("wayang.ml.tuple.average-size")); + Assertions.assertEquals("/var/www/html/data/executions.txt", contextConfig.getStringProperty("wayang.ml.executions.file")); + } + + @Test + public void testPluginRegistrationPreservesShorthandEncodeIds() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Configuration contextConfig = wayangContext.getConfiguration(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(contextConfig)); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testShorthandEncodeIdsAfterPluginRegistration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration config = new Configuration(); + + final WayangContext wayangContext = new WayangContext(config); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + final Configuration contextConfig = wayangContext.getConfiguration(); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(contextConfig)); + + // Setting shorthand after registration must not be shadowed by canonical default false + contextConfig.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(contextConfig)); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job wayangJob = wayangContext.createJob("", wayangPlan, ""); + wayangJob.buildInitialExecutionPlan(); + + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings()); + final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertNotEquals(0L, encoded.encoded[0]); + } + + @Test + public void testPluginRegistrationDirectSetPropertiesPreservesUserSettings() { + final Configuration config = new Configuration(); + config.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + config.setProperty("wayang.ml.model.file", "/my/custom/model.onnx"); + config.setProperty("wayang.ml.experience.enabled", "true"); + config.setProperty("wayang.core.optimizer.reoptimize", "true"); + + org.apache.wayang.ml.MachineLearning.plugin().setProperties(config); + + Assertions.assertEquals("/my/custom/model.onnx", config.getStringProperty("wayang.ml.model.file")); + Assertions.assertTrue(config.getBooleanProperty("wayang.ml.experience.enabled")); + Assertions.assertTrue(config.getBooleanProperty("wayang.core.optimizer.reoptimize")); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config)); + Assertions.assertEquals(100L, config.getLongProperty("wayang.ml.tuple.average-size")); + } + + @Test + public void testResolveEncodeIdsMatrix() { + // 1. Both absent + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(new Configuration())); + + // 2. Canonical true, shorthand absent + final Configuration config1 = new Configuration(); + config1.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config1)); + + // 3. Canonical false, shorthand absent + final Configuration config2 = new Configuration(); + config2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config2)); + + // 4. Canonical absent, shorthand true + final Configuration config3 = new Configuration(); + config3.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config3)); + + // 5. Canonical absent, shorthand false + final Configuration config4 = new Configuration(); + config4.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config4)); + + // 6. Canonical false, shorthand true on the same level: canonical takes precedence + final Configuration config5 = new Configuration(); + config5.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + config5.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config5)); + + // 7. Canonical true, shorthand false + final Configuration config6 = new Configuration(); + config6.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config6.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config6)); + + // 8. Canonical true, shorthand true + final Configuration config7 = new Configuration(); + config7.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + config7.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(config7)); + + // 9. Canonical false, shorthand false + final Configuration config8 = new Configuration(); + config8.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + config8.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(config8)); + } + + @Test + public void testChildShorthandFalseOverridesInheritedCanonicalTrue() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(parent)); + } + + @Test + public void testChildCanonicalFalseOverridesInheritedShorthandTrue() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(parent)); + } + + @Test + public void testChildTrueOverridesInheritedFalseAcrossSpellings() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + final Configuration child = parent.fork(); + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + final Configuration parent2 = new Configuration(); + parent2.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + final Configuration child2 = parent2.fork(); + child2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child2)); + } + + @Test + public void testChildOverrideAfterCachedParentRead() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + + // Reading through the child caches the inherited value locally; this must not look like an override. + Assertions.assertTrue(child.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + + final Configuration parent2 = new Configuration(); + parent2.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration child2 = parent2.fork(); + Assertions.assertTrue(child2.getBooleanProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY)); + child2.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child2)); + } + + @Test + public void testChildInheritsFromGrandparentWhenNotOverridden() { + final Configuration grandparent = new Configuration(); + grandparent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration parent = grandparent.fork(); + final Configuration child = parent.fork(); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testJobLevelOverrideAfterPluginRegistration() throws IOException, URISyntaxException { + final List> collector = new LinkedList<>(); + final Configuration userConfig = new Configuration(); + userConfig.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + + final WayangContext wayangContext = new WayangContext(userConfig); + wayangContext.register(org.apache.wayang.ml.MachineLearning.plugin()); + wayangContext.register(Java.basicPlugin()); + wayangContext.register(Spark.basicPlugin()); + + // The plugin must not inject a default that shadows the user's inherited shorthand. + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(wayangContext.getConfiguration())); + + final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString(); + final WayangPlan wayangPlan = createWayangPlan(filePath, collector); + final Job inheritingJob = wayangContext.createJob("", wayangPlan, ""); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(inheritingJob.getConfiguration())); + + final WayangPlan overridingPlan = createWayangPlan(filePath, collector); + final Job overridingJob = wayangContext.createJob("", overridingPlan, ""); + overridingJob.getConfiguration().setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(overridingJob.getConfiguration())); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(wayangContext.getConfiguration())); + + overridingJob.buildInitialExecutionPlan(); + final TreeEncoder encoder = new TreeEncoder(new OneHotMappings(), overridingJob.getConfiguration()); + final TreeNode encoded = encoder.encode(overridingPlan, overridingJob.getOptimizationContext()); + Assertions.assertNotNull(encoded); + Assertions.assertEquals(0L, encoded.encoded[0]); + } + + @Test + public void testChildExplicitCanonicalFalseAndShorthandTrueWithParentCanonicalFalse() { + // Parent canonical false; child explicitly sets canonical false and shorthand true => resolves to false (canonical wins on same level) + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + final Configuration child = parent.fork(); + child.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testChangingParentAfterCachedReadPreservesExplicitChildShorthand() { + // After caching an inherited value, changing the parent must not affect child whose shorthand is explicitly set + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + + // Reading through the child caches the inherited canonical value ("true") locally + Assertions.assertTrue(child.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + + // Child explicitly sets shorthand to true + child.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + + // Changing parent canonical to false must not make child resolve to false because child shorthand is explicitly true + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testChangingParentAfterCachedReadPreservesExplicitChildCanonical() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "true"); + final Configuration child = parent.fork(); + + // Reading caches shorthand ("true") + Assertions.assertTrue(child.getBooleanProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY)); + + // Child explicitly sets canonical to true + child.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + + // Changing parent shorthand to false + parent.setProperty(TreeEncoder.ENCODE_IDS_SHORTHAND_PROPERTY, "false"); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testChildTracksParentChangeWhenNotExplicitlyConfiguredDespiteCachedRead() { + final Configuration parent = new Configuration(); + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = parent.fork(); + + // Cache read on child + Assertions.assertTrue(child.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY)); + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + + // Parent changes to false; child has no explicit settings, so child must now resolve to false + parent.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "false"); + Assertions.assertFalse(TreeEncoder.resolveEncodeIds(child)); + } + + @Test + public void testPluginRegistrationPreservesGlobalDefaultModelFile() { + final Configuration globalConfig = Configuration.getDefaultConfiguration(); + final String modelKey = "wayang.ml.model.file"; + final String customModelPath = "/custom/global.onnx"; + + try { + globalConfig.setProperty(modelKey, customModelPath); + final Configuration child = new Configuration(); + MachineLearning.plugin().setProperties(child); + + // Child must inherit the explicitly configured global model path, not the bundled default + Assertions.assertEquals(customModelPath, child.getStringProperty(modelKey)); + } finally { + globalConfig.removeProperty(modelKey); + } + } + + @Test + public void testPluginRegistrationPreservesGlobalDefaultEncodeIds() { + final Configuration globalConfig = Configuration.getDefaultConfiguration(); + + try { + globalConfig.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true"); + final Configuration child = new Configuration(); + MachineLearning.plugin().setProperties(child); + + Assertions.assertTrue(TreeEncoder.resolveEncodeIds(child)); + } finally { + globalConfig.removeProperty(TreeEncoder.ENCODE_IDS_PROPERTY); + } + } + + @Test + public void testPluginRegistrationPreservesGlobalDefaultAcrossMultiLevelHierarchy() { + final Configuration globalConfig = Configuration.getDefaultConfiguration(); + final String modelKey = "wayang.ml.model.file"; + final String customModelPath = "/custom/global.onnx"; + + try { + globalConfig.setProperty(modelKey, customModelPath); + final Configuration parent = new Configuration(); + final Configuration child = parent.fork(); + MachineLearning.plugin().setProperties(child); + + Assertions.assertEquals(customModelPath, child.getStringProperty(modelKey)); + } finally { + globalConfig.removeProperty(modelKey); + } + } + + @Test + public void testChildCanOverrideGlobalDefault() { + final Configuration globalConfig = Configuration.getDefaultConfiguration(); + final String modelKey = "wayang.ml.model.file"; + final String customModelPath = "/custom/global.onnx"; + final String childModelPath = "/custom/child.onnx"; + + try { + globalConfig.setProperty(modelKey, customModelPath); + final Configuration child = new Configuration(); + child.setProperty(modelKey, childModelPath); + MachineLearning.plugin().setProperties(child); + + Assertions.assertEquals(childModelPath, child.getStringProperty(modelKey)); + } finally { + globalConfig.removeProperty(modelKey); + } } } +