Skip to content

[SYSTEMDS-3971] Refactor redundant Scuro tests #2614

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
christinadionysio merged 6 commits into from
Nov 4, 2026
Merged

[SYSTEMDS-3971] Refactor redundant Scuro tests #2614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7d62874
[MINOR] Merge the per-modality duplicates in the Scuro optimizer tests
shieru1214 Sep 12, 2026
4c29f1d
[MINOR] Merge redundant subtests in the Scuro window operation tests
shieru1214 Sep 12, 2026
929d9ff
[MINOR] Turn the Scuro fusion order tests into a property table
shieru1214 Sep 12, 2026
59b4101
Merge branch 'main' into scuro-v3
shieru1214 Oct 29, 2026
15095e0
[MINOR] Remove trailing whitespace left by the merge in the Scuro opt…
shieru1214 Oct 29, 2026
c0bb743
[MINOR] Address the review comments on the Scuro fusion and window tests
shieru1214 Oct 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 66 additions & 62 deletions src/main/python/tests/scuro/test_fusion_orders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,77 +19,81 @@
#
# -------------------------------------------------------------

import os
import shutil
import unittest
import numpy as np

from systemds.scuro import Concatenation, RowMax, Hadamard
from systemds.scuro.modality.unimodal_modality import UnimodalModality
from systemds.scuro.representations.bert import Bert
from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
from systemds.scuro.representations.average import Average
from systemds.scuro.drsearch.operator_registry import Registry
from tests.scuro.data_generator import ModalityRandomDataGenerator
from systemds.scuro.modality.type import ModalityType

# built-in fusions only: other test modules register their own, and a fusion
# that needs training cannot be combined without labels
FUSION_OPERATORS = [
op
for op in Registry().get_fusion_operators()
if op.__module__.startswith("systemds.") and not op().needs_training
]


class TestFusionOrders(unittest.TestCase):
"""
Tests the order properties of the fusion operators against the flags they
declare. Swapping the inputs of a pair, or the first pair of a chain, only
keeps the result for a commutative operator. A pairwise chain only matches
the n-ary call for an associative one.
"""

@classmethod
def setUpClass(cls):
cls.num_instances = 40
# These properties do not depend on the input shape, so the data can
# be small.
cls.num_instances = 4
cls.num_features = 8
cls.data_generator = ModalityRandomDataGenerator()
cls.r_1 = cls.data_generator.create1DModality(40, 100, ModalityType.AUDIO)
cls.r_2 = cls.data_generator.create1DModality(40, 100, ModalityType.TEXT)
cls.r_3 = cls.data_generator.create1DModality(40, 100, ModalityType.TEXT)

def test_fusion_order_avg(self):
r_1_r_2 = self.r_1.combine(self.r_2, Average())
r_2_r_1 = self.r_2.combine(self.r_1, Average())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Average())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Average())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Average())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_concat(self):
r_1_r_2 = self.r_1.combine(self.r_2, Concatenation())
r_2_r_1 = self.r_2.combine(self.r_1, Concatenation())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Concatenation())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Concatenation())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Concatenation())

self.assertFalse(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertFalse(np.array_equal(r_2_r_1.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_max(self):
r_1_r_2 = self.r_1.combine(self.r_2, RowMax())
r_2_r_1 = self.r_2.combine(self.r_1, RowMax())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, RowMax())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, RowMax())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], RowMax())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))

def test_fusion_order_hadamard(self):
r_1_r_2 = self.r_1.combine(self.r_2, Hadamard())
r_2_r_1 = self.r_2.combine(self.r_1, Hadamard())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Hadamard())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Hadamard())

r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Hadamard())

self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
def setUp(self):
self.r_1 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.AUDIO
)
self.r_2 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.TEXT
)
self.r_3 = self.data_generator.create1DModality(
self.num_instances, self.num_features, ModalityType.TEXT
)

@staticmethod
def _equal(left, right):
return np.array_equal(np.asarray(left.data), np.asarray(right.data))

def test_fusion_order_properties(self):
for fusion_operator in FUSION_OPERATORS:
name = fusion_operator.__name__
commutative = fusion_operator().commutative
associative = fusion_operator().associative
with self.subTest(fusion=name):
r_1_r_2 = self.r_1.combine(self.r_2, fusion_operator())
r_2_r_1 = self.r_2.combine(self.r_1, fusion_operator())
r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, fusion_operator())
r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, fusion_operator())
r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], fusion_operator())

self.assertEqual(
self._equal(r_1_r_2, r_2_r_1),
commutative,
f"{name}: swapping the two inputs",
)
self.assertEqual(
self._equal(r_1_r_2_r_3, r_2_r_1_r_3),
commutative,
f"{name}: swapping the first pair of a chain",
)
self.assertEqual(
self._equal(r_1_r_2_r_3, r1_r2_r3),
associative,
f"{name}: pairwise chain against the n-ary call",
)
self.assertFalse(
self._equal(r_1_r_2, r1_r2_r3),
f"{name}: a pair against all three inputs",
)
104 changes: 43 additions & 61 deletions src/main/python/tests/scuro/test_unimodal_optimizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,16 +140,51 @@ def setUpClass(cls):
TestTask("UnimodalRepresentationTask1", "Test1", cls.num_instances),
]

def test_unimodal_optimizer_for_text_modality(self):
text_data, text_md = ModalityRandomDataGenerator().create_text_data(
self.num_instances, 10
)
text = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.TEXT, text_data, str, text_md
# (label, [(modality type, generator keyword arguments)]). A set with two
# entries is passed to the optimizer as one multi-modality search, because
# optimize_unimodal_representation_for_modality loops over the list.
MODALITY_SETS = [
("text", [(ModalityType.TEXT, {})]),
("image", [(ModalityType.IMAGE, {})]),
("audio", [(ModalityType.AUDIO, {})]),
("video", [(ModalityType.VIDEO, {"num_frames": 10})]),
(
"text+image",
[(ModalityType.TEXT, {"num_sentences": 1}), (ModalityType.IMAGE, {})],
),
]

def _create_modality(self, modality_type, num_sentences=10, num_frames=1):
generator = ModalityRandomDataGenerator()
if modality_type is ModalityType.TEXT:
data, metadata = generator.create_text_data(
self.num_instances, num_sentences
)
data_type = str
elif modality_type is ModalityType.AUDIO:
data, metadata = generator.create_audio_data(self.num_instances, 3000)
data_type = np.float32
else:
# IMAGE and VIDEO use the same generator. The number of frames is
# the difference between them.
data, metadata = generator.create_visual_modality(
self.num_instances, num_frames, 10, 10
)
data_type = np.float32

return UnimodalModality(
TestDataLoader(self.indices, None, modality_type, data, data_type, metadata)
)
self.optimize_unimodal_representation_for_modality([text])

def test_unimodal_optimizer_per_modality_set(self):
for label, modality_specs in self.MODALITY_SETS:
with self.subTest(modalities=label):
self.optimize_unimodal_representation_for_modality(
[
self._create_modality(modality_type, **kwargs)
for modality_type, kwargs in modality_specs
]
)

def test_fitted_text_leaves_are_refit_per_fold_in_fusion(self):
text_data, text_md = ModalityRandomDataGenerator().create_text_data(
Expand Down Expand Up @@ -476,59 +511,6 @@ def test_windows_are_kept_for_representations_with_a_time_axis(self):
self.assertIs(operation, ResNet)
self.assertGreaterEqual(rank, 2)

def test_unimodal_optimizer_for_image_modality(self):
image_data, image_md = ModalityRandomDataGenerator().create_visual_modality(
self.num_instances, 1, 10, 10
)
image = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.IMAGE, image_data, np.float32, image_md
)
)
self.optimize_unimodal_representation_for_modality([image])

def test_unimodal_optimizer_for_multiple_modalities(self):
image_data, image_md = ModalityRandomDataGenerator().create_visual_modality(
self.num_instances, 1, 10, 10
)
image = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.IMAGE, image_data, np.float32, image_md
)
)
text_data, text_md = ModalityRandomDataGenerator().create_text_data(
self.num_instances
)
text = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.TEXT, text_data, str, text_md
)
)
self.optimize_unimodal_representation_for_modality([text, image])

def test_unimodal_optimizer_for_audio_modality(self):
audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
self.num_instances, 3000
)
audio = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md
)
)

self.optimize_unimodal_representation_for_modality([audio])

def test_unimodal_optimizer_for_video_modality(self):
video_data, video_md = ModalityRandomDataGenerator().create_visual_modality(
self.num_instances, 10, 10, 10
)
video = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.VIDEO, video_data, np.float32, video_md
)
)
self.optimize_unimodal_representation_for_modality([video])

# ------------------------------------------------------------------
# Every registered representation, run through the optimizer
# ------------------------------------------------------------------
Expand Down
12 changes: 1 addition & 11 deletions src/main/python/tests/scuro/test_unimodal_representations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,17 +205,7 @@ def test_audio_representations(self):
RMSE(),
Pitch(),
]
audio_data, audio_md = ModalityRandomDataGenerator().create_audio_data(
self.num_instances, 200
)

audio = UnimodalModality(
TestDataLoader(
self.indices, None, ModalityType.AUDIO, audio_data, np.float32, audio_md
)
)

audio.extract_raw_data()
audio = self._create_audio_modality(signal_length=200)
original_data = copy.deepcopy(audio.data)

for representation in audio_representations:
Expand Down
Loading
Loading