From 548f7015ba4e5803dfae6194605c3bd825cbb04e Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Sun, 18 Oct 2026 10:56:36 -0700 Subject: [PATCH 1/2] Remove `successful` fields --- monitoring/uss_qualifier/reports/report.py | 23 ------------------- .../uss_qualifier/scenarios/scenario.py | 18 --------------- monitoring/uss_qualifier/suites/suite.py | 3 --- 3 files changed, 44 deletions(-) diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 4945383da7..617f224b80 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -4,7 +4,6 @@ from datetime import UTC, datetime from typing import Any -import deprecation from implicitdict import ( ImplicitDict, Optional, @@ -133,10 +132,6 @@ class TestStepReport(ImplicitDict): def has_critical_problem(self) -> bool: return any(fc.severity == Severity.Critical for fc in self.failed_checks) - @deprecation.deprecated() - def successful(self) -> bool: - return False if self.failed_checks else True - def participants_with_failed_checks(self) -> set[str]: participants = set() for fc in self.failed_checks: @@ -305,9 +300,6 @@ class TestScenarioReport(ImplicitDict): end_time: Optional[StringBasedDateTime] """Time at which the test scenario completed or encountered an error""" - successful: bool = False - """DEPRECATED. True iff test scenario completed normally with no failed checks.""" - cases: list[TestCaseReport] """Reports for each of the test cases in this test scenario, in chronological order.""" @@ -415,9 +407,6 @@ class ActionGeneratorReport(ImplicitDict): end_time: Optional[StringBasedDateTime] """Time at which the action generator completed or encountered an error""" - successful: bool = False - """DEPRECATED. True iff all actions completed normally with no failed checks""" - def has_critical_problem(self) -> bool: return any(a.has_critical_problem() for a in self.actions) @@ -509,10 +498,6 @@ def _conditional( else: raise self.invalid_type_error - @deprecation.deprecated() - def successful(self) -> bool: - return self._conditional(lambda report: report.successful) - def has_critical_problem(self) -> bool: return self._conditional(lambda report: report.has_critical_problem()) @@ -728,11 +713,6 @@ class SkippedActionReport(ImplicitDict): declaration: TestSuiteActionDeclaration """Full declaration of the action that was skipped.""" - @property - @deprecation.deprecated() - def successful(self) -> bool: - return True - def has_critical_problem(self) -> bool: return False @@ -773,9 +753,6 @@ class TestSuiteReport(ImplicitDict): end_time: Optional[StringBasedDateTime] """Time at which the test suite completed""" - successful: bool = False - """DEPRECATED. True iff test suite completed normally with no failed checks""" - capability_evaluations: list[ParticipantCapabilityEvaluationReport] """List of capabilities defined in this test suite, evaluated for each participant.""" diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 729d392d5a..2fa535c4cf 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -694,7 +694,6 @@ def record_execution_error( self._make_scenario_report() assert self._scenario_report is not None self._scenario_report.execution_error = ErrorReport.create_from_exception(e) - self._scenario_report.successful = False self._phase = ScenarioPhase.Complete def get_report(self) -> TestScenarioReport: @@ -714,23 +713,6 @@ def get_report(self) -> TestScenarioReport: arrow.utcnow().datetime ) - # Evaluate success - self._scenario_report.successful = ( - "execution_error" not in self._scenario_report - ) - for case_report in self._scenario_report.cases: - for step_report in case_report.steps: - for failed_check in step_report.failed_checks: - if failed_check.severity != Severity.Low: - self._scenario_report.successful = False - if ( - "cleanup" in self._scenario_report - and self._scenario_report.cleanup is not None - ): - for failed_check in self._scenario_report.cleanup.failed_checks: - if failed_check.severity != Severity.Low: - self._scenario_report.successful = False - return self._scenario_report def sleep(self, duration: float | timedelta, reason: str) -> None: diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a772150427..6a367b965b 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -436,9 +436,6 @@ def run_action( result.result() if isinstance(result, Future) else result for result in results ) - report.successful = all( - not a.has_critical_problem() and a.successful() for a in report.actions - ) report.end_time = StringBasedDateTime(datetime.now(UTC)) From 42b190a18d6f3a024e4c76d873758a5fd530149a Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Sun, 18 Oct 2026 11:43:09 -0700 Subject: [PATCH 2/2] Fix unit tests and `make format` --- .../common_dictionary_evaluator_test.py | 12 +- .../astm/netrid/display_data_evaluator.py | 2 +- .../netrid/display_data_evaluator_test.py | 8 +- .../generic_test_scenario_test.py | 103 ++++++++++++------ .../reports/report/ActionGeneratorReport.json | 4 - .../reports/report/TestScenarioReport.json | 4 - .../reports/report/TestSuiteReport.json | 4 - 7 files changed, 88 insertions(+), 49 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py index 3a5cfbfdae..e745267707 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py @@ -43,7 +43,9 @@ def step_under_test(self: UnitTestScenario): evaluator._evaluate_operator_id(value_inj, value_obs, []) unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() - assert unit_test_scenario.get_report().successful == outcome + report = unit_test_scenario.get_report() + assert report.execution_error is None + assert (not list(report.query_failed_checks())) == outcome def test_operator_id_non_ascii(): @@ -371,7 +373,9 @@ def step_under_test(self: UnitTestScenario): unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() - assert unit_test_scenario.get_report().successful == outcome + report = unit_test_scenario.get_report() + assert report.execution_error is None + assert (not list(report.query_failed_checks())) == outcome if not outcome: found_correct_reason = False @@ -545,7 +549,9 @@ def step_under_test(self: UnitTestScenario): unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() - assert unit_test_scenario.get_report().successful == outcome + report = unit_test_scenario.get_report() + assert report.execution_error is None + assert (not list(report.query_failed_checks())) == outcome if wanted_fail: found_correct_reason = False diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index 21dd7edd76..a964788344 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -367,7 +367,7 @@ def evaluate_system_instantaneously( self._evaluate_sp_observation(rect, sp_observation, mapping_by_injection_id) step_report = self._test_scenario.end_test_step() - perform_observation = step_report.successful() + perform_observation = not step_report.failed_checks verified_sps = { obs.participant_id for obs in observers diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator_test.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator_test.py index 4a66164fb6..6f58c20558 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator_test.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator_test.py @@ -127,7 +127,9 @@ def step_under_test(self: UnitTestScenario): ) unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() - assert unit_test_scenario.get_report().successful == outcome + report = unit_test_scenario.get_report() + assert report.execution_error is None + assert (not list(report.query_failed_checks())) == outcome def test_evaluate_sp_flight_recent_positions(): @@ -161,7 +163,9 @@ def step_under_test(self: UnitTestScenario): ) unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test() - assert unit_test_scenario.get_report().successful == outcome + report = unit_test_scenario.get_report() + assert report.execution_error is None + assert (not list(report.query_failed_checks())) == outcome def test_evaluate_sp_flight_recent_positions_crossing_area_boundary(): diff --git a/monitoring/uss_qualifier/scenarios/scenario_test/generic_test_scenario_test.py b/monitoring/uss_qualifier/scenarios/scenario_test/generic_test_scenario_test.py index 73d451e4e5..d336a5fb67 100644 --- a/monitoring/uss_qualifier/scenarios/scenario_test/generic_test_scenario_test.py +++ b/monitoring/uss_qualifier/scenarios/scenario_test/generic_test_scenario_test.py @@ -1,5 +1,6 @@ import itertools +from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.dev.noop import NoOpResource, NoOpSpecification from monitoring.uss_qualifier.resources.resource import MissingResourceError from monitoring.uss_qualifier.scenarios.scenario import ( @@ -818,10 +819,9 @@ def test_get_report_state_machine(): report = gtsi.get_report() if "end_cleanup" in steps[:steps_to_do]: # Meaning the scenario is done - assert report.successful assert "execution_error" not in report + assert not list(report.query_failed_checks()) else: - assert not report.successful assert report.execution_error assert report.execution_error.type == "RuntimeError" assert ( @@ -839,7 +839,6 @@ def test_record_execution_error(): gtsi.record_execution_error(ErrorForTests("test-exception")) report = gtsi.get_report() - assert not report.successful assert report.execution_error assert report.execution_error.type.endswith(".ErrorForTests") assert report.execution_error.message == "test-exception" @@ -882,7 +881,6 @@ def test_record_execution_error_during_flow(): gtsi.record_execution_error(ErrorForTests("test-exception")) report = gtsi.get_report() - assert not report.successful assert report.execution_error assert report.execution_error.type.endswith(".ErrorForTests") assert report.execution_error.message == "test-exception" @@ -897,7 +895,8 @@ def test_get_report_return_cases_ran_no_cases(): terminate_gtsi_during_scenario(gtsi) report = gtsi.get_report() - assert report.successful + assert report.execution_error is None + assert not list(report.query_failed_checks()) assert not report.cases @@ -912,7 +911,8 @@ def test_get_report_return_cases_ran_one_cases(): terminate_gtsi_during_scenario(gtsi) report = gtsi.get_report() - assert report.successful + assert report.execution_error is None + assert not list(report.query_failed_checks()) assert report.cases names = [c.name for c in report.cases] assert "test-case-1" in names @@ -932,7 +932,8 @@ def test_get_report_return_cases_ran_two_cases(): terminate_gtsi_during_scenario(gtsi) report = gtsi.get_report() - assert report.successful + assert report.execution_error is None + assert not list(report.query_failed_checks()) assert report.cases names = [c.name for c in report.cases] assert "test-case-1" in names @@ -1526,8 +1527,9 @@ def test_stop_fast_enabled(): pass -def test_report_status_should_be_ok_if_low_failled(): - """Test report status should be maked as successfull if a low check failled""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_low_check(): + """Test that the report records a failed check with low severity.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_to_step(gtsi) @@ -1537,11 +1539,16 @@ def test_report_status_should_be_ok_if_low_failled(): report = gtsi.get_report() - assert report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-1-1-2-low" + assert failed_checks[0].severity == Severity.Low -def test_report_status_should_be_unsuccessfull_if_medium_failled(): - """Test report status should be maked as unsuccessfull if a medium check failled""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_medium_check(): + """Test that the report records a failed check with medium severity.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_to_step(gtsi) @@ -1551,11 +1558,16 @@ def test_report_status_should_be_unsuccessfull_if_medium_failled(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-1-1-3-medium" + assert failed_checks[0].severity == Severity.Medium -def test_report_status_should_be_unsuccessfull_if_high_failled(): - """Test report status should be maked as unsuccessfull if a high check failled""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_high_check(): + """Test that the report records a failed check with high severity.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_to_step(gtsi) @@ -1569,11 +1581,16 @@ def test_report_status_should_be_unsuccessfull_if_high_failled(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-1-1-4-high" + assert failed_checks[0].severity == Severity.High -def test_report_status_should_be_unsuccessfull_if_critical_failled(): - """Test report status should be maked as unsuccessfull if a critical check failled""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_critical_check(): + """Test that the report records a failed check with critical severity.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_to_step(gtsi) @@ -1587,11 +1604,16 @@ def test_report_status_should_be_unsuccessfull_if_critical_failled(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-1-1-5-critical" + assert failed_checks[0].severity == Severity.Critical -def test_report_status_should_be_ok_if_low_failled_in_cleanup(): - """Test report status should be maked as successfull if a low check failled during cleanup""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_low_check_in_cleanup(): + """Test that the report records a failed check with low severity during cleanup.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_during_cleanup(gtsi) @@ -1601,11 +1623,16 @@ def test_report_status_should_be_ok_if_low_failled_in_cleanup(): report = gtsi.get_report() - assert report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-c-2-low" + assert failed_checks[0].severity == Severity.Low -def test_report_status_should_be_unsuccessfull_if_medium_failled_in_cleanup(): - """Test report status should be maked as unsuccessfull if a medium check failled during cleanup""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_medium_check_in_cleanup(): + """Test that the report records a failed check with medium severity during cleanup.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_during_cleanup(gtsi) @@ -1615,11 +1642,16 @@ def test_report_status_should_be_unsuccessfull_if_medium_failled_in_cleanup(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-c-3-medium" + assert failed_checks[0].severity == Severity.Medium -def test_report_status_should_be_unsuccessfull_if_high_failled_in_cleanup(): - """Test report status should be maked as unsuccessfull if a high check failled during cleanup""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_high_check_in_cleanup(): + """Test that the report records a failed check with high severity during cleanup.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_during_cleanup(gtsi) @@ -1633,11 +1665,16 @@ def test_report_status_should_be_unsuccessfull_if_high_failled_in_cleanup(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-c-4-high" + assert failed_checks[0].severity == Severity.High -def test_report_status_should_be_unsuccessfull_if_critical_failled_in_cleanup(): - """Test report status should be maked as unsuccessfull if a critical check failled during cleanup""" +# This test is AI-generated and has not been closely inspected by a human. +def test_report_records_failed_critical_check_in_cleanup(): + """Test that the report records a failed check with critical severity during cleanup.""" gtsi = _build_generic_test_scenario_instance() advance_new_gtsi_during_cleanup(gtsi) @@ -1651,7 +1688,11 @@ def test_report_status_should_be_unsuccessfull_if_critical_failled_in_cleanup(): report = gtsi.get_report() - assert not report.successful + assert report.execution_error is None + failed_checks = [check for _, check in report.query_failed_checks()] + assert len(failed_checks) == 1 + assert failed_checks[0].name == "test-check-c-5-critical" + assert failed_checks[0].severity == Severity.Critical # TODO Ensure unkown participan or query type is tested in record query diff --git a/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json b/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json index 1d2f8048e5..f50d5abe03 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/ActionGeneratorReport.json @@ -30,10 +30,6 @@ "description": "Time at which the action generator started", "format": "date-time", "type": "string" - }, - "successful": { - "description": "DEPRECATED. True iff all actions completed normally with no failed checks", - "type": "boolean" } }, "required": [ diff --git a/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json b/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json index 074e8c7137..8b7648e18c 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/TestScenarioReport.json @@ -102,10 +102,6 @@ "description": "Time at which the test scenario started", "format": "date-time", "type": "string" - }, - "successful": { - "description": "DEPRECATED. True iff test scenario completed normally with no failed checks.", - "type": "boolean" } }, "required": [ diff --git a/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json b/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json index 449e40fe3c..1a90c0fe21 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json @@ -42,10 +42,6 @@ "format": "date-time", "type": "string" }, - "successful": { - "description": "DEPRECATED. True iff test suite completed normally with no failed checks", - "type": "boolean" - }, "suite_type": { "description": "Type/location of this test suite", "type": "string"