[uss_qualifier/reports] Finish removing successful property

monitoring/uss_qualifier/reports/report.py

@@ -4,7 +4,6 @@
 from datetime import UTC, datetime
 from typing import Any
 
-import deprecation
 from implicitdict import (
     ImplicitDict,
     Optional,
@@ -133,10 +132,6 @@ class TestStepReport(ImplicitDict):
     def has_critical_problem(self) -> bool:
         return any(fc.severity == Severity.Critical for fc in self.failed_checks)
 
-    @deprecation.deprecated()
-    def successful(self) -> bool:
-        return False if self.failed_checks else True
-
     def participants_with_failed_checks(self) -> set[str]:
         participants = set()
         for fc in self.failed_checks:
@@ -308,9 +303,6 @@ class TestScenarioReport(ImplicitDict):
     end_time: Optional[StringBasedDateTime]
     """Time at which the test scenario completed or encountered an error"""
 
-    successful: bool = False
-    """DEPRECATED.  True iff test scenario completed normally with no failed checks."""
-
     cases: list[TestCaseReport]
     """Reports for each of the test cases in this test scenario, in chronological order."""
 
@@ -425,9 +417,6 @@ class ActionGeneratorReport(ImplicitDict):
     end_time: Optional[StringBasedDateTime]
     """Time at which the action generator completed or encountered an error"""
 
-    successful: bool = False
-    """DEPRECATED.  True iff all actions completed normally with no failed checks"""
-
     def has_critical_problem(self) -> bool:
         return any(a.has_critical_problem() for a in self.actions)
 
@@ -519,10 +508,6 @@ def _conditional(
         else:
             raise self.invalid_type_error
 
-    @deprecation.deprecated()
-    def successful(self) -> bool:
-        return self._conditional(lambda report: report.successful)
-
     def has_critical_problem(self) -> bool:
         return self._conditional(lambda report: report.has_critical_problem())
 
@@ -738,11 +723,6 @@ class SkippedActionReport(ImplicitDict):
     declaration: TestSuiteActionDeclaration
     """Full declaration of the action that was skipped."""
 
-    @property
-    @deprecation.deprecated()
-    def successful(self) -> bool:
-        return True
-
     def has_critical_problem(self) -> bool:
         return False
 
@@ -783,9 +763,6 @@ class TestSuiteReport(ImplicitDict):
     end_time: Optional[StringBasedDateTime]
     """Time at which the test suite completed"""
 
-    successful: bool = False
-    """DEPRECATED.  True iff test suite completed normally with no failed checks"""
-
     capability_evaluations: list[ParticipantCapabilityEvaluationReport]
     """List of capabilities defined in this test suite, evaluated for each participant."""
 

monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator_test.py

@@ -43,7 +43,9 @@ def step_under_test(self: UnitTestScenario):
         evaluator._evaluate_operator_id(value_inj, value_obs, [])
 
     unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test()
-    assert unit_test_scenario.get_report().successful == outcome
+    report = unit_test_scenario.get_report()
+    assert report.execution_error is None
+    assert (not list(report.query_failed_checks())) == outcome
 
 
 def test_operator_id_non_ascii():
@@ -371,7 +373,9 @@ def step_under_test(self: UnitTestScenario):
 
     unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test()
 
-    assert unit_test_scenario.get_report().successful == outcome
+    report = unit_test_scenario.get_report()
+    assert report.execution_error is None
+    assert (not list(report.query_failed_checks())) == outcome
 
     if not outcome:
         found_correct_reason = False
@@ -545,7 +549,9 @@ def step_under_test(self: UnitTestScenario):
 
     unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test()
 
-    assert unit_test_scenario.get_report().successful == outcome
+    report = unit_test_scenario.get_report()
+    assert report.execution_error is None
+    assert (not list(report.query_failed_checks())) == outcome
 
     if wanted_fail:
         found_correct_reason = False

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py

@@ -367,7 +367,7 @@ def evaluate_system_instantaneously(
             self._evaluate_sp_observation(rect, sp_observation, mapping_by_injection_id)
 
             step_report = self._test_scenario.end_test_step()
-            perform_observation = step_report.successful()
+            perform_observation = not step_report.failed_checks
             verified_sps = {
                 obs.participant_id
                 for obs in observers

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator_test.py

@@ -127,7 +127,9 @@ def step_under_test(self: UnitTestScenario):
         )
 
     unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test()
-    assert unit_test_scenario.get_report().successful == outcome
+    report = unit_test_scenario.get_report()
+    assert report.execution_error is None
+    assert (not list(report.query_failed_checks())) == outcome
 
 
 def test_evaluate_sp_flight_recent_positions():
@@ -161,7 +163,9 @@ def step_under_test(self: UnitTestScenario):
         )
 
     unit_test_scenario = UnitTestScenario(step_under_test).execute_unit_test()
-    assert unit_test_scenario.get_report().successful == outcome
+    report = unit_test_scenario.get_report()
+    assert report.execution_error is None
+    assert (not list(report.query_failed_checks())) == outcome
 
 
 def test_evaluate_sp_flight_recent_positions_crossing_area_boundary():

monitoring/uss_qualifier/scenarios/scenario.py

@@ -704,7 +704,6 @@ def record_execution_error(
             self._make_scenario_report()
             assert self._scenario_report is not None
         self._scenario_report.execution_error = ErrorReport.create_from_exception(e)
-        self._scenario_report.successful = False
         self._phase = ScenarioPhase.Complete
 
     def get_report(self) -> TestScenarioReport:
@@ -724,23 +723,6 @@ def get_report(self) -> TestScenarioReport:
                 arrow.utcnow().datetime
             )
 
-        # Evaluate success
-        self._scenario_report.successful = (
-            "execution_error" not in self._scenario_report
-        )
-        for case_report in self._scenario_report.cases:
-            for step_report in case_report.steps:
-                for failed_check in step_report.failed_checks:
-                    if failed_check.severity != Severity.Low:
-                        self._scenario_report.successful = False
-        if (
-            "cleanup" in self._scenario_report
-            and self._scenario_report.cleanup is not None
-        ):
-            for failed_check in self._scenario_report.cleanup.failed_checks:
-                if failed_check.severity != Severity.Low:
-                    self._scenario_report.successful = False
-
         return self._scenario_report
 
     def sleep(self, duration: float | timedelta, reason: str) -> None:

monitoring/uss_qualifier/scenarios/scenario_test/generic_test_scenario_test.py

@@ -1,5 +1,6 @@
 import itertools
 
+from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.dev.noop import NoOpResource, NoOpSpecification
 from monitoring.uss_qualifier.resources.resource import MissingResourceError
 from monitoring.uss_qualifier.scenarios.scenario import (
@@ -818,10 +819,9 @@ def test_get_report_state_machine():
         report = gtsi.get_report()
 
         if "end_cleanup" in steps[:steps_to_do]:  # Meaning the scenario is done
-            assert report.successful
             assert "execution_error" not in report
+            assert not list(report.query_failed_checks())
         else:
-            assert not report.successful
             assert report.execution_error
             assert report.execution_error.type == "RuntimeError"
             assert (
@@ -839,7 +839,6 @@ def test_record_execution_error():
     gtsi.record_execution_error(ErrorForTests("test-exception"))
 
     report = gtsi.get_report()
-    assert not report.successful
     assert report.execution_error
     assert report.execution_error.type.endswith(".ErrorForTests")
     assert report.execution_error.message == "test-exception"
@@ -882,7 +881,6 @@ def test_record_execution_error_during_flow():
             gtsi.record_execution_error(ErrorForTests("test-exception"))
             report = gtsi.get_report()
 
-            assert not report.successful
             assert report.execution_error
             assert report.execution_error.type.endswith(".ErrorForTests")
             assert report.execution_error.message == "test-exception"
@@ -897,7 +895,8 @@ def test_get_report_return_cases_ran_no_cases():
     terminate_gtsi_during_scenario(gtsi)
 
     report = gtsi.get_report()
-    assert report.successful
+    assert report.execution_error is None
+    assert not list(report.query_failed_checks())
     assert not report.cases
 
 
@@ -912,7 +911,8 @@ def test_get_report_return_cases_ran_one_cases():
     terminate_gtsi_during_scenario(gtsi)
 
     report = gtsi.get_report()
-    assert report.successful
+    assert report.execution_error is None
+    assert not list(report.query_failed_checks())
     assert report.cases
     names = [c.name for c in report.cases]
     assert "test-case-1" in names
@@ -932,7 +932,8 @@ def test_get_report_return_cases_ran_two_cases():
     terminate_gtsi_during_scenario(gtsi)
 
     report = gtsi.get_report()
-    assert report.successful
+    assert report.execution_error is None
+    assert not list(report.query_failed_checks())
     assert report.cases
     names = [c.name for c in report.cases]
     assert "test-case-1" in names
@@ -1526,8 +1527,9 @@ def test_stop_fast_enabled():
             pass
 
 
-def test_report_status_should_be_ok_if_low_failled():
-    """Test report status should be maked as successfull if a low check failled"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_low_check():
+    """Test that the report records a failed check with low severity."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_to_step(gtsi)
@@ -1537,11 +1539,16 @@ def test_report_status_should_be_ok_if_low_failled():
 
     report = gtsi.get_report()
 
-    assert report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-1-1-2-low"
+    assert failed_checks[0].severity == Severity.Low
 
 
-def test_report_status_should_be_unsuccessfull_if_medium_failled():
-    """Test report status should be maked as unsuccessfull if a medium check failled"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_medium_check():
+    """Test that the report records a failed check with medium severity."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_to_step(gtsi)
@@ -1551,11 +1558,16 @@ def test_report_status_should_be_unsuccessfull_if_medium_failled():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-1-1-3-medium"
+    assert failed_checks[0].severity == Severity.Medium
 
 
-def test_report_status_should_be_unsuccessfull_if_high_failled():
-    """Test report status should be maked as unsuccessfull if a high check failled"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_high_check():
+    """Test that the report records a failed check with high severity."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_to_step(gtsi)
@@ -1569,11 +1581,16 @@ def test_report_status_should_be_unsuccessfull_if_high_failled():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-1-1-4-high"
+    assert failed_checks[0].severity == Severity.High
 
 
-def test_report_status_should_be_unsuccessfull_if_critical_failled():
-    """Test report status should be maked as unsuccessfull if a critical check failled"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_critical_check():
+    """Test that the report records a failed check with critical severity."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_to_step(gtsi)
@@ -1587,11 +1604,16 @@ def test_report_status_should_be_unsuccessfull_if_critical_failled():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-1-1-5-critical"
+    assert failed_checks[0].severity == Severity.Critical
 
 
-def test_report_status_should_be_ok_if_low_failled_in_cleanup():
-    """Test report status should be maked as successfull if a low check failled during cleanup"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_low_check_in_cleanup():
+    """Test that the report records a failed check with low severity during cleanup."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_during_cleanup(gtsi)
@@ -1601,11 +1623,16 @@ def test_report_status_should_be_ok_if_low_failled_in_cleanup():
 
     report = gtsi.get_report()
 
-    assert report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-c-2-low"
+    assert failed_checks[0].severity == Severity.Low
 
 
-def test_report_status_should_be_unsuccessfull_if_medium_failled_in_cleanup():
-    """Test report status should be maked as unsuccessfull if a medium check failled during cleanup"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_medium_check_in_cleanup():
+    """Test that the report records a failed check with medium severity during cleanup."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_during_cleanup(gtsi)
@@ -1615,11 +1642,16 @@ def test_report_status_should_be_unsuccessfull_if_medium_failled_in_cleanup():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-c-3-medium"
+    assert failed_checks[0].severity == Severity.Medium
 
 
-def test_report_status_should_be_unsuccessfull_if_high_failled_in_cleanup():
-    """Test report status should be maked as unsuccessfull if a high check failled during cleanup"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_high_check_in_cleanup():
+    """Test that the report records a failed check with high severity during cleanup."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_during_cleanup(gtsi)
@@ -1633,11 +1665,16 @@ def test_report_status_should_be_unsuccessfull_if_high_failled_in_cleanup():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-c-4-high"
+    assert failed_checks[0].severity == Severity.High
 
 
-def test_report_status_should_be_unsuccessfull_if_critical_failled_in_cleanup():
-    """Test report status should be maked as unsuccessfull if a critical check failled during cleanup"""
+# This test is AI-generated and has not been closely inspected by a human.
+def test_report_records_failed_critical_check_in_cleanup():
+    """Test that the report records a failed check with critical severity during cleanup."""
 
     gtsi = _build_generic_test_scenario_instance()
     advance_new_gtsi_during_cleanup(gtsi)
@@ -1651,7 +1688,11 @@ def test_report_status_should_be_unsuccessfull_if_critical_failled_in_cleanup():
 
     report = gtsi.get_report()
 
-    assert not report.successful
+    assert report.execution_error is None
+    failed_checks = [check for _, check in report.query_failed_checks()]
+    assert len(failed_checks) == 1
+    assert failed_checks[0].name == "test-check-c-5-critical"
+    assert failed_checks[0].severity == Severity.Critical
 
 
 # TODO Ensure unkown participan or query type is tested in record query