diff --git a/NEXT_RELEASE_NOTES.md b/NEXT_RELEASE_NOTES.md index 753d47f946..044f999e13 100644 --- a/NEXT_RELEASE_NOTES.md +++ b/NEXT_RELEASE_NOTES.md @@ -41,6 +41,7 @@ The release notes should contain at least the following sections: * A new required resource named `conflicting_flights_provider` must be provided. It must be a `resources.resource.ResourceProvidingResource` providing `resources.flight_planning.FlightIntentsResource` instances with spatially separated flight intents for each planner combination used by `ConflictEqualPriorityNotPermitted`. One suitable implementation is `resources.geospatial.TriangularCascadeSoutheastResource` with `conflicting_flights` as its `base_resource`; see [f3548_self_contained](./monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml) for an example. * A new required resource named `non_conflicting_flights_provider` must be provided. It must be a `resources.resource.ResourceProvidingResource[resources.flight_planning.FlightIntentsResource]` providing spatially separated flight intents for each DSS instance used in the `dss_probing` test suite. One suitable implementation is `resources.geospatial.TriangularCascadeSoutheastResource` with `non_conflicting_flights` as its `base_resource`; see [f3548_self_contained](./monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml) for an example. * A new required resource named `planning_area_provider` must be provided to replace `planning_area`. It must be a `resources.resource.ResourceProvidingResource` providing `resources.planning_area.PlanningAreaResource` instances with spatially separated planning areas for each DSS instance used in the `dss_probing` test suite. One suitable implementation is `resources.geospatial.TriangularCascadeSoutheastResource` with `planning_area` as its `base_resource`; see [f3548_self_contained](./monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml) for an example. The existing `planning_area` suite input is no longer used except by `suites.uspace.required_services`, which still requires it for NetRID tests. Keep any underlying planning area resource used as the provider's `base_resource`. +* For a uss_qualifier test configuration outside this repo using [`validation`](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/reports/validation/definitions.py), the structure of some test suites has changed and may require an update to validation criteria. ## Optional migration tasks diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index 1c1b546b6b..188d613078 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -325,7 +325,7 @@ class USSQualifierConfigurationV1(ImplicitDict): """If specified, configuration describing the artifacts related to the test run""" validation: Optional[ValidationConfiguration] = None - """If specified, configuration describing how to validate the output report (and return an error code if validation fails)""" + """If specified, configuration describing how to validate the output report to return an error code if validation fails.""" class USSQualifierConfiguration(ImplicitDict): diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index 76e7f7ad39..d3b300d426 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -316,15 +316,24 @@ v1: # Allow certain actions to execute their sub-actions in parallel. parallelism_rules: - # The DSS probing generator (action 3 in f3548_21) isn't ready to run its actions in parallel yet + # The DSS probing generator isn't ready to run its actions in parallel yet - max_parallelism: 1 conditions: - is_action_generator: types: - action_generators.resource_environments.ResourceEnvironmentsActionGenerator - defined_at: - # TODO: add ability to condition on potential actions - - test_suite.actions[3].action_generator + potentially_generates_any_of: + - test_suite: + suite_type: suites.astm.utm.dss_probing + # The single-planner generator isn't ready to run its actions in parallel yet + - max_parallelism: 1 + conditions: + - is_action_generator: + types: + - action_generators.resource_environments.ResourceEnvironmentsActionGenerator + potentially_generates_any_of: + - test_suite: + suite_type: suites.astm.utm.single_planner # ...but other ResourceEnvironmentsActionGenerators can have their actions run in parallel - max_parallelism: 2 conditions: diff --git a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml index a921107360..f40bef8984 100644 --- a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml @@ -82,4 +82,6 @@ v1: pass_condition: elements: count: - equal_to: 7 # 2 DatastoreAccess, 1 PoolInfo, 1 GetSystemVersions, 1 SCDAuthenticationValidation, 1 MakeUssReport, 1 EvaluateSystemVersions + # 2 DatastoreAccess, 2 SCDAuthenticationValidation (one per planner), + # 1 PoolInfo, 1 GetSystemVersions, 1 MakeUssReport, 1 EvaluateSystemVersions + equal_to: 8 diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index dfd89e20b6..f61620da5d 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -9,26 +9,16 @@ 3. Scenario: [ASTM F3548 flight planners preparation](../../../scenarios/astm/utm/prep_planners.md) ([`scenarios.astm.utm.PrepareFlightPlanners`](../../../scenarios/astm/utm/prep_planners.py)) 4. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) 1. Suite: [DSS testing for ASTM F3548-21](dss_probing.md) ([`suites.astm.utm.dss_probing`](dss_probing.yaml)) -5. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) -6. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) +5. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) -7. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) +6. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) -8. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Data Validation of GET operational intents by USS](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md) ([`scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS`](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py)) -9. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Awareness of relevant operational intents](../../../scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md) ([`scenarios.astm.utm.subscription_notifications.ReceiveNotificationsForAwareness`](../../../scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py)) -10. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [ASTM SCD USS: Interfaces authentication](../../../scenarios/astm/utm/uss_authentication/scd_authentication_validation.md) ([`scenarios.astm.utm.uss_authentication.scd_authentication_validation.SCDAuthenticationValidation`](../../../scenarios/astm/utm/uss_authentication/scd_authentication_validation.py)) -11. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Off-Nominal planning: down USS](../../../scenarios/astm/utm/off_nominal_planning/down_uss.md) ([`scenarios.astm.utm.DownUSS`](../../../scenarios/astm/utm/off_nominal_planning/down_uss.py)) -12. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) - 1. Scenario: [Off-Nominal planning: down USS with equal priority conflicts not permitted](../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md) ([`scenarios.astm.utm.DownUSSEqualPriorityNotPermitted`](../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py)) -13. Scenario: [ASTM F3548 flight planners preparation](../../../scenarios/astm/utm/prep_planners.md) ([`scenarios.astm.utm.PrepareFlightPlanners`](../../../scenarios/astm/utm/prep_planners.py)) -14. Scenario: [ASTM F3548 makeUssReport](../../../scenarios/astm/utm/make_uss_report.md) ([`scenarios.astm.utm.make_uss_report.MakeUssReport`](../../../scenarios/astm/utm/make_uss_report.py)) -15. Scenario: [ASTM F3548-21 evaluate system versions](../../../scenarios/astm/utm/versioning/evaluate_system_versions.md) ([`scenarios.astm.utm.versioning.evaluate_system_versions.EvaluateSystemVersions`](../../../scenarios/astm/utm/versioning/evaluate_system_versions.py)) -16. Scenario: [ASTM F3548 UTM aggregate checks](../../../scenarios/astm/utm/aggregate_checks.md) ([`scenarios.astm.utm.AggregateChecks`](../../../scenarios/astm/utm/aggregate_checks.py)) +7. Action generator: [`action_generators.resource_environments.ResourceEnvironmentsActionGenerator`](../../../action_generators/resource_environments.py) + 1. Suite: [Single flight planner testing for ASTM F3548-21](single_planner.md) ([`suites.astm.utm.single_planner`](single_planner.yaml)) +8. Scenario: [ASTM F3548 flight planners preparation](../../../scenarios/astm/utm/prep_planners.md) ([`scenarios.astm.utm.PrepareFlightPlanners`](../../../scenarios/astm/utm/prep_planners.py)) +9. Scenario: [ASTM F3548 makeUssReport](../../../scenarios/astm/utm/make_uss_report.md) ([`scenarios.astm.utm.make_uss_report.MakeUssReport`](../../../scenarios/astm/utm/make_uss_report.py)) +10. Scenario: [ASTM F3548-21 evaluate system versions](../../../scenarios/astm/utm/versioning/evaluate_system_versions.md) ([`scenarios.astm.utm.versioning.evaluate_system_versions.EvaluateSystemVersions`](../../../scenarios/astm/utm/versioning/evaluate_system_versions.py)) +11. Scenario: [ASTM F3548 UTM aggregate checks](../../../scenarios/astm/utm/aggregate_checks.md) ([`scenarios.astm.utm.AggregateChecks`](../../../scenarios/astm/utm/aggregate_checks.py)) ## [Checked requirements](../../README.md#checked-requirements) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index 5adf44e1ce..07b2b57644 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -106,6 +106,11 @@ local_resources: priority_preemption: priority_preemption_environments? equal_priority_conflict: equal_priority_conflict_environments specification: {} + single_planner_environments: + resource_type: resources.combinations.ResourceCombinationsResource + dependencies: + tested_uss: flight_planners + specification: {} actions: - test_scenario: scenario_type: scenarios.versioning.GetSystemVersions @@ -151,25 +156,6 @@ actions: planning_area: planning_area problematically_big_area: problematically_big_area test_exclusions: test_exclusions? -- action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations - resources: - flight_planners: flight_planners - flight_intent_validation_selector: flight_intent_validation_selector? - invalid_flight_intents: invalid_flight_intents - dss: dss - specification: - action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.FlightIntentValidation - resources: - flight_intents: invalid_flight_intents - tested_uss: uss1 - dss: dss - combination_selector_source: flight_intent_validation_selector? - flight_planners_source: flight_planners - roles: - - uss1 - action_generator: generator_type: action_generators.resource_environments.ResourceEnvironmentsActionGenerator resources: @@ -201,100 +187,28 @@ actions: control_uss: uss2 dss: dss - action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations - resources: - flight_planners: flight_planners - non_conflicting_flights: non_conflicting_flights - dss: dss - mock_uss: mock_uss - specification: - action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS - resources: - tested_uss: tested_uss - mock_uss: mock_uss - dss: dss - flight_intents: non_conflicting_flights - flight_planners_source: flight_planners - roles: - - tested_uss -- action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations + generator_type: action_generators.resource_environments.ResourceEnvironmentsActionGenerator resources: - flight_planners: flight_planners + single_planner_environments: single_planner_environments + invalid_flight_intents: invalid_flight_intents non_conflicting_flights: non_conflicting_flights + conflicting_flights: conflicting_flights dss: dss - mock_uss: mock_uss - specification: - action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.subscription_notifications.ReceiveNotificationsForAwareness - resources: - tested_uss: tested_uss - mock_uss: mock_uss - dss: dss - flight_intents: non_conflicting_flights - flight_planners_source: flight_planners - roles: - - tested_uss -- action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations - resources: - flight_planners: flight_planners + mock_uss: mock_uss? utm_auth: utm_auth? - dss: dss - non_conflicting_flights: non_conflicting_flights specification: + resource_environments_source: single_planner_environments action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.uss_authentication.scd_authentication_validation.SCDAuthenticationValidation + test_suite: + suite_type: suites.astm.utm.single_planner resources: tested_uss: tested_uss - utm_auth: utm_auth - dss: dss - flight_intents: non_conflicting_flights - flight_planners_source: flight_planners - roles: - - tested_uss -- action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations - resources: - flight_planners: flight_planners - nominal_planning_selector: nominal_planning_selector? - conflicting_flights: conflicting_flights - dss: dss - specification: - action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.DownUSS - resources: - flight_intents: conflicting_flights - tested_uss: uss1 - dss: dss - combination_selector_source: nominal_planning_selector - flight_planners_source: flight_planners - roles: - - uss1 -- action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations - resources: - flight_planners: flight_planners - nominal_planning_selector: nominal_planning_selector? - conflicting_flights: conflicting_flights - dss: dss - specification: - action_to_repeat: - test_scenario: - scenario_type: scenarios.astm.utm.DownUSSEqualPriorityNotPermitted - resources: - flight_intents: conflicting_flights - tested_uss: uss1 + invalid_flight_intents: invalid_flight_intents + non_conflicting_flights: non_conflicting_flights + conflicting_flights: conflicting_flights dss: dss - combination_selector_source: nominal_planning_selector - flight_planners_source: flight_planners - roles: - - uss1 + mock_uss: mock_uss? + utm_auth: utm_auth? - test_scenario: scenario_type: scenarios.astm.utm.PrepareFlightPlanners # TODO: Convert this to a scenario which only clears the area (does not check readiness) resources: diff --git a/monitoring/uss_qualifier/suites/astm/utm/single_planner.md b/monitoring/uss_qualifier/suites/astm/utm/single_planner.md new file mode 100644 index 0000000000..227d799698 --- /dev/null +++ b/monitoring/uss_qualifier/suites/astm/utm/single_planner.md @@ -0,0 +1,157 @@ + +# Single flight planner testing for ASTM F3548-21 test suite +[`suites.astm.utm.single_planner`](./single_planner.yaml) + +## [Actions](../../README.md#actions) + +1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) +2. Scenario: [Data Validation of GET operational intents by USS](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md) ([`scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS`](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py)) +3. Scenario: [Awareness of relevant operational intents](../../../scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.md) ([`scenarios.astm.utm.subscription_notifications.ReceiveNotificationsForAwareness`](../../../scenarios/astm/utm/subscription_notifications/receive_notifications_for_awareness/receive_notifications_for_awareness.py)) +4. Scenario: [ASTM SCD USS: Interfaces authentication](../../../scenarios/astm/utm/uss_authentication/scd_authentication_validation.md) ([`scenarios.astm.utm.uss_authentication.scd_authentication_validation.SCDAuthenticationValidation`](../../../scenarios/astm/utm/uss_authentication/scd_authentication_validation.py)) +5. Scenario: [Off-Nominal planning: down USS](../../../scenarios/astm/utm/off_nominal_planning/down_uss.md) ([`scenarios.astm.utm.DownUSS`](../../../scenarios/astm/utm/off_nominal_planning/down_uss.py)) +6. Scenario: [Off-Nominal planning: down USS with equal priority conflicts not permitted](../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.md) ([`scenarios.astm.utm.DownUSSEqualPriorityNotPermitted`](../../../scenarios/astm/utm/off_nominal_planning/down_uss_equal_priority_not_permitted.py)) + +## [Checked requirements](../../README.md#checked-requirements) + +