From 3b3b618f1d56235c76710b44b0a4f8abe1317772 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Sun, 18 Oct 2026 15:06:58 -0700 Subject: [PATCH] Remove participant verifiable capabilities --- AGENTS.md | 1 + github_pages/static/index.md | 1 - monitoring/uss_qualifier/README.md | 8 - .../configurations/dev/uspace.yaml | 3 - .../uss_qualifier/reports/capabilities.py | 282 --------- .../reports/capability_definitions.py | 104 ---- monitoring/uss_qualifier/reports/report.py | 204 +------ .../suites/astm/netrid/f3411_19.yaml | 25 - .../suites/astm/netrid/f3411_22a.yaml | 573 ------------------ .../suites/astm/utm/f3548_21.yaml | 15 - .../uss_qualifier/suites/definitions.py | 6 - monitoring/uss_qualifier/suites/suite.py | 33 - .../suites/uspace/flight_auth.yaml | 24 - .../suites/uspace/network_identification.yaml | 31 - .../suites/uspace/required_services.yaml | 12 - .../capability_definitions/AllConditions.json | 21 - .../capability_definitions/AnyCondition.json | 21 - .../CapabilityVerificationCondition.json | 62 -- .../CapabilityVerifiedCondition.json | 29 - .../NoFailedChecksCondition.json | 12 - .../ParticipantCapabilityDefinition.json | 34 -- .../RequirementsCheckedCondition.json | 19 - .../report/AllConditionsEvaluationReport.json | 30 - .../report/AnyConditionEvaluationReport.json | 30 - ...lityVerifiedConditionEvaluationReport.json | 38 -- .../reports/report/CheckedCapability.json | 34 -- .../reports/report/CheckedRequirement.json | 35 -- ...FailedChecksConditionEvaluationReport.json | 22 - ...ntCapabilityConditionEvaluationReport.json | 74 --- ...ParticipantCapabilityEvaluationReport.json | 34 -- ...mentsCheckedConditionEvaluationReport.json | 38 -- .../reports/report/SpuriousReportMatch.json | 24 - .../reports/report/TestSuiteReport.json | 8 - .../definitions/TestSuiteDefinition.json | 10 - 34 files changed, 14 insertions(+), 1883 deletions(-) delete mode 100644 monitoring/uss_qualifier/reports/capabilities.py delete mode 100644 monitoring/uss_qualifier/reports/capability_definitions.py delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/AllConditions.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/AnyCondition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerificationCondition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerifiedCondition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/NoFailedChecksCondition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/ParticipantCapabilityDefinition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/capability_definitions/RequirementsCheckedCondition.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/AllConditionsEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/AnyConditionEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/CapabilityVerifiedConditionEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/CheckedCapability.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/CheckedRequirement.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/NoFailedChecksConditionEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityConditionEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/RequirementsCheckedConditionEvaluationReport.json delete mode 100644 schemas/monitoring/uss_qualifier/reports/report/SpuriousReportMatch.json diff --git a/AGENTS.md b/AGENTS.md index 14454910b1..ff94f05cde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,7 @@ This document contains key context, nuances, and troubleshooting tips specifical ## 4. Local Testing constraints - **Unit-test working directory**: Run `uss_qualifier` unit tests from `monitoring/uss_qualifier/` with the repository root on `PYTHONPATH` (for example, `PYTHONPATH=../.. ../../.venv/bin/python -m pytest scenarios/scenario_test resources/flight_planning`). Some resource fixtures resolve test-data paths relative to the working directory and fail when run from the repository root. - **Qualifier Working Directory**: Run resource-construction checks and qualifier resource tests from `monitoring/uss_qualifier` with the repository root on `PYTHONPATH`. External resource paths such as `file://./test_data/...` resolve relative to the current working directory, as they do in `run_locally.sh`. +- **Local Python 3.14 artifact generation**: Python 3.14 defaults to `forkserver` on Linux, which cannot pickle the local worker functions in `reports/artifacts.py`. If a local qualifier run fails there with `PicklingError`, use the project's Docker runtime or a local launcher that calls `multiprocessing.set_start_method("fork")` before running `monitoring.uss_qualifier.main`. - **Docker Dependency**: Mock USS and DSS environments require active Docker containers. Standard testing commands are typically structured via bash scripts like `./monitoring/uss_qualifier/run_locally.sh `. If container-building fails due to `Authentication` or package registry issues in the agent's environment, gracefully halt and ask the human user to run the script instead. ## 5. Continuous Improvement of this Guide diff --git a/github_pages/static/index.md b/github_pages/static/index.md index 2b7fdd0224..1b646735ae 100644 --- a/github_pages/static/index.md +++ b/github_pages/static/index.md @@ -11,7 +11,6 @@ These reports were generated during continuous integration for the most recent P * [Sequence view](./artifacts/uss_qualifier/reports/uspace/sequence) * [Tested requirements](./artifacts/uss_qualifier/reports/uspace/requirements) * [Timing report](./artifacts/uss_qualifier/reports/uspace/timing) -* [Demonstrated capabilities](./artifacts/uss_qualifier/reports/uspace/capabilities.html) * [Raw report](./artifacts/uss_qualifier/reports/uspace/report.json) (large) ### [No-op test configuration](https://github.com/interuss/monitoring/blob/main/monitoring/uss_qualifier/configurations/dev/noop.yaml) diff --git a/monitoring/uss_qualifier/README.md b/monitoring/uss_qualifier/README.md index 6df5981b84..9e012eb5ea 100644 --- a/monitoring/uss_qualifier/README.md +++ b/monitoring/uss_qualifier/README.md @@ -130,11 +130,3 @@ Any artifacts specified in test configuration are generated. #### Validate test report Test report is evaluated in order to return the appropriate "success or error" execution code (if specified in test configuration). - -### Verifiable capabilities - -A test suite may define one or more verifiable "capabilities" and the criteria necessary to verify them for a participant. These verifiable capabilities are intended to be used to indicate whether a participant has satisfied the requirements for an optional capability. - -For instance, ASTM F3411-22a network remote identification includes the concept of an "intent-based network participant" who provides information about their overall flight intent, but not real-time telemetry. A NetRID Service Provider is not required to support intent-based network participants, but if they do choose to support them, then they must follow a few requirements. In this case, an "intent-based network participant support" capability may be defined to be verified only if uss_qualifier confirms compliance with the requirements regarding intent-based network participants. If compliance to the intent-based network participants cannot be confirmed, the NetRID Service Provider under test may still be fully compliant with the standard; they just elected not to support this particular optional capability and would therefore be standard-compliant without verifying the "intent-based network participant support" capability. - -As another example, ASTM F3411-22a does not require a NetRID Display Provider to provide Display Application clients with the serial number of an identified aircraft. However, a particular jurisdiction may accept F3411-22a remote identification as a means of compliance only if Display Applications are capable of providing the viewer with the aircraft serial number. In this case, the ASTM F3411-22a test suite could define a "serial number" capability indicating that a Display Provider provides the correct aircraft serial number to its Display Application clients, and then the test suite for the jurisdiction may define a "qualified in jurisdiction" capability which is not verified unless the ASTM F3411-22a test suite's "serial number" capability is verified (among other criteria). diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index d6b02b9643..0ebb690ace 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -130,9 +130,6 @@ v1: stop_fast: true artifacts: raw_report: {} - templated_reports: - - template_url: https://github.com/Orbitalize/reports/releases/download/v0.0.18/app-v0.0.18.zip - report_name: capabilities tested_requirements: - report_name: requirements requirement_collections: diff --git a/monitoring/uss_qualifier/reports/capabilities.py b/monitoring/uss_qualifier/reports/capabilities.py deleted file mode 100644 index 8f395a6a89..0000000000 --- a/monitoring/uss_qualifier/reports/capabilities.py +++ /dev/null @@ -1,282 +0,0 @@ -from collections.abc import Callable -from typing import Any, TypeVar - -from bc_jsonpath_ng.ext.parser import parse as bc_jsonpath_ng_ext_parse - -from monitoring.uss_qualifier.configurations.configuration import ParticipantID -from monitoring.uss_qualifier.reports.capability_definitions import ( - AllConditions, - AnyCondition, - CapabilityVerificationCondition, - CapabilityVerifiedCondition, - NoFailedChecksCondition, - RequirementsCheckedCondition, - SpecificCondition, -) -from monitoring.uss_qualifier.reports.report import ( - AllConditionsEvaluationReport, - AnyConditionEvaluationReport, - CapabilityVerifiedConditionEvaluationReport, - CheckedCapability, - CheckedRequirement, - NoFailedChecksConditionEvaluationReport, - ParticipantCapabilityConditionEvaluationReport, - RequirementsCheckedConditionEvaluationReport, - SpuriousReportMatch, - TestSuiteReport, -) -from monitoring.uss_qualifier.requirements.definitions import RequirementID -from monitoring.uss_qualifier.requirements.documentation import ( - resolve_requirements_collection, -) - -SpecificConditionType = TypeVar("SpecificConditionType", bound=SpecificCondition) -ConditionEvaluator = Callable[ - [SpecificConditionType, ParticipantID, TestSuiteReport], - ParticipantCapabilityConditionEvaluationReport, -] -_capability_condition_evaluators: dict[type[SpecificCondition], ConditionEvaluator] = {} - - -def capability_condition_evaluator[SpecificConditionType: SpecificCondition]( - condition_type: type[SpecificConditionType], -): - """Decorator to label a function that evaluates a specific condition for verifying a capability. - - Args: - condition_type: A Type that inherits from capability_definitions.SpecificCondition. - """ - - def register_evaluator(func: ConditionEvaluator) -> ConditionEvaluator: - _capability_condition_evaluators[condition_type] = func - return func - - return register_evaluator - - -def evaluate_condition_for_test_suite( - grant_condition: CapabilityVerificationCondition, - participant_id: ParticipantID, - report: TestSuiteReport, -) -> ParticipantCapabilityConditionEvaluationReport: - """Determine if a condition for verifying a capability is satisfied based on a Test Suite report. - - Args: - grant_condition: Capability-verifying condition to check. - participant_id: Participant for which the capability would be verified. - report: Test Suite report upon which the capability (and verification condition) are based. - - Returns: True if the condition was satisfied, False if not. - """ - populated_fields = [ - field_name - for field_name in grant_condition - if grant_condition[field_name] is not None - ] - if not populated_fields: - raise ValueError( - "No specific condition specified for grant_condition in CapabilityVerificationCondition" - ) - if len(populated_fields) > 1: - raise ValueError( - "Multiple conditions specified for grant_condition in CapabilityVerificationCondition: " - + ", ".join(populated_fields) - ) - specific_condition = grant_condition[populated_fields[0]] - condition_evaluator = _capability_condition_evaluators.get( - type(specific_condition), None - ) - if condition_evaluator is None: - raise RuntimeError( - f"Could not find evaluator for condition type {type(specific_condition).__name__}" - ) - return condition_evaluator(specific_condition, participant_id, report) - - -@capability_condition_evaluator(AllConditions) -def evaluate_all_conditions_condition( - condition: AllConditions, participant_id: ParticipantID, report: TestSuiteReport -) -> ParticipantCapabilityConditionEvaluationReport: - satisfied_conditions: list[ParticipantCapabilityConditionEvaluationReport] = [] - unsatisfied_conditions: list[ParticipantCapabilityConditionEvaluationReport] = [] - for subcondition in condition.conditions: - subreport = evaluate_condition_for_test_suite( - subcondition, participant_id, report - ) - if subreport.condition_satisfied: - satisfied_conditions.append(subreport) - else: - unsatisfied_conditions.append(subreport) - return ParticipantCapabilityConditionEvaluationReport( - condition_satisfied=len(satisfied_conditions) > 0 - and len(unsatisfied_conditions) == 0, - all_conditions=AllConditionsEvaluationReport( - satisfied_conditions=satisfied_conditions, - unsatisfied_conditions=unsatisfied_conditions, - ), - ) - - -@capability_condition_evaluator(AnyCondition) -def evaluate_any_condition_condition( - condition: AnyCondition, participant_id: ParticipantID, report: TestSuiteReport -) -> ParticipantCapabilityConditionEvaluationReport: - satisfied_options: list[ParticipantCapabilityConditionEvaluationReport] = [] - unsatisfied_options: list[ParticipantCapabilityConditionEvaluationReport] = [] - for subcondition in condition.conditions: - subreport = evaluate_condition_for_test_suite( - subcondition, participant_id, report - ) - if subreport.condition_satisfied: - satisfied_options.append(subreport) - else: - unsatisfied_options.append(subreport) - return ParticipantCapabilityConditionEvaluationReport( - condition_satisfied=len(satisfied_options) > 0, - all_conditions=AnyConditionEvaluationReport( - satisfied_options=satisfied_options, unsatisfied_options=unsatisfied_options - ), - ) - - -@capability_condition_evaluator(NoFailedChecksCondition) -def evaluate_no_failed_checks_condition( - condition: NoFailedChecksCondition, - participant_id: ParticipantID, - report: TestSuiteReport, -) -> ParticipantCapabilityConditionEvaluationReport: - failed_check_paths = [ - "$." + path for path, _ in report.query_failed_checks(participant_id) - ] - return ParticipantCapabilityConditionEvaluationReport( - condition_satisfied=len(failed_check_paths) == 0, - no_failed_checks=NoFailedChecksConditionEvaluationReport( - failed_checks=failed_check_paths - ), - ) - - -@capability_condition_evaluator(RequirementsCheckedCondition) -def evaluate_requirements_checked_conditions( - condition: RequirementsCheckedCondition, - participant_id: ParticipantID, - report: TestSuiteReport, -) -> ParticipantCapabilityConditionEvaluationReport: - req_checks: dict[RequirementID, CheckedRequirement] = { - req_id: CheckedRequirement( - requirement_id=req_id, passed_checks=[], failed_checks=[] - ) - for req_id in resolve_requirements_collection(condition.checked) - } - for path, passed_check in report.query_passed_checks(participant_id): - for req_id in passed_check.requirements: - if req_id in req_checks: - req_checks[req_id].passed_checks.append("$." + path) - for path, failed_check in report.query_failed_checks(participant_id): - for req_id in failed_check.requirements: - if req_id in req_checks: - req_checks[req_id].failed_checks.append("$." + path) - passed = [ - cr for cr in req_checks.values() if cr.passed_checks and not cr.failed_checks - ] - failed = [cr for cr in req_checks.values() if cr.failed_checks] - untested = [ - cr.requirement_id - for cr in req_checks.values() - if not cr.passed_checks and not cr.failed_checks - ] - return ParticipantCapabilityConditionEvaluationReport( - condition_satisfied=all(cr.passed_checks for cr in req_checks.values()) - and not any(cr.failed_checks for cr in req_checks.values()), - requirements_checked=RequirementsCheckedConditionEvaluationReport( - passed_requirements=passed, - failed_requirements=failed, - untested_requirements=untested, - ), - ) - - -def _jsonpath_of(descendant: Any, ancestor: Any) -> str: - """Construct a relative JSONPath to descendant from ancestor by exhaustive reference equality search. - - One would think this functionality would be part of the jsonpath_ng package when producing matches, but one would - apparently be wrong. This approach is monstrously inefficient, but easy to write and easy to understand. - """ - - def __jsonpath_of(descendant: Any, ancestor: Any) -> str | None: - if ancestor is descendant: - return "" - elif isinstance(ancestor, dict): - for k, v in ancestor.items(): - subpath = __jsonpath_of(descendant, v) - if subpath is not None: - return f".{k}{subpath}" - elif isinstance(ancestor, list): - for i, v in enumerate(ancestor): - subpath = __jsonpath_of(descendant, v) - if subpath is not None: - return f"[{i}]{subpath}" - - return None - - result = __jsonpath_of(descendant, ancestor) - - if result is None: - raise ValueError(f"No path to {descendant} from {ancestor}") - - return result - - -@capability_condition_evaluator(CapabilityVerifiedCondition) -def evaluate_capability_verified_condition( - condition: CapabilityVerifiedCondition, - participant_id: ParticipantID, - report: TestSuiteReport, -) -> ParticipantCapabilityConditionEvaluationReport: - path = ( - condition.capability_location - if "capability_location" in condition - and condition.capability_location is not None - else "$" - ) - matching_reports = bc_jsonpath_ng_ext_parse(path).find(report) - checked_capabilities = [] - spurious_matches = [] - for matching_report in matching_reports: - if isinstance(matching_report.value, TestSuiteReport): - for i, capability_eval in enumerate( - matching_report.value.capability_evaluations - ): - if capability_eval.participant_id != participant_id: - continue - if capability_eval.capability_id not in condition.capability_ids: - continue - report_path = "$" + _jsonpath_of(matching_report.value, report) - checked_capabilities.append( - CheckedCapability( - report_location=report_path, - capability_id=capability_eval.capability_id, - capability_location=f"{report_path}.capability_evaluations[{i}]", - capability_verified=capability_eval.verified, - ) - ) - else: - spurious_matches.append( - SpuriousReportMatch( - location="$" + _jsonpath_of(matching_report.value, report), - type=type(matching_report.value).__name__, - ) - ) - found_capabilities = {cc.capability_id for cc in checked_capabilities} - missing_capabilities = [ - c for c in condition.capability_ids if c not in found_capabilities - ] - return ParticipantCapabilityConditionEvaluationReport( - condition_satisfied=all(cc.capability_verified for cc in checked_capabilities) - and len(missing_capabilities) == 0, - capability_verified=CapabilityVerifiedConditionEvaluationReport( - checked_capabilities=checked_capabilities, - missing_capabilities=missing_capabilities, - spurious_matches=spurious_matches, - ), - ) diff --git a/monitoring/uss_qualifier/reports/capability_definitions.py b/monitoring/uss_qualifier/reports/capability_definitions.py deleted file mode 100644 index a21e715894..0000000000 --- a/monitoring/uss_qualifier/reports/capability_definitions.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -from implicitdict import ImplicitDict, Optional - -from monitoring.uss_qualifier.requirements.definitions import RequirementCollection - -CapabilityID = str -"""Identifier of a capability that uss_qualifier can verify.""" - -JSONPathExpression = str -"""JsonPath expression; see https://pypi.org/project/jsonpath-ng/""" -# TODO: Change type to JSONAddress where applicable following merging of #171 - - -class SpecificCondition(ImplicitDict): - pass - - -class AllConditions(SpecificCondition): - """Condition will only be satisfied when all specified conditions are satisfied. - - Note that an empty list of conditions will result in a successful evaluation.""" - - conditions: list[CapabilityVerificationCondition] - - -class AnyCondition(SpecificCondition): - """Condition will be satisfied when any of the specified conditions are satisfied. - - Note that an empty list of conditions will result in an unsuccessful evaluation.""" - - conditions: list[CapabilityVerificationCondition] - - -class RequirementsCheckedCondition(SpecificCondition): - """Condition will only be satisfied if at least one successful check exists for all specified requirements. - - Note that an empty collection of requirements will result in an unsuccessful evaluation. - """ - - checked: RequirementCollection - """Each requirement contained within this collection must be covered by at least one successful check.""" - - -class NoFailedChecksCondition(SpecificCondition): - """Condition will only be satisfied if there are no applicable failed checks. - - For a capability to be verified for a participant, only checks including the participant's ID will be considered. - """ - - pass - - -class CapabilityVerifiedCondition(SpecificCondition): - """Condition will be satisfied when the specified capability is verified. - - Note that a capability which do not declare any requirement will result in an unsuccessful evaluation. - """ - - capability_ids: list[CapabilityID] - """List of identifier of capability that must be verified for this condition to be satisfied.""" - - capability_location: Optional[JSONPathExpression] - """Location of report to inspect for the verification of the specified capability, relative to the report in which - the capability is defined. Implicit default value is "$" (look for verified capability in the report in which the - dependant capability is defined). - - If this location resolves to multiple TestSuiteReports, then the capability must be verified in all resolved reports - in order for this condition to be satisfied. When this location resolves to artifacts that are not - TestSuiteReports, those artifacts will be ignored. - - Note that capabilities are verified in the order they are defined. So, if capability B defined in a particular - location depends on whether capability A in that same location is granted, capability A must be defined before - capability B is defined. Also note that capability verifications are computed as test components are completed. - Since a parent test component (e.g., test suite) is not complete until all of its child components are complete, a - descendant test component's capability condition cannot depend on whether an ancestor's (e.g., parent's) capability - is verified. - """ - - -class CapabilityVerificationCondition(ImplicitDict): - """Specification of a single condition used to determine whether a capability should be verified. - - Exactly one field must be specified.""" - - all_conditions: Optional[AllConditions] - any_conditions: Optional[AnyCondition] - no_failed_checks: Optional[NoFailedChecksCondition] - requirements_checked: Optional[RequirementsCheckedCondition] - capability_verified: Optional[CapabilityVerifiedCondition] - - -class ParticipantCapabilityDefinition(ImplicitDict): - id: CapabilityID - """Identifier of this capability, unique at the level in which this capability is defined.""" - - name: str - """Human-readable name of the capability.""" - - description: str - """Human-readable description of the capability.""" - - verification_condition: CapabilityVerificationCondition - """Condition required in order to verify the capability.""" diff --git a/monitoring/uss_qualifier/reports/report.py b/monitoring/uss_qualifier/reports/report.py index 4945383da7..72da64eeb2 100644 --- a/monitoring/uss_qualifier/reports/report.py +++ b/monitoring/uss_qualifier/reports/report.py @@ -13,6 +13,7 @@ ) from monitoring.monitorlib import fetch, inspection +from monitoring.monitorlib.dicts import JSONAddress from monitoring.monitorlib.errors import stacktrace_string from monitoring.monitorlib.fileio import FileReference from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName @@ -21,10 +22,6 @@ ParticipantID, USSQualifierConfiguration, ) -from monitoring.uss_qualifier.reports.capability_definitions import ( - CapabilityID, - JSONPathExpression, -) from monitoring.uss_qualifier.requirements.definitions import RequirementID from monitoring.uss_qualifier.resources.definitions import ResourceID from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName @@ -144,26 +141,16 @@ def participants_with_failed_checks(self) -> set[str]: participants.add(p) return participants - def all_participants(self) -> set[ParticipantID]: - participants = set() - for pc in self.passed_checks: - for p in pc.participants: - participants.add(p) - for fc in self.failed_checks: - for p in fc.participants: - participants.add(p) - return participants - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: for i, pc in enumerate(self.passed_checks): if participant_id is None or participant_id in pc.participants: yield f"passed_checks[{i}]", pc def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: for i, fc in enumerate(self.failed_checks): if participant_id is None or participant_id in fc.participants: yield f"failed_checks[{i}]", fc @@ -215,22 +202,16 @@ class TestCaseReport(ImplicitDict): def has_critical_problem(self): return any(s.has_critical_problem() for s in self.steps) - def all_participants(self) -> set[ParticipantID]: - participants = set() - for step in self.steps: - participants = participants.union(step.all_participants()) - return participants - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: for i, step in enumerate(self.steps): for path, pc in step.query_passed_checks(participant_id): yield f"steps[{i}].{path}", pc def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: for i, step in enumerate(self.steps): for path, fc in step.query_failed_checks(participant_id): yield f"steps[{i}].{path}", fc @@ -326,17 +307,9 @@ def has_critical_problem(self) -> bool: return True return False - def all_participants(self) -> set[ParticipantID]: - participants = set() - for case in self.cases: - participants = participants.union(case.all_participants()) - if "cleanup" in self and self.cleanup: - participants = participants.union(self.cleanup.all_participants()) - return participants - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: for i, case in enumerate(self.cases): for path, pc in case.query_passed_checks(participant_id): yield f"cases[{i}].{path}", pc @@ -346,7 +319,7 @@ def query_passed_checks( def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: for i, case in enumerate(self.cases): for path, fc in case.query_failed_checks(participant_id): yield f"cases[{i}].{path}", fc @@ -421,22 +394,16 @@ class ActionGeneratorReport(ImplicitDict): def has_critical_problem(self) -> bool: return any(a.has_critical_problem() for a in self.actions) - def all_participants(self) -> set[ParticipantID]: - participants = set() - for action in self.actions: - participants = participants.union(action.all_participants()) - return participants - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: for i, action in enumerate(self.actions): for path, pc in action.query_passed_checks(participant_id): yield f"actions[{i}].{path}", pc def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: for i, action in enumerate(self.actions): for path, fc in action.query_failed_checks(participant_id): yield f"actions[{i}].{path}", fc @@ -516,12 +483,9 @@ def successful(self) -> bool: def has_critical_problem(self) -> bool: return self._conditional(lambda report: report.has_critical_problem()) - def all_participants(self) -> set[ParticipantID]: - return self._conditional(lambda report: report.all_participants()) - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: if "test_suite" in self and self.test_suite: report = self.test_suite prefix = "test_suite" @@ -539,7 +503,7 @@ def query_passed_checks( def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: if "test_suite" in self and self.test_suite: report = self.test_suite prefix = "test_suite" @@ -588,136 +552,6 @@ def get_action_type_name(self): return "unknown" -class AllConditionsEvaluationReport(ImplicitDict): - """Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied.""" - - satisfied_conditions: list[ParticipantCapabilityConditionEvaluationReport] - """All of the conditions that were satisfied (there must be at least one).""" - - unsatisfied_conditions: list[ParticipantCapabilityConditionEvaluationReport] - """All of the conditions that were unsatisfied (if any, then this condition will not be satisfied).""" - - -class AnyConditionEvaluationReport(ImplicitDict): - """Result of an evaluation of AnyCondition determined by whether any of the subconditions are satisfied.""" - - satisfied_options: list[ParticipantCapabilityConditionEvaluationReport] - """Which of the specified options were satisfied (if any were satisfied, then this condition should be satisfied).""" - - unsatisfied_options: list[ParticipantCapabilityConditionEvaluationReport] - """Which of the specified options were not satisfied (these are informational only and do not affect the evaluation).""" - - -class NoFailedChecksConditionEvaluationReport(ImplicitDict): - """Result of an evaluation of NoFailedChecksCondition dependent on whether any checks failed within the scope of the test suite in which this condition is located.""" - - failed_checks: list[JSONPathExpression] - """The location of each FailedCheck, relative to the TestSuiteReport in which this report is located.""" - - -class CheckedRequirement(ImplicitDict): - """A single requirement being checked for participant-verifiable capability verification.""" - - requirement_id: RequirementID - """The requirement being checked.""" - - passed_checks: list[JSONPathExpression] - """The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located.""" - - failed_checks: list[JSONPathExpression] - """The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located.""" - - -class RequirementsCheckedConditionEvaluationReport(ImplicitDict): - """Result of an evaluation of RequirementsCheckedCondition dependent on whether a set of requirements were successfully checked.""" - - passed_requirements: list[CheckedRequirement] - """Requirements with only PassedChecks.""" - - failed_requirements: list[CheckedRequirement] - """Requirements with FailedChecks.""" - - untested_requirements: list[RequirementID] - """Requirements that didn't have any PassedChecks or FailedChecks within the scope of the test suite in which this condition is located.""" - - -class CheckedCapability(ImplicitDict): - """Existing/previous participant-verifiable capability upon which a CapabilityVerifiedCondition depends.""" - - report_location: JSONPathExpression - """Location of the ParticipantCapabilityEvaluationReport for the existing/previous capability, relative to the TestSuiteReport in which the CapabilityVerifiedConditionEvaluationReport containing this CheckedCapability is located.""" - - capability_id: CapabilityID - """ID of the existing/previous participant-verifiable capability.""" - - capability_location: JSONPathExpression - """The location of the ParticipantCapabilityConditionEvaluationReport for the capability, relative to the TestSuiteReport in which this checked requirement is located.""" - - capability_verified: bool - """Whether this capability was successfully verified""" - - -class SpuriousReportMatch(ImplicitDict): - """Participant-verifiable capability evaluations are only present in TestSuiteReports. If a CapabilityVerifiedCondition points to a report element that is not a TestSuiteReport, an instance of this class will be generated.""" - - location: JSONPathExpression - """Location of the non-TestSuiteReport report element matching the CapabilityVerifiedCondition's `capability_location`, relative to the TestSuiteReport in which this condition is located.""" - - type: str - """Data type of the report element (not TestSuiteReport).""" - - -class CapabilityVerifiedConditionEvaluationReport(ImplicitDict): - """Result of an evaluation of a CapabilityVerifiedCondition dependent on whether other capabilities were verified.""" - - checked_capabilities: list[CheckedCapability] - """All capability evaluations checked for this condition.""" - - missing_capabilities: list[CapabilityID] - """Capabilities specified for this condition but not found in the report.""" - - spurious_matches: list[SpuriousReportMatch] - """Report elements matching the condition's `capability_location`, but not of the type TestSuiteReport.""" - - -class ParticipantCapabilityConditionEvaluationReport(ImplicitDict): - """Result of an evaluation of a condition related to whether a participant capability should be verified. - - Exactly one field other than `condition_satisfied` must be specified.""" - - condition_satisfied: bool - """Whether the condition was satisfied for the relevant participant.""" - - all_conditions: Optional[AllConditionsEvaluationReport] - """When specified, the condition evaluated was AllConditions.""" - - any_conditions: Optional[AnyConditionEvaluationReport] - """When specified, the condition evaluated was AnyCondition.""" - - no_failed_checks: Optional[NoFailedChecksConditionEvaluationReport] - """When specified, the condition evaluated was NoFailedChecksCondition.""" - - requirements_checked: Optional[RequirementsCheckedConditionEvaluationReport] - """When specified, the condition evaluated was RequirementsCheckedCondition.""" - - capability_verified: Optional[CapabilityVerifiedConditionEvaluationReport] - """When specified, the condition evaluated was CapabilityVerifiedCondition.""" - - -class ParticipantCapabilityEvaluationReport(ImplicitDict): - capability_id: CapabilityID - """ID of capability being evaluated.""" - - participant_id: ParticipantID - """ID of participant for which capability is being evaluated.""" - - verified: bool - """Whether the capability was successfully verified.""" - - condition_evaluation: ParticipantCapabilityConditionEvaluationReport - """Report produced by evaluating the condition for verifying this capability.""" - - class SkippedActionReport(ImplicitDict): timestamp: StringBasedDateTime """The time at which the action was skipped.""" @@ -736,9 +570,6 @@ def successful(self) -> bool: def has_critical_problem(self) -> bool: return False - def all_participants(self) -> set[ParticipantID]: - return set() - def queries(self) -> list[fetch.Query]: return [] @@ -776,28 +607,19 @@ class TestSuiteReport(ImplicitDict): successful: bool = False """DEPRECATED. True iff test suite completed normally with no failed checks""" - capability_evaluations: list[ParticipantCapabilityEvaluationReport] - """List of capabilities defined in this test suite, evaluated for each participant.""" - def has_critical_problem(self) -> bool: return any(a.has_critical_problem() for a in self.actions) - def all_participants(self) -> set[ParticipantID]: - participants = set() - for action in self.actions: - participants = participants.union(action.all_participants()) - return participants - def query_passed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, PassedCheck]]: + ) -> Iterator[tuple[JSONAddress, PassedCheck]]: for i, action in enumerate(self.actions): for path, pc in action.query_passed_checks(participant_id): yield f"actions[{i}].{path}", pc def query_failed_checks( self, participant_id: str | None = None - ) -> Iterator[tuple[JSONPathExpression, FailedCheck]]: + ) -> Iterator[tuple[JSONAddress, FailedCheck]]: for i, action in enumerate(self.actions): for path, fc in action.query_failed_checks(participant_id): yield f"actions[{i}].{path}", fc diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml index f289af2216..766a87269f 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_19.yaml @@ -103,28 +103,3 @@ actions: observers: observers dss_instances: dss_instances test_exclusions: test_exclusions? -participant_verifiable_capabilities: - - id: service_provider - name: NetRID Service Provider - description: Participant fulfills testable requirements necessary to act as a NetRID Service Provider according to F3411-19. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v19.service_provider#Tested by automated tests" - - id: display_provider - name: NetRID Display Provider - description: Participant fulfills testable requirements necessary to act as a NetRID Display Provider according to F3411-19. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v19.display_provider#Automated verification" - - id: dss_operator - name: NetRID DSS Operator - description: Participant fulfills testable requirements necessary to act operate a NetRID Discovery and Synchronization Service instance according to F3411-19. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v19.dss_provider" diff --git a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml index 7f4064f22a..b62521236a 100644 --- a/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml +++ b/monitoring/uss_qualifier/suites/astm/netrid/f3411_22a.yaml @@ -113,576 +113,3 @@ actions: observers: observers? dss_instances: dss_instances test_exclusions: test_exclusions? -participant_verifiable_capabilities: - - id: service_provider - name: NetRID Service Provider - description: Participant fulfills testable requirements necessary to act as a NetRID Service Provider according to F3411-22a. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Mandatory requirements" - - id: service_provider_intent_based - name: NetRID Service Provider with intent-based network participant support capability - description: Participant fulfills testable requirements necessary to provide Intent-based participant support capability according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Intent-based participant support" - - id: service_provider_extrapolation - name: NetRID Service Provider with position extrapolation capability - description: Participant fulfills testable requirements necessary to provide position extrapolation capability according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Position extrapolation" - - id: service_provider_operator_id_provider - name: NetRID Service Provider providing operator id data field - description: Participant fulfills testable requirements necessary to provide operator id data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Operator ID provider" - - id: service_provider_uas_id_serial_number_provider - name: NetRID Service Provider providing UAS ID serial number data field - description: Participant fulfills testable requirements necessary to provide UAS ID serial number (option 1) data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#UAS ID Serial Number provider" - - id: service_provider_height_provider - name: NetRID Service Provider providing height data field - description: Participant fulfills testable requirements necessary to provide height data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Height provider" - - id: service_provider_operator_position_provider - name: NetRID Service Provider providing operator latitude and longitude data fields - description: Participant fulfills testable requirements necessary to provide operator latitude and longitude data fields according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Operator Position provider" - - id: service_provider_operator_altitude_provider - name: NetRID Service Provider providing operator altitude data field - description: Participant fulfills testable requirements necessary to provide operator altitude data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Operator Altitude provider" - - id: service_provider_operator_location_type_provider - name: NetRID Service Provider providing operator location type data field - description: Participant fulfills testable requirements necessary to provide operator location type data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Operator Location Type provider" - - id: service_provider_operational_status_provider - name: NetRID Service Provider providing operational status data field - description: Participant fulfills testable requirements necessary to provide operational status data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - service_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.service_provider#Operational Status provider" - - - id: display_provider - name: NetRID Display Provider - description: NetRID Display Provider - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Mandatory requirements" - - id: display_provider_uas_id_serial_number_transmitter - name: NetRID Display Provider transmitting UAS ID serial number data field - description: Participant fulfills testable requirements necessary to transmit UAS ID serial number (option 1) data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UAS ID Serial Number transmitter" - - id: display_provider_uas_id_registration_id_transmitter - name: NetRID Display Provider transmitting UAS ID Registration ID data field - description: Participant fulfills testable requirements necessary to transmit UAS ID Registration ID (option 2) data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UAS ID Registration ID transmitter" - - id: display_provider_uas_id_utm_uuid_transmitter - name: NetRID Display Provider transmitting UAS ID UTM (UUID) data field - description: Participant fulfills testable requirements necessary to transmit UAS ID UTM (UUID) (option 3) data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UAS ID UTM (UUID) transmitter" - - id: display_provider_uas_id_specific_session_transmitter - name: NetRID Display Provider transmitting UAS ID Specific Session data field - description: Participant fulfills testable requirements necessary to transmit UAS ID Specific Session (option 4) data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UAS ID Specific Session ID transmitter" - - id: display_provider_ua_type_transmitter - name: NetRID Display Provider transmitting UA Type data field - description: Participant fulfills testable requirements necessary to transmit UA Type data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UA Type transmitter" - - id: display_provider_ua_classification_transmitter - name: NetRID Display Provider transmitting UA Classification data field - description: Participant fulfills testable requirements necessary to transmit UA Classification data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UA Classification transmitter" - - id: display_provider_ua_classification_type_transmitter - name: NetRID Display Provider transmitting UA Classification Type data field - description: Participant fulfills testable requirements necessary to transmit UA Classification Type data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#UA Classification Type transmitter" - - id: display_provider_timestamp_transmitter - name: NetRID Display Provider transmitting Timestamp data field - description: Participant fulfills testable requirements necessary to transmit Timestamp data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Timestamp transmitter" - - id: display_provider_timestamp_accuracy_transmitter - name: NetRID Display Provider transmitting Timestamp Accuracy data field - description: Participant fulfills testable requirements necessary to transmit Timestamp Accuracy data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Timestamp Accuracy transmitter" - - id: display_provider_operational_status_transmitter - name: NetRID Display Provider transmitting operational status data field - description: Participant fulfills testable requirements necessary to transmit operational status data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operational Status transmitter" - - id: display_provider_operational_description_transmitter - name: NetRID Display Provider transmitting operational description data field - description: Participant fulfills testable requirements necessary to transmit operational description data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operational Description transmitter" - - id: display_provider_operator_id_transmitter - name: NetRID Display Provider transmitting operator id data field - description: Participant fulfills testable requirements necessary to transmit operator id data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operator ID transmitter" - - id: display_provider_current_position_transmitter - name: NetRID Display Provider transmitting uas latitude and longitude data fields - description: Participant fulfills testable requirements necessary to transmit uas latitude and longitude data fields according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Current Position transmitter" - - id: display_provider_geodetic_altitude_transmitter - name: NetRID Display Provider transmitting geodetic altitude data field - description: Participant fulfills testable requirements necessary to transmit geodetic altitude data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Geodetic Altitude transmitter" - - id: display_provider_pressure_altitude_transmitter - name: NetRID Display Provider transmitting pressure altitude data field - description: Participant fulfills testable requirements necessary to transmit pressure altitude data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Pressure Altitude transmitter" - - id: display_provider_height_transmitter - name: NetRID Display Provider transmitting height data field - description: Participant fulfills testable requirements necessary to transmit height data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Height transmitter" - - id: display_provider_geodetic_vertical_accuracy_transmitter - name: NetRID Display Provider transmitting geodetic vertical accuracy data field - description: Participant fulfills testable requirements necessary to transmit geodetic vertical accuracy data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Geodetic Vertical Accuracy transmitter" - - id: display_provider_horizontal_accuracy_transmitter - name: NetRID Display Provider transmitting horizontal accuracy data field - description: Participant fulfills testable requirements necessary to transmit horizontal accuracy data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Horizontal Accuracy transmitter" - - id: display_provider_speed_accuracy_transmitter - name: NetRID Display Provider transmitting speed accuracy data field - description: Participant fulfills testable requirements necessary to transmit speed accuracy data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Speed Accuracy transmitter" - - id: display_provider_track_direction_transmitter - name: NetRID Display Provider transmitting track direction data field - description: Participant fulfills testable requirements necessary to transmit track direction data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Track Direction transmitter" - - id: display_provider_speed_transmitter - name: NetRID Display Provider transmitting speed data field - description: Participant fulfills testable requirements necessary to transmit speed data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Speed transmitter" - - id: display_provider_vertical_speed_transmitter - name: NetRID Display Provider transmitting vertical speed data field - description: Participant fulfills testable requirements necessary to transmit vertical speed data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Vertical Speed transmitter" - - id: display_provider_auth_data_transmitter - name: NetRID Display Provider transmitting additional authentication data field - description: Participant fulfills testable requirements necessary to transmit additional authentication data data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Auth Data transmitter" - - id: display_provider_operator_position_transmitter - name: NetRID Display Provider transmitting operator latitude and longitude data fields - description: Participant fulfills testable requirements necessary to transmit operator latitude and longitude data fields according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operator Position transmitter" - - id: display_provider_operator_altitude_transmitter - name: NetRID Display Provider transmitting operator altitude data field - description: Participant fulfills testable requirements necessary to transmit operator altitude data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operator Altitude transmitter" - - id: display_provider_operator_location_type_transmitter - name: NetRID Display Provider transmitting operator location type data field - description: Participant fulfills testable requirements necessary to transmit operator location type data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operator Location Type transmitter" - - id: display_provider_operating_area_radius_transmitter - name: NetRID Display Provider transmitting operating area radius data field - description: Participant fulfills testable requirements necessary to transmit operating area radius data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Radius transmitter" - - id: display_provider_operating_area_polygon_transmitter - name: NetRID Display Provider transmitting operating area polygon data field - description: Participant fulfills testable requirements necessary to transmit operating area polygon data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Polygon transmitter" - - id: display_provider_operating_area_count_transmitter - name: NetRID Display Provider transmitting operating area count data field - description: Participant fulfills testable requirements necessary to transmit operating area count data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Count transmitter" - - id: display_provider_operating_area_floor_transmitter - name: NetRID Display Provider transmitting operating area floor data field - description: Participant fulfills testable requirements necessary to transmit operating area floor data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Floor transmitter" - - id: display_provider_operating_area_ceiling_transmitter - name: NetRID Display Provider transmitting operating area ceiling data field - description: Participant fulfills testable requirements necessary to transmit operating area ceiling data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Ceiling transmitter" - - id: display_provider_operating_area_start_transmitter - name: NetRID Display Provider transmitting operating area start data field - description: Participant fulfills testable requirements necessary to transmit operating area start data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area Start transmitter" - - id: display_provider_operating_area_end_transmitter - name: NetRID Display Provider transmitting operating area end data field - description: Participant fulfills testable requirements necessary to transmit operating area end data field according to F3411-22a. - verification_condition: - all_conditions: - conditions: - - capability_verified: - capability_ids: - - display_provider - - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.display_provider#Operating Area End transmitter" - - - id: dss_operator - name: NetRID DSS Operator - description: Participant fulfills testable requirements necessary to act operate a NetRID Discovery and Synchronization Service instance according to F3411-22a. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3411.v22a.dss_provider" diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index b3141e7692..d646431ad1 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -298,18 +298,3 @@ actions: scenario_type: scenarios.astm.utm.AggregateChecks resources: flight_planners: flight_planners - -participant_verifiable_capabilities: - - id: scd - name: Strategic Conflict Detection - description: Participant fulfills testable requirements necessary to perform the Strategic Conflict Detection role. - verification_condition: - requirements_checked: - checked: - requirement_sets: - - "astm.f3548.v21.scd#Automated verification" - - id: no_failures - name: Fails No Existing Tests - description: Until testing coverage is sufficiently complete to verify all intended requirements for particular roles, this capability may be useful in communicating that the participant has passed all existing applicable tests for their role and capabilities. This capability should be removed once test coverage extends to all intended requirements. - verification_condition: - no_failed_checks: {} diff --git a/monitoring/uss_qualifier/suites/definitions.py b/monitoring/uss_qualifier/suites/definitions.py index 78156d2ad0..3aa7719cb6 100644 --- a/monitoring/uss_qualifier/suites/definitions.py +++ b/monitoring/uss_qualifier/suites/definitions.py @@ -7,9 +7,6 @@ from monitoring.uss_qualifier.action_generators.definitions import ( ActionGeneratorDefinition, ) -from monitoring.uss_qualifier.reports.capability_definitions import ( - ParticipantCapabilityDefinition, -) from monitoring.uss_qualifier.resources.definitions import ( ResourceDeclaration, ResourceID, @@ -130,9 +127,6 @@ class TestSuiteDefinition(ImplicitDict): actions: list[TestSuiteActionDeclaration] """The actions to take when running the test suite. Components will be executed in order.""" - participant_verifiable_capabilities: Optional[list[ParticipantCapabilityDefinition]] - """Definitions of capabilities verified by this test suite for individual participants.""" - @staticmethod def load_from_declaration( declaration: TestSuiteDeclaration, diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a772150427..c4d8614141 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -34,13 +34,9 @@ FullyQualifiedCheck, TestSuiteActionSelectionCondition, ) -from monitoring.uss_qualifier.reports.capabilities import ( - evaluate_condition_for_test_suite, -) from monitoring.uss_qualifier.reports.report import ( ActionGeneratorReport, FailedCheck, - ParticipantCapabilityEvaluationReport, Severity, SkippedActionReport, TestScenarioReport, @@ -320,7 +316,6 @@ def run(self, context: ExecutionContext) -> TestSuiteReport: documentation_url=self.documentation_url, start_time=StringBasedDateTime(datetime.now(UTC)), actions=[], - capability_evaluations=[], ) def actions() -> Iterator[TestSuiteAction | SkippedActionReport]: @@ -328,34 +323,6 @@ def actions() -> Iterator[TestSuiteAction | SkippedActionReport]: _run_actions(actions(), context, report) - # Evaluate participants' capabilities - if ( - "participant_verifiable_capabilities" in self.definition - and self.definition.participant_verifiable_capabilities - ): - all_participants = report.all_participants() - for capability in self.definition.participant_verifiable_capabilities: - for participant_id in all_participants: - cond_eval_report = evaluate_condition_for_test_suite( - capability.verification_condition, participant_id, report - ) - report.capability_evaluations.append( - ParticipantCapabilityEvaluationReport( - capability_id=capability.id, - participant_id=participant_id, - verified=cond_eval_report.condition_satisfied, - condition_evaluation=cond_eval_report, - ) - ) - if cond_eval_report.condition_satisfied: - logger.info( - "Test suite {} verified {} capability '{}' for {}", - self.declaration.type_name, - capability.id, - capability.name, - participant_id, - ) - return report diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml index bf78a6739d..3606d58aee 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml @@ -75,27 +75,3 @@ actions: flight_planners_source: flight_planners roles: - flight_planner -participant_verifiable_capabilities: - - id: no_failures - name: Fails No Existing Tests - description: Until testing coverage is sufficiently complete to verify all intended requirements for particular roles, this capability may be useful in communicating that the participant has passed all existing applicable tests for their role and capabilities. This capability should be removed once test coverage extends to all intended requirements. - verification_condition: - all_conditions: - conditions: - - no_failed_checks: {} - - capability_verified: - capability_ids: - - no_failures - # Look for the no_failures capability in any child action that is a suites.astm.utm.f3548_21 test suite - capability_location: '$.actions[?(@.test_suite)].test_suite[?(@.suite_type=="suites.astm.utm.f3548_21")]' - # ^ Root element is the report for the test suite in which this capability is defined (the test suite defined by this YAML file) - # ^ Select the value corresponding to the "actions" key - # ^ Keep only the items (in "actions") which have a "test_suite" key - # ^ Select the value corresponding to the "test_suite" key (type is TestSuiteReport) - # ^ Keep only the items whose "suite_type" key has a value of "suites.astm.utm.f3548_21" - - # Alternately, look for the no_failures capability in any *descendant* test suite that is a suites.astm.utm.f3548_21 type - # capability_location: '$..*[?(@.suite_type=="suites.astm.utm.f3548_21")]' - - # Alternately, look for the no_failures capability in the specific location where the F3548-21 test suite is - # capability_location: '$.actions[0].test_suite' diff --git a/monitoring/uss_qualifier/suites/uspace/network_identification.yaml b/monitoring/uss_qualifier/suites/uspace/network_identification.yaml index 1b367d3709..ee2b6529d8 100644 --- a/monitoring/uss_qualifier/suites/uspace/network_identification.yaml +++ b/monitoring/uss_qualifier/suites/uspace/network_identification.yaml @@ -42,34 +42,3 @@ actions: scenario_type: scenarios.uspace.netrid.msl.MSLAltitude resources: observers: observers -participant_verifiable_capabilities: -- id: uspace_netrid_service_provider - name: U-Space NetRID Service Provider - description: Participant fulfills testable requirements necessary to act as a provider of the mandatory NetRID U-space service via the use of ASTM F3411-22a as the primary means of compliance. - verification_condition: - all_conditions: - conditions: - - no_failed_checks: {} - - capability_verified: - capability_ids: - - service_provider - - display_provider - - service_provider_operator_id_provider # Operator Registration - - service_provider_uas_id_serial_number_provider # Serial Number - - service_provider_height_provider # Height - - service_provider_operator_position_provider # Operator Location - - service_provider_operator_altitude_provider # Operator Altitude - - service_provider_operator_location_type_provider # Operator Location Type - - service_provider_operational_status_provider # Operational Status - - display_provider_operator_id_transmitter # Operator Registration - - display_provider_uas_id_serial_number_transmitter # Serial Number - - display_provider_current_position_transmitter # UA Position - - display_provider_height_transmitter # Height - - display_provider_timestamp_transmitter # Timestamp - - display_provider_track_direction_transmitter # Track - - display_provider_speed_transmitter # Speed - - display_provider_operator_position_transmitter # Operator Location - - display_provider_operator_altitude_transmitter # Operator Altitude - - display_provider_operator_location_type_transmitter # Operator Location Type - - display_provider_operational_status_transmitter # Operational Status - capability_location: '$..*[?(@.suite_type=="suites.astm.netrid.f3411_22a")]' diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.yaml b/monitoring/uss_qualifier/suites/uspace/required_services.yaml index f01b954a89..5b4859b8fa 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.yaml +++ b/monitoring/uss_qualifier/suites/uspace/required_services.yaml @@ -90,15 +90,3 @@ actions: mock_uss_dp: mock_uss_dp? mock_uss_sp: mock_uss_sp? uss_identification: uss_identification? -participant_verifiable_capabilities: - - id: required_services - name: U-space Required Services (work in progress) - description: Participant fulfills all implemented tests for all testable requirements necessary to provide all required services in U-space. Note that this test suite is not yet complete, so this capability is not yet a good indication that all testable requirements have been met. - verification_condition: - all_conditions: - conditions: - - no_failed_checks: {} - - capability_verified: - capability_ids: - - uspace_netrid_service_provider - capability_location: '$.actions[?(@.test_suite)].test_suite[?(@.suite_type=="suites.uspace.network_identification")]' diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/AllConditions.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/AllConditions.json deleted file mode 100644 index 2ef50616d5..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/AllConditions.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/AllConditions.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Condition will only be satisfied when all specified conditions are satisfied.\n\nNote that an empty list of conditions will result in a successful evaluation.\n\nmonitoring.uss_qualifier.reports.capability_definitions.AllConditions, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "conditions": { - "items": { - "$ref": "CapabilityVerificationCondition.json" - }, - "type": "array" - } - }, - "required": [ - "conditions" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/AnyCondition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/AnyCondition.json deleted file mode 100644 index 9a6a00eea3..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/AnyCondition.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/AnyCondition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Condition will be satisfied when any of the specified conditions are satisfied.\n\nNote that an empty list of conditions will result in an unsuccessful evaluation.\n\nmonitoring.uss_qualifier.reports.capability_definitions.AnyCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "conditions": { - "items": { - "$ref": "CapabilityVerificationCondition.json" - }, - "type": "array" - } - }, - "required": [ - "conditions" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerificationCondition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerificationCondition.json deleted file mode 100644 index 8f8fd2200b..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerificationCondition.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerificationCondition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Specification of a single condition used to determine whether a capability should be verified.\n\nExactly one field must be specified.\n\nmonitoring.uss_qualifier.reports.capability_definitions.CapabilityVerificationCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "all_conditions": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "AllConditions.json" - } - ] - }, - "any_conditions": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "AnyCondition.json" - } - ] - }, - "capability_verified": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "CapabilityVerifiedCondition.json" - } - ] - }, - "no_failed_checks": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "NoFailedChecksCondition.json" - } - ] - }, - "requirements_checked": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "RequirementsCheckedCondition.json" - } - ] - } - }, - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerifiedCondition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerifiedCondition.json deleted file mode 100644 index dbba84ca25..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerifiedCondition.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/CapabilityVerifiedCondition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Condition will be satisfied when the specified capability is verified.\n\nNote that a capability which do not declare any requirement will result in an unsuccessful evaluation.\n\nmonitoring.uss_qualifier.reports.capability_definitions.CapabilityVerifiedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "capability_ids": { - "description": "List of identifier of capability that must be verified for this condition to be satisfied.", - "items": { - "type": "string" - }, - "type": "array" - }, - "capability_location": { - "description": "Location of report to inspect for the verification of the specified capability, relative to the report in which\nthe capability is defined. Implicit default value is \"$\" (look for verified capability in the report in which the\ndependant capability is defined).\n\nIf this location resolves to multiple TestSuiteReports, then the capability must be verified in all resolved reports\nin order for this condition to be satisfied. When this location resolves to artifacts that are not\nTestSuiteReports, those artifacts will be ignored.\n\nNote that capabilities are verified in the order they are defined. So, if capability B defined in a particular\nlocation depends on whether capability A in that same location is granted, capability A must be defined before\ncapability B is defined. Also note that capability verifications are computed as test components are completed.\nSince a parent test component (e.g., test suite) is not complete until all of its child components are complete, a\ndescendant test component's capability condition cannot depend on whether an ancestor's (e.g., parent's) capability\nis verified.", - "type": [ - "string", - "null" - ] - } - }, - "required": [ - "capability_ids" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/NoFailedChecksCondition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/NoFailedChecksCondition.json deleted file mode 100644 index ad16251e7a..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/NoFailedChecksCondition.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/NoFailedChecksCondition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Condition will only be satisfied if there are no applicable failed checks.\n\nFor a capability to be verified for a participant, only checks including the participant's ID will be considered.\n\nmonitoring.uss_qualifier.reports.capability_definitions.NoFailedChecksCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - } - }, - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/ParticipantCapabilityDefinition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/ParticipantCapabilityDefinition.json deleted file mode 100644 index f8e1d0643c..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/ParticipantCapabilityDefinition.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/ParticipantCapabilityDefinition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.uss_qualifier.reports.capability_definitions.ParticipantCapabilityDefinition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "description": { - "description": "Human-readable description of the capability.", - "type": "string" - }, - "id": { - "description": "Identifier of this capability, unique at the level in which this capability is defined.", - "type": "string" - }, - "name": { - "description": "Human-readable name of the capability.", - "type": "string" - }, - "verification_condition": { - "$ref": "CapabilityVerificationCondition.json", - "description": "Condition required in order to verify the capability." - } - }, - "required": [ - "description", - "id", - "name", - "verification_condition" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/capability_definitions/RequirementsCheckedCondition.json b/schemas/monitoring/uss_qualifier/reports/capability_definitions/RequirementsCheckedCondition.json deleted file mode 100644 index 9f08ba19fd..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/capability_definitions/RequirementsCheckedCondition.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/capability_definitions/RequirementsCheckedCondition.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Condition will only be satisfied if at least one successful check exists for all specified requirements.\n\nNote that an empty collection of requirements will result in an unsuccessful evaluation.\n\nmonitoring.uss_qualifier.reports.capability_definitions.RequirementsCheckedCondition, as defined in monitoring/uss_qualifier/reports/capability_definitions.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "checked": { - "$ref": "../../requirements/definitions/RequirementCollection.json", - "description": "Each requirement contained within this collection must be covered by at least one successful check." - } - }, - "required": [ - "checked" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/AllConditionsEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/AllConditionsEvaluationReport.json deleted file mode 100644 index f5532226bb..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/AllConditionsEvaluationReport.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/AllConditionsEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of AllConditions determined by whether all the subconditions are satisfied.\n\nmonitoring.uss_qualifier.reports.report.AllConditionsEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "satisfied_conditions": { - "description": "All of the conditions that were satisfied (there must be at least one).", - "items": { - "$ref": "ParticipantCapabilityConditionEvaluationReport.json" - }, - "type": "array" - }, - "unsatisfied_conditions": { - "description": "All of the conditions that were unsatisfied (if any, then this condition will not be satisfied).", - "items": { - "$ref": "ParticipantCapabilityConditionEvaluationReport.json" - }, - "type": "array" - } - }, - "required": [ - "satisfied_conditions", - "unsatisfied_conditions" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/AnyConditionEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/AnyConditionEvaluationReport.json deleted file mode 100644 index b665ee7a0c..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/AnyConditionEvaluationReport.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/AnyConditionEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of AnyCondition determined by whether any of the subconditions are satisfied.\n\nmonitoring.uss_qualifier.reports.report.AnyConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "satisfied_options": { - "description": "Which of the specified options were satisfied (if any were satisfied, then this condition should be satisfied).", - "items": { - "$ref": "ParticipantCapabilityConditionEvaluationReport.json" - }, - "type": "array" - }, - "unsatisfied_options": { - "description": "Which of the specified options were not satisfied (these are informational only and do not affect the evaluation).", - "items": { - "$ref": "ParticipantCapabilityConditionEvaluationReport.json" - }, - "type": "array" - } - }, - "required": [ - "satisfied_options", - "unsatisfied_options" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/CapabilityVerifiedConditionEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/CapabilityVerifiedConditionEvaluationReport.json deleted file mode 100644 index 4281914709..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/CapabilityVerifiedConditionEvaluationReport.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/CapabilityVerifiedConditionEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of a CapabilityVerifiedCondition dependent on whether other capabilities were verified.\n\nmonitoring.uss_qualifier.reports.report.CapabilityVerifiedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "checked_capabilities": { - "description": "All capability evaluations checked for this condition.", - "items": { - "$ref": "CheckedCapability.json" - }, - "type": "array" - }, - "missing_capabilities": { - "description": "Capabilities specified for this condition but not found in the report.", - "items": { - "type": "string" - }, - "type": "array" - }, - "spurious_matches": { - "description": "Report elements matching the condition's `capability_location`, but not of the type TestSuiteReport.", - "items": { - "$ref": "SpuriousReportMatch.json" - }, - "type": "array" - } - }, - "required": [ - "checked_capabilities", - "missing_capabilities", - "spurious_matches" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/CheckedCapability.json b/schemas/monitoring/uss_qualifier/reports/report/CheckedCapability.json deleted file mode 100644 index 45786a5471..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/CheckedCapability.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/CheckedCapability.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Existing/previous participant-verifiable capability upon which a CapabilityVerifiedCondition depends.\n\nmonitoring.uss_qualifier.reports.report.CheckedCapability, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "capability_id": { - "description": "ID of the existing/previous participant-verifiable capability.", - "type": "string" - }, - "capability_location": { - "description": "The location of the ParticipantCapabilityConditionEvaluationReport for the capability, relative to the TestSuiteReport in which this checked requirement is located.", - "type": "string" - }, - "capability_verified": { - "description": "Whether this capability was successfully verified", - "type": "boolean" - }, - "report_location": { - "description": "Location of the ParticipantCapabilityEvaluationReport for the existing/previous capability, relative to the TestSuiteReport in which the CapabilityVerifiedConditionEvaluationReport containing this CheckedCapability is located.", - "type": "string" - } - }, - "required": [ - "capability_id", - "capability_location", - "capability_verified", - "report_location" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/CheckedRequirement.json b/schemas/monitoring/uss_qualifier/reports/report/CheckedRequirement.json deleted file mode 100644 index b3bfdc0374..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/CheckedRequirement.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/CheckedRequirement.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "A single requirement being checked for participant-verifiable capability verification.\n\nmonitoring.uss_qualifier.reports.report.CheckedRequirement, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "failed_checks": { - "description": "The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located.", - "items": { - "type": "string" - }, - "type": "array" - }, - "passed_checks": { - "description": "The location of each PassedCheck involving the requirement of interest, relative to the TestSuiteReport in which the RequirementsCheckedConditionEvaluationReport containing this checked requirement is located.", - "items": { - "type": "string" - }, - "type": "array" - }, - "requirement_id": { - "description": "The requirement being checked.", - "type": "string" - } - }, - "required": [ - "failed_checks", - "passed_checks", - "requirement_id" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/NoFailedChecksConditionEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/NoFailedChecksConditionEvaluationReport.json deleted file mode 100644 index 7bb9b8a051..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/NoFailedChecksConditionEvaluationReport.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/NoFailedChecksConditionEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of NoFailedChecksCondition dependent on whether any checks failed within the scope of the test suite in which this condition is located.\n\nmonitoring.uss_qualifier.reports.report.NoFailedChecksConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "failed_checks": { - "description": "The location of each FailedCheck, relative to the TestSuiteReport in which this report is located.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "failed_checks" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityConditionEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityConditionEvaluationReport.json deleted file mode 100644 index fec24e6bca..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityConditionEvaluationReport.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityConditionEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of a condition related to whether a participant capability should be verified.\n\nExactly one field other than `condition_satisfied` must be specified.\n\nmonitoring.uss_qualifier.reports.report.ParticipantCapabilityConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "all_conditions": { - "description": "When specified, the condition evaluated was AllConditions.", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "AllConditionsEvaluationReport.json" - } - ] - }, - "any_conditions": { - "description": "When specified, the condition evaluated was AnyCondition.", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "AnyConditionEvaluationReport.json" - } - ] - }, - "capability_verified": { - "description": "When specified, the condition evaluated was CapabilityVerifiedCondition.", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "CapabilityVerifiedConditionEvaluationReport.json" - } - ] - }, - "condition_satisfied": { - "description": "Whether the condition was satisfied for the relevant participant.", - "type": "boolean" - }, - "no_failed_checks": { - "description": "When specified, the condition evaluated was NoFailedChecksCondition.", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "NoFailedChecksConditionEvaluationReport.json" - } - ] - }, - "requirements_checked": { - "description": "When specified, the condition evaluated was RequirementsCheckedCondition.", - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "RequirementsCheckedConditionEvaluationReport.json" - } - ] - } - }, - "required": [ - "condition_satisfied" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityEvaluationReport.json deleted file mode 100644 index a60c0050a9..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityEvaluationReport.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/ParticipantCapabilityEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "monitoring.uss_qualifier.reports.report.ParticipantCapabilityEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "capability_id": { - "description": "ID of capability being evaluated.", - "type": "string" - }, - "condition_evaluation": { - "$ref": "ParticipantCapabilityConditionEvaluationReport.json", - "description": "Report produced by evaluating the condition for verifying this capability." - }, - "participant_id": { - "description": "ID of participant for which capability is being evaluated.", - "type": "string" - }, - "verified": { - "description": "Whether the capability was successfully verified.", - "type": "boolean" - } - }, - "required": [ - "capability_id", - "condition_evaluation", - "participant_id", - "verified" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/RequirementsCheckedConditionEvaluationReport.json b/schemas/monitoring/uss_qualifier/reports/report/RequirementsCheckedConditionEvaluationReport.json deleted file mode 100644 index eb89d59a1a..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/RequirementsCheckedConditionEvaluationReport.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/RequirementsCheckedConditionEvaluationReport.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Result of an evaluation of RequirementsCheckedCondition dependent on whether a set of requirements were successfully checked.\n\nmonitoring.uss_qualifier.reports.report.RequirementsCheckedConditionEvaluationReport, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "failed_requirements": { - "description": "Requirements with FailedChecks.", - "items": { - "$ref": "CheckedRequirement.json" - }, - "type": "array" - }, - "passed_requirements": { - "description": "Requirements with only PassedChecks.", - "items": { - "$ref": "CheckedRequirement.json" - }, - "type": "array" - }, - "untested_requirements": { - "description": "Requirements that didn't have any PassedChecks or FailedChecks within the scope of the test suite in which this condition is located.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "failed_requirements", - "passed_requirements", - "untested_requirements" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/SpuriousReportMatch.json b/schemas/monitoring/uss_qualifier/reports/report/SpuriousReportMatch.json deleted file mode 100644 index 2347f18862..0000000000 --- a/schemas/monitoring/uss_qualifier/reports/report/SpuriousReportMatch.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$id": "https://github.com/interuss/monitoring/blob/main/schemas/monitoring/uss_qualifier/reports/report/SpuriousReportMatch.json", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "description": "Participant-verifiable capability evaluations are only present in TestSuiteReports. If a CapabilityVerifiedCondition points to a report element that is not a TestSuiteReport, an instance of this class will be generated.\n\nmonitoring.uss_qualifier.reports.report.SpuriousReportMatch, as defined in monitoring/uss_qualifier/reports/report.py", - "properties": { - "$ref": { - "description": "Path to content that replaces the $ref", - "type": "string" - }, - "location": { - "description": "Location of the non-TestSuiteReport report element matching the CapabilityVerifiedCondition's `capability_location`, relative to the TestSuiteReport in which this condition is located.", - "type": "string" - }, - "type": { - "description": "Data type of the report element (not TestSuiteReport).", - "type": "string" - } - }, - "required": [ - "location", - "type" - ], - "type": "object" -} \ No newline at end of file diff --git a/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json b/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json index 449e40fe3c..680dd6bfe7 100644 --- a/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json +++ b/schemas/monitoring/uss_qualifier/reports/report/TestSuiteReport.json @@ -14,13 +14,6 @@ }, "type": "array" }, - "capability_evaluations": { - "description": "List of capabilities defined in this test suite, evaluated for each participant.", - "items": { - "$ref": "ParticipantCapabilityEvaluationReport.json" - }, - "type": "array" - }, "documentation_url": { "description": "URL at which this test suite is described", "type": "string" @@ -53,7 +46,6 @@ }, "required": [ "actions", - "capability_evaluations", "documentation_url", "name", "start_time", diff --git a/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteDefinition.json b/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteDefinition.json index 418f224424..b7cf6655de 100644 --- a/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteDefinition.json +++ b/schemas/monitoring/uss_qualifier/suites/definitions/TestSuiteDefinition.json @@ -34,16 +34,6 @@ "description": "Name of the test suite", "type": "string" }, - "participant_verifiable_capabilities": { - "description": "Definitions of capabilities verified by this test suite for individual participants.", - "items": { - "$ref": "../../reports/capability_definitions/ParticipantCapabilityDefinition.json" - }, - "type": [ - "array", - "null" - ] - }, "resources": { "additionalProperties": { "type": "string"