[NAT.11] Image family: functional PNG/TIFF/EXR codecs over OpenImageIO and the managed ImageReader binding

.github/workflows/native-abi.yml

@@ -153,7 +153,7 @@ jobs:
             cmake --install artifacts/cmake/win-x64/%%p || exit /b 1
           )
 
-          ctest --test-dir artifacts/cmake/win-x64/shim-static --output-on-failure -R "^(arc_native_abi_c17_layout_tests|arc_native_abi_cpp20_tests)$" || exit /b 1
+          ctest --test-dir artifacts/cmake/win-x64/shim-static --output-on-failure -R "^(arc_native_abi_c17_layout_tests|arc_native_abi_cpp20_tests|arcslate_image_abi\.hello|arcslate_image_abi\.codec\.[a-z0-9_]+)$" || exit /b 1
           dotnet test --project tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj --filter Category=NativeAbiLayout --no-restore || exit /b 1
 
           sccache --show-stats || exit /b 1

eng/packaging/native.py

@@ -16,13 +16,24 @@
 import native_provenance
 import build_identity
 VCPKG_COMMIT = "36677bbd0b3bf11da7376e62e14bffcc54d2eaeb"
+# Every owned library exports the three probes. ArcImageNative (ABI 1.1, Annex 06) adds its functional surface.
+PROBE_EXPORTS = ("_get_abi_version", "_get_build_info", "_get_last_error")
+FUNCTIONAL_EXPORTS = {"ArcImageNative": ("_open", "_read", "_close")}
 
 
 def require(condition, message):
     if not condition:
         raise ValueError(message)
 
 
+def admitted_exports(entry):
+    return {entry["prefix"] + suffix for suffix in PROBE_EXPORTS + FUNCTIONAL_EXPORTS.get(entry["library"], ())}
+
+
+def require_owned_exports(entry, exports):
+    require(set(exports) == admitted_exports(entry), "Owned native export set differs from the admitted ABI.")
+
+
 def digest(path):
     with path.open("rb") as stream:
         return hashlib.file_digest(stream, "sha256").hexdigest()
@@ -279,8 +290,7 @@ def stage(directory, vcpkg, installed_root):
                 if not system_dependency(dependency):
                     pending.append(dependency)
         owned = selected[entry["library"].lower() + ".dll"]
-        require(set(owned["exports"]) == {entry["prefix"] + suffix for suffix in ["_get_abi_version", "_get_build_info", "_get_last_error"]},
-                "Owned native export set differs from the admitted ABI.")
+        require_owned_exports(entry, owned["exports"])
         for original, relative in [(ROOT / entry["header"], "include/arc/" + Path(entry["header"]).name),
                                    (ROOT / "native/shared/include/arc/arc_native_abi.h", "include/arc/arc_native_abi.h"),
                                    (binary_root / "lib" / (entry["library"] + ".lib"), "sdk/win-x64/lib/" + entry["library"] + ".lib")]:

eng/packaging/native_consumer.py

@@ -12,6 +12,15 @@
 
 import packages
 
+# Functional libraries advertise a higher minor ABI and a closed capability manifest in their build info
+# (Annex 06 section 1 and decision D2). Their build info is JSON whose identity is a field, not a suffix.
+IMAGE_CAPABILITIES = ["image.open", "image.read", "image.close"]
+IMAGE_FORMATS = ["png", "tiff", "exr"]
+
+
+def abi_minor(entry):
+    return 1 if entry["library"] == "ArcImageNative" else 0
+
 
 def execute(executable, directory, env, failure=False):
     runtime_env = {key: value for key, value in env.items() if not key.upper().startswith(("VCPKG", "DOTNET_ROOT"))}
@@ -80,18 +89,30 @@ def consume(directory, version, commit):
             + '</ItemGroup></Project>\n', encoding="utf-8")
         program = '''using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Text.Json;
 using ArcForges.Native.Abstractions;
 [assembly: DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory | DllImportSearchPath.System32)]
+static bool StringsEqual(JsonElement element, params string[] expected) => element.EnumerateArray().Select(item => item.GetString()).SequenceEqual(expected);
 '''
         faults = []
+        identity_fields = expected_suffix[1:]
         for entry in selected:
             name = entry["dependencies"][0]
             family = name.split(".")[-1]
             api = f"{name}.{family}Abi"
-            program += f'''if ({api}.GetAbiVersion() != new NativeAbiVersion(1, 0)) throw new Exception("ABI mismatch");
+            minor = abi_minor(entry)
+            if minor == 0:
+                identity_check = f'if (!{api}.GetBuildInfo().EndsWith({json.dumps(expected_suffix)}, StringComparison.Ordinal)) throw new Exception("Native build identity mismatch");\n'
+            else:
+                capabilities = ", ".join(json.dumps(item) for item in IMAGE_CAPABILITIES)
+                formats = ", ".join(json.dumps(item) for item in IMAGE_FORMATS)
+                identity_check = f'''using var manifest{family} = JsonDocument.Parse({api}.GetBuildInfo());
+var build{family} = manifest{family}.RootElement;
+if (build{family}.GetProperty("library").GetString() != {json.dumps(entry["library"])} || build{family}.GetProperty("abi").GetProperty("minor").GetInt32() != {minor} || !StringsEqual(build{family}.GetProperty("capabilities"), {capabilities}) || !StringsEqual(build{family}.GetProperty("formats"), {formats}) || build{family}.GetProperty("identity").GetString() != {json.dumps(identity_fields)}) throw new Exception("Native capability manifest or build identity mismatch");
+'''
+            program += f'''if ({api}.GetAbiVersion() != new NativeAbiVersion(1, {minor})) throw new Exception("ABI mismatch");
 Console.WriteLine({api}.GetBuildInfo());
-if (!{api}.GetBuildInfo().EndsWith({json.dumps(expected_suffix)}, StringComparison.Ordinal)) throw new Exception("Native build identity mismatch");
-var metadata{family} = System.Reflection.CustomAttributeExtensions.GetCustomAttributes<System.Reflection.AssemblyMetadataAttribute>(typeof({api}).Assembly).ToDictionary(a => a.Key, a => a.Value);
+{identity_check}var metadata{family} = System.Reflection.CustomAttributeExtensions.GetCustomAttributes<System.Reflection.AssemblyMetadataAttribute>(typeof({api}).Assembly).ToDictionary(a => a.Key, a => a.Value);
 if (metadata{family}["ArcForges.SourceCommit"] != "{commit}" || metadata{family}["ArcForges.BuildId"] != "{manifest['build']['buildId']}" || metadata{family}["ArcForges.PipelineRun"] != "{manifest['build']['pipelineRun'] or 'local'}" || metadata{family}["ArcForges.SourceDateEpoch"] != "{manifest['build']['sourceDateEpoch']}") throw new Exception("Managed build identity mismatch");
 if (System.Reflection.CustomAttributeExtensions.GetCustomAttribute<System.Reflection.AssemblyInformationalVersionAttribute>(typeof({api}).Assembly)?.InformationalVersion.Split('+')[0] != "{version}") throw new Exception("Managed release mismatch");
 if ({api}.GetLastError().Status != NativeStatus.Ok) throw new Exception("Initial error state");
@@ -183,15 +204,23 @@ def c_consumer(root, published, entries, version, env, expected_suffix):
         libraries.append('"' + str(installed / "sdk/win-x64/lib" / (entry["library"] + ".lib")) + '"')
         source += '#include <arc/' + Path(entry["header"]).name + '>\n'
         prefix = entry["prefix"]
+        minor = abi_minor(entry)
+        if minor == 0:
+            identity_expected = [expected_suffix]
+        else:
+            identity_expected = ['"capabilities":[' + ",".join(json.dumps(item) for item in IMAGE_CAPABILITIES) + "]",
+                                 '"formats":[' + ",".join(json.dumps(item) for item in IMAGE_FORMATS) + "]",
+                                 '"identity":"' + expected_suffix[1:] + '"']
+        identity_test = " || ".join(f"strstr(text, {json.dumps(item)}) == NULL" for item in identity_expected)
         body += f'''{{
   uint32_t major = 0, minor = 0;
-  if ({prefix}_get_abi_version(&major, &minor) != ARC_OK || major != 1 || minor != 0) return 1;
+  if ({prefix}_get_abi_version(&major, &minor) != ARC_OK || major != 1 || minor != {minor}) return 1;
   arc_mut_buffer_t query = {{0}};
   if ({prefix}_get_build_info(&query) != ARC_BUFFER_TOO_SMALL || query.required > 4096 || query.required == 0) return 2;
   char text[4096] = {{0}};
   arc_mut_buffer_t output = {{text, sizeof(text), 0}};
   if ({prefix}_get_build_info(&output) != ARC_OK || output.required != query.required) return 3;
-  if (output.required >= sizeof(text) || strstr(text, {json.dumps(expected_suffix)}) == NULL) return 7;
+  if (output.required >= sizeof(text) || {identity_test}) return 7;
   if ({prefix}_get_abi_version(NULL, &minor) != ARC_INVALID_ARGUMENT) return 4;
   arc_error_info_t error = {{0}};
   error.struct_size = sizeof(error); error.struct_version = 1;

eng/packaging/packages.py

@@ -157,8 +157,7 @@ def inspect(path, entry, expected_version, commit):
                 require(all(name in files or native.system_dependency(name) for name in details["imports"]),
                         "Missing non-system native dependency.")
             owned = files[entry["library"].lower() + ".dll"]
-            require(set(owned["exports"]) == {entry["prefix"] + suffix for suffix in
-                    ["_get_abi_version", "_get_build_info", "_get_last_error"]}, "Owned ABI exports mismatch.")
+            native.require_owned_exports(entry, owned["exports"])
             sbom = json.loads(archive.read("sbom.json"))
             require(sbom["sourceCommit"] == commit and sbom["binaryFiles"] == document["files"], "Native SBOM identity mismatch.")
             for dependency in sbom["buildDependencies"]:

eng/packaging/test_packages.py

@@ -11,6 +11,7 @@
 import xml.etree.ElementTree as ET
 import zipfile
 
+import native
 import packages
 
 
@@ -303,5 +304,44 @@ def tamper(files, _):
             packages.verify(target, manifest['version'], manifest['sourceCommit'])
 
 
+class OwnedExportGuards(unittest.TestCase):
+    image = {"id": "ArcForges.Native.Image.Runtime.win-x64", "kind": "native", "library": "ArcImageNative", "prefix": "arc_image"}
+    probes = ["arc_image_get_abi_version", "arc_image_get_build_info", "arc_image_get_last_error"]
+    functional = ["arc_image_open", "arc_image_read", "arc_image_close"]
+
+    def test_image_admits_exactly_the_abi_1_1_surface(self):
+        native.require_owned_exports(self.image, self.probes + self.functional)
+
+    def test_image_missing_export_fails(self):
+        for name in self.probes + self.functional:
+            with self.subTest(missing=name), self.assertRaisesRegex(ValueError, "admitted ABI"):
+                native.require_owned_exports(self.image, [item for item in self.probes + self.functional if item != name])
+
+    def test_image_extra_export_fails(self):
+        for extra in ["arc_image_finish", "arc_image_open_ex", "arc_pdf_open"]:
+            with self.subTest(extra=extra), self.assertRaisesRegex(ValueError, "admitted ABI"):
+                native.require_owned_exports(self.image, self.probes + self.functional + [extra])
+
+    def test_image_probe_only_surface_is_no_longer_admitted(self):
+        with self.assertRaisesRegex(ValueError, "admitted ABI"):
+            native.require_owned_exports(self.image, self.probes)
+
+    def test_other_libraries_keep_the_probe_only_surface(self):
+        other = {"id": "ArcForges.Other.Runtime.win-x64", "kind": "native", "library": "ArcOtherNative", "prefix": "arc_other"}
+        probes = [name.replace("arc_image", "arc_other") for name in self.probes]
+        native.require_owned_exports(other, probes)
+        with self.assertRaisesRegex(ValueError, "admitted ABI"):
+            native.require_owned_exports(other, probes + ["arc_other_open"])
+
+    def test_admitted_set_matches_the_windows_def_file(self):
+        document = json.loads((packages.ROOT / "eng/packaging/packages.json").read_text())
+        entry = next(item for item in document["packages"] if item.get("library") == "ArcImageNative")
+        lines = (packages.ROOT / "native/arcimage-abi/exports/windows.def").read_text().splitlines()
+        start = next(index for index, line in enumerate(lines) if line.strip() == "EXPORTS") + 1
+        defined = {line.strip() for line in lines[start:] if line.strip()}
+        self.assertEqual(defined, native.admitted_exports(entry))
+        self.assertEqual(defined, set(self.probes + self.functional))
+
+
 if __name__ == "__main__":
     unittest.main()

eng/policy/architecture-contract-tests.json

@@ -689,6 +689,41 @@
     "testProject": "tests/ArchitectureTests/ArcForges.Tests.ArchitectureTests.csproj",
     "testMethod": "ArcForges.Tests.ArchitectureTests.PublicBoundaryContractTests.ImagePublicContractHasOnlyTypedManagedResults()"
   },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageByteSource.ReadAt(long, System.Span<byte>)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.CancellationDuringNativeReadKeepsCoverageConsistentWithOutcome()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageByteSource.FromStream(System.IO.Stream)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.PngRgba8StraightAlphaRoundTripsByteExactAndCompletes()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageReader.OpenAsync(ArcForges.Native.Image.ImageByteSource, ArcForges.Native.Image.ImageOpenOptions?, System.Threading.CancellationToken)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.PngRgba8StraightAlphaRoundTripsByteExactAndCompletes()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageReader.ProbeAsync(ArcForges.Native.Image.ImageByteSource, ArcForges.Native.Image.ImageOpenOptions?, System.Threading.CancellationToken)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.TiffRgba8UnassociatedAlphaRoundTripsByteExact()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageReader.ReadRegionAsync(ArcForges.Native.Image.ImageRegion, System.Threading.CancellationToken)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.PngRgba8StraightAlphaRoundTripsByteExactAndCompletes()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageReader.CompleteAsync(System.Threading.CancellationToken)",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.RegionsCoverEveryPixelOnlyWhenEveryBandIsRead()"
+  },
+  {
+    "apiSymbol": "ArcForges.Native.Image.ImageReader.DisposeAsync()",
+    "testProject": "tests/NativeAbiTests/ArcForges.Tests.NativeAbiTests.csproj",
+    "testMethod": "ArcForges.Tests.NativeAbiTests.ImageReaderTests.DisposeAsyncClosesTheHandleAndRefusesFurtherUse()"
+  },
   {
     "apiSymbol": "ArcForges.Security.ActorOperation<T>.Forward<TNext>(TNext)",
     "testProject": "src/BuildingBlocks/ArcForges.Security/Tests/ArcForges.Security.Tests.csproj",