feat!: Sample.elements, metadata groups and segment times of Velox single spectra

Sources/Formato/Formato.swift

@@ -87,7 +87,8 @@ public nonisolated enum FormatoLoader {
             func memory(_ s: FormatoSignal) -> Result<FormatoLazySignal, any Error> {
                 .success(FormatoLazySignal(title: s.title, signalType: s.signalType, axes: s.axes, metadata: s.metadata, array: FormatoInMemoryArray(s.data)))
             }
-            out += try VeloxSpectrumSignals(path: path).signals.map(memory)
+            // A refused spectrum (an element selection HyperSpy rejects, say) is its failure; the images still load.
+            do { out += try VeloxSpectrumSignals(path: path).signals.map(memory) } catch { out.append(.failure(error)) }
             let images = try VeloxImageSignals(path: path)
             out += images.images.map { image in Result { try images.signal(image) } }
             do { out += try VeloxSpectrumImageSignals(path: path).signals.map(memory) } catch { out.append(.failure(error)) }

Sources/FormatoCore/FormatoCore.docc/The data model.md

@@ -28,7 +28,7 @@ Two arrays are equal when their type, shape and bytes are equal. That equality i
 ``FormatoMetadata`` is an ordered list of ``FormatoMetadataEntry`` values. Each entry has:
 
 - a key, under RosettaSciIO's metadata name;
-- a value, a number or a text (``FormatoValue``);
+- a value: a number, a text, a flag or a list of texts (``FormatoValue``);
 - an optional unit;
 - a ``FormatoProvenance`` that names the file, the place in the file the value came from, and the rule that turned it into the entry.
 

Sources/FormatoCore/FormatoMetadata.swift

@@ -11,9 +11,12 @@ public nonisolated enum FormatoValue: Sendable, Equatable {
     case number(Double)
     case text(String)
     case flag(Bool)
+    /// A list of strings, in order (RosettaSciIO's `Sample.elements` of a Velox single spectrum).
+    case texts([String])
 
     public var number: Double? { if case .number(let v) = self { return v }; return nil }
     public var text: String? { if case .text(let v) = self { return v }; return nil }
+    public var texts: [String]? { if case .texts(let v) = self { return v }; return nil }
 }
 
 /// Where a metadata value came from.

Sources/FormatoEMD/VeloxEMDReader.swift

@@ -1014,6 +1014,31 @@ public nonisolated struct VeloxEMDReader: Sendable {
         }
     }
 
+    // MARK: Single spectra
+
+    /// The metadata of each single spectrum, column 0 of `Data/Spectrum/<id>/Metadata`, in the order `VeloxSpectrumSignals` returns the
+    /// spectra (empty where it returns none: no `Data/Spectrum`, or `Data/Image` without `Data/SpectrumImage`). The live and real time
+    /// of the segments a spectrum sums are `binaryResultSegmentTimes`; they are not in the signals' metadata, which stays RosettaSciIO's.
+    /// Throws `missingMetadata` for a spectrum without a Metadata JSON object (RosettaSciIO raises there too), and as `images()` does
+    /// for a damaged file.
+    public func spectrumMetadata(cancelled: @Sendable () -> Bool = FormatoCancellation.task) throws -> [VeloxMetadata] {
+        try HDF5Serial.run {
+            try FormatoCancellation.check(cancelled)
+            let file = try Self.openVeloxFile(path: path)
+            if file.exists("/Data/Image") && !file.exists("/Data/SpectrumImage") { return [] }
+            return try file.children(of: "/Data/Spectrum").map { id in
+                try FormatoCancellation.check(cancelled)
+                let group = "/Data/Spectrum/" + id
+                guard file.exists(group + "/Metadata") else { throw VeloxEMDError.missingMetadata("Data/Spectrum/\(id)/Metadata") }
+                let json = try file.readMetadataJSON(group: group, column: 0)
+                guard let metadata = try? VeloxMetadata.parse(jsonData: json) else {
+                    throw VeloxEMDError.missingMetadata("a JSON object in Data/Spectrum/\(id)/Metadata")
+                }
+                return metadata
+            }
+        }
+    }
+
     // MARK: Stored element selection
 
     /// Symbols by atomic number minus one (H to Lr), the same table rsciio's `atomic_number2name` spells for 1...103.

Sources/FormatoEMD/VeloxMetadata.swift

@@ -183,6 +183,19 @@ public nonisolated struct VeloxMetadata: Sendable {
     public var alphaDegrees: Double? { stageAlphaRadians.map { $0 * 180 / .pi } }
     public var betaDegrees: Double? { stageBetaRadians.map { $0 * 180 / .pi } }
 
+    /// The live and real time (s) of each X-ray segment that is not disabled and whose `DetectorName` contains `BinaryResult.Detector`
+    /// (the name match of RosettaSciIO's energy axis, `_get_dispersion_offset`), in the file's order, as stored. For a single spectrum
+    /// (`VeloxEMDReader.spectrumMetadata()`): RosettaSciIO's public `fei_emd_spectrum.emd` (Velox 4) names "SuperXG2", all four
+    /// segments, each stating the same times; a Velox 6 spectrum names one ("SuperXG24"). Unlike the energy axis, every matching
+    /// `AnalyticalDetector` entry is kept, not the first. Velox 4 to 13 store them as `Detectors.Detector-<n>.LiveTime` / `.RealTime`; RosettaSciIO maps only
+    /// `DetectorMetadata.LiveTime` / `.RealTime`, which no public single spectrum has. Empty when `BinaryResult.Detector` is absent.
+    public var binaryResultSegmentTimes: [VeloxSegmentTimes] {
+        guard let named = binaryResultDetector else { return [] }
+        return superXDetectors.filter { $0.enabled != false && (named.isEmpty || $0.name.contains(named)) }.map {
+            VeloxSegmentTimes(name: $0.name, liveTimeSeconds: $0.liveTimeColumn0, realTimeSeconds: $0.realTimeColumn0)
+        }
+    }
+
     /// The grid `ScanSize x ScanArea` implies, `(ny, nx)`; nil unless both are
     /// integers to within 1/100 of a pixel (the Velox rasters are exact binary fractions).
     public var derivedGrid: (ny: Int, nx: Int)? {

Sources/FormatoEMD/VeloxSampleElements.swift

@@ -0,0 +1,120 @@
+//
+//  VeloxSampleElements.swift
+//  Role: The four file-level metadata groups of a Velox file as RosettaSciIO 049e7d70's `read_file` parses them
+//        (`_parse_metadata_group`), and `Sample.elements` as RosettaSciIO's mapping gives it once HyperSpy applies it
+//        (`_get_mapping(map_selected_element=True)` with `_convert_element_list`, through `hyperspy/io.py` `dict2signal`).
+//        `_convert_element_list` indexes `d.keys()[0]`, which only HyperSpy's `DictionaryTreeBrowser` allows: its `keys()` is the
+//        sorted list of the slugified keys (`slugify(key, valid_variable_name=True)`), so the entry taken is the first in that order.
+//        Wherever Python raises (a key missing, a value without `.keys()`, `int()` refusing, an atomic number outside 1...119), the
+//        read is refused, as HyperSpy's `load` raises. Pinned by tools/velox-elements/pins.py.
+//
+
+import Foundation
+import FormatoCore
+
+/// The children of `Displays`, `Operations`, `SharedProperties` and `Features`, as `_parse_metadata_group` reads them.
+nonisolated enum VeloxFileGroups {
+    /// A child of one of the four groups: a group's datasets by name (as `{name: json}`), or one dataset's JSON.
+    struct Entry { let path: String; let isGroup: Bool; let value: Any }
+
+    static let names = ["Displays", "Operations", "SharedProperties", "Features"]
+
+    /// The groups the file has, each child parsed. Must run inside `HDF5Serial.run`. Throws `refuse(reason)` where RosettaSciIO raises:
+    /// a group stored as a dataset, an entry that is not one variable-length string of JSON. (An empty dataset, where RosettaSciIO
+    /// logs a warning and keeps what it read, is refused here too: the reader cannot tell it from a damaged one.)
+    static func read(file: VeloxFile, refuse: (String) -> FormatoReadError) throws -> [String: [String: Entry]] {
+        func json(_ path: String) throws -> Any {
+            guard let text = file.readString(path) else { throw refuse("\(path) is not one variable-length string") }
+            guard let value = try? JSONSerialization.jsonObject(with: Data(text.utf8), options: .fragmentsAllowed) else {
+                throw refuse("\(path) is not JSON")
+            }
+            return value
+        }
+        func isDataset(_ path: String) -> Bool { (try? file.open(path)).map { file.close($0); return true } ?? false }
+        var groups: [String: [String: Entry]] = [:]
+        for name in names where file.exists("/" + name) {
+            // `_parse_metadata_group` on a dataset: `_get_keys_from_group` raises AttributeError.
+            guard !isDataset("/" + name) else { throw refuse("/\(name) is a dataset, not a group") }
+            var entries: [String: Entry] = [:]
+            for child in file.children(of: "/" + name) {
+                let path = "/\(name)/\(child)"
+                if isDataset(path) {
+                    entries[child] = Entry(path: path, isGroup: false, value: try json(path))
+                } else {
+                    var sub: [String: Any] = [:]
+                    for leaf in file.children(of: path) { sub[leaf] = try json(path + "/" + leaf) }
+                    entries[child] = Entry(path: path, isGroup: true, value: sub)
+                }
+            }
+            groups[name] = entries
+        }
+        return groups
+    }
+}
+
+/// `Sample.elements` of a Velox single spectrum.
+package nonisolated enum VeloxSampleElements {
+    /// RosettaSciIO's `atomic_number2name` (`rsciio/utils/_elements.py`), 1 to 119, its spellings kept (112 is "Cp", 113 to 119 the
+    /// placeholder names).
+    static let symbols = ("H He Li Be B C N O F Ne Na Mg Al Si P S Cl Ar K Ca Sc Ti V Cr Mn Fe Co Ni Cu Zn Ga Ge As Se Br Kr "
+        + "Rb Sr Y Zr Nb Mo Tc Ru Rh Pd Ag Cd In Sn Sb Te I Xe Cs Ba La Ce Pr Nd Pm Sm Eu Gd Tb Dy Ho Er Tm Yb Lu Hf Ta W Re Os Ir Pt Au Hg "
+        + "Tl Pb Bi Po At Rn Fr Ra Ac Th Pa U Np Pu Am Cm Bk Cf Es Fm Md No Lr Rf Db Sg Bh Hs Mt Ds Rg Cp Uut Uuq Uup Uuh Uus Uuo Uue")
+        .split(separator: " ").map(String.init)
+
+    /// HyperSpy's `slugify(value, valid_variable_name=True)`: NFKD, non-ASCII dropped, ASCII punctuation dropped (not "_" or " "),
+    /// stripped, spaces to "_", and "Number_" before a result that is not an identifier.
+    package static func slugify(_ key: String) -> String {
+        let ascii = key.decomposedStringWithCompatibilityMapping.unicodeScalars.filter(\.isASCII)
+        let kept = ascii.filter { $0 == "_" || $0 == " " || CharacterSet.alphanumerics.contains($0) }
+        var text = String(String.UnicodeScalarView(kept))
+        text = text.trimmingCharacters(in: .whitespaces).replacingOccurrences(of: " ", with: "_")
+        let isIdentifier = text.first.map { $0 == "_" || $0.isLetter } == true
+        return isIdentifier ? text : "Number_" + text
+    }
+
+    /// The mapping's source key for this file's version: `int(self.version)` >= 11 reads `SharedProperties`, else `Operations`.
+    static func settingsPath(version: Int) -> (group: String, entry: String) {
+        version >= 11 ? ("SharedProperties", "EDSSpectrumQuantificationSettings") : ("Operations", "ImageQuantificationOperation")
+    }
+
+    /// `_convert_element_list` on what HyperSpy's `get_item` returns for the settings path. `value` is the parsed entry (a dict of
+    /// the group's datasets, or one dataset's JSON). nil is never returned: Python returns a list or raises (`refuse`).
+    static func convert(_ value: Any, path: String, refuse: (String) -> FormatoReadError) throws -> (elements: [String], source: String) {
+        // `d.keys()[0]`: only a dict has `keys()`; HyperSpy sorts the slugified keys and leaves out those starting with "_". Keys
+        // that slugify alike are one attribute whose values HyperSpy merges: refused here (no file has them).
+        guard let settings = value as? [String: Any] else { throw refuse("\(path) has no keys() (it is not a group of entries)") }
+        var bySlug: [String: String] = [:]
+        for key in settings.keys {
+            let slug = slugify(key)
+            guard bySlug.updateValue(key, forKey: slug) == nil else { throw refuse("\(path) has two entries that read as \(slug) (unsupported here)") }
+        }
+        guard let first = bySlug.keys.filter({ !$0.hasPrefix("_") }).sorted().first, let key = bySlug[first], let entry = settings[key] else {
+            throw refuse("\(path) has no entry (keys()[0])")
+        }
+        guard let inner = entry as? [String: Any] else { throw refuse("\(path)/\(key) has no keys() (it is not a JSON object)") }
+        let source = "\(path)/\(key).elementSelection"
+        // "elementSelection" in `_d.keys()`, absent: no element selected, [].
+        let matches = inner.keys.filter { slugify($0) == "elementSelection" }
+        guard !matches.isEmpty else { return ([], source) }
+        guard matches.count == 1, let selection = inner[matches[0]] else {
+            throw refuse("\(path)/\(key) has several keys that read as elementSelection (unsupported here)")
+        }
+        // `for atomic_number in atomic_number_list`: a list, a string (its characters), an empty object; anything else raises.
+        let items: [Any]
+        switch selection {
+        case let list as [Any]: items = list
+        case let text as String: items = text.unicodeScalars.map { String($0) }   // Python iterates code points
+        // A dict iterates over its keys that do not slugify to "_..." (none: []); any such key gives a tuple, which int() refuses.
+        case let object as [String: Any] where object.keys.allSatisfy({ slugify($0).hasPrefix("_") }): items = []
+        default: throw refuse("\(source) is not a list or a string")
+        }
+        var out: [String] = []
+        for item in items {
+            if item is [Any] || item is [String: Any] { throw refuse("\(source) holds a list or an object (int() raises)") }
+            guard let z = pythonIntValue(item) else { throw refuse("\(source): int() refuses \(item)") }
+            guard z >= 1, z <= symbols.count else { throw refuse("\(source): \(z) is not in atomic_number2name") }
+            out.append(symbols[z - 1])
+        }
+        return (out, source)
+    }
+}

Sources/FormatoEMD/VeloxSpectrumSignals.swift

@@ -7,9 +7,10 @@
 //        and the mapping HyperSpy applies to `original_metadata` (`_get_mapping`) for every key the spectrum's metadata holds.
 //        RosettaSciIO reads spectra only from a file whose `Data` has no `Image` group (type `Spectrum`) or has both `Image` and
 //        `SpectrumImage` (type `SpectrumStream`); in any other file it returns none, and so does this.
-//        Eager: a spectrum is a few thousand counts. Registered deviations (the SI path's, docs/parity.md): the four metadata
-//        groups (`Displays`, `Operations`, `SharedProperties`, `Features`) are not parsed, so a malformed entry there does not
-//        refuse the read; `Sample.elements` (the element selection) is not mapped.
+//        Eager: a spectrum is a few thousand counts. The four metadata groups (`Displays`, `Operations`, `SharedProperties`,
+//        `Features`) are parsed as `read_file` parses them (a malformed entry refuses the read), and `Sample.elements` is mapped as
+//        HyperSpy applies RosettaSciIO's `_convert_element_list` (VeloxSampleElements.swift). Registered deviation (docs/parity.md):
+//        `_parse_image_display`, which RosettaSciIO runs for a SpectrumStream file before reading its spectra, is not run here.
 //
 
 import Foundation
@@ -60,6 +61,9 @@ private nonisolated struct VeloxSpectrumRead {
         guard let versionText = file.readString("/Version"),
               let version = (try? JSONSerialization.jsonObject(with: Data(versionText.utf8))) as? [String: Any],
               veloxAcceptsFormat(version["format"] as? String) else { throw VeloxEMDError.notVeloxFile(fileName) }
+        // `read_file`: `_parse_json(f["Version"][0])["version"]` (a KeyError without it), then the four metadata groups.
+        guard let versionValue = version["version"] else { throw raises("its Version has no version key") }
+        let groups = try VeloxFileGroups.read(file: file) { raises($0) }
         guard file.exists("/Data"), !isDataset("/Data") else { throw raises("no Data group") }
         // `_check_im_type`: Image + SpectrumImage is a stream, Image alone is images, no Image is a spectrum.
         let hasImage = file.exists("/Data/Image"), hasSpectrumImage = file.exists("/Data/SpectrumImage")
@@ -69,12 +73,12 @@ private nonisolated struct VeloxSpectrumRead {
         var out: [FormatoSignal] = []
         for key in file.children(of: "/Data/Spectrum") {
             try FormatoCancellation.check(cancelled)
-            out.append(try spectrum(key))
+            out.append(try spectrum(key, version: versionValue, groups: groups))
         }
         return out
     }
 
-    func spectrum(_ key: String) throws -> FormatoSignal {
+    func spectrum(_ key: String, version: Any, groups: [String: [String: VeloxFileGroups.Entry]]) throws -> FormatoSignal {
         let group = "/Data/Spectrum/" + key
         guard file.exists(group + "/Data") else { throw raises("\(group) has no Data") }
         guard file.exists(group + "/Metadata") else { throw raises("\(group) has no Metadata") }
@@ -108,7 +112,16 @@ private nonisolated struct VeloxSpectrumRead {
         axes.append(FormatoAxis(name: "Energy", size: data.bins, scale: energy.scale, offset: energy.offset, units: "keV", navigate: false))
         let shape = data.stack > 1 ? [data.stack, data.bins] : [data.bins]
         let array = try FormatoArray(dataType: data.type, shape: shape, bytes: data.bytes)
-        let md = try metadata(om: om, source: source)
+        var md = try metadata(om: om, source: source)
+        // `_get_mapping()`: `int(self.version)` picks the settings group; `original_metadata.update(self.original_metadata)` put the
+        // four groups beside the spectrum's own keys, so the mapping finds the group's entries there.
+        guard let versionNumber = pythonIntValue(version) else { throw raises("int() refuses its Version's version") }
+        let (groupName, entryName) = VeloxSampleElements.settingsPath(version: versionNumber)
+        if let settings = groups[groupName]?[entryName] {
+            let (elements, elementsSource) = try VeloxSampleElements.convert(settings.value, path: String(settings.path.dropFirst())) { raises($0) }
+            md.set(entry("Sample.elements", .texts(elements), source: elementsSource,
+                         rule: "rsciio emd velox mapping _convert_element_list, as HyperSpy applies it: the first of keys() (sorted), atomic_number2name[int(z)]"))
+        }
         return FormatoSignal(title: md["General.title"]?.value.text ?? "EDS", signalType: "EDS_TEM", axes: axes, metadata: md, data: array)
     }