diff --git a/Sources/Formato/Formato.swift b/Sources/Formato/Formato.swift index 0a6410c..d995087 100644 --- a/Sources/Formato/Formato.swift +++ b/Sources/Formato/Formato.swift @@ -87,7 +87,8 @@ public nonisolated enum FormatoLoader { func memory(_ s: FormatoSignal) -> Result { .success(FormatoLazySignal(title: s.title, signalType: s.signalType, axes: s.axes, metadata: s.metadata, array: FormatoInMemoryArray(s.data))) } - out += try VeloxSpectrumSignals(path: path).signals.map(memory) + // A refused spectrum (an element selection HyperSpy rejects, say) is its failure; the images still load. + do { out += try VeloxSpectrumSignals(path: path).signals.map(memory) } catch { out.append(.failure(error)) } let images = try VeloxImageSignals(path: path) out += images.images.map { image in Result { try images.signal(image) } } do { out += try VeloxSpectrumImageSignals(path: path).signals.map(memory) } catch { out.append(.failure(error)) } diff --git a/Sources/FormatoCore/FormatoCore.docc/The data model.md b/Sources/FormatoCore/FormatoCore.docc/The data model.md index e8bdd96..aa203fc 100644 --- a/Sources/FormatoCore/FormatoCore.docc/The data model.md +++ b/Sources/FormatoCore/FormatoCore.docc/The data model.md @@ -28,7 +28,7 @@ Two arrays are equal when their type, shape and bytes are equal. That equality i ``FormatoMetadata`` is an ordered list of ``FormatoMetadataEntry`` values. Each entry has: - a key, under RosettaSciIO's metadata name; -- a value, a number or a text (``FormatoValue``); +- a value: a number, a text, a flag or a list of texts (``FormatoValue``); - an optional unit; - a ``FormatoProvenance`` that names the file, the place in the file the value came from, and the rule that turned it into the entry. diff --git a/Sources/FormatoCore/FormatoMetadata.swift b/Sources/FormatoCore/FormatoMetadata.swift index 0a5be6f..f101a4e 100644 --- a/Sources/FormatoCore/FormatoMetadata.swift +++ b/Sources/FormatoCore/FormatoMetadata.swift @@ -11,9 +11,12 @@ public nonisolated enum FormatoValue: Sendable, Equatable { case number(Double) case text(String) case flag(Bool) + /// A list of strings, in order (RosettaSciIO's `Sample.elements` of a Velox single spectrum). + case texts([String]) public var number: Double? { if case .number(let v) = self { return v }; return nil } public var text: String? { if case .text(let v) = self { return v }; return nil } + public var texts: [String]? { if case .texts(let v) = self { return v }; return nil } } /// Where a metadata value came from. diff --git a/Sources/FormatoEMD/VeloxEMDReader.swift b/Sources/FormatoEMD/VeloxEMDReader.swift index d1a7eb8..481fdaf 100644 --- a/Sources/FormatoEMD/VeloxEMDReader.swift +++ b/Sources/FormatoEMD/VeloxEMDReader.swift @@ -1014,6 +1014,31 @@ public nonisolated struct VeloxEMDReader: Sendable { } } + // MARK: Single spectra + + /// The metadata of each single spectrum, column 0 of `Data/Spectrum//Metadata`, in the order `VeloxSpectrumSignals` returns the + /// spectra (empty where it returns none: no `Data/Spectrum`, or `Data/Image` without `Data/SpectrumImage`). The live and real time + /// of the segments a spectrum sums are `binaryResultSegmentTimes`; they are not in the signals' metadata, which stays RosettaSciIO's. + /// Throws `missingMetadata` for a spectrum without a Metadata JSON object (RosettaSciIO raises there too), and as `images()` does + /// for a damaged file. + public func spectrumMetadata(cancelled: @Sendable () -> Bool = FormatoCancellation.task) throws -> [VeloxMetadata] { + try HDF5Serial.run { + try FormatoCancellation.check(cancelled) + let file = try Self.openVeloxFile(path: path) + if file.exists("/Data/Image") && !file.exists("/Data/SpectrumImage") { return [] } + return try file.children(of: "/Data/Spectrum").map { id in + try FormatoCancellation.check(cancelled) + let group = "/Data/Spectrum/" + id + guard file.exists(group + "/Metadata") else { throw VeloxEMDError.missingMetadata("Data/Spectrum/\(id)/Metadata") } + let json = try file.readMetadataJSON(group: group, column: 0) + guard let metadata = try? VeloxMetadata.parse(jsonData: json) else { + throw VeloxEMDError.missingMetadata("a JSON object in Data/Spectrum/\(id)/Metadata") + } + return metadata + } + } + } + // MARK: Stored element selection /// Symbols by atomic number minus one (H to Lr), the same table rsciio's `atomic_number2name` spells for 1...103. diff --git a/Sources/FormatoEMD/VeloxMetadata.swift b/Sources/FormatoEMD/VeloxMetadata.swift index 0506780..48482aa 100644 --- a/Sources/FormatoEMD/VeloxMetadata.swift +++ b/Sources/FormatoEMD/VeloxMetadata.swift @@ -183,6 +183,19 @@ public nonisolated struct VeloxMetadata: Sendable { public var alphaDegrees: Double? { stageAlphaRadians.map { $0 * 180 / .pi } } public var betaDegrees: Double? { stageBetaRadians.map { $0 * 180 / .pi } } + /// The live and real time (s) of each X-ray segment that is not disabled and whose `DetectorName` contains `BinaryResult.Detector` + /// (the name match of RosettaSciIO's energy axis, `_get_dispersion_offset`), in the file's order, as stored. For a single spectrum + /// (`VeloxEMDReader.spectrumMetadata()`): RosettaSciIO's public `fei_emd_spectrum.emd` (Velox 4) names "SuperXG2", all four + /// segments, each stating the same times; a Velox 6 spectrum names one ("SuperXG24"). Unlike the energy axis, every matching + /// `AnalyticalDetector` entry is kept, not the first. Velox 4 to 13 store them as `Detectors.Detector-.LiveTime` / `.RealTime`; RosettaSciIO maps only + /// `DetectorMetadata.LiveTime` / `.RealTime`, which no public single spectrum has. Empty when `BinaryResult.Detector` is absent. + public var binaryResultSegmentTimes: [VeloxSegmentTimes] { + guard let named = binaryResultDetector else { return [] } + return superXDetectors.filter { $0.enabled != false && (named.isEmpty || $0.name.contains(named)) }.map { + VeloxSegmentTimes(name: $0.name, liveTimeSeconds: $0.liveTimeColumn0, realTimeSeconds: $0.realTimeColumn0) + } + } + /// The grid `ScanSize x ScanArea` implies, `(ny, nx)`; nil unless both are /// integers to within 1/100 of a pixel (the Velox rasters are exact binary fractions). public var derivedGrid: (ny: Int, nx: Int)? { diff --git a/Sources/FormatoEMD/VeloxSampleElements.swift b/Sources/FormatoEMD/VeloxSampleElements.swift new file mode 100644 index 0000000..11fd644 --- /dev/null +++ b/Sources/FormatoEMD/VeloxSampleElements.swift @@ -0,0 +1,120 @@ +// +// VeloxSampleElements.swift +// Role: The four file-level metadata groups of a Velox file as RosettaSciIO 049e7d70's `read_file` parses them +// (`_parse_metadata_group`), and `Sample.elements` as RosettaSciIO's mapping gives it once HyperSpy applies it +// (`_get_mapping(map_selected_element=True)` with `_convert_element_list`, through `hyperspy/io.py` `dict2signal`). +// `_convert_element_list` indexes `d.keys()[0]`, which only HyperSpy's `DictionaryTreeBrowser` allows: its `keys()` is the +// sorted list of the slugified keys (`slugify(key, valid_variable_name=True)`), so the entry taken is the first in that order. +// Wherever Python raises (a key missing, a value without `.keys()`, `int()` refusing, an atomic number outside 1...119), the +// read is refused, as HyperSpy's `load` raises. Pinned by tools/velox-elements/pins.py. +// + +import Foundation +import FormatoCore + +/// The children of `Displays`, `Operations`, `SharedProperties` and `Features`, as `_parse_metadata_group` reads them. +nonisolated enum VeloxFileGroups { + /// A child of one of the four groups: a group's datasets by name (as `{name: json}`), or one dataset's JSON. + struct Entry { let path: String; let isGroup: Bool; let value: Any } + + static let names = ["Displays", "Operations", "SharedProperties", "Features"] + + /// The groups the file has, each child parsed. Must run inside `HDF5Serial.run`. Throws `refuse(reason)` where RosettaSciIO raises: + /// a group stored as a dataset, an entry that is not one variable-length string of JSON. (An empty dataset, where RosettaSciIO + /// logs a warning and keeps what it read, is refused here too: the reader cannot tell it from a damaged one.) + static func read(file: VeloxFile, refuse: (String) -> FormatoReadError) throws -> [String: [String: Entry]] { + func json(_ path: String) throws -> Any { + guard let text = file.readString(path) else { throw refuse("\(path) is not one variable-length string") } + guard let value = try? JSONSerialization.jsonObject(with: Data(text.utf8), options: .fragmentsAllowed) else { + throw refuse("\(path) is not JSON") + } + return value + } + func isDataset(_ path: String) -> Bool { (try? file.open(path)).map { file.close($0); return true } ?? false } + var groups: [String: [String: Entry]] = [:] + for name in names where file.exists("/" + name) { + // `_parse_metadata_group` on a dataset: `_get_keys_from_group` raises AttributeError. + guard !isDataset("/" + name) else { throw refuse("/\(name) is a dataset, not a group") } + var entries: [String: Entry] = [:] + for child in file.children(of: "/" + name) { + let path = "/\(name)/\(child)" + if isDataset(path) { + entries[child] = Entry(path: path, isGroup: false, value: try json(path)) + } else { + var sub: [String: Any] = [:] + for leaf in file.children(of: path) { sub[leaf] = try json(path + "/" + leaf) } + entries[child] = Entry(path: path, isGroup: true, value: sub) + } + } + groups[name] = entries + } + return groups + } +} + +/// `Sample.elements` of a Velox single spectrum. +package nonisolated enum VeloxSampleElements { + /// RosettaSciIO's `atomic_number2name` (`rsciio/utils/_elements.py`), 1 to 119, its spellings kept (112 is "Cp", 113 to 119 the + /// placeholder names). + static let symbols = ("H He Li Be B C N O F Ne Na Mg Al Si P S Cl Ar K Ca Sc Ti V Cr Mn Fe Co Ni Cu Zn Ga Ge As Se Br Kr " + + "Rb Sr Y Zr Nb Mo Tc Ru Rh Pd Ag Cd In Sn Sb Te I Xe Cs Ba La Ce Pr Nd Pm Sm Eu Gd Tb Dy Ho Er Tm Yb Lu Hf Ta W Re Os Ir Pt Au Hg " + + "Tl Pb Bi Po At Rn Fr Ra Ac Th Pa U Np Pu Am Cm Bk Cf Es Fm Md No Lr Rf Db Sg Bh Hs Mt Ds Rg Cp Uut Uuq Uup Uuh Uus Uuo Uue") + .split(separator: " ").map(String.init) + + /// HyperSpy's `slugify(value, valid_variable_name=True)`: NFKD, non-ASCII dropped, ASCII punctuation dropped (not "_" or " "), + /// stripped, spaces to "_", and "Number_" before a result that is not an identifier. + package static func slugify(_ key: String) -> String { + let ascii = key.decomposedStringWithCompatibilityMapping.unicodeScalars.filter(\.isASCII) + let kept = ascii.filter { $0 == "_" || $0 == " " || CharacterSet.alphanumerics.contains($0) } + var text = String(String.UnicodeScalarView(kept)) + text = text.trimmingCharacters(in: .whitespaces).replacingOccurrences(of: " ", with: "_") + let isIdentifier = text.first.map { $0 == "_" || $0.isLetter } == true + return isIdentifier ? text : "Number_" + text + } + + /// The mapping's source key for this file's version: `int(self.version)` >= 11 reads `SharedProperties`, else `Operations`. + static func settingsPath(version: Int) -> (group: String, entry: String) { + version >= 11 ? ("SharedProperties", "EDSSpectrumQuantificationSettings") : ("Operations", "ImageQuantificationOperation") + } + + /// `_convert_element_list` on what HyperSpy's `get_item` returns for the settings path. `value` is the parsed entry (a dict of + /// the group's datasets, or one dataset's JSON). nil is never returned: Python returns a list or raises (`refuse`). + static func convert(_ value: Any, path: String, refuse: (String) -> FormatoReadError) throws -> (elements: [String], source: String) { + // `d.keys()[0]`: only a dict has `keys()`; HyperSpy sorts the slugified keys and leaves out those starting with "_". Keys + // that slugify alike are one attribute whose values HyperSpy merges: refused here (no file has them). + guard let settings = value as? [String: Any] else { throw refuse("\(path) has no keys() (it is not a group of entries)") } + var bySlug: [String: String] = [:] + for key in settings.keys { + let slug = slugify(key) + guard bySlug.updateValue(key, forKey: slug) == nil else { throw refuse("\(path) has two entries that read as \(slug) (unsupported here)") } + } + guard let first = bySlug.keys.filter({ !$0.hasPrefix("_") }).sorted().first, let key = bySlug[first], let entry = settings[key] else { + throw refuse("\(path) has no entry (keys()[0])") + } + guard let inner = entry as? [String: Any] else { throw refuse("\(path)/\(key) has no keys() (it is not a JSON object)") } + let source = "\(path)/\(key).elementSelection" + // "elementSelection" in `_d.keys()`, absent: no element selected, []. + let matches = inner.keys.filter { slugify($0) == "elementSelection" } + guard !matches.isEmpty else { return ([], source) } + guard matches.count == 1, let selection = inner[matches[0]] else { + throw refuse("\(path)/\(key) has several keys that read as elementSelection (unsupported here)") + } + // `for atomic_number in atomic_number_list`: a list, a string (its characters), an empty object; anything else raises. + let items: [Any] + switch selection { + case let list as [Any]: items = list + case let text as String: items = text.unicodeScalars.map { String($0) } // Python iterates code points + // A dict iterates over its keys that do not slugify to "_..." (none: []); any such key gives a tuple, which int() refuses. + case let object as [String: Any] where object.keys.allSatisfy({ slugify($0).hasPrefix("_") }): items = [] + default: throw refuse("\(source) is not a list or a string") + } + var out: [String] = [] + for item in items { + if item is [Any] || item is [String: Any] { throw refuse("\(source) holds a list or an object (int() raises)") } + guard let z = pythonIntValue(item) else { throw refuse("\(source): int() refuses \(item)") } + guard z >= 1, z <= symbols.count else { throw refuse("\(source): \(z) is not in atomic_number2name") } + out.append(symbols[z - 1]) + } + return (out, source) + } +} diff --git a/Sources/FormatoEMD/VeloxSpectrumSignals.swift b/Sources/FormatoEMD/VeloxSpectrumSignals.swift index e1725fa..9d03748 100644 --- a/Sources/FormatoEMD/VeloxSpectrumSignals.swift +++ b/Sources/FormatoEMD/VeloxSpectrumSignals.swift @@ -7,9 +7,10 @@ // and the mapping HyperSpy applies to `original_metadata` (`_get_mapping`) for every key the spectrum's metadata holds. // RosettaSciIO reads spectra only from a file whose `Data` has no `Image` group (type `Spectrum`) or has both `Image` and // `SpectrumImage` (type `SpectrumStream`); in any other file it returns none, and so does this. -// Eager: a spectrum is a few thousand counts. Registered deviations (the SI path's, docs/parity.md): the four metadata -// groups (`Displays`, `Operations`, `SharedProperties`, `Features`) are not parsed, so a malformed entry there does not -// refuse the read; `Sample.elements` (the element selection) is not mapped. +// Eager: a spectrum is a few thousand counts. The four metadata groups (`Displays`, `Operations`, `SharedProperties`, +// `Features`) are parsed as `read_file` parses them (a malformed entry refuses the read), and `Sample.elements` is mapped as +// HyperSpy applies RosettaSciIO's `_convert_element_list` (VeloxSampleElements.swift). Registered deviation (docs/parity.md): +// `_parse_image_display`, which RosettaSciIO runs for a SpectrumStream file before reading its spectra, is not run here. // import Foundation @@ -60,6 +61,9 @@ private nonisolated struct VeloxSpectrumRead { guard let versionText = file.readString("/Version"), let version = (try? JSONSerialization.jsonObject(with: Data(versionText.utf8))) as? [String: Any], veloxAcceptsFormat(version["format"] as? String) else { throw VeloxEMDError.notVeloxFile(fileName) } + // `read_file`: `_parse_json(f["Version"][0])["version"]` (a KeyError without it), then the four metadata groups. + guard let versionValue = version["version"] else { throw raises("its Version has no version key") } + let groups = try VeloxFileGroups.read(file: file) { raises($0) } guard file.exists("/Data"), !isDataset("/Data") else { throw raises("no Data group") } // `_check_im_type`: Image + SpectrumImage is a stream, Image alone is images, no Image is a spectrum. let hasImage = file.exists("/Data/Image"), hasSpectrumImage = file.exists("/Data/SpectrumImage") @@ -69,12 +73,12 @@ private nonisolated struct VeloxSpectrumRead { var out: [FormatoSignal] = [] for key in file.children(of: "/Data/Spectrum") { try FormatoCancellation.check(cancelled) - out.append(try spectrum(key)) + out.append(try spectrum(key, version: versionValue, groups: groups)) } return out } - func spectrum(_ key: String) throws -> FormatoSignal { + func spectrum(_ key: String, version: Any, groups: [String: [String: VeloxFileGroups.Entry]]) throws -> FormatoSignal { let group = "/Data/Spectrum/" + key guard file.exists(group + "/Data") else { throw raises("\(group) has no Data") } guard file.exists(group + "/Metadata") else { throw raises("\(group) has no Metadata") } @@ -108,7 +112,16 @@ private nonisolated struct VeloxSpectrumRead { axes.append(FormatoAxis(name: "Energy", size: data.bins, scale: energy.scale, offset: energy.offset, units: "keV", navigate: false)) let shape = data.stack > 1 ? [data.stack, data.bins] : [data.bins] let array = try FormatoArray(dataType: data.type, shape: shape, bytes: data.bytes) - let md = try metadata(om: om, source: source) + var md = try metadata(om: om, source: source) + // `_get_mapping()`: `int(self.version)` picks the settings group; `original_metadata.update(self.original_metadata)` put the + // four groups beside the spectrum's own keys, so the mapping finds the group's entries there. + guard let versionNumber = pythonIntValue(version) else { throw raises("int() refuses its Version's version") } + let (groupName, entryName) = VeloxSampleElements.settingsPath(version: versionNumber) + if let settings = groups[groupName]?[entryName] { + let (elements, elementsSource) = try VeloxSampleElements.convert(settings.value, path: String(settings.path.dropFirst())) { raises($0) } + md.set(entry("Sample.elements", .texts(elements), source: elementsSource, + rule: "rsciio emd velox mapping _convert_element_list, as HyperSpy applies it: the first of keys() (sorted), atomic_number2name[int(z)]")) + } return FormatoSignal(title: md["General.title"]?.value.text ?? "EDS", signalType: "EDS_TEM", axes: axes, metadata: md, data: array) } diff --git a/Sources/FormatoEMD/VeloxTitles.swift b/Sources/FormatoEMD/VeloxTitles.swift index be38b26..39b061e 100644 --- a/Sources/FormatoEMD/VeloxTitles.swift +++ b/Sources/FormatoEMD/VeloxTitles.swift @@ -23,8 +23,7 @@ nonisolated struct VeloxTitles: Sendable { let byImage: [String: Title] private struct Fallback: Error {} // Python's KeyError inside `_get_detector_name` - /// A child of one of the four file-level groups: a group's datasets by name (as `{name: json}`), or one dataset's JSON. - private struct Entry { let path: String; let isGroup: Bool; let value: Any } + private typealias Entry = VeloxFileGroups.Entry private static let kinds = ["CameraInputOperation", "StemInputOperation", "SurfaceReconstructionOperation", "MathematicsOperation", "DpcOperation", "IntegrationOperation", "FftOperation"] @@ -50,23 +49,7 @@ nonisolated struct VeloxTitles: Sendable { version.keys.contains("version") else { throw refuse("its Version has no version key") } - var groups: [String: [String: Entry]] = [:] - for name in ["Displays", "Operations", "SharedProperties", "Features"] where file.exists("/" + name) { - // `_parse_metadata_group` on a dataset: `_get_keys_from_group` raises AttributeError. - guard !isDataset("/" + name) else { throw refuse("/\(name) is a dataset, not a group") } - var entries: [String: Entry] = [:] - for child in file.children(of: "/" + name) { - let path = "/\(name)/\(child)" - if isDataset(path) { - entries[child] = Entry(path: path, isGroup: false, value: try json(path)) - } else { - var sub: [String: Any] = [:] - for leaf in file.children(of: path) { sub[leaf] = try json(path + "/" + leaf) } - entries[child] = Entry(path: path, isGroup: true, value: sub) - } - } - groups[name] = entries - } + let groups = try VeloxFileGroups.read(file: file, refuse: refuse) let allImages = file.children(of: "/Data/Image") let images = firstImageOnly ? Array(allImages.prefix(1)) : allImages let spectrumStream = file.exists("/Data/Image") && file.exists("/Data/SpectrumImage") diff --git a/Sources/FormatoMSA/FormatoMSA.swift b/Sources/FormatoMSA/FormatoMSA.swift index 79d20ff..26c58ca 100644 --- a/Sources/FormatoMSA/FormatoMSA.swift +++ b/Sources/FormatoMSA/FormatoMSA.swift @@ -491,6 +491,8 @@ public nonisolated enum MSAWriter { case .text(let t): set(keyword.name, .text(t)) case .number(let d): set(keyword.name, .number(d)) case .flag(let b): set(keyword.name, .text(b ? "True" : "False")) + // Python's `str(list)` of plain strings; no MSA keyword maps to a list today. + case .texts(let l): set(keyword.name, .text("[" + l.map { "'\($0)'" }.joined(separator: ", ") + "]")) } } } diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-bad-json.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-bad-json.emd new file mode 100644 index 0000000..81db917 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-bad-json.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-bad-number.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-bad-number.emd new file mode 100644 index 0000000..12bc4b4 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-bad-number.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-digit-string.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-digit-string.emd new file mode 100644 index 0000000..3d1cd6e Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-digit-string.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-empty-group.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-group.emd new file mode 100644 index 0000000..f23d9ae Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-group.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-empty-selection.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-selection.emd new file mode 100644 index 0000000..e3d3d20 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-selection.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-empty-string.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-string.emd new file mode 100644 index 0000000..d506c0b Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-empty-string.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-entry-not-object.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-entry-not-object.emd new file mode 100644 index 0000000..21ffd0b Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-entry-not-object.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-no-version-key.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-no-version-key.emd new file mode 100644 index 0000000..9032761 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-no-version-key.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-none.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-none.emd new file mode 100644 index 0000000..ccb1a6a Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-none.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-null-selection.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-null-selection.emd new file mode 100644 index 0000000..8dfeb32 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-null-selection.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-settings-list.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-settings-list.emd new file mode 100644 index 0000000..8a32d85 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-settings-list.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-slug-order.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-slug-order.emd new file mode 100644 index 0000000..d2877e0 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-slug-order.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-v11-operations-only.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-v11-operations-only.emd new file mode 100644 index 0000000..c7d521a Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-v11-operations-only.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-v11-two-settings.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-v11-two-settings.emd new file mode 100644 index 0000000..ac47376 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-v11-two-settings.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-v13-disabled.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-v13-disabled.emd new file mode 100644 index 0000000..ba2b53c Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-v13-disabled.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-v4.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-v4.emd new file mode 100644 index 0000000..fd49ab4 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-v4.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-v6-one-segment.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-v6-one-segment.emd new file mode 100644 index 0000000..405713f Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-v6-one-segment.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-version-text.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-version-text.emd new file mode 100644 index 0000000..84b7ae3 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-version-text.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-z-0.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-z-0.emd new file mode 100644 index 0000000..d1e3d61 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-z-0.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/el-z-120.emd b/Tests/FormatoEMDTests/Fixtures/elements/el-z-120.emd new file mode 100644 index 0000000..8e726f8 Binary files /dev/null and b/Tests/FormatoEMDTests/Fixtures/elements/el-z-120.emd differ diff --git a/Tests/FormatoEMDTests/Fixtures/elements/velox-elements-pins.json b/Tests/FormatoEMDTests/Fixtures/elements/velox-elements-pins.json new file mode 100644 index 0000000..12825d6 --- /dev/null +++ b/Tests/FormatoEMDTests/Fixtures/elements/velox-elements-pins.json @@ -0,0 +1,1560 @@ +{ + "files": { + "el-bad-json": { + "raises": "JSONDecodeError" + }, + "el-bad-number": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "ValueError" + } + } + ], + "el-digit-string": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "O", + "He" + ] + } + } + ], + "el-empty-group": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "IndexError" + } + } + ], + "el-empty-selection": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [] + } + } + ], + "el-empty-string": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [] + } + } + ], + "el-entry-not-object": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "AttributeError" + } + } + ], + "el-no-version-key": { + "raises": "KeyError" + }, + "el-none": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ] + } + ], + "el-null-selection": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "TypeError" + } + } + ], + "el-settings-list": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "AttributeError" + } + } + ], + "el-slug-order": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "Fe" + ] + } + } + ], + "el-v11-operations-only": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ] + } + ], + "el-v11-two-settings": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "Ca", + "Cu", + "H", + "Uue" + ] + } + }, + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "Ca", + "Cu", + "H", + "Uue" + ] + } + } + ], + "el-v13-disabled": [ + { + "binary_result_detector": "UltraX", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": "true", + "key": "Detector-1", + "live_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "name": "UltraX1", + "real_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": "true", + "key": "Detector-2", + "live_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "name": "UltraX2", + "real_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": "false", + "key": "Detector-3", + "live_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "name": "UltraX3", + "real_time_s": { + "bits": "400741d664254386", + "value": 2.9071472 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": "true", + "key": "Detector-4", + "live_time_s": { + "bits": "400a000000000000", + "value": 3.25 + }, + "name": "UltraX4", + "real_time_s": { + "bits": "400c000000000000", + "value": 3.5 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + } + ], + "el-v4": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "Nb", + "O" + ] + } + } + ], + "el-v6-one-segment": [ + { + "binary_result_detector": "SuperXG24", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "403b951bcb368b5c", + "value": 27.58245534974718 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4040624eb42d9996", + "value": 32.76802685 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "403bd1befa684e54", + "value": 27.819320345368695 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4040624d759ee88e", + "value": 32.767988875 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "403be81d4ff9cfd3", + "value": 27.906697271813595 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "40406245260f5e42", + "value": 32.76773525 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "403bb335b231a35b", + "value": 27.700038087004355 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4040624f10088bc0", + "value": 32.7680378 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "value": [ + "Fe", + "Co", + "O", + "Si" + ] + } + } + ], + "el-version-text": { + "raises": "ValueError" + }, + "el-z-0": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "KeyError" + } + } + ], + "el-z-120": [ + { + "binary_result_detector": "SuperXG2", + "detectors": [ + { + "enabled": null, + "key": "Detector-0", + "live_time_s": null, + "name": "HAADF", + "real_time_s": null, + "type": "ScanningDetector" + }, + { + "enabled": null, + "key": "Detector-1", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG21", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-2", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG22", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-3", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG23", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + }, + { + "enabled": null, + "key": "Detector-4", + "live_time_s": { + "bits": "40241f74c46cc830", + "value": 10.061437738684702 + }, + "name": "SuperXG24", + "real_time_s": { + "bits": "4026bace71b2cac3", + "value": 11.36485629375 + }, + "type": "AnalyticalDetector" + } + ], + "elements": { + "raises": "KeyError" + } + } + ] + }, + "generator": "tools/velox-elements/pins.py", + "public": { + "fei_emd_files.zip/fei_SI_EDS-HAADF-4detectors_2frames.emd": [ + { + "elements": { + "value": [] + } + }, + { + "elements": { + "value": [] + } + }, + { + "elements": { + "value": [] + } + }, + { + "elements": { + "value": [] + } + } + ], + "fei_emd_files.zip/fei_SI_SuperX-HAADF_10frames_10x50.emd": [], + "fei_emd_files.zip/fei_emd_image.emd": [], + "fei_emd_files.zip/fei_emd_si.emd": [], + "fei_emd_files.zip/fei_emd_spectrum.emd": [ + { + "elements": null + } + ], + "velox_EELS_EDS.zip/example_velox_EELS_EDS.emd": [ + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + }, + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + }, + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + }, + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + }, + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + }, + { + "elements": { + "value": [ + "O", + "Fe", + "Co" + ] + } + } + ], + "velox_emd_version11.zip/Test SI 16x16 215 kx.emd": [ + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + } + ], + "velox_emd_version11.zip/Test SI 16x16 ReducedData 215 kx.emd": [ + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + }, + { + "elements": { + "value": [ + "C", + "O", + "Ca", + "Cu" + ] + } + } + ] + }, + "reference": "RosettaSciIO 049e7d7070e84779b499adaf3295beee9facb004 (rsciio.emd.file_reader, select_type=single_spectrum); the mapping applied with HyperSpy 2.4.0's DictionaryTreeBrowser as hyperspy/io.py dict2signal applies it", + "sha256": { + "el-bad-json": "a481682ba9e75395fc3e079b8671b0c7295ca5361e5756bb58dd53400b4bec01", + "el-bad-number": "09ea351789d4684ae9458e8108b5e10bc2d9c786d4cbe93d1c200cf48dc15470", + "el-digit-string": "bf371a15c9a40e0c06c9aabb789a001f4726562d0864025339fc0380b05e917c", + "el-empty-group": "be8f2088e8b02e336a9ec476fd5a4432ff9305fdd9b78ab2603ae432a457cf73", + "el-empty-selection": "9941130daf9c9c45888620abde34a32eb1c6247e60cc75bd01df9bd641bf55f2", + "el-empty-string": "14d48db91d3b9dd015b875be67198bff8385d0afb20258cd5bae53b6093c0f21", + "el-entry-not-object": "bb2fe6dc477e39e3ae300a6eda340550c3a1db522ab9d014dc1cc44b9324fa5b", + "el-no-version-key": "376936a40bb4cf1c2b31363f4d66842602b3e02fb8a9b13e3b3ebdcf9bf334d3", + "el-none": "5055061c2b67e6879129f622a219ac35671a490ebee5ac807fdec14b8e3704e1", + "el-null-selection": "81077aa6f8334982284cee730b8d990e36d97d68ab8b590c9ee66bbd771d4bbe", + "el-settings-list": "72ada0cdab1892078e02d14ddad6d5f7f64f43f41293f9e825589e6c2f34a68f", + "el-slug-order": "721699a97a1b1552e493d020f75348f4e77724e70152d822827664845eb3f841", + "el-v11-operations-only": "0f21b2736a352c02a4c7b4d6e128ac2f264571cc0e9cd02428d6134094712b3c", + "el-v11-two-settings": "0f899f3896fcbd2d5ffe69dbecba5e1dfa2f8d43c36580e4b652cf35bd84e3de", + "el-v13-disabled": "c8cb608d494cc80c70f79b3e55946a8773eba82545a559236c58f973fe2a2af3", + "el-v4": "77316431d966e8c65866e52a41f14873a22803ffc9855199d1eed8fae2fe1a79", + "el-v6-one-segment": "0211f7301c3de36fe54078dbf7ede73cd0637fd719e88f2f93ff21efcaa05fe6", + "el-version-text": "6510421008157b7a51aa4d69e7ad07f7ef6ded76ac7df7bd64733c44aae39731", + "el-z-0": "e809b8792c4eeb581fc663c295bf7447721b1346fa62d88516262e18d09097e5", + "el-z-120": "f7ec224ace72419b6567ee9cf3958205417aa8c6b61d6409b51050e9ea3a10a0" + } +} diff --git a/Tests/FormatoEMDTests/VeloxSpectrumElementsTests.swift b/Tests/FormatoEMDTests/VeloxSpectrumElementsTests.swift new file mode 100644 index 0000000..6a7dafb --- /dev/null +++ b/Tests/FormatoEMDTests/VeloxSpectrumElementsTests.swift @@ -0,0 +1,151 @@ +// +// VeloxSpectrumElementsTests.swift +// `Sample.elements`, the four file-level metadata groups and the segment times of Velox single spectra. +// Files: Fixtures/elements (tools/velox-elements/pins.py: planted with h5py, read by RosettaSciIO 049e7d70, the mapping applied with +// HyperSpy 2.4.0's DictionaryTreeBrowser as `dict2signal` applies it). Exact: the element list as HyperSpy sets it, or the refusal +// where RosettaSciIO or HyperSpy raises; the times to the bit against `float()` of RosettaSciIO's `original_metadata`. RosettaSciIO +// has no rule for which segments a spectrum's times are: the test re-derives the kit's documented rule from the pinned entries, so +// it pins the stored values, not the rule. +// + +import Foundation +import Testing +import FormatoCore +import FormatoEMD + +extension VeloxEMDSerialized { + @Suite struct VeloxSpectrumElementsTests { + static let dir = URL(fileURLWithPath: #filePath).deletingLastPathComponent().appendingPathComponent("Fixtures/elements") + + struct Number: Decodable { let value: Double?; let bits: String } + struct Detector: Decodable { + let name: String?; let type: String?; let enabled: String? + let liveTimeS: Number?; let realTimeS: Number? + } + struct Elements: Decodable { let value: [String]?; let raises: String? } + struct Spectrum: Decodable { let binaryResultDetector: String?; let detectors: [Detector]; let elements: Elements? } + enum File: Decodable { + case raises(String), spectra([Spectrum]) + init(from decoder: Decoder) throws { + if let list = try? [Spectrum](from: decoder) { self = .spectra(list); return } + struct Raise: Decodable { let raises: String } + self = .raises(try Raise(from: decoder).raises) + } + } + struct Pins: Decodable { let files: [String: File] } + + static func pins() throws -> Pins { + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + return try decoder.decode(Pins.self, from: Data(contentsOf: dir.appendingPathComponent("velox-elements-pins.json"))) + } + static func path(_ name: String) -> String { dir.appendingPathComponent(name + ".emd").path } + private func bits(_ x: Double?) -> String? { x.map { String(format: "%016llx", $0.bitPattern) } } + + @Test func testThePinsCoverEveryCase() throws { + let files = try Self.pins().files + #expect(files.count == 20) + let read = files.values.filter { if case .spectra(let s) = $0 { return s.allSatisfy { $0.elements?.raises == nil } }; return false } + #expect(read.count == 10) + for name in files.keys { #expect(FileManager.default.fileExists(atPath: Self.path(name)), "\(name)") } + } + + /// The refusal `VeloxSpectrumSignals` gives where RosettaSciIO or HyperSpy raises: `unsupported`, naming RosettaSciIO. + private func expectRefused(_ name: String, _ why: String) { + #expect(FileManager.default.fileExists(atPath: Self.path(name)), "\(name)") + do { + _ = try VeloxSpectrumSignals(path: Self.path(name)) + Issue.record("\(name): read, where \(why)") + } catch let FormatoReadError.unsupported(message) { + #expect(message.contains("RosettaSciIO raises"), "\(name): \(message)") + } catch { + Issue.record("\(name): \(error), not the refusal, where \(why)") + } + } + + @Test func testSlugifyAsHyperSpy() { + let table = ["0b4c": "Number_0b4c", "B111": "B111", "element-Selection": "elementSelection", "_a": "_a", " a b ": "a_b", + "": "Number_", "é1": "e1", "1-2": "Number_12", "a.b": "ab"] + // Checked against HyperSpy 2.4.0's slugify(key, valid_variable_name=True). + for (key, slug) in table { #expect(VeloxSampleElements.slugify(key) == slug, "\(key)") } + } + + @Test func testSampleElementsAsHyperSpySetsThem() throws { + for (name, file) in try Self.pins().files.sorted(by: { $0.key < $1.key }) { + switch file { + case .raises(let why): + expectRefused(name, "RosettaSciIO raises \(why)") + case .spectra(let spectra) where spectra.contains(where: { $0.elements?.raises != nil }): + expectRefused(name, "HyperSpy raises in the mapping") + case .spectra(let spectra): + let signals = try VeloxSpectrumSignals(path: Self.path(name)).signals + #expect(signals.count == spectra.count, "\(name)") + for (signal, pin) in zip(signals, spectra) { + #expect(signal.metadata["Sample.elements"]?.value.texts == pin.elements?.value, "\(name)") + #expect(signal.metadata["Sample.elements"].map { $0.value.texts != nil } ?? true, "\(name)") + } + } + } + } + + @Test func testTheProvenanceNamesTheEntryHyperSpyTakes() throws { + let signal = try #require(try VeloxSpectrumSignals(path: Self.path("el-v11-two-settings")).signals.first) + let entry = try #require(signal.metadata["Sample.elements"]) + #expect(entry.value == .texts(["Ca", "Cu", "H", "Uue"])) + #expect(entry.provenance.source == "SharedProperties/EDSSpectrumQuantificationSettings/0b4c.elementSelection") + let v4 = try #require(try VeloxSpectrumSignals(path: Self.path("el-v4")).signals.first) + #expect(v4.metadata["Sample.elements"]?.provenance.source == "Operations/ImageQuantificationOperation/80e138836ebd4b7aad5be7d3dd071f47.elementSelection") + } + + @Test func testTheSegmentTimesOfEachSpectrum() throws { + for (name, file) in try Self.pins().files.sorted(by: { $0.key < $1.key }) { + guard case .spectra(let spectra) = file else { continue } + let metadata = try VeloxEMDReader(path: Self.path(name)).spectrumMetadata() + #expect(metadata.count == spectra.count, "\(name)") + for (m, pin) in zip(metadata, spectra) { + #expect(m.binaryResultDetector == pin.binaryResultDetector) + let named = pin.binaryResultDetector ?? "" + let expected = pin.detectors.filter { $0.type == "AnalyticalDetector" && $0.enabled != "false" && ($0.name ?? "").contains(named) } + let times = m.binaryResultSegmentTimes + #expect(times.map(\.name) == expected.map { $0.name ?? "" }, "\(name)") + #expect(times.map { bits($0.liveTimeSeconds) } == expected.map { $0.liveTimeS?.bits }, "\(name)") + #expect(times.map { bits($0.realTimeSeconds) } == expected.map { $0.realTimeS?.bits }, "\(name)") + } + } + // Velox 4, "SuperXG2": all four segments, one common time; Velox 6, "SuperXG24": the one segment; Velox 13: the disabled one left out. + let v4 = try VeloxEMDReader(path: Self.path("el-v4")).spectrumMetadata()[0] + #expect(v4.binaryResultSegmentTimes.count == 4 && v4.commonLiveTimeSeconds == 10.061437738684702) + let v6 = try VeloxEMDReader(path: Self.path("el-v6-one-segment")).spectrumMetadata()[0] + #expect(v6.binaryResultSegmentTimes.map(\.name) == ["SuperXG24"] && v6.commonLiveTimeSeconds == nil) + let v13 = try VeloxEMDReader(path: Self.path("el-v13-disabled")).spectrumMetadata()[0] + #expect(v13.binaryResultSegmentTimes.map(\.name) == ["UltraX1", "UltraX2", "UltraX4"]) + } + + /// RosettaSciIO's public Velox files, unzipped by tools/test.sh: the elements of each single spectrum. + @Test func testThePublicFiles() throws { + struct PublicSpectrum: Decodable { let elements: Elements? } + struct Public: Decodable { let `public`: [String: [PublicSpectrum]] } + let decoder = JSONDecoder() + decoder.keyDecodingStrategy = .convertFromSnakeCase + let pins = try decoder.decode(Public.self, from: Data(contentsOf: Self.dir.appendingPathComponent("velox-elements-pins.json"))) + #expect(pins.public["velox_emd_version11.zip/Test SI 16x16 215 kx.emd"]?.first?.elements?.value == ["C", "O", "Ca", "Cu"]) + let root = Self.dir.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + let present = pins.public.filter { + FileManager.default.fileExists(atPath: root.appendingPathComponent(".build/rsciio-unzipped/emd/" + $0.key).path) + } + guard !present.isEmpty else { return } // not unzipped (a clone without References): the planted files stand + #expect(present.count == pins.public.count, "every public file is unzipped or none") + for (name, spectra) in present.sorted(by: { $0.key < $1.key }) { + let signals = try VeloxSpectrumSignals(path: root.appendingPathComponent(".build/rsciio-unzipped/emd/" + name).path).signals + #expect(signals.map { $0.metadata["Sample.elements"]?.value.texts } == spectra.map { $0.elements?.value }, "\(name)") + } + } + + @Test func testAFileWhoseSpectraRosettaSciIODoesNotReadGivesNoMetadata() throws { + let spectra = Self.dir.deletingLastPathComponent().appendingPathComponent("spectra") + #expect(try VeloxEMDReader(path: spectra.appendingPathComponent("sp-images-only.emd").path).spectrumMetadata().isEmpty) + #expect(try VeloxEMDReader(path: spectra.appendingPathComponent("sp-none.emd").path).spectrumMetadata().isEmpty) + #expect(try VeloxEMDReader(path: spectra.appendingPathComponent("sp-two.emd").path).spectrumMetadata().count == 2) + } + } +} diff --git a/Tests/FormatoEMDTests/VeloxSpectrumSignalsTests.swift b/Tests/FormatoEMDTests/VeloxSpectrumSignalsTests.swift index 604b94b..2c80d7c 100644 --- a/Tests/FormatoEMDTests/VeloxSpectrumSignalsTests.swift +++ b/Tests/FormatoEMDTests/VeloxSpectrumSignalsTests.swift @@ -131,7 +131,8 @@ extension VeloxEMDSerialized { default: Issue.record("unexpected pin value for \(key)") } } - #expect(Set(signal.metadata.keys) == wantedKeys, "\(label): the metadata keys") + // `Sample.elements` is HyperSpy's step on RosettaSciIO's mapping: pinned apart (VeloxSpectrumElementsTests). + #expect(Set(signal.metadata.keys).subtracting(["Sample.elements"]) == wantedKeys, "\(label): the metadata keys") } } } diff --git a/docs/open-items.md b/docs/open-items.md index 7b9baf9..42fcce5 100644 --- a/docs/open-items.md +++ b/docs/open-items.md @@ -15,4 +15,4 @@ Not done from the task list (each is a change that needs its own pins): - TIFF series still refused by name: SubIFD series, OME, generic pyramids; a shaped description in a second ImageDescription tag (`description1`). - Header-less MIB; `small_datacube` rank-3 reshape; DM4 lazy 4D gather speed (the 4D row above is the eager load; the whole-scan Float32 tile case is the app's) and rank-3 or more than four dimensions; EMPAD `mode="acquire"` scan block; HDF5 version attributes int64 against int32; TIFF vendor branches (Zeiss SEM, TVIPS, Olympus SIS, JEOL SightX, Hamamatsu), ImageJ "nm", FEI2 extended MRC header; unbounded allocation from file sizes in Velox (`markersPerFrame`, frame transformations, detector times). - Then-list: HSPY/ZSPY, compressed TIFF, NCEM/Berkeley EMD, SER/EMI, Bruker BCF, other writers. -- `Sample.elements` (element selection) is not mapped for Velox single spectra; the four metadata groups are not parsed on that path. +- `Sample.elements` (element selection) is not mapped for Velox spectrum images (single spectra map it since #10); `_parse_image_display` is not run on the single-spectrum path of a SpectrumStream file. diff --git a/docs/parity.md b/docs/parity.md index ae8e38d..46eda04 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -28,7 +28,7 @@ every one of these, as D17 prescribes for readers. | Plugin | Extensions | Writes | Kit symbol | State | RosettaSciIO test and tolerance | Priority | Note | |---|---|---|---|---|---|---|---| | DigitalMicrograph | dm3, dm4 | no | `FormatoDM` `DMFile`, `DM4Datacube` | **partial** | test_digitalmicrograph.py: allclose, approx, array_equal, explicit rtol/atol/decimal in places | study | complex, RGB, bool, big-endian refused; metadata mapping partial; scan pixel size of EELS/EDS spectrum images (`xAxis`, `yAxis`, 0.11.0) and of 4D scans pinned to the bit | -| EMD | emd, de5 | yes | `FormatoEMD` `VeloxImageSignals`, `VeloxSpectrumImageSignals`, `VeloxSpectrumSignals` (0.9.0) | **partial** | test_emd_ncem.py, test_emd_prismatic.py, test_emd_velox.py: allclose, almost_equal, array_equal, deep_almost_equal (allclose), equal, explicit rtol/atol/decimal in places | study | Velox: images, spectrum images (live and real time per detector from the SI group, 0.11.0), single spectra, "DevelopersKit" accepted; compound FFT/DPC, `lazy=True` SI, `load_SI_image_stack`, `Sample.elements` not. NCEM/Berkeley EMD, prismatic and DE5: missing (useful) | +| EMD | emd, de5 | yes | `FormatoEMD` `VeloxImageSignals`, `VeloxSpectrumImageSignals`, `VeloxSpectrumSignals` (0.9.0) | **partial** | test_emd_ncem.py, test_emd_prismatic.py, test_emd_velox.py: allclose, almost_equal, array_equal, deep_almost_equal (allclose), equal, explicit rtol/atol/decimal in places | study | Velox: images, spectrum images (live and real time per detector from the SI group, 0.11.0), single spectra (with `Sample.elements`), "DevelopersKit" accepted; compound FFT/DPC, `lazy=True` SI, `load_SI_image_stack`, `Sample.elements` of a spectrum image not. NCEM/Berkeley EMD, prismatic and DE5: missing (useful) | | TIFF | tif, tiff | yes (some dims) | `FormatoTIFF` `TIFFSignals`, `TIFFAppImage` | **partial** | test_tiff.py: allclose, deep_almost_equal (allclose), equal, explicit rtol/atol/decimal in places | study | uncompressed strips; FEI, Gatan, ImageJ, plain, tifffile shaped series (0.11.0); OME, SubIFD series, generic pyramids, Zeiss SEM, TVIPS, Olympus SIS, JEOL SightX, Hamamatsu, compression, tiles refused by name | | MRC | mrc, ali | no | `FormatoMRC` `MRCReader`, `MRCAppImage` | **partial** | test_mrc.py, test_mrcz.py: almost_equal, array_almost_equal, array_equal, deep_almost_equal (allclose), equal, explicit rtol/atol/decimal in places | study | modes 0,1,2,6,12 and 4 refused (complex); DE movie options (`virtual_images`, `external_images`, `scan_file`), `navigation_shape`, `metadata_file` missing | | EMPAD | xml | no | `FormatoEMPAD` `EMPADReader` | **partial** | test_empad.py: allclose | study | `remove_nans` default false only | @@ -166,7 +166,20 @@ never written (RosettaSciIO tests `"General.item"`, a typo, as RosettaSciIO does as RosettaSciIO's `json.loads`, so the energy axis and the detector-metadata match take the same entry (since 0.0.3). Left: a key written twice in one object reads its first value here and its last in Python (no file seen has one). - **Velox metadata mapping:** the image path maps the file name and the beam energy; `VeloxSpectrumSignals` (0.9.0) maps every field - RosettaSciIO's `_get_mapping` maps for a single spectrum except `Sample.elements`. The images and spectrum images keep the narrower + RosettaSciIO's `_get_mapping` maps for a single spectrum. **`Sample.elements` of a single spectrum:** a `FormatoValue.texts` list, as + HyperSpy 2.4.0 sets it when it applies RosettaSciIO's `_convert_element_list` (`d.keys()[0]` works only on HyperSpy's + `DictionaryTreeBrowser`, whose `keys()` sorts the slugified keys; the version picks `SharedProperties/EDSSpectrumQuantificationSettings` + from 11 on, `Operations/ImageQuantificationOperation` before; `atomic_number2name` 1 to 119). Where Python raises (no entry, an entry + that is not an object, `int()` refusing, an atomic number outside the table, `int(version)` refusing) the read is refused, as HyperSpy's + `load` raises. The four file-level groups are parsed as `read_file` parses them, so a malformed entry refuses the read. Exact against + planted files and, when unzipped by `tools/test.sh`, the public ones (`tools/velox-elements/pins.py`). Left, refused here where Python + reads (no file has one): two keys of the settings group or of one entry that slugify alike (HyperSpy merges them); an empty entry + dataset in the four groups (RosettaSciIO logs a warning and keeps what it read); `NaN`/`Infinity` in their JSON (`json.loads` accepts + them, `JSONSerialization` does not). `_parse_image_display` is not run for a SpectrumStream file's spectra. **Live and real time of a single + spectrum:** RosettaSciIO maps only `DetectorMetadata.LiveTime`/`.RealTime`, which no public single spectrum holds; Velox 4 to 13 store + them per segment in `Detectors`. `VeloxEMDReader.spectrumMetadata()` gives each spectrum's metadata and + `VeloxMetadata.binaryResultSegmentTimes` the segments whose name contains `BinaryResult.Detector` (not disabled), to the bit against + `float()` of RosettaSciIO's `original_metadata`; not in the signals' metadata. The images and spectrum images keep the narrower mapping (camera length, magnification, instrument and stage are in `AcquisitionMetadata.velox`; for a spectrum image `AcquisitionMetadata.velox(spectrumImage:)` from `VeloxEMDReader.spectrumImageMetadata()`, 0.11.0, camera length, magnification and beam energy to the bit against RosettaSciIO's returned mapping). **Live and real time of a spectrum image (0.11.0):** RosettaSciIO maps none diff --git a/tools/velox-elements/pins.py b/tools/velox-elements/pins.py new file mode 100644 index 0000000..09e9f63 --- /dev/null +++ b/tools/velox-elements/pins.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Planted Velox single-spectrum files for `Sample.elements`, the four file-level metadata groups and the segment times. + + HYPERSPY_SITE_PACKAGES= \ + References/rsciio-venv/bin/python -I -B tools/velox-elements/pins.py + +writes Tests/FormatoEMDTests/Fixtures/elements/*.emd and Tests/FormatoEMDTests/Fixtures/elements/velox-elements-pins.json. + +Every file is a Velox `Spectrum` file (no `Data/Image`), built with the helpers of tools/velox-spectrum/pins.py, read by +RosettaSciIO 049e7d70 (`rsciio.emd.file_reader(select_type="single_spectrum")`). `Sample.elements` is a HyperSpy step: RosettaSciIO +returns `_convert_element_list` in its `mapping`, and that function indexes `d.keys()[0]`, which only HyperSpy's +`DictionaryTreeBrowser` allows (its `keys()` is a sorted list of the slugified keys). So the mapping is applied here with HyperSpy's +own class, in the loop of `hyperspy/io.py` `dict2signal` (`opattr in original_metadata`, `get_item`, the function, `set_item` when +not None), on an `original_metadata` built as `BaseSignal` builds it (`DictionaryTreeBrowser().add_dictionary(...)`). The class is +imported from HYPERSPY_SITE_PACKAGES, appended after the venv's own packages, so RosettaSciIO and numpy stay the venv's. + +Per file: RosettaSciIO's exception, or per spectrum the elements (a list, absent when the mapping does not apply, or the exception +the mapping raises: HyperSpy's `load` raises it too), `BinaryResult.Detector` and every `Detectors` entry of the spectrum's +`original_metadata` in written order (name, type, `Enabled`, `float()` of `LiveTime` and `RealTime` with their bits). RosettaSciIO +maps no live or real time for these files: their spectra hold no `DetectorMetadata` (asserted). +Built twice in the process, the outputs compared. +""" +import hashlib +import importlib.util +import json +import logging +import os +import shutil +import sys +import tempfile +import warnings +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +OUT = ROOT / "Tests" / "FormatoEMDTests" / "Fixtures" / "elements" +HYPERSPY_VERSION = "2.4.0" + + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +step1 = module("step1_pins", ROOT / "tools" / "step-1" / "pins.py") +SP = module("velox_spectrum_pins", ROOT / "tools" / "velox-spectrum" / "pins.py") + + +def tree_class(): + site = os.environ.get("HYPERSPY_SITE_PACKAGES") + if not site: + sys.exit("velox-elements/pins.py: set HYPERSPY_SITE_PACKAGES to the site-packages of a HyperSpy " + HYPERSPY_VERSION + " install") + sys.path.append(site) + from importlib.metadata import version + import hyperspy + if Path(hyperspy.__file__).resolve().parents[1] != Path(site).resolve(): + sys.exit(f"velox-elements/pins.py: hyperspy imported from {hyperspy.__file__}, not from {site}") + found = version("hyperspy") + if found != HYPERSPY_VERSION: + sys.exit(f"velox-elements/pins.py: HyperSpy is {found}, not {HYPERSPY_VERSION}") + from hyperspy.misc._utils import DictionaryTreeBrowser + return DictionaryTreeBrowser + + +V11_SETTINGS = "SharedProperties/EDSSpectrumQuantificationSettings" +V4_SETTINGS = "Operations/ImageQuantificationOperation" + + +def times_doc(names_times, binary): + """A spectrum's Detectors: one scanning detector, then X-ray segments (name, live, real, enabled) as Velox 4 to 13 write them.""" + det = {"Detector-0": {"DetectorName": "HAADF", "DetectorType": "ScanningDetector"}} + for i, (name, live, real, enabled) in enumerate(names_times, start=1): + d = {"DetectorName": name, "DetectorType": "AnalyticalDetector", "Dispersion": "5", "OffsetEnergy": "-480", + "PulseProcessTime": "3.0000000000000001e-006", "RealTime": real, "LiveTime": live} + if enabled is not None: + d["Enabled"] = enabled + det[f"Detector-{i}"] = d + return SP.spectrum_doc(Detectors=det, BinaryResult={"Detector": binary, "DetectorIndex": 1}) + + +FOUR = [("SuperXG21", "10.061437738684702", "11.36485629375", None), ("SuperXG22", "10.061437738684702", "11.36485629375", None), + ("SuperXG23", "10.061437738684702", "11.36485629375", None), ("SuperXG24", "10.061437738684702", "11.36485629375", None)] +SEGMENTS = [("SuperXG21", "27.582455349747178", "32.768026849999998", None), ("SuperXG22", "27.819320345368695", "32.767988875", None), + ("SuperXG23", "27.906697271813595", "32.767735250000001", None), ("SuperXG24", "27.700038087004355", "32.768037800000002", None)] +ULTRA = [("UltraX1", "2.9071471999999998", "2.9071471999999998", "true"), ("UltraX2", "2.9071471999999998", "2.9071471999999998", "true"), + ("UltraX3", "2.9071471999999998", "2.9071471999999998", "false"), ("UltraX4", "3.25", "3.5", "true")] + + +def write_group(f, path, entries): + """`path` a group of variable-length JSON strings, one per entry name (a value that is a str is written as is).""" + g = f.require_group(path) + for name, doc in entries: + text = doc if isinstance(doc, str) else json.dumps(doc) + g.create_dataset(name, data=np.array([text], dtype=object), dtype=SP.S, track_times=False) + + +def build(out): + ramp = np.arange(12, dtype=np.uint32).reshape(12, 1) * 3 + 2 + + def make(name, version, settings=None, extra=None, doc=None, spectra=1): + with SP.new_file(str(out / f"{name}.emd"), version) as f: + for k in range(spectra): + SP.add_spectrum(f, f"s{k}", ramp + k, doc if doc is not None else times_doc(FOUR, "SuperXG2")) + if settings is not None: + write_group(f, settings[0], settings[1]) + for path, entries in (extra or []): + write_group(f, path, entries) + + # Velox 4: elements in Operations, four segments named by "SuperXG2", every one matching. The public fei_emd_spectrum.emd's times. + make("el-v4", 4, (V4_SETTINGS, [("80e138836ebd4b7aad5be7d3dd071f47", {"elementSelection": ["41", "8"], "kind": "q"})]), + extra=[("Features/SIFeature", [("f1", {"x": 1})]), ("Displays/Other", [("d1", {"y": 2})])]) + # Velox 6: one segment named, the others state other times; elements stored as numbers. + make("el-v6-one-segment", 6, (V4_SETTINGS, [("fab3", {"elementSelection": [26, 27.9, "+8", " 1_4 "]})]), + doc=times_doc(SEGMENTS, "SuperXG24")) + # Velox 11: elements in SharedProperties; two entries, the first in HyperSpy's sorted order taken ("Number_0b..." before "a2..."). + make("el-v11-two-settings", 11, (V11_SETTINGS, [("a24a64ef30da49beb12dabaa598d9f16", {"elementSelection": ["6", "8"]}), + ("0b4c", {"elementSelection": ["20", "29", "1", "119"]})]), + extra=[("Operations/ImageQuantificationOperation", [("zz", {"elementSelection": ["79"]})])], spectra=2) + # The slugified order differs from the raw one: "B111" sorts before "Number_0b4c" (raw: "0b4c" first); "_a" is left out of + # HyperSpy's keys(); the inner key "element-Selection" reads as elementSelection. + make("el-slug-order", 11, (V11_SETTINGS, [("0b4c", {"elementSelection": ["8"]}), ("B111", {"element-Selection": ["26"]}), + ("_a", {"elementSelection": ["1"]})])) + # Velox 13: a disabled segment, and a segment the binary result does not name ("UltraX4" holds "UltraX"; all four match). + make("el-v13-disabled", 13, (V11_SETTINGS, [("34d7", {"elementSelection": [8, 26, 27]})]), doc=times_doc(ULTRA, "UltraX")) + # No elementSelection: the mapping gives []; an empty string gives [] too (iterating "" in Python). + make("el-empty-selection", 11, (V11_SETTINGS, [("0ea5", {"other": "x"})])) + make("el-empty-string", 4, (V4_SETTINGS, [("80e1", {"elementSelection": ""})])) + make("el-digit-string", 4, (V4_SETTINGS, [("80e1", {"elementSelection": "82"})])) + # The version picks the group: a Velox 11 file with Operations only maps nothing; a file with no group maps nothing. + make("el-v11-operations-only", 11, extra=[(V4_SETTINGS, [("80e1", {"elementSelection": ["8"]})])]) + make("el-none", 11) + # Refusals (HyperSpy raises in the mapping, or RosettaSciIO raises while reading the groups). + make("el-bad-number", 11, (V11_SETTINGS, [("e1", {"elementSelection": ["8", "abc"]})])) + make("el-z-120", 11, (V11_SETTINGS, [("e1", {"elementSelection": ["120"]})])) + make("el-z-0", 11, (V11_SETTINGS, [("e1", {"elementSelection": [0]})])) + make("el-null-selection", 11, (V11_SETTINGS, [("e1", {"elementSelection": None})])) + make("el-settings-list", 11, extra=[("SharedProperties", [("EDSSpectrumQuantificationSettings", ["8"])])]) + make("el-entry-not-object", 11, (V11_SETTINGS, [("e1", ["8"])])) + make("el-bad-json", 11, (V11_SETTINGS, [("e1", {"elementSelection": ["8"]})]), extra=[("Features/SIFeature", [("f1", "{not json")])]) + with SP.new_file(str(out / "el-empty-group.emd"), 11) as f: + SP.add_spectrum(f, "s0", ramp, times_doc(FOUR, "SuperXG2")) + f.require_group(V11_SETTINGS) + with SP.new_file(str(out / "el-version-text.emd"), 11) as f: # int("11.0") raises in _get_mapping + del f["Version"] + SP.write_json(f, "Version", {"version": "11.0", "format": "Velox"}) + SP.add_spectrum(f, "s0", ramp, times_doc(FOUR, "SuperXG2")) + with SP.new_file(str(out / "el-no-version-key.emd"), 11) as f: + del f["Version"] + SP.write_json(f, "Version", {"format": "Velox"}) + SP.add_spectrum(f, "s0", ramp, times_doc(FOUR, "SuperXG2")) + + +def float_record(raw): + if raw is None: + return None + return {"value": step1.jfloat(float(raw)), "bits": step1.bits(float(raw))} + + +def record(path, tree): + from rsciio.emd import file_reader + try: + dicts = file_reader(str(path), select_type="single_spectrum", lazy=False) + except Exception as e: # noqa: BLE001 + return {"raises": type(e).__name__} + out = [] + for d in dicts: + om = d["original_metadata"] + assert "DetectorMetadata" not in om, path + rec = {"binary_result_detector": om.get("BinaryResult", {}).get("Detector"), + "detectors": [{"key": k, "name": v.get("DetectorName"), "type": v.get("DetectorType"), "enabled": v.get("Enabled"), + "live_time_s": float_record(v.get("LiveTime")), "real_time_s": float_record(v.get("RealTime"))} + for k, v in om["Detectors"].items()]} + browser = tree() + browser.add_dictionary(om) + md = tree() + try: # hyperspy/io.py dict2signal, the mapping loop + for opattr, (mpattr, function) in d["mapping"].items(): + if opattr in browser: + value = browser.get_item(opattr) + if function is not None: + value = function(value) + if value is not None: + md.set_item(mpattr, value) + except Exception as e: # noqa: BLE001 + rec["elements"] = {"raises": type(e).__name__} + else: + if md.has_item("Sample.elements"): + rec["elements"] = {"value": list(md.get_item("Sample.elements"))} + out.append(rec) + return out + + +def public(tree): + """RosettaSciIO's public Velox files with single spectra (tools/velox-spectrum/pins.py's list), unzipped into a fresh folder: + the elements of each spectrum. These files are SpectrumStream or Spectrum files; sparse is in the venv.""" + import zipfile + data = step1.CLONE / "rsciio" / "tests" / "data" / "emd" + out = {} + with tempfile.TemporaryDirectory() as tmp: + for zname in sorted({z for z, _ in SP.PUBLIC}): + zipfile.ZipFile(data / zname).extractall(Path(tmp) / zname) + for zname, fname in SP.PUBLIC: + recs = record(Path(tmp) / zname / fname, tree) + out[f"{zname}/{fname}"] = recs if isinstance(recs, dict) else [{"elements": r.get("elements")} for r in recs] + return out + + +def run(out, tree): + warnings.simplefilter("ignore") + logging.disable(logging.CRITICAL) + step1.load_rsciio() + out.mkdir(parents=True, exist_ok=True) + build(out) + names = sorted(p.stem for p in out.glob("*.emd")) + pins = {"generator": "tools/velox-elements/pins.py", + "public": public(tree), + "reference": f"RosettaSciIO {step1.PIN} (rsciio.emd.file_reader, select_type=single_spectrum); the mapping applied with " + f"HyperSpy {HYPERSPY_VERSION}'s DictionaryTreeBrowser as hyperspy/io.py dict2signal applies it", + "files": {n: record(out / f"{n}.emd", tree) for n in names}, + "sha256": {n: hashlib.sha256((out / f"{n}.emd").read_bytes()).hexdigest() for n in names}} + return json.dumps(pins, indent=1, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" + + +def main(): + tree = tree_class() + with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b: + first, second = run(Path(a), tree), run(Path(b), tree) + if first != second: + sys.exit("velox-elements/pins.py: two builds differ") + if OUT.exists(): + shutil.rmtree(OUT) + shutil.copytree(a, OUT) + (OUT / "velox-elements-pins.json").write_text(first, encoding="utf-8") + print("wrote", OUT / "velox-elements-pins.json") + + +if __name__ == "__main__": + main()